+endcodespacerange"""
+TOUNICODE_TAIL = """\
+endcmap
+CMapName currentdict /CMap defineresource pop
+end
+end"""
+
+
+def make_tounicode(cmap, used):
+ short = []
+ for x in used:
+ if x in cmap:
+ short.append((x, cmap[x]))
+ line = [TOUNICODE_HEAD]
+ for block in batched(short, 100):
+ line.append(f"{len(block)} beginbfchar")
+ for glyph, code in block:
+ if code < 0x10000:
+ line.append(f"<{glyph:04x}><{code:04x}>")
+ else:
+ code -= 0x10000
+ high = 0xD800 + (code >> 10)
+ low = 0xDC00 + (code & 0b1111111111)
+ line.append(f"<{glyph:04x}><{high:04x}{low:04x}>")
+ line.append("endbfchar")
+ line.append(TOUNICODE_TAIL)
+ return "\n".join(line)
+
+
+def reproduce_one_font(doc, index):
+ m = doc.xref_get_key(index, "ToUnicode")
+ f = doc.xref_get_key(index, "DescendantFonts")
+ if m[0] == "xref" and f[0] == "array":
+ mi = to_int(m[1])
+ fi = to_int(f[1])
+ ff = doc.xref_get_key(fi, "FontDescriptor/FontFile2")
+ ms = doc.xref_stream(mi)
+ fs = doc.xref_stream(to_int(ff[1]))
+ cmap = parse_tounicode_cmap(ms)
+ used = parse_truetype_data(fs)
+ text = make_tounicode(cmap, used)
+ doc.update_stream(mi, bytes(text, "U8"))
+
+
+def reproduce_cmap(doc):
+ assert doc
+ font_set = set()
+ for page in doc:
+ font_list = page.get_fonts()
+ for font in font_list:
+ if font[1] == "ttf" and font[3] in FONT_NAMES and ".ttf" in font[4]:
+ font_set.add(font)
+ for font in font_set:
+ reproduce_one_font(doc, font[0])
+ return doc
+
+
+def _subset_fonts_process(pdf_path, output_path):
+ """Function to run in subprocess for font subsetting.
+
+ Args:
+ pdf_path: Path to the PDF file to subset
+ output_path: Path where to save the result
+ """
+ try:
+ pdf = pymupdf.open(pdf_path)
+ pdf.subset_fonts(fallback=False)
+ pdf.save(output_path)
+ # 返回 0 表示æˆåŠŸ
+ os._exit(0)
+ except Exception as e:
+ logger.error(f"Error in font subsetting subprocess: {e}")
+ # 返回 1 表示失败
+ os._exit(1)
+
+
+def _save_pdf_clean_process(
+ pdf_path,
+ output_path,
+ garbage=1,
+ deflate=True,
+ clean=True,
+ deflate_fonts=True,
+ linear=False,
+):
+ """Function to run in subprocess for saving PDF with clean=True which can be time-consuming.
+
+ Args:
+ pdf_path: Path to the PDF file to save
+ output_path: Path where to save the result
+ garbage: Garbage collection level (0, 1, 2, 3, 4)
+ deflate: Whether to deflate the PDF
+ clean: Whether to clean the PDF
+ deflate_fonts: Whether to deflate fonts
+ linear: Whether to linearize the PDF
+ """
+ try:
+ pdf = pymupdf.open(pdf_path)
+ pdf.save(
+ output_path,
+ garbage=garbage,
+ deflate=deflate,
+ clean=clean,
+ deflate_fonts=deflate_fonts,
+ linear=linear,
+ )
+ # 返回 0 表示æˆåŠŸ
+ os._exit(0)
+ except Exception as e:
+ logger.error(f"Error in save PDF with clean=True subprocess: {e}")
+ # 返回 1 表示失败
+ os._exit(1)
+
+
+class PDFCreater:
+ stage_name = "Generate drawing instructions"
+
+ def __init__(
+ self,
+ original_pdf_path: str,
+ document: il_version_1.Document,
+ translation_config: TranslationConfig,
+ mediabox_data: dict,
+ ):
+ self.original_pdf_path = original_pdf_path
+ self.docs = document
+ self.font_path = translation_config.font
+ self.font_mapper = FontMapper(translation_config)
+ self.translation_config = translation_config
+ self.mediabox_data = mediabox_data
+ self.detailed_logger = None
+
+ def render_graphic_state(
+ self,
+ draw_op: BitStream,
+ graphic_state: il_version_1.GraphicState,
+ ):
+ if graphic_state is None:
+ return
+ # if graphic_state.stroking_color_space_name:
+ # draw_op.append(
+ # f"/{graphic_state.stroking_color_space_name} CS \n".encode()
+ # )
+ # if graphic_state.non_stroking_color_space_name:
+ # draw_op.append(
+ # f"/{graphic_state.non_stroking_color_space_name}"
+ # f" cs \n".encode()
+ # )
+ # if graphic_state.ncolor is not None:
+ # if len(graphic_state.ncolor) == 1:
+ # draw_op.append(f"{graphic_state.ncolor[0]} g \n".encode())
+ # elif len(graphic_state.ncolor) == 3:
+ # draw_op.append(
+ # f"{' '.join((str(x) for x in graphic_state.ncolor))} sc \n".encode()
+ # )
+ # if graphic_state.scolor is not None:
+ # if len(graphic_state.scolor) == 1:
+ # draw_op.append(f"{graphic_state.scolor[0]} G \n".encode())
+ # elif len(graphic_state.scolor) == 3:
+ # draw_op.append(
+ # f"{' '.join((str(x) for x in graphic_state.scolor))} SC \n".encode()
+ # )
+
+ if graphic_state.passthrough_per_char_instruction:
+ draw_op.append(
+ f"{graphic_state.passthrough_per_char_instruction} \n".encode(),
+ )
+
+ def render_paragraph_to_char(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ ) -> list[il_version_1.PdfCharacter]:
+ chars = []
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_character:
+ chars.append(composition.pdf_character)
+ elif composition.pdf_formula:
+ # Flatten formula: extract all characters from the formula
+ chars.extend(composition.pdf_formula.pdf_character)
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"This type only appears in the IL "
+ f"after the translation is completed."
+ f"During pdf rendering, this type is not supported."
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ if not chars and paragraph.unicode and paragraph.debug_id:
+ logger.error(
+ f"Unable to export paragraphs that have "
+ f"not yet been formatted: {paragraph}",
+ )
+ return chars
+ return chars
+
+ def create_render_units_for_page(
+ self,
+ page: il_version_1.Page,
+ translation_config: TranslationConfig,
+ ) -> list[RenderUnit]:
+ """Convert all renderable objects in a page to render units."""
+ render_units = []
+
+ # Collect all characters (from page and paragraphs)
+ chars = []
+ if page.pdf_character:
+ chars.extend(page.pdf_character)
+ for paragraph in page.pdf_paragraph:
+ chars.extend(self.render_paragraph_to_char(paragraph))
+
+ # Convert characters to render units
+ for i, char in enumerate(chars):
+ render_order = getattr(char, "render_order", 100) # Default render order
+ sub_render_order = getattr(char, "sub_render_order", i)
+ render_units.append(
+ CharacterRenderUnit(char, render_order, sub_render_order)
+ )
+
+ # Collect forms from formulas within paragraphs
+ formula_forms = []
+ for paragraph in page.pdf_paragraph:
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_formula:
+ formula_forms.extend(composition.pdf_formula.pdf_form)
+
+ # Convert forms to render units (page-level forms + forms from formulas)
+ if not translation_config.skip_form_render:
+ all_forms = list(page.pdf_form) + formula_forms
+ for i, form in enumerate(all_forms):
+ render_order = getattr(
+ form, "render_order", 50
+ ) # Forms render before characters
+ sub_render_order = getattr(form, "sub_render_order", i)
+ render_units.append(
+ FormRenderUnit(form, render_order, sub_render_order)
+ )
+
+ # Convert rectangles to render units (only for OCR workaround or debug)
+ for i, rect in enumerate(page.pdf_rectangle):
+ if (
+ translation_config.ocr_workaround
+ and not rect.debug_info
+ and rect.fill_background
+ ) or (translation_config.debug and rect.debug_info):
+ render_order = getattr(
+ rect, "render_order", 10
+ ) # Rectangles render first
+ sub_render_order = getattr(rect, "sub_render_order", i)
+ line_width = 0.1 if translation_config.ocr_workaround else 0.4
+ render_units.append(
+ RectangleRenderUnit(
+ rect, render_order, sub_render_order, line_width
+ )
+ )
+
+ # Collect curves from formulas within paragraphs
+ formula_curves = []
+ for paragraph in page.pdf_paragraph:
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_formula:
+ formula_curves.extend(composition.pdf_formula.pdf_curve)
+
+ # Convert curves to render units (page-level curves + curves from formulas, only for debug)
+ if not translation_config.skip_curve_render:
+ all_curves = list(page.pdf_curve) + formula_curves
+ for i, curve in enumerate(all_curves):
+ if curve.debug_info or translation_config.debug:
+ render_order = getattr(
+ curve, "render_order", 20
+ ) # Curves render after rectangles
+ sub_render_order = getattr(curve, "sub_render_order", i)
+ render_units.append(
+ CurveRenderUnit(curve, render_order, sub_render_order)
+ )
+
+ return render_units
+
+ def render_units_to_stream(
+ self,
+ render_units: list[RenderUnit],
+ context: RenderContext,
+ page_op: BitStream,
+ xobj_draw_ops: dict[str, BitStream],
+ ) -> None:
+ """Render sorted render units to appropriate draw streams."""
+ # Sort render units by (render_order, sub_render_order)
+ sorted_units = sorted(render_units, key=lambda unit: unit.get_sort_key())
+
+ for unit in sorted_units:
+ # Determine which draw_op to use based on xobj_id
+ if unit.xobj_id in xobj_draw_ops:
+ draw_op = xobj_draw_ops[unit.xobj_id]
+ else:
+ draw_op = page_op
+
+ # Render the unit
+ unit.render(draw_op, context)
+
+ def get_available_font_list(self, pdf, page):
+ page_xref_id = pdf[page.page_number].xref
+ return self.get_xobj_available_fonts(page_xref_id, pdf)
+
+ def get_xobj_available_fonts(self, page_xref_id, pdf):
+ try:
+ resources_type, r_id = pdf.xref_get_key(page_xref_id, "Resources")
+ if resources_type == "xref":
+ resource_xref_id = re.search("(\\d+) 0 R", r_id).group(1)
+ r_id = pdf.xref_object(int(resource_xref_id))
+ resources_type = "dict"
+ if resources_type == "dict":
+ xref_id = re.search("/Font (\\d+) 0 R", r_id)
+ if xref_id is not None:
+ xref_id = xref_id.group(1)
+ font_dict = pdf.xref_object(int(xref_id))
+ else:
+ search = re.search("/Font *<<(.+?)>>", r_id.replace("\n", " "))
+ if search is None:
+ # Have resources but no fonts
+ return set()
+ font_dict = search.group(1)
+ else:
+ r_id = int(r_id.split(" ")[0])
+ _, font_dict = pdf.xref_get_key(r_id, "Font")
+ fonts = re.findall("/([^ ]+?) ", font_dict)
+ return set(fonts)
+ except Exception:
+ return set()
+
+ def _render_rectangle(
+ self,
+ draw_op: BitStream,
+ rectangle: il_version_1.PdfRectangle,
+ line_width: float = 0.4,
+ ):
+ """Draw a rectangle in PDF for visualization purposes.
+
+ Args:
+ draw_op: BitStream to append PDF drawing operations
+ rectangle: Rectangle object containing position information
+ line_width: Line width
+ """
+ x1 = rectangle.box.x
+ y1 = rectangle.box.y
+ x2 = rectangle.box.x2
+ y2 = rectangle.box.y2
+ width = x2 - x1
+ height = y2 - y1
+ # Save graphics state
+ draw_op.append(b"q ")
+
+ # Set green color for debug visibility
+ draw_op.append(
+ rectangle.graphic_state.passthrough_per_char_instruction.encode(),
+ ) # Green stroke
+ if rectangle.line_width is not None:
+ line_width = rectangle.line_width
+ if line_width > 0:
+ draw_op.append(f" {line_width:.6f} w ".encode()) # Line width
+ draw_op.append(f"{x1:.6f} {y1:.6f} {width:.6f} {height:.6f} re ".encode())
+ if rectangle.fill_background:
+ draw_op.append(b" f ")
+ else:
+ draw_op.append(b" S ")
+
+ # Restore graphics state
+ draw_op.append(b" n Q\n")
+
+ def create_side_by_side_dual_pdf(
+ self,
+ original_pdf: pymupdf.Document,
+ translated_pdf: pymupdf.Document,
+ dual_out_path: str,
+ translation_config: TranslationConfig,
+ ) -> pymupdf.Document:
+ """Create a dual PDF with side-by-side pages (original and translation).
+
+ Args:
+ original_pdf: Original PDF document
+ translated_pdf: Translated PDF document
+ dual_out_path: Output path for the dual PDF
+ translation_config: Translation configuration
+
+ Returns:
+ The created dual PDF document
+ """
+ # Create a new PDF for side-by-side pages
+ dual = pymupdf.open()
+ page_count = min(original_pdf.page_count, translated_pdf.page_count)
+
+ for page_id in range(page_count):
+ # Get pages from both PDFs
+ orig_page = original_pdf[page_id]
+ trans_page = translated_pdf[page_id]
+ rotate_angle = orig_page.rotation
+ total_width = orig_page.rect.width + trans_page.rect.width
+ max_height = max(orig_page.rect.height, trans_page.rect.height)
+ left_width = (
+ orig_page.rect.width
+ if not translation_config.dual_translate_first
+ else trans_page.rect.width
+ )
+
+ orig_page.set_rotation(0)
+ trans_page.set_rotation(0)
+
+ # Create new page with combined width
+ dual_page = dual.new_page(width=total_width, height=max_height)
+
+ # Define rectangles for left and right sides
+ rect_left = pymupdf.Rect(0, 0, left_width, max_height)
+ rect_right = pymupdf.Rect(left_width, 0, total_width, max_height)
+
+ # Show pages according to dual_translate_first setting
+ if translation_config.dual_translate_first:
+ # Show translated page on left and original on right
+ rect_left, rect_right = rect_right, rect_left
+ try:
+ # Show original page on left and translated on right (default)
+ dual_page.show_pdf_page(
+ rect_left,
+ original_pdf,
+ page_id,
+ keep_proportion=True,
+ rotate=-rotate_angle,
+ )
+ except Exception as e:
+ logger.warning(
+ f"Failed to show original page on left and translated on right (default). "
+ f"Page ID: {page_id}. "
+ f"Original PDF: {self.original_pdf_path}. "
+ f"Translated PDF: {translation_config.input_file}. ",
+ exc_info=e,
+ )
+ try:
+ dual_page.show_pdf_page(
+ rect_right,
+ translated_pdf,
+ page_id,
+ keep_proportion=True,
+ rotate=-rotate_angle,
+ )
+ except Exception as e:
+ logger.warning(
+ f"Failed to show translated page on left and original on right. "
+ f"Page ID: {page_id}. "
+ f"Original PDF: {self.original_pdf_path}. "
+ f"Translated PDF: {translation_config.input_file}. ",
+ exc_info=e,
+ )
+ return dual
+
+ def create_alternating_pages_dual_pdf(
+ self,
+ original_pdf: pymupdf.Document,
+ translated_pdf: pymupdf.Document,
+ translation_config: TranslationConfig,
+ ) -> pymupdf.Document:
+ """Create a dual PDF with alternating pages (original and translation).
+
+ Args:
+ original_pdf_path: Path to the original PDF
+ translated_pdf: Translated PDF document
+ translation_config: Translation configuration
+
+ Returns:
+ The created dual PDF document
+ """
+ # Open the original PDF and insert translated PDF
+ dual = original_pdf
+ dual.insert_file(translated_pdf)
+
+ # Rearrange pages to alternate between original and translated
+ page_count = translated_pdf.page_count
+ for page_id in range(page_count):
+ if translation_config.dual_translate_first:
+ dual.move_page(page_count + page_id, page_id * 2)
+ else:
+ dual.move_page(page_count + page_id, page_id * 2 + 1)
+
+ return dual
+
+ def write_debug_info(
+ self,
+ pdf: pymupdf.Document,
+ translation_config: TranslationConfig,
+ ):
+ self.font_mapper.add_font(pdf, self.docs)
+
+ for page in self.docs.page:
+ _, r_id = pdf.xref_get_key(pdf[page.page_number].xref, "Contents")
+ resource_xref_id = re.search("(\\d+) 0 R", r_id).group(1)
+ base_op = pdf.xref_stream(int(resource_xref_id))
+ translation_config.raise_if_cancelled()
+ xobj_available_fonts = {}
+ xobj_draw_ops = {}
+ xobj_encoding_length_map = {}
+ available_font_list = self.get_available_font_list(pdf, page)
+
+ page_encoding_length_map = {
+ f.font_id: f.encoding_length for f in page.pdf_font
+ }
+ page_op = BitStream()
+ # q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}
+ page_op.append(b"q ")
+ if base_op is not None:
+ page_op.append(base_op)
+ page_op.append(b" Q ")
+ page_op.append(
+ f"q Q 1 0 0 1 {page.cropbox.box.x:.6f} {page.cropbox.box.y:.6f} cm \n".encode(),
+ )
+ # 收集所有å—符
+ chars = []
+ # é¦–å…ˆæ·»åŠ é¡µé¢çº§åˆ«çš„å—符
+ if page.pdf_character:
+ chars.extend(page.pdf_character)
+ # 然åŽæ·»åŠ 段è½ä¸çš„å—符
+ for paragraph in page.pdf_paragraph:
+ chars.extend(self.render_paragraph_to_char(paragraph))
+
+ # 渲染所有å—符
+ for char in chars:
+ if not getattr(char, "debug_info", False):
+ continue
+ if char.char_unicode == "\n":
+ continue
+ if char.pdf_character_id is None:
+ # dummy char
+ continue
+ char_size = char.pdf_style.font_size
+ font_id = char.pdf_style.font_id
+
+ if font_id not in available_font_list:
+ continue
+ draw_op = page_op
+ encoding_length_map = page_encoding_length_map
+
+ draw_op.append(b"q ")
+ self.render_graphic_state(draw_op, char.pdf_style.graphic_state)
+ if char.vertical:
+ draw_op.append(
+ f"BT /{font_id} {char_size:f} Tf 0 1 -1 0 {char.box.x2:f} {char.box.y:f} Tm ".encode(),
+ )
+ else:
+ draw_op.append(
+ f"BT /{font_id} {char_size:f} Tf 1 0 0 1 {char.box.x:f} {char.box.y:f} Tm ".encode(),
+ )
+
+ encoding_length = encoding_length_map[font_id]
+ # pdf32000-2008 page14:
+ # As hexadecimal data enclosed in angle brackets < >
+ # see 7.3.4.3, "Hexadecimal Strings."
+ draw_op.append(
+ f"<{char.pdf_character_id:0{encoding_length * 2}x}>".upper().encode(),
+ )
+
+ draw_op.append(b" Tj ET Q \n")
+ for rect in page.pdf_rectangle:
+ if not rect.debug_info:
+ continue
+ self._render_rectangle(page_op, rect)
+ draw_op = page_op
+ # Since this is a draw instruction container,
+ # no additional information is needed
+ pdf.update_stream(int(resource_xref_id), draw_op.tobytes())
+ translation_config.raise_if_cancelled()
+
+ # 使用å进程进行å—体å集化
+ if not translation_config.skip_clean:
+ pdf = self.subset_fonts_in_subprocess(pdf, translation_config, tag="debug")
+ return pdf
+
+ @staticmethod
+ def subset_fonts_in_subprocess(
+ pdf: pymupdf.Document, translation_config: TranslationConfig, tag: str
+ ) -> pymupdf.Document:
+ """Run font subsetting in a subprocess with timeout.
+
+ Args:
+ pdf: The PDF document object
+ translation_config: Translation configuration
+
+ Returns:
+ Path to the PDF with subsetted fonts, or original path if subsetting failed or timed out
+ """
+ original_pdf = pdf
+ # Create temporary file paths
+ temp_input = str(
+ translation_config.get_working_file_path(f"temp_subset_input_{tag}.pdf")
+ )
+ temp_output = str(
+ translation_config.get_working_file_path(f"temp_subset_output_{tag}.pdf")
+ )
+
+ # Save PDF to temporary file without subsetting
+ pdf.save(temp_input)
+
+ # Create and start subprocess
+ process = Process(target=_subset_fonts_process, args=(temp_input, temp_output))
+ process.start()
+
+ # Wait for subprocess with timeout (1 minute)
+ timeout = 60 # 1 minutes in seconds
+ start_time = time.time()
+
+ while process.is_alive():
+ if time.time() - start_time > timeout:
+ logger.warning(
+ f"Font subsetting timeout after {timeout} seconds, terminating subprocess"
+ )
+ process.terminate()
+ try:
+ process.join(5) # Give it 5 seconds to clean up
+ if process.is_alive():
+ logger.warning("Subprocess did not terminate, killing it")
+ process.kill()
+ process.terminate()
+ process.kill()
+ process.terminate()
+ process.kill()
+ process.terminate()
+ except Exception as e:
+ logger.error(f"Error terminating font subsetting process: {e}")
+
+ return original_pdf
+
+ time.sleep(0.5) # Check every half second
+
+ # Process completed, check exit code
+ exit_code = process.exitcode
+ success = exit_code == 0
+
+ # Check if subsetting was successful
+ if (
+ success
+ and Path(temp_output).exists()
+ and Path(temp_output).stat().st_size > 0
+ ):
+ logger.info("Font subsetting completed successfully")
+ return pymupdf.open(temp_output)
+ else:
+ logger.warning(
+ f"Font subsetting failed with exit code {exit_code} or produced empty file"
+ )
+ return original_pdf
+
+ @staticmethod
+ def save_pdf_with_timeout(
+ pdf: pymupdf.Document,
+ output_path: str,
+ translation_config: TranslationConfig,
+ garbage: int = 1,
+ deflate: bool = True,
+ clean: bool = True,
+ deflate_fonts: bool = True,
+ linear: bool = False,
+ timeout: int = 120,
+ tag: str = "",
+ ) -> bool:
+ """Save a PDF document with a timeout for the clean=True operation.
+
+ Args:
+ pdf: The PDF document object
+ output_path: Path where to save the PDF
+ translation_config: Translation configuration
+ garbage: Garbage collection level (0, 1, 2, 3, 4)
+ deflate: Whether to deflate the PDF
+ clean: Whether to clean the PDF
+ deflate_fonts: Whether to deflate fonts
+ linear: Whether to linearize the PDF
+ timeout: Timeout in seconds (default: 2 minutes)
+
+ Returns:
+ True if saved with clean=True successfully, False if fallback to clean=False was used
+ """
+ # Create temporary file paths
+ temp_input = str(
+ translation_config.get_working_file_path(f"temp_save_input_{tag}.pdf")
+ )
+ temp_output = str(
+ translation_config.get_working_file_path(f"temp_save_output_{tag}.pdf")
+ )
+
+ # Save PDF to temporary file first
+ pdf.save(temp_input)
+
+ # Try to save with clean=True in a subprocess
+ process = Process(
+ target=_save_pdf_clean_process,
+ args=(
+ temp_input,
+ temp_output,
+ garbage,
+ deflate,
+ clean,
+ deflate_fonts,
+ linear,
+ ),
+ )
+ process.start()
+
+ # Wait for subprocess with timeout
+ start_time = time.time()
+
+ while process.is_alive():
+ if time.time() - start_time > timeout:
+ logger.warning(
+ f"PDF save with clean={clean} timeout after {timeout} seconds, terminating subprocess"
+ )
+ process.terminate()
+ try:
+ process.join(5) # Give it 5 seconds to clean up
+ if process.is_alive():
+ logger.warning("Subprocess did not terminate, killing it")
+ process.kill()
+ process.terminate()
+ process.kill()
+ process.terminate()
+ process.kill()
+ process.terminate()
+ except Exception as e:
+ logger.error(f"Error terminating PDF save process: {e}")
+
+ # Fallback to save without clean parameter
+ logger.info("Falling back to save with clean=False")
+ try:
+ pdf.save(
+ output_path,
+ garbage=garbage,
+ deflate=deflate,
+ clean=False,
+ deflate_fonts=deflate_fonts,
+ linear=linear,
+ )
+ return False
+ except Exception as e:
+ logger.error(f"Error in fallback save: {e}")
+ # Last resort: basic save
+ pdf.save(output_path)
+ return False
+
+ time.sleep(0.5) # Check every half second
+
+ # Process completed, check exit code
+ exit_code = process.exitcode
+ success = exit_code == 0
+
+ # Check if save was successful
+ if (
+ success
+ and Path(temp_output).exists()
+ and Path(temp_output).stat().st_size > 0
+ ):
+ logger.info(f"PDF save with clean={clean} completed successfully")
+ # Copy the successfully created file to the target path
+ try:
+ import shutil
+
+ shutil.copy2(temp_output, output_path)
+ return True
+ except Exception as e:
+ logger.error(f"Error copying saved PDF: {e}")
+ pdf.save(output_path) # Fallback to direct save
+ return False
+ finally:
+ Path(temp_input).unlink()
+ Path(temp_output).unlink()
+ else:
+ logger.warning(
+ f"PDF save with clean={clean} failed with exit code {exit_code} or produced empty file"
+ )
+ # Fallback to save without clean parameter
+ try:
+ pdf.save(
+ output_path,
+ garbage=garbage,
+ deflate=deflate,
+ clean=False,
+ deflate_fonts=deflate_fonts,
+ linear=linear,
+ )
+ except Exception as e:
+ logger.error(f"Error in fallback save: {e}")
+ # Last resort: basic save
+ pdf.save(output_path)
+
+ return False
+
+ def restore_media_box(self, doc: pymupdf.Document, mediabox_data: dict) -> None:
+ for xref, page_box_data in mediabox_data.items():
+ for name, box in page_box_data.items():
+ try:
+ doc.xref_set_key(xref, name, box)
+ except Exception:
+ logger.debug(f"Error restoring media box {name} from PDF")
+
+ def write(
+ self,
+ translation_config: TranslationConfig,
+ check_font_exists: bool = False,
+ ) -> TranslateResult:
+ # Add detailed logging at the start
+ if self.detailed_logger:
+ self.detailed_logger.start_stage("Generate Drawing Instructions")
+ self.detailed_logger.log_step(
+ "PDF Generation Started",
+ f"Total pages: {len(self.docs.page)}"
+ )
+
+ try:
+ basename = Path(translation_config.input_file).stem
+ debug_suffix = ".debug" if translation_config.debug else ""
+ if (
+ translation_config.watermark_output_mode
+ != WatermarkOutputMode.Watermarked
+ ):
+ debug_suffix += ".no_watermark"
+ mono_out_path = translation_config.get_output_file_path(
+ f"{basename}{debug_suffix}.{translation_config.lang_out}.mono.pdf",
+ )
+ pdf = pymupdf.open(self.original_pdf_path)
+ self.font_mapper.add_font(pdf, self.docs)
+
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(self.docs.page),
+ ) as pbar:
+ # Add detailed logging for each page being rendered
+ for i, page in enumerate(self.docs.page):
+ if self.detailed_logger:
+ char_count = len(page.pdf_character) if hasattr(page, 'pdf_character') else 0
+ para_count = len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0
+
+ self.detailed_logger.log_step(
+ f"Rendering Page {i+1}",
+ f"Characters: {char_count}, Paragraphs: {para_count}"
+ )
+
+ self.update_page_content_stream(
+ check_font_exists, page, pdf, translation_config
+ )
+ pbar.advance()
+
+ translation_config.raise_if_cancelled()
+ gc_level = 1
+ if self.translation_config.ocr_workaround:
+ gc_level = 4
+
+ # Add detailed logging for font subsetting
+ if self.detailed_logger:
+ self.detailed_logger.start_stage("Subset Font")
+ self.detailed_logger.log_step("Font subsetting started")
+
+ with self.translation_config.progress_monitor.stage_start(
+ SUBSET_FONT_STAGE_NAME,
+ 1,
+ ) as pbar:
+ if not translation_config.skip_clean:
+ pdf = self.subset_fonts_in_subprocess(
+ pdf, translation_config, tag="mono"
+ )
+
+ pbar.advance()
+
+ # Add detailed logging after font subsetting
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Font subsetting complete")
+ self.detailed_logger.end_stage("Subset Font")
+
+ try:
+ self.restore_media_box(pdf, self.mediabox_data)
+ except Exception:
+ logger.exception("restore media box failed")
+
+ if translation_config.only_include_translated_page:
+ total_page = set(range(0, len(pdf)))
+
+ pages_to_translate = {
+ page.page_number
+ for page in self.docs.page
+ if self.translation_config.should_translate_page(
+ page.page_number + 1
+ )
+ }
+
+ should_removed_page = list(total_page - pages_to_translate)
+
+ pdf.delete_pages(should_removed_page)
+
+ # Add detailed logging before saving
+ if self.detailed_logger:
+ self.detailed_logger.start_stage("Save PDF")
+ self.detailed_logger.log_step("Saving PDF files")
+
+ with self.translation_config.progress_monitor.stage_start(
+ SAVE_PDF_STAGE_NAME,
+ 2,
+ ) as pbar:
+ if not translation_config.no_mono:
+ if translation_config.debug:
+ translation_config.raise_if_cancelled()
+ pdf.save(
+ f"{mono_out_path}.decompressed.pdf",
+ expand=True,
+ pretty=True,
+ )
+ translation_config.raise_if_cancelled()
+ self.save_pdf_with_timeout(
+ pdf,
+ mono_out_path,
+ translation_config,
+ garbage=gc_level,
+ deflate=True,
+ clean=not translation_config.skip_clean,
+ deflate_fonts=True,
+ linear=False,
+ tag="mono",
+ )
+ pbar.advance()
+ dual_out_path = None
+ if not translation_config.no_dual:
+ dual_out_path = translation_config.get_output_file_path(
+ f"{basename}{debug_suffix}.{translation_config.lang_out}.dual.pdf",
+ )
+ if translation_config.use_alternating_pages_dual:
+ dual = self.create_alternating_pages_dual_pdf(
+ pymupdf.open(self.original_pdf_path),
+ pdf,
+ translation_config,
+ )
+ else:
+ dual = self.create_side_by_side_dual_pdf(
+ pymupdf.open(self.original_pdf_path),
+ pdf,
+ dual_out_path,
+ translation_config,
+ )
+ self.save_pdf_with_timeout(
+ dual,
+ dual_out_path,
+ translation_config,
+ garbage=gc_level,
+ deflate=True,
+ clean=not translation_config.skip_clean,
+ deflate_fonts=True,
+ linear=False,
+ tag="dual",
+ )
+ if translation_config.debug:
+ translation_config.raise_if_cancelled()
+ dual.save(
+ f"{dual_out_path}.decompressed.pdf",
+ expand=True,
+ pretty=True,
+ )
+ pbar.advance()
+
+ if self.translation_config.no_mono:
+ mono_out_path = None
+ if self.translation_config.no_dual:
+ dual_out_path = None
+
+ auto_extracted_glossary_path = None
+ if (
+ self.translation_config.save_auto_extracted_glossary
+ and self.translation_config.shared_context_cross_split_part.auto_extracted_glossary
+ ):
+ auto_extracted_glossary_path = self.translation_config.get_output_file_path(
+ f"{basename}{debug_suffix}.{translation_config.lang_out}.glossary.csv"
+ )
+ with auto_extracted_glossary_path.open("w", encoding="utf-8") as f:
+ logger.info(
+ f"save auto extracted glossary to {auto_extracted_glossary_path}"
+ )
+ f.write(
+ self.translation_config.shared_context_cross_split_part.auto_extracted_glossary.to_csv()
+ )
+
+ # Add detailed logging after saving is complete
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ "PDF Save Complete",
+ f"Mono PDF: {mono_out_path}\n"
+ f"Dual PDF: {dual_out_path}"
+ )
+ self.detailed_logger.end_stage("Save PDF")
+ self.detailed_logger.end_stage("Generate Drawing Instructions")
+
+ return TranslateResult(
+ mono_out_path, dual_out_path, auto_extracted_glossary_path
+ )
+ except Exception:
+ logger.exception(
+ "Failed to create PDF: %s",
+ translation_config.input_file,
+ )
+ if not check_font_exists:
+ return self.write(translation_config, True)
+ raise
+
+ def update_page_content_stream(
+ self, check_font_exists, page, pdf, translation_config, skip_char: bool = False
+ ):
+ assert page.cropbox is not None and page.cropbox.box is not None
+ page_crop_box = page.cropbox.box
+ ctm_for_ops = (
+ 1,
+ 0,
+ 0,
+ 1,
+ -page_crop_box.x,
+ -page_crop_box.y,
+ )
+ ctm_for_ops = f" {' '.join(f'{x:f}' for x in ctm_for_ops)} cm ".encode()
+ translation_config.raise_if_cancelled()
+ xobj_available_fonts = {}
+ xobj_draw_ops = {}
+ xobj_encoding_length_map = {}
+ available_font_list = self.get_available_font_list(pdf, page)
+ page_encoding_length_map: dict[str | None, int | None] = {
+ f.font_id: f.encoding_length for f in page.pdf_font
+ }
+ all_encoding_length_map = page_encoding_length_map.copy()
+ for xobj in page.pdf_xobject:
+ xobj_available_fonts[xobj.xobj_id] = available_font_list.copy()
+ try:
+ xobj_available_fonts[xobj.xobj_id].update(
+ self.get_xobj_available_fonts(xobj.xref_id, pdf),
+ )
+ except Exception:
+ pass
+ xobj_encoding_length_map[xobj.xobj_id] = {
+ f.font_id: f.encoding_length for f in xobj.pdf_font
+ }
+ all_encoding_length_map.update(xobj_encoding_length_map[xobj.xobj_id])
+ xobj_encoding_length_map[xobj.xobj_id].update(page_encoding_length_map)
+ xobj_op = BitStream()
+ base_op = xobj.base_operations.value
+ base_op = zstd_decompress(base_op)
+ xobj_op.append(base_op.encode())
+ xobj_draw_ops[xobj.xobj_id] = xobj_op
+ page_op = BitStream()
+ # q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}
+ # page_op.append(b"q ")
+ # base_op = page.base_operations.value
+ # base_op = zstd_decompress(base_op)
+ # page_op.append(base_op.encode())
+ # page_op.append(b" \n")
+ page_op.append(ctm_for_ops)
+ page_op.append(b" \n")
+ # Create render context
+ context = RenderContext(
+ pdf_creator=self,
+ page=page,
+ available_font_list=available_font_list,
+ page_encoding_length_map=page_encoding_length_map,
+ all_encoding_length_map=all_encoding_length_map,
+ xobj_available_fonts=xobj_available_fonts,
+ xobj_encoding_length_map=xobj_encoding_length_map,
+ ctm_for_ops=ctm_for_ops,
+ check_font_exists=check_font_exists,
+ )
+ # Create render units for all renderable objects
+ render_units = self.create_render_units_for_page(page, translation_config)
+ if skip_char:
+ render_units = [
+ unit
+ for unit in render_units
+ if not isinstance(unit, CharacterRenderUnit)
+ ]
+ # Render all units to their appropriate streams
+ self.render_units_to_stream(render_units, context, page_op, xobj_draw_ops)
+ # Update xobject streams
+ for xobj in page.pdf_xobject:
+ draw_op = xobj_draw_ops[xobj.xobj_id]
+ try:
+ pdf.update_stream(xobj.xref_id, draw_op.tobytes())
+ except Exception:
+ logger.warning(f"update xref {xobj.xref_id} stream fail, continue")
+ draw_op = page_op
+ op_container = pdf.get_new_xref()
+ # Since this is a draw instruction container,
+ # no additional information is needed
+ pdf.update_object(op_container, "<<>>")
+ pdf.update_stream(op_container, draw_op.tobytes())
+ pdf[page.page_number].set_contents(op_container)
\ No newline at end of file
diff --git a/babeldoc/format/pdf/document_il/frontend/__init__.py b/babeldoc/format/pdf/document_il/frontend/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/babeldoc/format/pdf/document_il/frontend/il_creater.py b/babeldoc/format/pdf/document_il/frontend/il_creater.py
new file mode 100644
index 0000000000000000000000000000000000000000..4dbf49d78df7f162545b7611a2373c0c26be66b6
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/frontend/il_creater.py
@@ -0,0 +1,1310 @@
+import base64
+import functools
+import logging
+import math
+import re
+from io import BytesIO
+from itertools import islice
+from typing import Literal
+
+import freetype
+import pymupdf
+
+import babeldoc.pdfminer.pdfinterp
+from babeldoc.format.pdf.babelpdf.base14 import get_base14_bbox
+from babeldoc.format.pdf.babelpdf.cidfont import get_cidfont_bbox
+from babeldoc.format.pdf.babelpdf.encoding import WinAnsiEncoding
+from babeldoc.format.pdf.babelpdf.encoding import get_type1_encoding
+from babeldoc.format.pdf.babelpdf.utils import guarded_bbox
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.utils import zstd_helper
+from babeldoc.format.pdf.document_il.utils.matrix_helper import decompose_ctm
+from babeldoc.format.pdf.document_il.utils.style_helper import BLACK
+from babeldoc.format.pdf.document_il.utils.style_helper import YELLOW
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from babeldoc.pdfminer.layout import LTChar
+from babeldoc.pdfminer.layout import LTFigure
+from babeldoc.pdfminer.pdffont import PDFCIDFont
+from babeldoc.pdfminer.pdffont import PDFFont
+
+# from babeldoc.pdfminer.pdfpage import PDFPage as PDFMinerPDFPage
+# from babeldoc.pdfminer.pdftypes import PDFObjRef as PDFMinerPDFObjRef
+# from babeldoc.pdfminer.pdftypes import resolve1 as pdftypes_resolve1
+from babeldoc.pdfminer.psparser import PSLiteral
+from babeldoc.pdfminer.utils import apply_matrix_pt
+from babeldoc.pdfminer.utils import get_bound
+from babeldoc.pdfminer.utils import mult_matrix
+
+
+def invert_matrix(
+ ctm: tuple[float, float, float, float, float, float],
+) -> tuple[float, float, float, float, float, float]:
+ """
+ Calculate the inverse of a 2D transformation matrix.
+ Matrix format: (a, b, c, d, e, f) representing:
+ [a c e]
+ [b d f]
+ [0 0 1]
+ """
+ a, b, c, d, e, f = ctm
+
+ # Calculate determinant
+ det = a * d - b * c
+
+ if abs(det) < 1e-10:
+ # Matrix is singular, return identity matrix
+ return (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)
+
+ # Calculate inverse matrix elements
+ inv_a = d / det
+ inv_b = -b / det
+ inv_c = -c / det
+ inv_d = a / det
+ inv_e = (c * f - d * e) / det
+ inv_f = (b * e - a * f) / det
+
+ return (inv_a, inv_b, inv_c, inv_d, inv_e, inv_f)
+
+
+def batched(iterable, n, *, strict=False):
+ # batched('ABCDEFG', 3) → ABC DEF G
+ if n < 1:
+ raise ValueError("n must be at least one")
+ iterator = iter(iterable)
+ while batch := tuple(islice(iterator, n)):
+ if strict and len(batch) != n:
+ raise ValueError("batched(): incomplete batch")
+ yield batch
+
+
+logger = logging.getLogger(__name__)
+
+#
+# def create_hook(func, hook):
+# @wraps(func)
+# def wrapper(*args, **kwargs):
+# hook(*args, **kwargs)
+# return func(*args, **kwargs)
+#
+# return wrapper
+#
+#
+# def hook_pdfminer_pdf_page_init(*args):
+# attrs = args[3]
+# try:
+# while isinstance(attrs["MediaBox"], PDFMinerPDFObjRef):
+# attrs["MediaBox"] = pdftypes_resolve1(attrs["MediaBox"])
+# except Exception:
+# logger.exception(f"try to fix mediabox failed: {attrs}")
+#
+#
+# PDFMinerPDFPage.__init__ = create_hook(
+# PDFMinerPDFPage.__init__, hook_pdfminer_pdf_page_init
+# )
+
+
+def indirect(obj):
+ if isinstance(obj, tuple) and obj[0] == "xref":
+ return int(obj[1].split(" ")[0])
+
+
+def get_glyph_cbox(face, g):
+ face.load_glyph(g, freetype.FT_LOAD_NO_SCALE)
+ cbox = face.glyph.outline.get_bbox()
+ return cbox.xMin, cbox.yMin, cbox.xMax, cbox.yMax
+
+
+def get_char_cbox(face, idx):
+ g = face.get_char_index(idx)
+ return get_glyph_cbox(face, g)
+
+
+def get_name_cbox(face, name):
+ if name:
+ if isinstance(name, str):
+ name = name.encode("utf-8")
+ g = face.get_name_index(name)
+ return get_glyph_cbox(face, g)
+ return (0, 0, 0, 0)
+
+
+def font_encoding_lookup(doc, idx, key):
+ obj = doc.xref_get_key(idx, key)
+ if obj[0] == "name":
+ enc_name = obj[1][1:]
+ if enc_vector := get_type1_encoding(enc_name):
+ return enc_name, enc_vector
+
+
+def parse_font_encoding(doc, idx):
+ if encoding := font_encoding_lookup(doc, idx, "Encoding/BaseEncoding"):
+ return encoding
+ if encoding := font_encoding_lookup(doc, idx, "Encoding"):
+ return encoding
+ return ("Custom", get_type1_encoding("StandardEncoding"))
+
+
+def get_truetype_ansi_bbox_list(face):
+ scale = 1000 / face.units_per_EM
+ bbox_list = [get_char_cbox(face, code) for code in WinAnsiEncoding]
+ bbox_list = [[v * scale for v in bbox] for bbox in bbox_list]
+ return bbox_list
+
+
+def collect_face_cmap(face):
+ umap = [] # unicode maps
+ lmap = [] # legacy maps
+ for cmap in face.charmaps:
+ if cmap.encoding_name == "FT_ENCODING_UNICODE":
+ umap.append(cmap)
+ else:
+ lmap.append(cmap)
+ return umap, lmap
+
+
+def get_truetype_custom_bbox_list(face):
+ umap, lmap = collect_face_cmap(face)
+ if umap:
+ face.set_charmap(umap[0])
+ elif lmap:
+ face.set_charmap(lmap[0])
+ else:
+ return []
+ scale = 1000 / face.units_per_EM
+ bbox_list = [get_char_cbox(face, code) for code in range(256)]
+ bbox_list = [[v * scale for v in bbox] for bbox in bbox_list]
+ return bbox_list
+
+
+def parse_font_file(doc, idx, encoding, differences):
+ bbox_list = []
+ data = doc.xref_stream(idx)
+ face = freetype.Face(BytesIO(data))
+ if face.get_format() == b"TrueType":
+ if encoding[0] == "WinAnsiEncoding":
+ return get_truetype_ansi_bbox_list(face)
+ elif encoding[0] == "Custom":
+ return get_truetype_custom_bbox_list(face)
+ glyph_name_set = set()
+ for x in range(0, face.num_glyphs):
+ glyph_name_set.add(face.get_glyph_name(x).decode("U8"))
+ scale = 1000 / face.units_per_EM
+ enc_name, enc_vector = encoding
+ _, lmap = collect_face_cmap(face)
+ abbr = enc_name.removesuffix("Encoding")
+ if lmap and abbr in ["Custom", "MacRoman", "Standard", "WinAnsi", "MacExpert"]:
+ face.set_charmap(lmap[0])
+ for i, x in enumerate(enc_vector):
+ if x in glyph_name_set:
+ v = get_name_cbox(face, x.encode("U8"))
+ else:
+ v = get_char_cbox(face, i)
+ bbox_list.append(v)
+ if differences:
+ for code, name in differences:
+ bbox_list[code] = get_name_cbox(face, name.encode("U8"))
+ norm_bbox_list = [[v * scale for v in box] for box in bbox_list]
+ return norm_bbox_list
+
+
+def parse_encoding(obj_str):
+ delta = []
+ current = 0
+ for x in re.finditer(
+ r"(?P[\[\]])|(?P\d+)|(?P/[^\s/\[\]()<>]+)|(?P.)", obj_str
+ ):
+ key = x.lastgroup
+ val = x.group()
+ if key == "c":
+ current = int(val)
+ if key == "n":
+ delta.append((current, val[1:]))
+ current += 1
+ return delta
+
+
+def parse_mapping(text):
+ mapping = []
+ for x in re.finditer(r"<(?P[a-fA-F0-9]+)>", text):
+ mapping.append(x.group("num"))
+ return mapping
+
+
+def update_cmap_pair(cmap, data):
+ for start_str, stop_str, value_str in batched(data, 3):
+ start = int(start_str, 16)
+ stop = int(stop_str, 16)
+ try:
+ value = base64.b16decode(value_str, True).decode("UTF-16-BE")
+ for code in range(start, stop + 1):
+ cmap[code] = value
+ except Exception:
+ pass # to skip surrogate pairs (D800-DFFF)
+
+
+def update_cmap_code(cmap, data):
+ for code_str, value_str in batched(data, 2):
+ code = int(code_str, 16)
+ try:
+ value = base64.b16decode(value_str, True).decode("UTF-16-BE")
+ cmap[code] = value
+ except Exception:
+ pass # to skip surrogate pairs (D800-DFFF)
+
+
+def parse_cmap(cmap_str):
+ cmap = {}
+ for x in re.finditer(
+ r"\s+beginbfrange\s*(?P(<[0-9a-fA-F]+>\s*)+)endbfrange\s+", cmap_str
+ ):
+ update_cmap_pair(cmap, parse_mapping(x.group("r")))
+ for x in re.finditer(
+ r"\s+beginbfchar\s*(?P(<[0-9a-fA-F]+>\s*)+)endbfchar", cmap_str
+ ):
+ update_cmap_code(cmap, parse_mapping(x.group("c")))
+ return cmap
+
+
+def get_code(cmap, c):
+ for k, v in cmap.items():
+ if v == c:
+ return k
+ return -1
+
+
+def get_bbox(bbox, size, c, x, y):
+ x_min, y_min, x_max, y_max = bbox[c]
+ factor = 1 / 1000 * size
+ x_min = x_min * factor
+ y_min = -y_min * factor
+ x_max = x_max * factor
+ y_max = -y_max * factor
+ ll = (x + x_min, y + y_min)
+ lr = (x + x_max, y + y_min)
+ ul = (x + x_min, y + y_max)
+ ur = (x + x_max, y + y_max)
+ return pymupdf.Quad(ll, lr, ul, ur)
+
+
+# 常见 Unicode 空格字符的代码点
+unicode_spaces = [
+ "\u0020", # 半角空格
+ "\u00a0", # 不间断空格
+ "\u1680", # Ogham 空格标记
+ "\u2000", # En Quad
+ "\u2001", # Em Quad
+ "\u2002", # En Space
+ "\u2003", # Em Space
+ "\u2004", # 三分之一 Em 空格
+ "\u2005", # 四分之一 Em 空格
+ "\u2006", # 六分之一 Em 空格
+ "\u2007", # 数样间距
+ "\u2008", # 行首前导空格
+ "\u2009", # 瘦弱空格
+ "\u200a", # hair space
+ "\u202f", # 窄不间断空格
+ "\u205f", # 数学中等空格
+ "\u3000", # 全角空格
+ "\u200b", # 零宽度空格
+ "\u2060", # 零宽度非断空格
+ "\t", # 水平制表符
+]
+
+# 构建正则表达式
+pattern = "^[" + "".join(unicode_spaces) + "]+$"
+
+# 编译正则
+space_regex = re.compile(pattern)
+
+
+def get_rotation_angle(matrix):
+ """
+ 根据 PDF 的字符矩阵计算旋转角度（单位：度）
+ matrix: tuple/list, 格式 (a, b, c, d, e, f)
+ """
+ a, b, c, d, e, f = matrix
+ # 旋转角度：arctan2(b, a)
+ angle_rad = math.atan2(b, a)
+ angle_deg = math.degrees(angle_rad)
+ return angle_deg
+
+
+class ILCreater:
+ stage_name = "Parse PDF and Create Intermediate Representation"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.detailed_logger = None # Will be set from high_level.py
+ self.progress = None
+ self.current_page: il_version_1.Page = None
+ self.mupdf: pymupdf.Document = None
+ self.model = translation_config.doc_layout_model
+ self.docs = il_version_1.Document(page=[])
+ self.stroking_color_space_name = None
+ self.non_stroking_color_space_name = None
+ self.passthrough_per_char_instruction: list[tuple[str, str]] = []
+ self.translation_config = translation_config
+ self.passthrough_per_char_instruction_stack: list[list[tuple[str, str]]] = []
+ self.xobj_id = 0
+ self.xobj_inc = 0
+ self.xobj_map: dict[int, il_version_1.PdfXobject] = {}
+ self.xobj_stack = []
+ self.current_page_font_name_id_map = {}
+ self.current_page_font_char_bounding_box_map = {}
+ self.current_available_fonts = {}
+ self.mupdf_font_map: dict[int, pymupdf.Font] = {}
+ self.graphic_state_pool = {}
+ self.enable_graphic_element_process = (
+ translation_config.enable_graphic_element_process
+ )
+ self.render_order = 0
+ self.current_clip_paths: list[tuple] = []
+ self.clip_paths_stack: list[list[tuple]] = []
+
+ def transform_clip_path(
+ self,
+ clip_path,
+ source_ctm: tuple[float, float, float, float, float, float],
+ target_ctm: tuple[float, float, float, float, float, float],
+ ):
+ """Transform clip path coordinates from source CTM to target CTM."""
+ if source_ctm == target_ctm:
+ return clip_path
+
+ # Calculate transformation matrix: inverse(target_ctm) * source_ctm
+ inv_target_ctm = invert_matrix(target_ctm)
+ transform_matrix = mult_matrix(source_ctm, inv_target_ctm)
+
+ transformed_path = []
+ for path_element in clip_path:
+ if len(path_element) == 1:
+ # Path operation without coordinates (e.g., 'h' for close path)
+ transformed_path.append(path_element)
+ else:
+ # Path operation with coordinates
+ op = path_element[0]
+ coords = path_element[1:]
+ transformed_coords = []
+
+ # Transform coordinate pairs
+ for i in range(0, len(coords), 2):
+ if i + 1 < len(coords):
+ x, y = coords[i], coords[i + 1]
+ transformed_point = apply_matrix_pt(transform_matrix, (x, y))
+ transformed_coords.extend(transformed_point)
+ else:
+ # Handle odd number of coordinates (shouldn't happen in well-formed paths)
+ transformed_coords.append(coords[i])
+
+ transformed_path.append([op] + transformed_coords)
+
+ return transformed_path
+
+ def get_render_order_and_increase(self):
+ self.render_order += 1
+ return self.render_order
+
+ def get_render_order(self):
+ return self.render_order
+
+ def on_finish(self):
+ self.progress.__exit__(None, None, None)
+
+ def is_graphic_operation(self, operator: str):
+ if not self.enable_graphic_element_process:
+ return False
+
+ return re.match(
+ "^(m|l|c|v|y|re|h|S|s|f|f*|F|B|B*|b|b*|n|Do)$",
+ operator,
+ )
+
+ def is_passthrough_per_char_operation(self, operator: str):
+ return re.match(
+ "^(sc|SC|sh|scn|SCN|g|G|rg|RG|k|K|cs|CS|gs|ri|w|J|j|M|i)$",
+ operator,
+ )
+
+ def can_remove_old_passthrough_per_char_instruction(self, operator: str):
+ return re.match(
+ "^(sc|SC|sh|scn|SCN|g|G|rg|RG|k|K|cs|CS|ri|w|J|j|M|i|d)$",
+ operator,
+ )
+
+ def on_line_dash(self, dash, phase):
+ dash_str = f"[{' '.join(f'{arg}' for arg in dash)}]"
+ self.on_passthrough_per_char("d", [dash_str, str(phase)])
+
+ def on_passthrough_per_char(self, operator: str, args: list[str]):
+ if not self.is_passthrough_per_char_operation(operator) and operator not in (
+ "W n",
+ "W* n",
+ "d",
+ "W",
+ "W*",
+ ):
+ logger.error("Unknown passthrough_per_char operation: %s", operator)
+ return
+ # logger.debug("xobj_id: %d, on_passthrough_per_char: %s ( %s )", self.xobj_id, operator, args)
+ args = [self.parse_arg(arg) for arg in args]
+ if self.can_remove_old_passthrough_per_char_instruction(operator):
+ for _i, value in enumerate(self.passthrough_per_char_instruction.copy()):
+ op, arg = value
+ if op == operator:
+ self.passthrough_per_char_instruction.remove(value)
+ break
+ self.passthrough_per_char_instruction.append((operator, " ".join(args)))
+ pass
+
+ def remove_latest_passthrough_per_char_instruction(self):
+ if self.passthrough_per_char_instruction:
+ self.passthrough_per_char_instruction.pop()
+
+ def parse_arg(self, arg: str):
+ if isinstance(arg, PSLiteral):
+ return f"/{arg.name}"
+ if not isinstance(arg, str):
+ return str(arg)
+ return arg
+
+ def pop_passthrough_per_char_instruction(self):
+ if self.passthrough_per_char_instruction_stack:
+ self.passthrough_per_char_instruction = (
+ self.passthrough_per_char_instruction_stack.pop()
+ )
+ else:
+ self.passthrough_per_char_instruction = []
+ logging.error(
+ "pop_passthrough_per_char_instruction error on page: %s",
+ self.current_page.page_number,
+ )
+
+ if self.clip_paths_stack:
+ self.current_clip_paths = self.clip_paths_stack.pop()
+ else:
+ self.current_clip_paths = []
+
+ def push_passthrough_per_char_instruction(self):
+ self.passthrough_per_char_instruction_stack.append(
+ self.passthrough_per_char_instruction.copy(),
+ )
+ self.clip_paths_stack.append(self.current_clip_paths.copy())
+
+ # pdf32000 page 171
+ def on_stroking_color_space(self, color_space_name):
+ self.stroking_color_space_name = color_space_name
+
+ def on_non_stroking_color_space(self, color_space_name):
+ self.non_stroking_color_space_name = color_space_name
+
+ def on_new_stream(self):
+ self.stroking_color_space_name = None
+ self.non_stroking_color_space_name = None
+ self.passthrough_per_char_instruction = []
+ self.current_clip_paths = []
+
+ def push_xobj(self):
+ self.xobj_stack.append(
+ (
+ self.xobj_id,
+ self.current_clip_paths.copy(),
+ self.current_available_fonts.copy(),
+ ),
+ )
+ self.current_clip_paths = []
+
+ def pop_xobj(self):
+ (self.xobj_id, self.current_clip_paths, self.current_available_fonts) = (
+ self.xobj_stack.pop()
+ )
+
+ def on_xobj_begin(self, bbox, xref_id):
+ logger.debug(f"on_xobj_begin: {bbox} @ {xref_id}")
+ self.push_passthrough_per_char_instruction()
+ self.push_xobj()
+ self.xobj_inc += 1
+ self.xobj_id = self.xobj_inc
+ xobject = il_version_1.PdfXobject(
+ box=il_version_1.Box(
+ x=float(bbox[0]),
+ y=float(bbox[1]),
+ x2=float(bbox[2]),
+ y2=float(bbox[3]),
+ ),
+ xobj_id=self.xobj_id,
+ xref_id=xref_id,
+ pdf_font=[],
+ )
+ self.current_page.pdf_xobject.append(xobject)
+ self.xobj_map[self.xobj_id] = xobject
+ xobject.pdf_font.extend(self.current_available_fonts.values())
+ return self.xobj_id
+
+ def on_xobj_end(self, xobj_id, base_op):
+ self.pop_passthrough_per_char_instruction()
+ self.pop_xobj()
+ xobj = self.xobj_map[xobj_id]
+ base_op = zstd_helper.zstd_compress(base_op)
+ xobj.base_operations = il_version_1.BaseOperations(value=base_op)
+ self.xobj_inc += 1
+
+ def on_page_start(self):
+ self.current_page = il_version_1.Page(
+ pdf_font=[],
+ pdf_character=[],
+ page_layout=[],
+ pdf_curve=[],
+ pdf_form=[],
+ # currently don't support UserUnit page parameter
+ # pdf32000 page 79
+ unit="point",
+ )
+ self.current_page_font_name_id_map = {}
+ self.current_page_font_char_bounding_box_map = {}
+ self.passthrough_per_char_instruction_stack = []
+ self.xobj_stack = []
+ self.non_stroking_color_space_name = None
+ self.stroking_color_space_name = None
+ self.current_clip_paths = []
+ self.clip_paths_stack = []
+ self.docs.page.append(self.current_page)
+
+ def on_page_end(self):
+ self.progress.advance(1)
+
+ def on_page_crop_box(
+ self,
+ x0: float | int,
+ y0: float | int,
+ x1: float | int,
+ y1: float | int,
+ ):
+ box = il_version_1.Box(x=float(x0), y=float(y0), x2=float(x1), y2=float(y1))
+ self.current_page.cropbox = il_version_1.Cropbox(box=box)
+
+ def on_page_media_box(
+ self,
+ x0: float | int,
+ y0: float | int,
+ x1: float | int,
+ y1: float | int,
+ ):
+ box = il_version_1.Box(x=float(x0), y=float(y0), x2=float(x1), y2=float(y1))
+ self.current_page.mediabox = il_version_1.Mediabox(box=box)
+
+ def on_page_number(self, page_number: int):
+ assert isinstance(page_number, int)
+ assert page_number >= 0
+ self.current_page.page_number = page_number
+
+ def on_page_base_operation(self, operation: str):
+ operation = zstd_helper.zstd_compress(operation)
+ self.current_page.base_operations = il_version_1.BaseOperations(value=operation)
+
+ def on_page_resource_font(self, font: PDFFont, xref_id: int, font_id: str):
+ font_name = font.fontname
+ logger.debug(f"handle font {font_name} @ {xref_id} in {self.xobj_id}")
+ if isinstance(font_name, bytes):
+ try:
+ font_name = font_name.decode("utf-8")
+ except UnicodeDecodeError:
+ font_name = "BASE64:" + base64.b64encode(font_name).decode("utf-8")
+ encoding_length = 1
+ if isinstance(font, PDFCIDFont):
+ try:
+ # pdf 32000:2008 page 273
+ # Table 118 - Predefined CJK CMap names
+ _, encoding = self.mupdf.xref_get_key(xref_id, "Encoding")
+ if encoding == "/Identity-H" or encoding == "/Identity-V":
+ encoding_length = 2
+ elif encoding == "/WinAnsiEncoding":
+ encoding_length = 1
+ else:
+ _, to_unicode_id = self.mupdf.xref_get_key(xref_id, "ToUnicode")
+ if to_unicode_id is not None:
+ to_unicode_bytes = self.mupdf.xref_stream(
+ int(to_unicode_id.split(" ")[0]),
+ )
+ code_range = re.search(
+ b"begincodespacerange\n?.*<(\\d+?)>.*",
+ to_unicode_bytes,
+ ).group(1)
+ encoding_length = len(code_range) // 2
+ except Exception:
+ if (
+ font.unicode_map
+ and font.unicode_map.cid2unichr
+ and max(font.unicode_map.cid2unichr.keys()) > 255
+ ):
+ encoding_length = 2
+ else:
+ encoding_length = 1
+ try:
+ if xref_id in self.mupdf_font_map:
+ mupdf_font = self.mupdf_font_map[xref_id]
+ else:
+ mupdf_font = pymupdf.Font(
+ fontbuffer=self.mupdf.extract_font(xref_id)[3]
+ )
+ mupdf_font.has_glyph = functools.lru_cache(maxsize=10240, typed=True)(
+ mupdf_font.has_glyph,
+ )
+ bold = mupdf_font.is_bold
+ italic = mupdf_font.is_italic
+ monospaced = mupdf_font.is_monospaced
+ serif = mupdf_font.is_serif
+ self.mupdf_font_map[xref_id] = mupdf_font
+ except Exception:
+ bold = None
+ italic = None
+ monospaced = None
+ serif = None
+ il_font_metadata = il_version_1.PdfFont(
+ name=font_name,
+ xref_id=xref_id,
+ font_id=font_id,
+ encoding_length=encoding_length,
+ bold=bold,
+ italic=italic,
+ monospace=monospaced,
+ serif=serif,
+ ascent=font.ascent,
+ descent=font.descent,
+ pdf_font_char_bounding_box=[],
+ )
+ try:
+ if xref_id is None:
+ logger.warning("xref_id is None for font %s", font_name)
+ raise ValueError("xref_id is None for font %s", font_name)
+ bbox_list, cmap = self.parse_font_xobj_id(xref_id)
+ font_char_bounding_box_map = {}
+ if not cmap:
+ cmap = {x: x for x in range(257)}
+ for char_id, char_bbox in enumerate(bbox_list):
+ font_char_bounding_box_map[char_id] = char_bbox
+ for char_id in cmap:
+ if char_id < 0 or char_id >= len(bbox_list):
+ continue
+ bbox = bbox_list[char_id]
+ x, y, x2, y2 = bbox
+ if (
+ x == 0
+ and y == 0
+ and x2 == 500
+ and y2 == 698
+ or x == 0
+ and y == 0
+ and x2 == 0
+ and y2 == 0
+ ):
+ # ignore default bounding box
+ continue
+ il_font_metadata.pdf_font_char_bounding_box.append(
+ il_version_1.PdfFontCharBoundingBox(
+ x=x,
+ y=y,
+ x2=x2,
+ y2=y2,
+ char_id=char_id,
+ )
+ )
+ font_char_bounding_box_map[char_id] = bbox
+ if self.xobj_id in self.xobj_map:
+ if self.xobj_id not in self.current_page_font_char_bounding_box_map:
+ self.current_page_font_char_bounding_box_map[self.xobj_id] = {}
+ self.current_page_font_char_bounding_box_map[self.xobj_id][xref_id] = (
+ font_char_bounding_box_map
+ )
+ else:
+ self.current_page_font_char_bounding_box_map[xref_id] = (
+ font_char_bounding_box_map
+ )
+ except Exception as e:
+ if xref_id is None:
+ logger.error("failed to parse font xobj id None: %s", e)
+ else:
+ logger.error("failed to parse font xobj id %d: %s", xref_id, e)
+ self.current_page_font_name_id_map[xref_id] = font_id
+ self.current_available_fonts[font_id] = il_font_metadata
+
+ fonts = self.current_page.pdf_font
+ if self.xobj_id in self.xobj_map:
+ fonts = self.xobj_map[self.xobj_id].pdf_font
+ should_remove = []
+ for f in fonts:
+ if f.font_id == font_id:
+ should_remove.append(f)
+ for sr in should_remove:
+ fonts.remove(sr)
+ fonts.append(il_font_metadata)
+
+ def parse_font_xobj_id(self, xobj_id: int):
+ if xobj_id is None:
+ return [], {}
+
+ bbox_list = []
+ encoding = parse_font_encoding(self.mupdf, xobj_id)
+ differences = []
+ font_differences = self.mupdf.xref_get_key(xobj_id, "Encoding/Differences")
+ if font_differences:
+ differences = parse_encoding(font_differences[1])
+ for file_key in ["FontFile", "FontFile2", "FontFile3"]:
+ font_file = self.mupdf.xref_get_key(xobj_id, f"FontDescriptor/{file_key}")
+ if file_idx := indirect(font_file):
+ bbox_list = parse_font_file(
+ self.mupdf,
+ file_idx,
+ encoding,
+ differences,
+ )
+ cmap = {}
+ to_unicode = self.mupdf.xref_get_key(xobj_id, "ToUnicode")
+ if to_unicode_idx := indirect(to_unicode):
+ cmap = parse_cmap(self.mupdf.xref_stream(to_unicode_idx).decode("U8"))
+ if not bbox_list:
+ obj_type, obj_val = self.mupdf.xref_get_key(xobj_id, "BaseFont")
+ if obj_type == "name":
+ bbox_list = get_base14_bbox(obj_val[1:])
+ if cid_bbox := get_cidfont_bbox(self.mupdf, xobj_id):
+ bbox_list = cid_bbox
+ return bbox_list, cmap
+
+ def create_graphic_state(
+ self,
+ gs: babeldoc.pdfminer.pdfinterp.PDFGraphicState | list[tuple[str, str]],
+ include_clipping: bool = False,
+ target_ctm: tuple[float, float, float, float, float, float] = None,
+ clip_paths=None,
+ ):
+ if clip_paths is None:
+ clip_paths = self.current_clip_paths
+ passthrough_instruction = getattr(gs, "passthrough_instruction", gs)
+
+ def filter_clipping(op):
+ return op not in ("W n", "W* n")
+
+ def pass_all(_op):
+ return True
+
+ if include_clipping:
+ filter_clipping = pass_all
+
+ passthrough_per_char_instruction_parts = [
+ f"{arg} {op}" for op, arg in passthrough_instruction if filter_clipping(op)
+ ]
+
+ # Add transformed clipping paths if requested and target CTM is provided
+ if include_clipping and target_ctm and clip_paths:
+ for clip_path, source_ctm, evenodd in clip_paths:
+ try:
+ # Transform clip path from source CTM to target CTM
+ transformed_path = self.transform_clip_path(
+ clip_path, source_ctm, target_ctm
+ )
+
+ # Generate clipping instruction
+ op = "W* n" if evenodd else "W n"
+ args = []
+ for p in transformed_path:
+ if len(p) == 1:
+ args.append(p[0])
+ elif len(p) > 1:
+ args.extend([f"{x:F}" for x in p[1:]])
+ args.append(p[0])
+
+ if args:
+ clipping_instruction = f"{' '.join(args)} {op}"
+ passthrough_per_char_instruction_parts.append(
+ clipping_instruction
+ )
+
+ except Exception as e:
+ logger.warning("Error transforming clip path: %s", e)
+
+ passthrough_per_char_instruction = " ".join(
+ passthrough_per_char_instruction_parts
+ )
+
+ # 可能会影响部分 graphic state 准确度。不过 BabelDOC 仅使用 passthrough_per_char_instruction
+ # 所以应该是没啥影响
+ # 但是池化 graphic state 后可以减少内存占用
+ if passthrough_per_char_instruction not in self.graphic_state_pool:
+ self.graphic_state_pool[passthrough_per_char_instruction] = (
+ il_version_1.GraphicState(
+ passthrough_per_char_instruction=passthrough_per_char_instruction
+ )
+ )
+ graphic_state = self.graphic_state_pool[passthrough_per_char_instruction]
+
+ return graphic_state
+
+ def on_lt_char(self, char: LTChar):
+ if char.aw_font_id is None:
+ return
+ try:
+ rotation_angle = get_rotation_angle(char.matrix)
+ if not (-0.1 <= rotation_angle <= 0.1 or 89.9 <= rotation_angle <= 90.1):
+ return
+ except Exception:
+ logger.warning(
+ "Failed to get rotation angle for char %s",
+ char.get_text(),
+ )
+ gs = self.create_graphic_state(char.graphicstate)
+ # Get font from current page or xobject
+ font = None
+ pdf_font = None
+ for pdf_font in self.xobj_map.get(char.xobj_id, self.current_page).pdf_font:
+ if pdf_font.font_id == char.aw_font_id:
+ font = pdf_font
+ break
+
+ # Get descent from font
+ descent = 0
+ if font and hasattr(font, "descent"):
+ descent = font.descent * char.size / 1000
+
+ char_id = char.cid
+
+ char_bounding_box = None
+ try:
+ if (
+ font_bounding_box_map
+ := self.current_page_font_char_bounding_box_map.get(
+ char.xobj_id, self.current_page_font_char_bounding_box_map
+ ).get(font.xref_id)
+ ):
+ char_bounding_box = font_bounding_box_map.get(char_id, None)
+ else:
+ char_bounding_box = None
+ except Exception:
+ # logger.debug(
+ # "Failed to get font bounding box for char %s",
+ # char.get_text(),
+ # )
+ char_bounding_box = None
+
+ char_unicode = char.get_text()
+ # if "(cid:" not in char_unicode and len(char_unicode) > 1:
+ # return
+ if space_regex.match(char_unicode):
+ char_unicode = " "
+ advance = char.adv
+ bbox = il_version_1.Box(
+ x=char.bbox[0],
+ y=char.bbox[1],
+ x2=char.bbox[2],
+ y2=char.bbox[3],
+ )
+ if bbox.x2 < bbox.x or bbox.y2 < bbox.y:
+ logger.warning(
+ "Invalid bounding box for character %s: %s",
+ char_unicode,
+ bbox,
+ )
+
+ if char.matrix[0] == 0 and char.matrix[3] == 0:
+ vertical = True
+ visual_bbox = il_version_1.Box(
+ x=char.bbox[0] - descent,
+ y=char.bbox[1],
+ x2=char.bbox[2] - descent,
+ y2=char.bbox[3],
+ )
+ else:
+ vertical = False
+ # Add descent to y coordinates
+ visual_bbox = il_version_1.Box(
+ x=char.bbox[0],
+ y=char.bbox[1] + descent,
+ x2=char.bbox[2],
+ y2=char.bbox[3] + descent,
+ )
+ visual_bbox = il_version_1.VisualBbox(box=visual_bbox)
+ pdf_style = il_version_1.PdfStyle(
+ font_id=char.aw_font_id,
+ font_size=char.size,
+ graphic_state=gs,
+ )
+
+ if font:
+ font_xref_id = font.xref_id
+ if font_xref_id in self.mupdf_font_map:
+ mupdf_font = self.mupdf_font_map[font_xref_id]
+ # if "(cid:" not in char_unicode:
+ # if mupdf_cid := mupdf_font.has_glyph(ord(char_unicode)):
+ # char_id = mupdf_cid
+
+ pdf_char = il_version_1.PdfCharacter(
+ box=bbox,
+ pdf_character_id=char_id,
+ advance=advance,
+ char_unicode=char_unicode,
+ vertical=vertical,
+ pdf_style=pdf_style,
+ xobj_id=char.xobj_id,
+ visual_bbox=visual_bbox,
+ render_order=char.render_order,
+ sub_render_order=0,
+ )
+ if self.translation_config.ocr_workaround:
+ pdf_char.pdf_style.graphic_state = BLACK
+ pdf_char.render_order = None
+ if pdf_style.font_size == 0.0:
+ logger.warning(
+ "Font size is 0.0 for character %s. Skip it.",
+ char_unicode,
+ )
+ return
+
+ # ===== ADD YOUR LOGGING CODE HERE =====
+ if self.detailed_logger and hasattr(char, 'bbox'):
+ char_data = {
+ 'unicode': char_unicode, # Use char_unicode which is already extracted
+ 'x': char.bbox[0],
+ 'y': char.bbox[1],
+ 'width': (char.bbox[2] - char.bbox[0]),
+ 'height': (char.bbox[3] - char.bbox[1]),
+ 'font_id': char.aw_font_id if hasattr(char, 'aw_font_id') else 'N/A',
+ 'font_size': char.size if hasattr(char, 'size') else 0
+ }
+ self.detailed_logger.log_character_extraction(
+ self.current_page.page_number if self.current_page and hasattr(self.current_page, 'page_number') else 0,
+ char_data
+ )
+ # ===== END OF LOGGING CODE =====
+
+ if char_bounding_box and len(char_bounding_box) == 4:
+ x_min, y_min, x_max, y_max = char_bounding_box
+ factor = 1 / 1000 * pdf_style.font_size
+ x_min = x_min * factor
+ y_min = y_min * factor
+ x_max = x_max * factor
+ y_max = y_max * factor
+ ll = (char.bbox[0] + x_min, char.bbox[1] + y_min)
+ ur = (char.bbox[0] + x_max, char.bbox[1] + y_max)
+
+ volume = (ur[0] - ll[0]) * (ur[1] - ll[1])
+ if volume > 1:
+ pdf_char.visual_bbox = il_version_1.VisualBbox(
+ il_version_1.Box(ll[0], ll[1], ur[0], ur[1])
+ )
+
+ self.current_page.pdf_character.append(pdf_char)
+
+ if self.translation_config.show_char_box:
+ self.current_page.pdf_rectangle.append(
+ il_version_1.PdfRectangle(
+ box=pdf_char.visual_bbox.box,
+ graphic_state=YELLOW,
+ debug_info=True,
+ line_width=0.2,
+ )
+ )
+
+ def on_lt_curve(self, curve: babeldoc.pdfminer.layout.LTCurve):
+ if not self.enable_graphic_element_process:
+ return
+ bbox = il_version_1.Box(
+ x=curve.bbox[0],
+ y=curve.bbox[1],
+ x2=curve.bbox[2],
+ y2=curve.bbox[3],
+ )
+ # Extract CTM from curve object if it exists
+ curve_ctm = getattr(curve, "ctm", None)
+ gs = self.create_graphic_state(
+ curve.passthrough_instruction,
+ include_clipping=True,
+ target_ctm=curve_ctm,
+ clip_paths=curve.clip_paths,
+ )
+ paths = []
+ for point in curve.original_path:
+ op = point[0]
+ if len(point) == 1:
+ paths.append(
+ il_version_1.PdfPath(
+ op=op,
+ x=None,
+ y=None,
+ has_xy=False,
+ )
+ )
+ continue
+ for p in point[1:-1]:
+ paths.append(
+ il_version_1.PdfPath(
+ op="",
+ x=p[0],
+ y=p[1],
+ has_xy=True,
+ )
+ )
+ paths.append(
+ il_version_1.PdfPath(
+ op=point[0],
+ x=point[-1][0],
+ y=point[-1][1],
+ has_xy=True,
+ )
+ )
+
+ fill_background = curve.fill
+ stroke_path = curve.stroke
+ evenodd = curve.evenodd
+ # Extract CTM from curve object if it exists
+ ctm = getattr(curve, "ctm", None)
+
+ # Extract raw path from curve object if it exists
+ raw_path = getattr(curve, "raw_path", None)
+ raw_pdf_paths = None
+ if raw_path is not None:
+ raw_pdf_paths = []
+ for path in raw_path:
+ if path[0] == "h": # h command (close path)
+ raw_pdf_paths.append(
+ il_version_1.PdfOriginalPath(
+ pdf_path=il_version_1.PdfPath(
+ x=0.0,
+ y=0.0,
+ op=path[0],
+ has_xy=False,
+ )
+ )
+ )
+ else: # commands with coordinates (m, l, c, v, y, etc.)
+ for p in batched(path[1:-2], 2, strict=True):
+ raw_pdf_paths.append(
+ il_version_1.PdfOriginalPath(
+ pdf_path=il_version_1.PdfPath(
+ x=float(p[0]),
+ y=float(p[1]),
+ op="",
+ has_xy=True,
+ )
+ )
+ )
+ # Last point in the path
+ raw_pdf_paths.append(
+ il_version_1.PdfOriginalPath(
+ pdf_path=il_version_1.PdfPath(
+ x=float(path[-2]),
+ y=float(path[-1]),
+ op=path[0],
+ has_xy=True,
+ )
+ )
+ )
+
+ curve_obj = il_version_1.PdfCurve(
+ box=bbox,
+ graphic_state=gs,
+ pdf_path=paths,
+ fill_background=fill_background,
+ stroke_path=stroke_path,
+ evenodd=evenodd,
+ debug_info="a",
+ xobj_id=curve.xobj_id,
+ render_order=curve.render_order,
+ ctm=list(ctm) if ctm is not None else None,
+ pdf_original_path=raw_pdf_paths,
+ )
+ self.current_page.pdf_curve.append(curve_obj)
+ pass
+
+ def on_xobj_form(
+ self,
+ ctm: tuple[float, float, float, float, float, float],
+ xobj_id: int,
+ xref_id: int,
+ form_type: Literal["image", "form"],
+ do_args: str,
+ bbox: tuple[float, float, float, float],
+ matrix: tuple[float, float, float, float, float, float],
+ ):
+ logger.debug(f"on_xobj_form: {do_args}[{bbox}] @ {xref_id} in {self.xobj_id}")
+ matrix = mult_matrix(matrix, ctm)
+ (x, y, w, h) = guarded_bbox(bbox)
+ bounds = ((x, y), (x + w, y), (x, y + h), (x + w, y + h))
+ bbox = get_bound(apply_matrix_pt(matrix, (p, q)) for (p, q) in bounds)
+
+ gs = self.create_graphic_state(
+ self.passthrough_per_char_instruction, include_clipping=True, target_ctm=ctm
+ )
+
+ figure_bbox = il_version_1.Box(
+ x=bbox[0],
+ y=bbox[1],
+ x2=bbox[2],
+ y2=bbox[3],
+ )
+ pdf_matrix = il_version_1.PdfMatrix(
+ a=ctm[0],
+ b=ctm[1],
+ c=ctm[2],
+ d=ctm[3],
+ e=ctm[4],
+ f=ctm[5],
+ )
+ affine_transform = decompose_ctm(ctm)
+ xobj_form = il_version_1.PdfXobjForm(
+ xref_id=xref_id,
+ do_args=do_args,
+ )
+ pdf_form_subtype = il_version_1.PdfFormSubtype(
+ pdf_xobj_form=xobj_form,
+ )
+ new_form = il_version_1.PdfForm(
+ xobj_id=xobj_id,
+ box=figure_bbox,
+ pdf_matrix=pdf_matrix,
+ graphic_state=gs,
+ pdf_affine_transform=affine_transform,
+ render_order=self.get_render_order_and_increase(),
+ form_type=form_type,
+ pdf_form_subtype=pdf_form_subtype,
+ ctm=list(ctm),
+ )
+ self.current_page.pdf_form.append(new_form)
+
+ def on_pdf_clip_path(
+ self,
+ clip_path,
+ evenodd: bool,
+ ctm: tuple[float, float, float, float, float, float],
+ ):
+ try:
+ self.current_clip_paths.append((clip_path.copy(), ctm, evenodd))
+ except Exception as e:
+ logger.warning("Error in on_pdf_clip_path: %s", e)
+
+ def create_il(self):
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ "Creating Intermediate Representation",
+ f"Total pages: {len(self.docs.page)}\n"
+ f"Total characters: {sum(len(p.pdf_character) for p in self.docs.page)}"
+ )
+ pages = [
+ page
+ for page in self.docs.page
+ if self.translation_config.should_translate_page(page.page_number + 1)
+ ]
+ self.docs.page = pages
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ "IL Creation Complete",
+ data={
+ 'total_pages': len(self.docs.page),
+ 'total_chars': sum(len(p.pdf_character) for p in self.docs.page),
+ 'total_fonts': len(set(f.font_id for p in self.docs.page for f in p.pdf_font))
+ }
+ )
+ return self.docs
+
+ def on_total_pages(self, total_pages: int):
+ assert isinstance(total_pages, int)
+ assert total_pages > 0
+ self.docs.total_pages = total_pages
+ total = 0
+ for page in range(total_pages):
+ if self.translation_config.should_translate_page(page + 1) is False:
+ continue
+ total += 1
+ self.progress = self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ total,
+ )
+
+ def on_pdf_figure(self, figure: LTFigure):
+ box = il_version_1.Box(
+ figure.bbox[0],
+ figure.bbox[1],
+ figure.bbox[2],
+ figure.bbox[3],
+ )
+ self.current_page.pdf_figure.append(il_version_1.PdfFigure(box=box))
+
+ def on_inline_image_begin(self):
+ """Begin processing inline image"""
+ # Store current state for inline image processing
+ self._inline_image_state = {
+ "ctm": None,
+ "parameters": {},
+ }
+
+ def on_inline_image_end(self, stream_obj, ctm):
+ """End processing inline image and create PdfForm"""
+ import base64
+ import json
+
+ from babeldoc.format.pdf.babelpdf.utils import guarded_bbox
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import decompose_ctm
+ from babeldoc.pdfminer.utils import apply_matrix_pt
+ from babeldoc.pdfminer.utils import get_bound
+
+ # Extract image parameters from stream dictionary
+ image_dict = stream_obj.attrs if hasattr(stream_obj, "attrs") else {}
+
+ # Build parameters dictionary
+ parameters = {}
+ for key, value in image_dict.items():
+ if hasattr(value, "name"):
+ parameters[key] = value.name
+ else:
+ parameters[key] = str(value)
+
+ # Get image data (encoded as base64)
+ image_data = ""
+ if hasattr(stream_obj, "data") and stream_obj.data is not None:
+ image_data = base64.b64encode(stream_obj.data).decode("ascii")
+ elif hasattr(stream_obj, "rawdata") and stream_obj.rawdata is not None:
+ image_data = base64.b64encode(stream_obj.rawdata).decode("ascii")
+
+ # Create inline form with parameters as JSON string
+ inline_form = il_version_1.PdfInlineForm(
+ form_data=image_data, image_parameters=json.dumps(parameters)
+ )
+
+ # Calculate bounding box - inline images are typically 1x1 unit square in user space
+ bbox = (0, 0, 1, 1)
+ (x, y, w, h) = guarded_bbox(bbox)
+ bounds = ((x, y), (x + w, y), (x, y + h), (x + w, y + h))
+ final_bbox = get_bound(apply_matrix_pt(ctm, (p, q)) for (p, q) in bounds)
+
+ # Create graphics state
+ gs = self.create_graphic_state(
+ self.passthrough_per_char_instruction, include_clipping=True, target_ctm=ctm
+ )
+
+ # Create PdfMatrix from CTM
+ pdf_matrix = il_version_1.PdfMatrix(
+ a=ctm[0], b=ctm[1], c=ctm[2], d=ctm[3], e=ctm[4], f=ctm[5]
+ )
+
+ # Create affine transform
+ affine_transform = decompose_ctm(ctm)
+
+ # Create PdfFormSubtype with inline form
+ pdf_form_subtype = il_version_1.PdfFormSubtype(pdf_inline_form=inline_form)
+
+ # Create PdfForm for the inline image
+ pdf_form = il_version_1.PdfForm(
+ box=il_version_1.Box(
+ x=final_bbox[0],
+ y=final_bbox[1],
+ x2=final_bbox[2],
+ y2=final_bbox[3],
+ ),
+ graphic_state=gs,
+ pdf_matrix=pdf_matrix,
+ pdf_affine_transform=affine_transform,
+ pdf_form_subtype=pdf_form_subtype,
+ xobj_id=self.xobj_id,
+ ctm=list(ctm),
+ render_order=self.get_render_order_and_increase(),
+ form_type="image",
+ )
+
+ # Add to current page
+ self.current_page.pdf_form.append(pdf_form)
diff --git a/babeldoc/format/pdf/document_il/il_version_1.py b/babeldoc/format/pdf/document_il/il_version_1.py
new file mode 100644
index 0000000000000000000000000000000000000000..cee64cc671f520e4abd2e09ca13c7b3435a54d10
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/il_version_1.py
@@ -0,0 +1,1323 @@
+from dataclasses import dataclass
+from dataclasses import field
+
+
+@dataclass(slots=True)
+class BaseOperations:
+ class Meta:
+ name = "baseOperations"
+
+ value: str = field(
+ default="",
+ metadata={
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class Box:
+ class Meta:
+ name = "box"
+
+ x: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ y: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ x2: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ y2: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class GraphicState:
+ class Meta:
+ name = "graphicState"
+
+ passthrough_per_char_instruction: str | None = field(
+ default=None,
+ metadata={
+ "name": "passthroughPerCharInstruction",
+ "type": "Attribute",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfAffineTransform:
+ class Meta:
+ name = "pdfAffineTransform"
+
+ translation_x: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ translation_y: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ rotation: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ scale_x: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ scale_y: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ shear: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfFontCharBoundingBox:
+ class Meta:
+ name = "pdfFontCharBoundingBox"
+
+ x: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ y: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ x2: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ y2: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ char_id: int | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfInlineForm:
+ class Meta:
+ name = "pdfInlineForm"
+
+ form_data: str | None = field(
+ default=None,
+ metadata={
+ "name": "formData",
+ "type": "Attribute",
+ },
+ )
+ image_parameters: str | None = field(
+ default=None,
+ metadata={
+ "name": "imageParameters",
+ "type": "Attribute",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfMatrix:
+ class Meta:
+ name = "pdfMatrix"
+
+ a: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ b: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ c: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ d: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ e: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ f: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfPath:
+ class Meta:
+ name = "pdfPath"
+
+ x: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ y: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ op: str | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ has_xy: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfXobjForm:
+ class Meta:
+ name = "pdfXobjForm"
+
+ xref_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "xrefId",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ do_args: str | None = field(
+ default=None,
+ metadata={
+ "name": "doArgs",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class Cropbox:
+ class Meta:
+ name = "cropbox"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class Mediabox:
+ class Meta:
+ name = "mediabox"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PageLayout:
+ class Meta:
+ name = "pageLayout"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ id: int | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ conf: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ class_name: str | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfFigure:
+ class Meta:
+ name = "pdfFigure"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfFont:
+ class Meta:
+ name = "pdfFont"
+
+ pdf_font_char_bounding_box: list[PdfFontCharBoundingBox] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfFontCharBoundingBox",
+ "type": "Element",
+ },
+ )
+ name: str | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ font_id: str | None = field(
+ default=None,
+ metadata={
+ "name": "fontId",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ xref_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "xrefId",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ encoding_length: int | None = field(
+ default=None,
+ metadata={
+ "name": "encodingLength",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ bold: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ italic: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ monospace: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ serif: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ ascent: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ descent: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfFormSubtype:
+ class Meta:
+ name = "pdfFormSubtype"
+
+ pdf_inline_form: PdfInlineForm | None = field(
+ default=None,
+ metadata={
+ "name": "pdfInlineForm",
+ "type": "Element",
+ },
+ )
+ pdf_xobj_form: PdfXobjForm | None = field(
+ default=None,
+ metadata={
+ "name": "pdfXobjForm",
+ "type": "Element",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfOriginalPath:
+ class Meta:
+ name = "pdfOriginalPath"
+
+ pdf_path: PdfPath | None = field(
+ default=None,
+ metadata={
+ "name": "pdfPath",
+ "type": "Element",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfRectangle:
+ class Meta:
+ name = "pdfRectangle"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ graphic_state: GraphicState | None = field(
+ default=None,
+ metadata={
+ "name": "graphicState",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ debug_info: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ fill_background: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ xobj_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "xobjId",
+ "type": "Attribute",
+ },
+ )
+ line_width: float | None = field(
+ default=None,
+ metadata={
+ "name": "lineWidth",
+ "type": "Attribute",
+ },
+ )
+ render_order: int | None = field(
+ default=None,
+ metadata={
+ "name": "renderOrder",
+ "type": "Attribute",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfStyle:
+ class Meta:
+ name = "pdfStyle"
+
+ graphic_state: GraphicState | None = field(
+ default=None,
+ metadata={
+ "name": "graphicState",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ font_id: str | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ font_size: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class VisualBbox:
+ class Meta:
+ name = "visual_bbox"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfCharacter:
+ class Meta:
+ name = "pdfCharacter"
+
+ pdf_style: PdfStyle | None = field(
+ default=None,
+ metadata={
+ "name": "pdfStyle",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ visual_bbox: VisualBbox | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ },
+ )
+ vertical: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ scale: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ pdf_character_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "pdfCharacterId",
+ "type": "Attribute",
+ },
+ )
+ char_unicode: str | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ advance: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ xobj_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "xobjId",
+ "type": "Attribute",
+ },
+ )
+ debug_info: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ formula_layout_id: int | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ render_order: int | None = field(
+ default=None,
+ metadata={
+ "name": "renderOrder",
+ "type": "Attribute",
+ },
+ )
+ sub_render_order: int | None = field(
+ default=None,
+ metadata={
+ "name": "subRenderOrder",
+ "type": "Attribute",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfCurve:
+ class Meta:
+ name = "pdfCurve"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ graphic_state: GraphicState | None = field(
+ default=None,
+ metadata={
+ "name": "graphicState",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_path: list[PdfPath] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfPath",
+ "type": "Element",
+ },
+ )
+ pdf_original_path: list[PdfOriginalPath] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfOriginalPath",
+ "type": "Element",
+ },
+ )
+ debug_info: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ fill_background: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ stroke_path: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ evenodd: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ xobj_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "xobjId",
+ "type": "Attribute",
+ },
+ )
+ render_order: int | None = field(
+ default=None,
+ metadata={
+ "name": "renderOrder",
+ "type": "Attribute",
+ },
+ )
+ ctm: list[object] = field(
+ default_factory=list,
+ metadata={
+ "type": "Attribute",
+ "length": 6,
+ "tokens": True,
+ },
+ )
+ relocation_transform: list[object] = field(
+ default_factory=list,
+ metadata={
+ "type": "Attribute",
+ "length": 6,
+ "tokens": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfForm:
+ class Meta:
+ name = "pdfForm"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ graphic_state: GraphicState | None = field(
+ default=None,
+ metadata={
+ "name": "graphicState",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_matrix: PdfMatrix | None = field(
+ default=None,
+ metadata={
+ "name": "pdfMatrix",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_affine_transform: PdfAffineTransform | None = field(
+ default=None,
+ metadata={
+ "name": "pdfAffineTransform",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_form_subtype: PdfFormSubtype | None = field(
+ default=None,
+ metadata={
+ "name": "pdfFormSubtype",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ xobj_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "xobjId",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ ctm: list[object] = field(
+ default_factory=list,
+ metadata={
+ "type": "Attribute",
+ "length": 6,
+ "tokens": True,
+ },
+ )
+ relocation_transform: list[object] = field(
+ default_factory=list,
+ metadata={
+ "type": "Attribute",
+ "length": 6,
+ "tokens": True,
+ },
+ )
+ render_order: int | None = field(
+ default=None,
+ metadata={
+ "name": "renderOrder",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ form_type: str | None = field(
+ default=None,
+ metadata={
+ "name": "formType",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfSameStyleUnicodeCharacters:
+ class Meta:
+ name = "pdfSameStyleUnicodeCharacters"
+
+ pdf_style: PdfStyle | None = field(
+ default=None,
+ metadata={
+ "name": "pdfStyle",
+ "type": "Element",
+ },
+ )
+ unicode: str | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ debug_info: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfXobject:
+ class Meta:
+ name = "pdfXobject"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_font: list[PdfFont] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfFont",
+ "type": "Element",
+ },
+ )
+ base_operations: BaseOperations | None = field(
+ default=None,
+ metadata={
+ "name": "baseOperations",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ xobj_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "xobjId",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ xref_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "xrefId",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfFormula:
+ class Meta:
+ name = "pdfFormula"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_character: list[PdfCharacter] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfCharacter",
+ "type": "Element",
+ "min_occurs": 1,
+ },
+ )
+ pdf_curve: list[PdfCurve] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfCurve",
+ "type": "Element",
+ },
+ )
+ pdf_form: list[PdfForm] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfForm",
+ "type": "Element",
+ },
+ )
+ x_offset: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ y_offset: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ x_advance: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ line_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "lineId",
+ "type": "Attribute",
+ },
+ )
+ is_corner_mark: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfLine:
+ class Meta:
+ name = "pdfLine"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_character: list[PdfCharacter] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfCharacter",
+ "type": "Element",
+ "min_occurs": 1,
+ },
+ )
+ render_order: int | None = field(
+ default=None,
+ metadata={
+ "name": "renderOrder",
+ "type": "Attribute",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfSameStyleCharacters:
+ class Meta:
+ name = "pdfSameStyleCharacters"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_style: PdfStyle | None = field(
+ default=None,
+ metadata={
+ "name": "pdfStyle",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_character: list[PdfCharacter] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfCharacter",
+ "type": "Element",
+ "min_occurs": 1,
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfParagraphComposition:
+ class Meta:
+ name = "pdfParagraphComposition"
+
+ pdf_line: PdfLine | None = field(
+ default=None,
+ metadata={
+ "name": "pdfLine",
+ "type": "Element",
+ },
+ )
+ pdf_formula: PdfFormula | None = field(
+ default=None,
+ metadata={
+ "name": "pdfFormula",
+ "type": "Element",
+ },
+ )
+ pdf_same_style_characters: PdfSameStyleCharacters | None = field(
+ default=None,
+ metadata={
+ "name": "pdfSameStyleCharacters",
+ "type": "Element",
+ },
+ )
+ pdf_character: PdfCharacter | None = field(
+ default=None,
+ metadata={
+ "name": "pdfCharacter",
+ "type": "Element",
+ },
+ )
+ pdf_same_style_unicode_characters: PdfSameStyleUnicodeCharacters | None = field(
+ default=None,
+ metadata={
+ "name": "pdfSameStyleUnicodeCharacters",
+ "type": "Element",
+ },
+ )
+
+
+@dataclass(slots=True)
+class PdfParagraph:
+ class Meta:
+ name = "pdfParagraph"
+
+ box: Box | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_style: PdfStyle | None = field(
+ default=None,
+ metadata={
+ "name": "pdfStyle",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_paragraph_composition: list[PdfParagraphComposition] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfParagraphComposition",
+ "type": "Element",
+ },
+ )
+ xobj_id: int | None = field(
+ default=None,
+ metadata={
+ "name": "xobjId",
+ "type": "Attribute",
+ },
+ )
+ unicode: str | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ scale: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ optimal_scale: float | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ vertical: bool | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ first_line_indent: bool | None = field(
+ default=None,
+ metadata={
+ "name": "FirstLineIndent",
+ "type": "Attribute",
+ },
+ )
+ debug_id: str | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ layout_label: str | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ layout_id: int | None = field(
+ default=None,
+ metadata={
+ "type": "Attribute",
+ },
+ )
+ render_order: int | None = field(
+ default=None,
+ metadata={
+ "name": "renderOrder",
+ "type": "Attribute",
+ },
+ )
+
+ text_direction: str | None = field(
+ default=None,
+ metadata={
+ "name": "textDirection",
+ "type": "Attribute",
+ },
+ )
+ text_align: str | None = field(
+ default=None,
+ metadata={
+ "name": "textAlign",
+ "type": "Attribute",
+ },
+ )
+
+
+@dataclass(slots=True)
+class Page:
+ class Meta:
+ name = "page"
+
+ mediabox: Mediabox | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ cropbox: Cropbox | None = field(
+ default=None,
+ metadata={
+ "type": "Element",
+ "required": True,
+ },
+ )
+ pdf_xobject: list[PdfXobject] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfXobject",
+ "type": "Element",
+ },
+ )
+ page_layout: list[PageLayout] = field(
+ default_factory=list,
+ metadata={
+ "name": "pageLayout",
+ "type": "Element",
+ },
+ )
+ pdf_rectangle: list[PdfRectangle] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfRectangle",
+ "type": "Element",
+ },
+ )
+ pdf_font: list[PdfFont] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfFont",
+ "type": "Element",
+ },
+ )
+ pdf_paragraph: list[PdfParagraph] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfParagraph",
+ "type": "Element",
+ },
+ )
+ pdf_figure: list[PdfFigure] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfFigure",
+ "type": "Element",
+ },
+ )
+ pdf_character: list[PdfCharacter] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfCharacter",
+ "type": "Element",
+ },
+ )
+ pdf_curve: list[PdfCurve] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfCurve",
+ "type": "Element",
+ },
+ )
+ pdf_form: list[PdfForm] = field(
+ default_factory=list,
+ metadata={
+ "name": "pdfForm",
+ "type": "Element",
+ },
+ )
+ base_operations: BaseOperations | None = field(
+ default=None,
+ metadata={
+ "name": "baseOperations",
+ "type": "Element",
+ "required": True,
+ },
+ )
+ page_number: int | None = field(
+ default=None,
+ metadata={
+ "name": "pageNumber",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+ unit: str | None = field(
+ default=None,
+ metadata={
+ "name": "Unit",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
+
+
+@dataclass(slots=True)
+class Document:
+ class Meta:
+ name = "document"
+
+ page: list[Page] = field(
+ default_factory=list,
+ metadata={
+ "type": "Element",
+ "min_occurs": 1,
+ },
+ )
+ total_pages: int | None = field(
+ default=None,
+ metadata={
+ "name": "totalPages",
+ "type": "Attribute",
+ "required": True,
+ },
+ )
diff --git a/babeldoc/format/pdf/document_il/il_version_1.rnc b/babeldoc/format/pdf/document_il/il_version_1.rnc
new file mode 100644
index 0000000000000000000000000000000000000000..0b66c299fbf314da481dbab65007e51db384bfa2
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/il_version_1.rnc
@@ -0,0 +1,239 @@
+start = Document
+Document =
+ element document {
+ Page+,
+ attribute totalPages { xsd:int }
+ }
+Page =
+ element page {
+ element mediabox { Box },
+ element cropbox { Box },
+ PDFXobject*,
+ PageLayout*,
+ PDFRectangle*,
+ PDFFont*,
+ PDFParagraph*,
+ PDFFigure*,
+ PDFCharacter*,
+ PDFCurve*,
+ PDFForm*,
+ attribute pageNumber { xsd:int },
+ attribute Unit { xsd:string },
+ element baseOperations { xsd:string }
+ }
+Box =
+ element box {
+ # from (x,y) to (x2,y2)
+ attribute x { xsd:float },
+ attribute y { xsd:float },
+ attribute x2 { xsd:float },
+ attribute y2 { xsd:float }
+ }
+PDFXrefId = xsd:int
+PDFFont =
+ element pdfFont {
+ attribute name { xsd:string },
+ attribute fontId { xsd:string },
+ attribute xrefId { PDFXrefId },
+ attribute encodingLength { xsd:int },
+ attribute bold { xsd:boolean }?,
+ attribute italic { xsd:boolean }?,
+ attribute monospace { xsd:boolean }?,
+ attribute serif { xsd:boolean }?,
+ attribute ascent { xsd:float }?,
+ attribute descent { xsd:float }?,
+ PDFFontCharBoundingBox*
+ }
+PDFFontCharBoundingBox =
+ element pdfFontCharBoundingBox {
+ attribute x { xsd:float },
+ attribute y { xsd:float },
+ attribute x2 { xsd:float },
+ attribute y2 { xsd:float },
+ attribute char_id { xsd:int }
+ }
+PDFXobject =
+ element pdfXobject {
+ attribute xobjId { xsd:int },
+ attribute xrefId { PDFXrefId },
+ Box,
+ PDFFont*,
+ element baseOperations { xsd:string }
+ }
+PDFCharacter =
+ element pdfCharacter {
+ attribute vertical { xsd:boolean }?,
+ attribute scale { xsd:float }?,
+ attribute pdfCharacterId { xsd:int }?,
+ attribute char_unicode { xsd:string },
+ attribute advance { xsd:float }?,
+ # xobject nesting depth
+ attribute xobjId { xsd:int }?,
+ attribute debug_info { xsd:boolean }?,
+ attribute formula_layout_id { xsd:int }?,
+ attribute renderOrder { xsd:int }?,
+ attribute subRenderOrder { xsd:int }?,
+ PDFStyle,
+ Box,
+ element visual_bbox { Box }?
+ }
+PageLayout =
+ element pageLayout {
+ attribute id { xsd:int },
+ attribute conf { xsd:float },
+ attribute class_name { xsd:string },
+ Box
+ }
+GraphicState =
+ element graphicState {
+ attribute passthroughPerCharInstruction { xsd:string }?
+ }
+PDFStyle =
+ element pdfStyle {
+ attribute font_id { xsd:string },
+ attribute font_size { xsd:float },
+ GraphicState
+ }
+PDFParagraph =
+ element pdfParagraph {
+ attribute xobjId { xsd:int }?,
+ attribute unicode { xsd:string },
+ attribute scale { xsd:float }?,
+ attribute optimal_scale { xsd:float }?,
+ attribute vertical { xsd:boolean }?,
+ attribute FirstLineIndent { xsd:boolean }?,
+ attribute debug_id { xsd:string }?,
+ attribute layout_label { xsd:string }?,
+ attribute layout_id { xsd:int }?,
+ attribute renderOrder { xsd:int }?,
+ Box,
+ PDFStyle,
+ PDFParagraphComposition*
+ }
+PDFParagraphComposition =
+ element pdfParagraphComposition {
+ PDFLine
+ | PDFFormula
+ | PDFSameStyleCharacters
+ | PDFCharacter
+ | PDFSameStyleUnicodeCharacters
+ }
+PDFLine =
+ element pdfLine {
+ Box,
+ PDFCharacter+,
+ attribute renderOrder { xsd:int }?
+ }
+PDFSameStyleCharacters =
+ element pdfSameStyleCharacters { Box, PDFStyle, PDFCharacter+ }
+PDFSameStyleUnicodeCharacters =
+ element pdfSameStyleUnicodeCharacters {
+ PDFStyle?,
+ attribute unicode { xsd:string },
+ attribute debug_info { xsd:boolean }?
+ }
+PDFFormula =
+ element pdfFormula {
+ Box,
+ PDFCharacter+,
+ PDFCurve*,
+ PDFForm*,
+ attribute x_offset { xsd:float },
+ attribute y_offset { xsd:float },
+ attribute x_advance { xsd:float }?,
+ attribute lineId { xsd:int }?,
+ attribute is_corner_mark { xsd:boolean }?
+ }
+PDFFigure = element pdfFigure { Box }
+PDFRectangle =
+ element pdfRectangle {
+ Box,
+ GraphicState,
+ attribute debug_info { xsd:boolean }?,
+ attribute fill_background { xsd:boolean }?,
+ attribute xobjId { xsd:int }?,
+ attribute lineWidth { xsd:float }?,
+ attribute renderOrder { xsd:int }?
+ }
+PDFCurve =
+ element pdfCurve {
+ Box,
+ GraphicState,
+ PDFPath*,
+ PDFOriginalPath*,
+ attribute debug_info { xsd:boolean }?,
+ attribute fill_background { xsd:boolean }?,
+ attribute stroke_path { xsd:boolean }?,
+ attribute evenodd { xsd:boolean }?,
+ attribute xobjId { xsd:int }?,
+ attribute renderOrder { xsd:int }?,
+ attribute ctm {
+ list {
+ xsd:float, xsd:float, xsd:float, xsd:float, xsd:float, xsd:float
+ }
+ }?,
+ attribute relocation_transform {
+ list {
+ xsd:float, xsd:float, xsd:float, xsd:float, xsd:float, xsd:float
+ }
+ }?
+ }
+PDFOriginalPath = element pdfOriginalPath { PDFPath }
+PDFPath =
+ element pdfPath {
+ attribute x { xsd:float },
+ attribute y { xsd:float },
+ attribute op { xsd:string },
+ attribute has_xy { xsd:boolean }?
+ }
+PDFForm =
+ element pdfForm {
+ attribute xobjId { xsd:int },
+ Box,
+ GraphicState,
+ PDFMatrix,
+ PDFAffineTransform,
+ attribute ctm {
+ list {
+ xsd:float, xsd:float, xsd:float, xsd:float, xsd:float, xsd:float
+ }
+ }?,
+ attribute relocation_transform {
+ list {
+ xsd:float, xsd:float, xsd:float, xsd:float, xsd:float, xsd:float
+ }
+ }?,
+ attribute renderOrder { xsd:int },
+ attribute formType { xsd:string },
+ PDFFormSubtype
+ }
+PDFFormSubtype = element pdfFormSubtype { PDFInlineForm | PDFXobjForm }
+PDFInlineForm =
+ element pdfInlineForm {
+ attribute formData { xsd:string }?,
+ attribute imageParameters { xsd:string }?
+ }
+PDFXobjForm =
+ element pdfXobjForm {
+ attribute xrefId { PDFXrefId },
+ attribute doArgs { xsd:string }
+ }
+PDFMatrix =
+ element pdfMatrix {
+ attribute a { xsd:float },
+ attribute b { xsd:float },
+ attribute c { xsd:float },
+ attribute d { xsd:float },
+ attribute e { xsd:float },
+ attribute f { xsd:float }
+ }
+# Decomposed transform parameters for a CTM
+PDFAffineTransform =
+ element pdfAffineTransform {
+ attribute translation_x { xsd:float },
+ attribute translation_y { xsd:float },
+ attribute rotation { xsd:float },
+ attribute scale_x { xsd:float },
+ attribute scale_y { xsd:float },
+ attribute shear { xsd:float }
+ }
diff --git a/babeldoc/format/pdf/document_il/il_version_1.rng b/babeldoc/format/pdf/document_il/il_version_1.rng
new file mode 100644
index 0000000000000000000000000000000000000000..b85074bdc971e6b22dbd81f7914440aaae3b2366
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/il_version_1.rng
@@ -0,0 +1,645 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ..., etc."
+ )
+ llm_prompt_parts.append(
+ " Formula or variable placeholders enclosed in curly braces: e.g., {v3}, {equation_1}, {name}, etc."
+ )
+ llm_prompt_parts.append(
+ " Any other placeholders like [[...]], %%...%%, %s, %d, etc."
+ )
+ llm_prompt_parts.append(
+ "2. Preserve the exact structure, position, and content of the above elements, do not modify spacing, punctuation, or formatting."
+ )
+ llm_prompt_parts.append(
+ "3. If the input contains:Proper nouns, code, or non-translatable technical terms, retain them in the original form."
+ )
+ llm_prompt_parts.append(
+ "4. If adjacent paragraphs are semantically coherent, you may appropriately adjust the word order, but you must keep the number of paragraphs unchanged and must not move placeholders from one paragraph to another."
+ )
+
+ # 4. ## Input/Output Format:
+ llm_prompt_parts.append("\n## Input/Output Format:")
+ llm_prompt_parts.append(
+ '1. You will receive a JSON object with entries containing "id" and "input" fields.'
+ )
+ llm_prompt_parts.append(
+ f'2. Your task is to translate the value of "input" into {self.translation_config.lang_out}, while applying the rules above.'
+ )
+ llm_prompt_parts.append(
+ '3. Return a new JSON object with the same "id" and the translated "output" field.'
+ )
+ llm_prompt_parts.append(
+ "Please return the translated json directly without wrapping ```json``` tag or include any additional information."
+ )
+
+ # 5. ##example (Renumbered from 5 to 4)
+ llm_prompt_parts.append("\n## Example:")
+ llm_prompt_parts.append("Here is an example of the expected format:")
+ llm_prompt_parts.append("") # Blank line
+ llm_prompt_parts.append("")
+ llm_prompt_parts.append("```json")
+ llm_prompt_parts.append("Input:")
+ llm_prompt_parts.append("{")
+ llm_prompt_parts.append(' "id": 0,')
+ llm_prompt_parts.append(
+ ' "input": "{v1},world!",'
+ )
+ llm_prompt_parts.append(' "layout_label": "list_item_hybrid"')
+ llm_prompt_parts.append("}")
+ llm_prompt_parts.append("```")
+ llm_prompt_parts.append("Output:")
+ llm_prompt_parts.append("```json")
+ llm_prompt_parts.append("{")
+ llm_prompt_parts.append(' "id": 0,')
+ llm_prompt_parts.append(
+ ' "output": "{v1}，世界ï¼"'
+ )
+ llm_prompt_parts.append("}")
+ llm_prompt_parts.append("```")
+ llm_prompt_parts.append("")
+
+ # 2. ##Contextual Hints for Better Translation
+ contextual_hints_section: list[str] = []
+ hint_idx = 1
+ if title_paragraph:
+ contextual_hints_section.append(
+ f"{hint_idx}. First title in full text: {title_paragraph.unicode}"
+ )
+ hint_idx += 1
+
+ if local_title_paragraph:
+ is_different_from_global = True
+ if title_paragraph:
+ if local_title_paragraph.debug_id == title_paragraph.debug_id:
+ is_different_from_global = False
+
+ if is_different_from_global:
+ contextual_hints_section.append(
+ f"{hint_idx}. The most recent title is: {local_title_paragraph.unicode}"
+ )
+ hint_idx += 1
+
+ # --- ADD GLOSSARY HINTS ---
+ batch_text_for_glossary_matching = "\n".join(
+ item.get("input", "") for item in json_format_input
+ )
+
+ active_glossary_markdown_blocks: list[str] = []
+ # Use cached glossaries
+ if self._cached_glossaries:
+ for glossary in self._cached_glossaries:
+ # Get active entries for the current batch_text_for_glossary_matching
+ active_entries = glossary.get_active_entries_for_text(
+ batch_text_for_glossary_matching
+ )
+
+ if active_entries:
+ current_glossary_md_entries: list[str] = []
+ for original_source, target_text in sorted(active_entries):
+ current_glossary_md_entries.append(
+ f"| {original_source} | {target_text} |"
+ )
+
+ if current_glossary_md_entries:
+ glossary_table_md = (
+ f"### Glossary: {glossary.name}\n\n"
+ "| Source Term | Target Term |\n"
+ "|-------------|-------------|\n"
+ + "\n".join(current_glossary_md_entries)
+ )
+ active_glossary_markdown_blocks.append(glossary_table_md)
+
+ if contextual_hints_section or active_glossary_markdown_blocks:
+ llm_prompt_parts.append("\n## Contextual Hints for Better Translation")
+ llm_prompt_parts.extend(contextual_hints_section)
+
+ if active_glossary_markdown_blocks:
+ llm_prompt_parts.append(
+ f"{hint_idx}. You MUST strictly adhere to the following glossaries. please give preference to other glossaries. If a source term from a table appears in the text, use the corresponding target term in your translation:"
+ )
+ # hint_idx += 1 # No need to increment if tables are part of this point
+ for md_block in active_glossary_markdown_blocks:
+ llm_prompt_parts.append(f"\n{md_block}\n")
+
+ # 6. ## Here is the input:
+ llm_prompt_parts.append("\n## Here is the input:")
+
+ # Combine all parts for the main prompt
+ main_prompt_content = "\n".join(llm_prompt_parts)
+
+ # Append the actual JSON input string at the end, without markdown fence
+ final_input = main_prompt_content + "\n\n" + json_format_input_str
+
+ for llm_translate_tracker in llm_translate_trackers:
+ llm_translate_tracker.set_input(final_input)
+ llm_output = self.translate_engine.llm_translate(
+ final_input,
+ rate_limit_params={
+ "paragraph_token_count": paragraph_token_count,
+ "request_json_mode": True,
+ },
+ )
+ for llm_translate_tracker in llm_translate_trackers:
+ llm_translate_tracker.set_output(llm_output)
+ llm_output = llm_output.strip()
+
+ llm_output = self._clean_json_output(llm_output)
+
+ parsed_output = json.loads(llm_output)
+
+ if isinstance(parsed_output, dict) and parsed_output.get(
+ "output", parsed_output.get("input", False)
+ ):
+ parsed_output = [parsed_output]
+
+ translation_results = {
+ item["id"]: item.get("output", item.get("input"))
+ for item in parsed_output
+ }
+
+ if len(translation_results) != len(inputs):
+ raise Exception(
+ f"Translation results length mismatch. Expected: {len(inputs)}, Got: {len(translation_results)}"
+ )
+
+ # Store translated texts for logging
+ translated_texts_for_logging = []
+
+ for id_, output in translation_results.items():
+ should_fallback = True
+ try:
+ if not isinstance(output, str):
+ logger.warning(
+ f"Translation result is not a string. Output: {output}"
+ )
+ continue
+
+ id_ = int(id_) # Ensure id is an integer
+ if id_ >= len(inputs):
+ logger.warning(f"Invalid id {id_}, skipping")
+ continue
+
+ # Clean up any excessive punctuation in the translated text
+ translated_text = re.sub(r"[. 。…，]{20,}", ".", output)
+
+ # Store for logging
+ translated_texts_for_logging.append(translated_text)
+
+ # Log the language configuration
+ lang_out = (self.translation_config.lang_out or "").lower()
+ logger.info(f"Output language configured as: '{lang_out}'")
+
+ # Apply Arabic shaping and BiDi processing if output language is Arabic
+ is_arabic = False
+ if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ is_arabic = True
+ logger.info(f"Arabic detected via direct match: {lang_out}")
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic = True
+ logger.info(f"Arabic detected via pattern match: {lang_out}")
+
+ if is_arabic:
+ logger.info("="*60)
+ logger.info(f"ARABIC SHAPING STARTED")
+ logger.info(f"BEFORE Arabic Shaping: {translated_text}")
+ try:
+ # Check if text is already shaped (contains presentation forms)
+ # Set RTL attributes for proper layout
+ inputs[id_][2].text_direction = "rtl"
+ inputs[id_][2].text_align = "right"
+ logger.info(f"Set RTL attributes: text_direction=rtl, text_align=right")
+ if not re.search(r'[\uFB50-\uFDFF\uFE70-\uFEFF]', translated_text):
+ logger.info("Text is not pre-shaped, applying reshape and bidi...")
+
+ # Extract inline tags before shaping to prevent corruption
+ tag_pattern = r'<[^>]+>'
+ tags = []
+ tag_positions = []
+ for match in re.finditer(tag_pattern, translated_text):
+ tags.append(match.group(0))
+ tag_positions.append((match.start(), match.end()))
+
+ if tags:
+ logger.info(f"Found {len(tags)} inline tags to protect")
+ text_without_tags = translated_text
+ placeholder_map = {}
+ for i in range(len(tags) - 1, -1, -1):
+ start, end = tag_positions[i]
+ placeholder = f"\u200D{i}\u200D"
+ placeholder_map[placeholder] = tags[i]
+ text_without_tags = text_without_tags[:start] + placeholder + text_without_tags[end:]
+
+ # Reshape Arabic text for proper character joining
+ reshaped_text = reshape(text_without_tags)
+ logger.info(f"AFTER Reshaping: {reshaped_text}")
+ # Apply bidirectional algorithm for proper text ordering
+ translated_text = get_display(reshaped_text, base_dir='R')
+
+ # Restore tags
+ for placeholder, tag in placeholder_map.items():
+ translated_text = translated_text.replace(placeholder, tag)
+ logger.info(f"Restored {len(tags)} inline tags")
+ else:
+ # No tags, process normally
+ # Reshape Arabic text for proper character joining
+ reshaped_text = reshape(translated_text)
+ logger.info(f"AFTER Reshaping: {reshaped_text}")
+ # Apply bidirectional algorithm for proper text ordering
+ translated_text = get_display(reshaped_text, base_dir='R')
+ logger.info(f"AFTER BiDi Display: {translated_text}")
+ logger.info("Arabic shaping completed successfully")
+ else:
+ logger.info("Text already contains Arabic presentation forms - skipping reshape")
+ logger.info("="*60)
+ except Exception as e:
+ logger.error(f"Failed to shape Arabic text: {e}", exc_info=True)
+ logger.info("="*60)
+ # Continue with original text if shaping fails
+ else:
+ logger.info(f"Not Arabic language, skipping Arabic shaping. Language: {lang_out}")
+
+ logger.info(f"Final Translated paragraph: {translated_text}")
+
+ # Get the original input for this translation
+ translate_input = inputs[id_][1]
+ llm_translate_tracker = inputs[id_][4]
+
+ input_unicode = inputs[id_][0]
+ output_unicode = translated_text
+
+ trimed_input = re.sub(r"[. 。…，]{20,}", ".", input_unicode)
+
+ input_token_count = self.calc_token_count(trimed_input)
+ output_token_count = self.calc_token_count(output_unicode)
+
+ if trimed_input == output_unicode and input_token_count > 10:
+ llm_translate_tracker.set_error_message(
+ "Translation result is the same as input, fallback."
+ )
+ logger.warning(
+ "Translation result is the same as input, fallback."
+ )
+ continue
+
+ if not (0.3 < output_token_count / input_token_count < 3):
+ llm_translate_tracker.set_error_message(
+ f"Translation result is too long or too short. Input: {input_token_count}, Output: {output_token_count}"
+ )
+ logger.warning(
+ f"Translation result is too long or too short. Input: {input_token_count}, Output: {output_token_count}"
+ )
+ continue
+
+ edit_distance = Levenshtein.distance(input_unicode, output_unicode)
+ if edit_distance < 5 and input_token_count > 20:
+ llm_translate_tracker.set_error_message(
+ f"Translation result edit distance is too small. distance: {edit_distance}, input: {input_unicode}, output: {output_unicode}"
+ )
+ logger.warning(
+ f"Translation result edit distance is too small. distance: {edit_distance}, input: {input_unicode}, output: {output_unicode}"
+ )
+ continue
+ # Apply the translation to the paragraph
+ self.il_translator.post_translate_paragraph(
+ inputs[id_][2],
+ inputs[id_][3],
+ translate_input,
+ translated_text,
+ )
+ should_fallback = False
+ if pbar:
+ pbar.advance(1)
+ except Exception as e:
+ error_message = f"Error translating paragraph. Error: {e}."
+ logger.exception(error_message)
+ # Ignore error and continue
+ for llm_translate_tracker in llm_translate_trackers:
+ llm_translate_tracker.set_error_message(error_message)
+ continue
+ finally:
+ self.total_count += 1
+ if should_fallback:
+ self.fallback_count += 1
+ inputs[id_][4].set_fallback_to_translate()
+ logger.warning(
+ f"Fallback to simple translation. paragraph id: {inputs[id_][2].debug_id}"
+ )
+ paragraph_token_count = self.calc_token_count(
+ inputs[id_][2].unicode
+ )
+ paragraph_unicodes = inputs[id_][5]
+ inputs[id_][2].unicode = paragraph_unicodes[id_]
+ executor.submit(
+ self.il_translator.translate_paragraph,
+ inputs[id_][2],
+ batch_paragraph.pages[id_],
+ pbar,
+ inputs[id_][3],
+ page_font_map,
+ xobj_font_map,
+ priority=1048576 - paragraph_token_count,
+ paragraph_token_count=paragraph_token_count,
+ title_paragraph=title_paragraph,
+ local_title_paragraph=local_title_paragraph,
+ )
+ else:
+ self.ok_count += 1
+
+ # Log translation batch completion with results
+ if hasattr(self, 'detailed_logger') and self.detailed_logger:
+ input_texts = [inp[0] for inp in inputs][:3] # First 3 input texts
+ self.detailed_logger.log_step(
+ f"Translation Batch {mp_id} Complete",
+ data={
+ 'batch_size': len(inputs),
+ 'translations_completed': len(translated_texts_for_logging),
+ 'sample_inputs': input_texts,
+ 'sample_outputs': translated_texts_for_logging[:3] if translated_texts_for_logging else []
+ }
+ )
+
+ except Exception as e:
+ # Log translation batch error
+ if hasattr(self, 'detailed_logger') and self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Translation Batch {mp_id} Error",
+ data={
+ 'error': str(e),
+ 'batch_size': len(batch_paragraph.paragraphs)
+ }
+ )
+
+ error_message = f"Error {e} during translation. try fallback"
+ logger.warning(error_message)
+ for llm_translate_tracker in llm_translate_trackers:
+ llm_translate_tracker.set_error_message(error_message)
+ llm_translate_tracker.set_fallback_to_translate()
+ self.total_count += len(llm_translate_trackers)
+ self.fallback_count += len(llm_translate_trackers)
+ for input_ in inputs:
+ input_[2].unicode = input_[5]
+ if not should_translate_paragraph:
+ should_translate_paragraph = list(
+ range(len(batch_paragraph.paragraphs))
+ )
+ for i in should_translate_paragraph:
+ paragraph = batch_paragraph.paragraphs[i]
+ tracker = batch_paragraph.trackers[i]
+ if paragraph.debug_id is None:
+ continue
+ paragraph_token_count = self.calc_token_count(paragraph.unicode)
+ executor.submit(
+ self.il_translator.translate_paragraph,
+ paragraph,
+ batch_paragraph.pages[i],
+ pbar,
+ tracker,
+ page_font_map,
+ xobj_font_map,
+ priority=1048576 - paragraph_token_count,
+ paragraph_token_count=paragraph_token_count,
+ title_paragraph=title_paragraph,
+ local_title_paragraph=local_title_paragraph,
+ )
+
+ def _clean_json_output(self, llm_output: str) -> str:
+ # Clean up JSON output by removing common wrapper tags
+ llm_output = llm_output.strip()
+ if llm_output.startswith(""):
+ llm_output = llm_output[6:]
+ if llm_output.endswith(""):
+ llm_output = llm_output[:-7]
+ if llm_output.startswith("```json"):
+ llm_output = llm_output[7:]
+ if llm_output.startswith("```"):
+ llm_output = llm_output[3:]
+ if llm_output.endswith("```"):
+ llm_output = llm_output[:-3]
+ return llm_output.strip()
\ No newline at end of file
diff --git a/babeldoc/format/pdf/document_il/midend/layout_parser.py b/babeldoc/format/pdf/document_il/midend/layout_parser.py
new file mode 100644
index 0000000000000000000000000000000000000000..484a44ecb1d497bbc1f5a006a62648c182f72d3a
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/layout_parser.py
@@ -0,0 +1,235 @@
+import logging
+import math
+import os
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import cv2
+import numpy as np
+from pymupdf import Document
+
+import babeldoc.format.pdf.document_il.utils.extract_char
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.utils.style_helper import GREEN
+from babeldoc.format.pdf.translation_config import TranslationConfig
+
+logger = logging.getLogger(__name__)
+
+
+class LayoutParser:
+ stage_name = "Parse Page Layout"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.detailed_logger = None
+ self.translation_config = translation_config
+ self.model = translation_config.doc_layout_model
+
+ def _save_debug_image(self, image: np.ndarray, layout, page_number: int):
+ """Save debug image with drawn boxes if debug mode is enabled."""
+ if not self.translation_config.debug:
+ return
+
+ debug_dir = Path(self.translation_config.get_working_file_path("ocr-box-image"))
+ debug_dir.mkdir(parents=True, exist_ok=True)
+
+ # Draw boxes on the image
+ debug_image = image.copy()
+ for box in layout.boxes:
+ x0, y0, x1, y1 = box.xyxy
+ cv2.rectangle(
+ debug_image,
+ (int(x0), int(y0)),
+ (int(x1), int(y1)),
+ (0, 255, 0),
+ 2,
+ )
+ # Add text label
+ cv2.putText(
+ debug_image,
+ layout.names[box.cls],
+ (int(x0), int(y0) - 5),
+ cv2.FONT_HERSHEY_SIMPLEX,
+ 0.5,
+ (0, 255, 0),
+ 1,
+ )
+ img_bgr = cv2.cvtColor(debug_image, cv2.COLOR_RGB2BGR)
+
+ # Save the image
+ output_path = debug_dir / f"{page_number}.jpg"
+ cv2.imwrite(str(output_path), img_bgr)
+
+ def _save_debug_box_to_page(self, page: il_version_1.Page):
+ """Save debug boxes and text labels to the PDF page."""
+ if not self.translation_config.debug:
+ return
+
+ color = GREEN
+
+ for layout in page.page_layout:
+ # Create a rectangle box
+ scale_factor = 1
+ if layout.class_name == "fallback_line":
+ scale_factor = 0.1
+ rect = il_version_1.PdfRectangle(
+ box=il_version_1.Box(
+ x=layout.box.x,
+ y=layout.box.y,
+ x2=layout.box.x2,
+ y2=layout.box.y2,
+ ),
+ graphic_state=color,
+ debug_info=True,
+ line_width=0.4 * scale_factor,
+ )
+ page.pdf_rectangle.append(rect)
+
+ # Create text label at top-left corner
+ # Note: PDF coordinates are from bottom-left,
+ # so we use y2 for top position
+ style = il_version_1.PdfStyle(
+ font_id="base",
+ font_size=4 * scale_factor,
+ graphic_state=color,
+ )
+ page.pdf_paragraph.append(
+ il_version_1.PdfParagraph(
+ first_line_indent=False,
+ box=il_version_1.Box(
+ x=layout.box.x,
+ y=layout.box.y2,
+ x2=layout.box.x2,
+ y2=layout.box.y2 + 5,
+ ),
+ vertical=False,
+ pdf_style=style,
+ unicode=layout.class_name,
+ pdf_paragraph_composition=[
+ il_version_1.PdfParagraphComposition(
+ pdf_same_style_unicode_characters=il_version_1.PdfSameStyleUnicodeCharacters(
+ unicode=layout.class_name,
+ pdf_style=style,
+ debug_info=True,
+ ),
+ ),
+ ],
+ xobj_id=-1,
+ ),
+ )
+
+ def process(self, docs: il_version_1.Document, mupdf_doc: Document):
+ """Generate layouts for all pages that need to be translated."""
+ # Get pages that need to be translated
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ "Layout Parsing Started",
+ f"Total pages to process: {len(docs.page)}"
+ )
+ total = len(docs.page)
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ total * 2,
+ ) as progress:
+ # Process predictions for each page
+ for page, layouts in self.model.handle_document(
+ docs.page,
+ mupdf_doc,
+ self.translation_config,
+ self._save_debug_image,
+ ):
+ page_layouts = []
+ for layout in layouts.boxes:
+ # Convert coordinate system from picture to il
+ # system to the il coordinate system
+ x0, y0, x1, y1 = layout.xyxy
+ # pix = get_no_rotation_img(mupdf_doc[page.page_number])
+ # pix = mupdf_doc[page.page_number].get_pixmap()
+ # h, w = pix.height, pix.width
+ box = mupdf_doc[page.page_number].mediabox_size
+ b_h = math.ceil(box.y)
+ b_w = math.ceil(box.x)
+ # if b_h != h or b_w != w:
+ # logger.warning(f"page {page.page_number} mediabox is not correct, b_h: {b_h}, h: {h}, b_w: {b_w}, w: {w}")
+ h, w = b_h, b_w
+ x0, y0, x1, y1 = (
+ np.clip(int(x0 - 1), 0, w - 1),
+ np.clip(int(h - y1 - 1), 0, h - 1),
+ np.clip(int(x1 + 1), 0, w - 1),
+ np.clip(int(h - y0 + 1), 0, h - 1),
+ )
+ page_layout = il_version_1.PageLayout(
+ id=len(page_layouts) + 1,
+ box=il_version_1.Box(
+ x0.item(),
+ y0.item(),
+ x1.item(),
+ y1.item(),
+ ),
+ conf=layout.conf.item(),
+ class_name=layouts.names[layout.cls],
+ )
+ page_layouts.append(page_layout)
+
+ page.page_layout = page_layouts
+ # self.generate_fallback_line_layout_for_page(page)
+ # self._save_debug_box_to_page(page)
+ progress.advance(1)
+ with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
+ for page in docs.page:
+ executor.submit(
+ self.generate_fallback_line_layout_for_page, page, progress
+ )
+ for i, page in enumerate(docs.page):
+ if self.detailed_logger:
+ layout_info = {
+ 'page_number': i + 1,
+ 'detected_elements': len(page.pdf_layout_element) if hasattr(page, 'pdf_layout_element') else 0,
+ 'element_types': {}
+ }
+
+ if hasattr(page, 'pdf_layout_element'):
+ for elem in page.pdf_layout_element:
+ elem_type = elem.layout_label if hasattr(elem, 'layout_label') else 'unknown'
+ layout_info['element_types'][elem_type] = layout_info['element_types'].get(elem_type, 0) + 1
+
+ self.detailed_logger.log_step(
+ f"Page {i+1} Layout Detection",
+ data=layout_info
+ )
+
+ return docs
+
+ def generate_fallback_line_layout_for_page(self, page: il_version_1.Page, progress):
+ try:
+ exists_page_layouts = page.page_layout
+ char_boxes = babeldoc.format.pdf.document_il.utils.extract_char.convert_page_to_char_boxes(
+ page
+ )
+ if not char_boxes:
+ return
+
+ clusters = babeldoc.format.pdf.document_il.utils.extract_char.process_page_chars_to_lines(
+ char_boxes
+ )
+ for cluster in clusters:
+ boxes = [c[0] for c in cluster.chars]
+ min_x = min(b.x for b in boxes)
+ max_x = max(b.x2 for b in boxes)
+ min_y = min(b.y for b in boxes)
+ max_y = max(b.y2 for b in boxes)
+ cluster.chars = il_version_1.Box(min_x, min_y, max_x, max_y)
+ page_layout = il_version_1.PageLayout(
+ id=len(exists_page_layouts) + 1,
+ box=il_version_1.Box(
+ min_x,
+ min_y,
+ max_x,
+ max_y,
+ ),
+ conf=1,
+ class_name="fallback_line",
+ )
+ exists_page_layouts.append(page_layout)
+ self._save_debug_box_to_page(page)
+ finally:
+ progress.advance(1)
diff --git a/babeldoc/format/pdf/document_il/midend/paragraph_finder.py b/babeldoc/format/pdf/document_il/midend/paragraph_finder.py
new file mode 100644
index 0000000000000000000000000000000000000000..39a7533b5189cc424231c3abd45943918488c006
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/paragraph_finder.py
@@ -0,0 +1,1074 @@
+import logging
+import random
+import re
+
+import numpy as np
+
+from babeldoc.babeldoc_exception.BabelDOCException import ExtractTextError
+from babeldoc.format.pdf.document_il import Box
+from babeldoc.format.pdf.document_il import Document
+from babeldoc.format.pdf.document_il import Page
+from babeldoc.format.pdf.document_il import PdfCharacter
+from babeldoc.format.pdf.document_il import PdfLine
+from babeldoc.format.pdf.document_il import PdfParagraph
+from babeldoc.format.pdf.document_il import PdfParagraphComposition
+from babeldoc.format.pdf.document_il import PdfRectangle
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.formular_helper import (
+ collect_page_formula_font_ids,
+)
+from babeldoc.format.pdf.document_il.utils.layout_helper import (
+ HEIGHT_NOT_USFUL_CHAR_IN_CHAR,
+)
+from babeldoc.format.pdf.document_il.utils.layout_helper import SPACE_REGEX
+from babeldoc.format.pdf.document_il.utils.layout_helper import Layout
+from babeldoc.format.pdf.document_il.utils.layout_helper import add_space_dummy_chars
+from babeldoc.format.pdf.document_il.utils.layout_helper import build_layout_index
+from babeldoc.format.pdf.document_il.utils.layout_helper import calculate_iou_for_boxes
+from babeldoc.format.pdf.document_il.utils.layout_helper import get_char_unicode_string
+from babeldoc.format.pdf.document_il.utils.layout_helper import get_character_layout
+from babeldoc.format.pdf.document_il.utils.layout_helper import is_bullet_point
+from babeldoc.format.pdf.document_il.utils.layout_helper import is_bullet_or_list_marker
+from babeldoc.format.pdf.document_il.utils.layout_helper import could_be_list_marker_start
+from babeldoc.format.pdf.document_il.utils.layout_helper import (
+ is_character_in_formula_layout,
+)
+from babeldoc.format.pdf.document_il.utils.layout_helper import is_text_layout
+from babeldoc.format.pdf.document_il.utils.paragraph_helper import is_cid_paragraph
+from babeldoc.format.pdf.document_il.utils.style_helper import INDIGO
+from babeldoc.format.pdf.document_il.utils.style_helper import WHITE
+from babeldoc.format.pdf.translation_config import TranslationConfig
+
+logger = logging.getLogger(__name__)
+
+# Base58 alphabet (Bitcoin style, without numbers 0, O, I, l)
+BASE58_ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
+
+
+def generate_base58_id(length: int = 5) -> str:
+ """Generate a random base58 ID of specified length."""
+ return "".join(random.choice(BASE58_ALPHABET) for _ in range(length))
+
+
+class ParagraphFinder:
+ stage_name = "Parse Paragraphs"
+
+ # 定义项目符å·çš„æ£åˆ™è¡¨è¾¾å¼æ¨¡å¼
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.translation_config = translation_config
+ self.detailed_logger = None
+ self.font_mapper = FontMapper(translation_config)
+
+ def _preprocess_formula_layouts(self, page: Page):
+ """
+ Identifies 'formula' layouts that do not significantly overlap with any text layouts
+ and re-labels them as 'isolate_formula'.
+ """
+ # Use a simplified Layout object for is_text_layout check
+ text_layouts = [
+ layout
+ for layout in page.page_layout
+ if is_text_layout(Layout(layout.id, layout.class_name))
+ ]
+ formula_layouts = [
+ layout for layout in page.page_layout if layout.class_name == "formula"
+ ]
+
+ if not text_layouts or not formula_layouts:
+ return
+
+ for formula_layout in formula_layouts:
+ is_isolated = True
+ for text_layout in text_layouts:
+ iou = calculate_iou_for_boxes(formula_layout.box, text_layout.box)
+ if iou >= 0.5:
+ is_isolated = False
+ break
+
+ if is_isolated:
+ formula_layout.class_name = "isolate_formula"
+
+ def add_text_fill_background(self, page: Page):
+ layout_map = {layout.id: layout for layout in page.page_layout}
+ for paragraph in page.pdf_paragraph:
+ layout_id = paragraph.layout_id
+ if layout_id is None:
+ continue
+ layout = layout_map[layout_id]
+ if paragraph.box is None:
+ continue
+ x1, y1, x2, y2 = (
+ paragraph.box.x,
+ paragraph.box.y,
+ paragraph.box.x2,
+ paragraph.box.y2,
+ )
+ layout_box = layout.box
+ if layout_box.x < x1:
+ x1 = layout_box.x
+ if layout_box.y < y1:
+ y1 = layout_box.y
+ if layout_box.x2 > x2:
+ x2 = layout_box.x2
+ if layout_box.y2 > y2:
+ y2 = layout_box.y2
+ assert x2 > x1 and y2 > y1
+ page.pdf_rectangle.append(
+ PdfRectangle(
+ box=Box(x1, y1, x2, y2),
+ fill_background=True,
+ graphic_state=WHITE,
+ debug_info=False,
+ xobj_id=paragraph.xobj_id,
+ )
+ )
+
+ def update_paragraph_data(self, paragraph: PdfParagraph, update_unicode=False):
+ if not paragraph.pdf_paragraph_composition:
+ return
+
+ chars = []
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_line:
+ chars.extend(composition.pdf_line.pdf_character)
+ elif composition.pdf_formula:
+ chars.extend(composition.pdf_formula.pdf_character)
+ elif composition.pdf_character:
+ chars.append(composition.pdf_character)
+ elif composition.pdf_same_style_unicode_characters:
+ continue
+ else:
+ logger.error(
+ "Unexpected composition type"
+ " in PdfParagraphComposition. "
+ "This type only appears in the IL "
+ "after the translation is completed.",
+ )
+ continue
+
+ if update_unicode and chars:
+ paragraph.unicode = get_char_unicode_string(chars)
+ if not chars:
+ return
+ # 更新边界框
+ min_x = min(char.visual_bbox.box.x for char in chars)
+ min_y = min(char.visual_bbox.box.y for char in chars)
+ max_x = max(char.visual_bbox.box.x2 for char in chars)
+ max_y = max(char.visual_bbox.box.y2 for char in chars)
+ paragraph.box = Box(min_x, min_y, max_x, max_y)
+ paragraph.vertical = chars[0].vertical
+ paragraph.xobj_id = chars[0].xobj_id
+
+ paragraph.first_line_indent = False
+ if (
+ paragraph.pdf_paragraph_composition
+ and paragraph.pdf_paragraph_composition[0].pdf_line
+ and paragraph.pdf_paragraph_composition[0]
+ .pdf_line.pdf_character[0]
+ .visual_bbox.box.x
+ - paragraph.box.x
+ > 1
+ ):
+ paragraph.first_line_indent = True
+
+ def update_line_data(self, line: PdfLine):
+ min_x = min(char.visual_bbox.box.x for char in line.pdf_character)
+ min_y = min(char.visual_bbox.box.y for char in line.pdf_character)
+ max_x = max(char.visual_bbox.box.x2 for char in line.pdf_character)
+ max_y = max(char.visual_bbox.box.y2 for char in line.pdf_character)
+ line.box = Box(min_x, min_y, max_x, max_y)
+
+ def add_debug_info(self, page: Page):
+ if not self.translation_config.debug:
+ return
+ for paragraph in page.pdf_paragraph:
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_line:
+ line = composition.pdf_line
+ page.pdf_rectangle.append(
+ PdfRectangle(
+ box=line.box,
+ fill_background=False,
+ graphic_state=INDIGO,
+ debug_info=True,
+ line_width=0.2,
+ )
+ )
+
+ def process(self, document):
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Paragraph Finding Started")
+
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(document.page),
+ ) as pbar:
+ if not document.page:
+ return
+ for page in document.page:
+ self.translation_config.raise_if_cancelled()
+ self.process_page(page)
+ pbar.advance()
+
+ total_paragraph_count = 0
+ for page in document.page:
+ total_paragraph_count += len(page.pdf_paragraph)
+ if total_paragraph_count == 0:
+ raise ExtractTextError("The document contains no paragraphs.")
+
+ if self.check_cid_paragraph(document):
+ raise ExtractTextError("The document contains too many CID paragraphs.")
+
+ for page_idx, page in enumerate(document.page):
+ if self.detailed_logger and hasattr(page, 'pdf_paragraph'):
+ for para_idx, para in enumerate(page.pdf_paragraph[:10]): # First 10
+ para_info = {
+ 'page': page_idx + 1,
+ 'paragraph_id': para_idx + 1,
+ 'text': para.unicode if hasattr(para, 'unicode') else '',
+ 'char_count': len(para.unicode) if hasattr(para, 'unicode') else 0,
+ 'layout_label': para.layout_label if hasattr(para, 'layout_label') else 'unknown',
+ 'box': str(para.box) if hasattr(para, 'box') else 'N/A'
+ }
+ self.detailed_logger.log_step(
+ f"Paragraph Detected (Page {page_idx+1}, Para {para_idx+1})",
+ data=para_info
+ )
+
+ def check_cid_paragraph(self, doc: Document):
+ cid_para_count = 0
+ para_total = 0
+ for page in doc.page:
+ para_total += len(page.pdf_paragraph)
+ for para in page.pdf_paragraph:
+ if is_cid_paragraph(para):
+ cid_para_count += 1
+ return cid_para_count / para_total > 0.8
+
+ def bbox_overlap(self, bbox1: Box, bbox2: Box) -> bool:
+ return (
+ bbox1.x < bbox2.x2
+ and bbox1.x2 > bbox2.x
+ and bbox1.y < bbox2.y2
+ and bbox1.y2 > bbox2.y
+ )
+
+ def process_page(self, page: Page):
+ layout_index, layout_map = build_layout_index(page)
+ # 预处ç†å…¬å¼å¸ƒå±€çš„æ ‡ç¾
+ self._preprocess_formula_layouts(page)
+
+ # 第一æ¥ï¼šæ ¹æ® layout 创建 paragraphs
+ # 在这一æ¥ä¸ï¼Œpage.pdf_character ä¸çš„å—符会被移除
+ paragraphs = self._group_characters_into_paragraphs(
+ page, layout_index, layout_map
+ )
+ page.pdf_paragraph = paragraphs
+
+ page_level_formula_font_ids, xobj_specific_formula_font_ids = (
+ collect_page_formula_font_ids(
+ page, self.translation_config.formular_font_pattern
+ )
+ )
+
+ # for para in paragraphs:
+ # if not para.debug_id:
+ # continue
+ # new_line = PdfLine(
+ # pdf_character=[x.pdf_character for x in para.pdf_paragraph_composition]
+ # )
+ # self.update_line_data(new_line)
+ # para.pdf_paragraph_composition = [
+ # PdfParagraphComposition(pdf_line=new_line)
+ # ]
+
+ # 第二æ¥ï¼šå°†æ®µè½å†…çš„å—符拆分为行
+ for paragraph in paragraphs:
+ if (
+ paragraph.xobj_id
+ and paragraph.xobj_id in xobj_specific_formula_font_ids
+ ):
+ current_formula_font_ids = xobj_specific_formula_font_ids[
+ paragraph.xobj_id
+ ]
+ else:
+ current_formula_font_ids = page_level_formula_font_ids
+ self._split_paragraph_into_lines(paragraph, current_formula_font_ids)
+
+ # 第三æ¥ï¼šå¤„ç†æ®µè½ä¸çš„ç©ºæ ¼
+ for paragraph in paragraphs:
+ add_space_dummy_chars(paragraph)
+ self.process_paragraph_spacing(paragraph)
+ self.update_paragraph_data(paragraph)
+
+ # 第四æ¥ï¼šè®¡ç®—所有行宽度的ä¸ä½æ•°
+ median_width = self.calculate_median_line_width(paragraphs)
+
+ # 第五æ¥ï¼šå¤„ç†ç‹¬ç«‹æ®µè½
+ self.process_independent_paragraphs(paragraphs, median_width)
+
+ # 新增åŽå¤„ç†ï¼šåˆå¹¶å¸¦è¡Œå·äº¤æ›¿çš„æ£æ–‡æ®µè½ï¼ˆa æ£æ–‡ã€b è¡Œå·ã€c æ£æ–‡ -> åˆå¹¶ a 与 c，ä¿ç•™ b）
+ if getattr(self.translation_config, "merge_alternating_line_numbers", True):
+ self.merge_alternating_line_number_paragraphs(paragraphs)
+
+ for paragraph in paragraphs:
+ self.update_paragraph_data(paragraph, update_unicode=True)
+
+ if self.translation_config.ocr_workaround:
+ self.add_text_fill_background(page)
+ # since this is ocr file,
+ # image characters are not needed
+ page.pdf_character = []
+
+ self.fix_overlapping_paragraphs(page)
+
+ # 第å…æ¥ï¼šå¯¹æ¯ä¸€è¡Œçš„å—符进行排åº
+ # self._sort_characters_in_lines(page)
+
+ self.add_debug_info(page)
+
+ # 新阶段：设置段è½çš„ renderorder 为所有组æˆéƒ¨åˆ†ä¸ renderorder 最å°çš„
+ self._set_paragraph_render_order(page)
+
+ def _set_paragraph_render_order(self, page: Page):
+ """
+ 设置段è½çš„ renderorder 为段è½æ‰€æœ‰ç»„æˆéƒ¨åˆ†ä¸ renderorder 最å°çš„值
+ """
+ for paragraph in page.pdf_paragraph:
+ min_render_order = 9999999999999999
+
+ # é历段è½çš„所有组æˆéƒ¨åˆ†
+ for composition in paragraph.pdf_paragraph_composition:
+ # 检查 PdfLine ä¸çš„å—符
+ if composition.pdf_line:
+ for char in composition.pdf_line.pdf_character:
+ if (
+ hasattr(char, "render_order")
+ and char.render_order is not None
+ ):
+ min_render_order = min(min_render_order, char.render_order)
+
+ # 检查å•ä¸ªå—符
+ elif composition.pdf_character:
+ char = composition.pdf_character
+ if hasattr(char, "render_order") and char.render_order is not None:
+ min_render_order = min(min_render_order, char.render_order)
+
+ # 检查公å¼ä¸çš„å—符
+ elif composition.pdf_formula:
+ for char in composition.pdf_formula.pdf_character:
+ if (
+ hasattr(char, "render_order")
+ and char.render_order is not None
+ ):
+ min_render_order = min(min_render_order, char.render_order)
+
+ # 如果找到了有效的 renderorder，设置段è½çš„ renderorder
+ if min_render_order != 9999999999999999:
+ paragraph.render_order = min_render_order
+
+ def is_isolated_formula(self, char: PdfCharacter):
+ return char.char_unicode in (
+ "(cid:122)",
+ "(cid:123)",
+ "(cid:124)",
+ "(cid:125)",
+ )
+
+ def _paragraph_text_ascii(self, p: PdfParagraph) -> str:
+ parts: list[str] = []
+ for comp in p.pdf_paragraph_composition or []:
+ if comp.pdf_line:
+ for ch in comp.pdf_line.pdf_character or []:
+ if ch.char_unicode is not None:
+ parts.append(ch.char_unicode)
+ elif comp.pdf_character and comp.pdf_character.char_unicode is not None:
+ parts.append(comp.pdf_character.char_unicode)
+ return "".join(parts)
+
+ def _is_ascii_digit_or_space_paragraph(self, p: PdfParagraph) -> bool:
+ text = self._paragraph_text_ascii(p)
+ if not text:
+ return True
+ has_digit = False
+ for c in text:
+ if c.isdigit() and ord(c) < 128:
+ has_digit = True
+ continue
+ if c.isspace():
+ continue
+ return False
+ return True if has_digit or text.strip() == "" else False
+
+ @staticmethod
+ def _same_layout_and_xobj(a: PdfParagraph, c: PdfParagraph) -> bool:
+ return (
+ a.layout_id is not None
+ and c.layout_id is not None
+ and a.layout_id == c.layout_id
+ and a.xobj_id is not None
+ and c.xobj_id is not None
+ and a.xobj_id == c.xobj_id
+ )
+
+ def merge_alternating_line_number_paragraphs(self, paragraphs: list[PdfParagraph]):
+ # a 代表æ£æ–‡
+ # l 代表行å·
+ if not paragraphs or len(paragraphs) < 3:
+ return
+ i = 0
+ while i < len(paragraphs) - 2:
+ a = paragraphs[i]
+ # åžæŽ‰ä¸€ä¸ªæˆ–多个连ç»çš„è¡Œå·æ®µ l
+ j = i + 1
+ saw_l = False
+ while j < len(paragraphs) and self._is_ascii_digit_or_space_paragraph(
+ paragraphs[j]
+ ):
+ saw_l = True
+ j += 1
+ # 现在 j 指å‘候选的 c
+ if saw_l and j < len(paragraphs):
+ c = paragraphs[j]
+ if self._same_layout_and_xobj(a, c):
+ a.pdf_paragraph_composition.extend(c.pdf_paragraph_composition)
+ self.update_paragraph_data(a)
+ del paragraphs[j]
+ # ä¸ç§»åŠ¨ i，继ç»å°è¯•æŠŠæ›´å¤šæ£æ–‡æŽ¥åˆ° a，实现 a l+ a l+ a ... 链å¼åˆå¹¶
+ continue
+ i += 1
+
+ def _group_characters_into_paragraphs(
+ self, page: Page, layout_index, layout_map
+ ) -> list[PdfParagraph]:
+ paragraphs: list[PdfParagraph] = []
+ if page.pdf_paragraph:
+ paragraphs.extend(page.pdf_paragraph)
+ page.pdf_paragraph = []
+
+ char_areas = [
+ (char.visual_bbox.box.x2 - char.visual_bbox.box.x)
+ * (char.visual_bbox.box.y2 - char.visual_bbox.box.y)
+ for char in page.pdf_character
+ ]
+ median_char_area = 0.0
+ if char_areas:
+ char_areas.sort()
+ mid = len(char_areas) // 2
+ median_char_area = (
+ char_areas[mid]
+ if len(char_areas) % 2 == 1
+ else (char_areas[mid - 1] + char_areas[mid]) / 2
+ )
+
+ current_paragraph: PdfParagraph | None = None
+ current_layout: Layout | None = None
+ skip_chars = []
+
+ for char in page.pdf_character:
+ char_layout = get_character_layout(char, layout_index, layout_map)
+ # Check if character is in any formula layout and set formula_layout_id
+ char.formula_layout_id = is_character_in_formula_layout(
+ char, page, layout_index, layout_map
+ )
+
+ if not is_text_layout(char_layout) or self.is_isolated_formula(char):
+ skip_chars.append(char)
+ continue
+
+ char_box = char.visual_bbox.box
+ # char_pdf_box = char.box
+ # if calculate_iou_for_boxes(char_box, char_pdf_box) < 0.2:
+ # char_box = char_pdf_box
+ char_area = (char_box.x2 - char_box.x) * (char_box.y2 - char_box.y)
+ is_small_char = char_area < median_char_area * 0.05
+
+ is_new_paragraph = False
+ if current_paragraph is None:
+ is_new_paragraph = True
+ elif (
+ not (
+ is_small_char
+ and current_paragraph.pdf_paragraph_composition
+ and char_layout.id == current_layout.id
+ )
+ and char.char_unicode not in HEIGHT_NOT_USFUL_CHAR_IN_CHAR
+ ):
+ if (
+ (
+ char_layout.id != current_layout.id
+ and not SPACE_REGEX.match(char.char_unicode)
+ )
+ or ( # not same xobject
+ current_paragraph.pdf_paragraph_composition
+ and current_paragraph.pdf_paragraph_composition[
+ -1
+ ].pdf_character.xobj_id
+ != char.xobj_id
+ )
+ or (
+ (is_bullet_point(char) or could_be_list_marker_start(char))
+ and not current_paragraph.pdf_paragraph_composition
+ )
+ ):
+ is_new_paragraph = True
+
+ if is_new_paragraph:
+ current_layout = char_layout
+ current_paragraph = PdfParagraph(
+ pdf_paragraph_composition=[],
+ layout_id=current_layout.id,
+ debug_id=generate_base58_id(),
+ layout_label=current_layout.name,
+ )
+ paragraphs.append(current_paragraph)
+
+ current_paragraph.pdf_paragraph_composition.append(
+ PdfParagraphComposition(pdf_character=char)
+ )
+
+ page.pdf_character = skip_chars
+ for para in paragraphs:
+ self.update_paragraph_data(para)
+ return paragraphs
+
+ def _merge_overlapping_clusters(
+ self, lines: dict[int, list[PdfCharacter]], char_height_average: float
+ ) -> dict[int, list[PdfCharacter]]:
+ """
+ Merge clusters that have significant y-axis overlap.
+ If y_intersection / min_height > 0.5 or the distance between y-midlines is less than char_height_average, merge the two clusters.
+ """
+ if len(lines) <= 1:
+ return lines
+
+ # Calculate y-axis ranges for each cluster
+ cluster_ranges = {}
+ cluster_midlines = {}
+ for label, chars in lines.items():
+ y_values = [char.visual_bbox.box.y for char in chars] + [
+ char.visual_bbox.box.y2 for char in chars
+ ]
+ y_min, y_max = min(y_values), max(y_values)
+ cluster_ranges[label] = (y_min, y_max)
+ cluster_midlines[label] = (y_min + y_max) / 2
+
+ # Keep merging until no more merges are possible
+ changed = True
+ while changed:
+ changed = False
+ labels_to_check = list(lines.keys())
+
+ for i in range(len(labels_to_check)):
+ if not changed: # Only continue if no merge happened in this iteration
+ for j in range(i + 1, len(labels_to_check)):
+ label1, label2 = labels_to_check[i], labels_to_check[j]
+
+ # Skip if either label has been merged away
+ if label1 not in lines or label2 not in lines:
+ continue
+
+ y1_min, y1_max = cluster_ranges[label1]
+ y2_min, y2_max = cluster_ranges[label2]
+
+ # Calculate intersection
+ intersection_start = max(y1_min, y2_min)
+ intersection_end = min(y1_max, y2_max)
+
+ # Calculate midline distance
+ midline_distance = abs(
+ cluster_midlines[label1] - cluster_midlines[label2]
+ )
+
+ should_merge = False
+ if (
+ intersection_end > intersection_start
+ ): # There is intersection
+ intersection_height = intersection_end - intersection_start
+ height1 = y1_max - y1_min
+ height2 = y2_max - y2_min
+ min_height = min(height1, height2)
+
+ # Check if intersection ratio exceeds threshold
+ if (
+ min_height > 0
+ and intersection_height / min_height > 0.3
+ ):
+ should_merge = True
+
+ # Check if midline distance is less than char_height_average
+ if midline_distance < char_height_average:
+ should_merge = True
+
+ if should_merge:
+ # Merge label2 into label1
+ lines[label1].extend(lines[label2])
+ del lines[label2]
+
+ # Update cluster range and midline for the merged cluster
+ new_y_min = min(y1_min, y2_min)
+ new_y_max = max(y1_max, y2_max)
+ cluster_ranges[label1] = (new_y_min, new_y_max)
+ cluster_midlines[label1] = (new_y_min + new_y_max) / 2
+ del cluster_ranges[label2]
+ del cluster_midlines[label2]
+
+ changed = True
+ break
+
+ return lines
+
+ def _get_effective_y_bounds(self, char: PdfCharacter) -> tuple[float, float]:
+ """
+ Determines the effective vertical boundaries (y1, y2) for a character.
+
+ It prioritizes the visual bounding box if its Intersection over Union (IoU)
+ with the PDF bounding box is high (>= 0.5), otherwise, it falls back to the
+ PDF bounding box. This helps use more accurate layout information when available.
+ """
+ visual_box = char.visual_bbox.box
+ return visual_box.y, visual_box.y2
+ pdf_box = char.box
+ if calculate_iou_for_boxes(visual_box, pdf_box) >= 0.5:
+ return visual_box.y, visual_box.y2
+ return pdf_box.y, pdf_box.y2
+
+ @staticmethod
+ def _compute_collision_counts_histogram(
+ y1_arr: np.ndarray,
+ y2_arr: np.ndarray,
+ para_y_min: float,
+ para_y_max: float,
+ step: float,
+ ) -> np.ndarray:
+ """Compute overlap counts at each scan line using a difference-array histogram.
+
+ Args:
+ y1_arr: 1-D array with lower y bounds of characters (inclusive).
+ y2_arr: 1-D array with upper y bounds of characters (exclusive).
+ para_y_min: Minimum y of the paragraph.
+ para_y_max: Maximum y of the paragraph.
+ step: Scan step size.
+
+ Returns:
+ 1-D NumPy int32 array where index i corresponds to y = para_y_max - i × step.
+ """
+ # Number of scan positions
+ m = int(np.ceil((para_y_max - para_y_min) / step))
+ if m <= 0:
+ return np.array([], dtype=np.int32)
+
+ # Map character bounds to discrete indices (top inclusive, bottom exclusive)
+ starts = np.floor((para_y_max - y2_arr) / step).astype(np.int32)
+ ends = np.floor((para_y_max - y1_arr) / step).astype(np.int32) + 1
+ # Clip ends to the valid range [0, m]
+ np.clip(ends, 0, m, out=ends)
+
+ hist = np.zeros(m + 1, dtype=np.int32)
+ np.add.at(hist, starts, 1)
+ np.add.at(hist, ends, -1)
+
+ return np.cumsum(hist[:-1])
+
+ def _split_paragraph_into_lines(
+ self, paragraph: PdfParagraph, formula_font_ids: set[str]
+ ):
+ """
+ Splits a paragraph into lines using a "line-threading" method.
+
+ This method works by scanning vertically across the paragraph's bounding
+ box and counting how many characters intersect with a horizontal line
+ at each y-coordinate. The regions with a low number of intersections
+ (less than 2) are identified as gaps between lines. The characters
+ are then partitioned into lines based on these identified gaps.
+ """
+ if not paragraph.pdf_paragraph_composition:
+ return
+
+ # 1. Extract all characters and other compositions from the paragraph.
+ all_chars: list[PdfCharacter] = []
+ other_compositions: list[PdfParagraphComposition] = []
+ for comp in paragraph.pdf_paragraph_composition:
+ if comp.pdf_character:
+ all_chars.append(comp.pdf_character)
+ else:
+ other_compositions.append(comp)
+
+ if not all_chars:
+ return
+
+ # 2. Determine effective y-bounds for each character and the paragraph's total vertical range.
+ char_y_bounds = [
+ {"char": char, "y1": y1, "y2": y2}
+ for char in all_chars
+ for y1, y2 in [self._get_effective_y_bounds(char)]
+ ]
+
+ if not char_y_bounds:
+ paragraph.pdf_paragraph_composition = other_compositions
+ self.update_paragraph_data(paragraph)
+ return
+
+ para_y_min = min(b["y1"] for b in char_y_bounds)
+ para_y_max = max(b["y2"] for b in char_y_bounds)
+
+ # If the paragraph is vertically flat, treat it as a single line.
+ if (para_y_max - para_y_min) < 5: # Using a small threshold
+ # all_chars.sort(key=lambda c: c.visual_bbox.box.x)
+ single_line_composition = self.create_line(all_chars)
+ paragraph.pdf_paragraph_composition = [
+ single_line_composition
+ ] + other_compositions
+ self.update_paragraph_data(paragraph)
+ return
+
+ # 3. Perform "threading" scan to create a collision histogram.
+ # Scan from top (max y) to bottom (min y) with a step of 0.5.
+ scan_y_min = para_y_min
+ scan_y_max = para_y_max
+ step = 0.25
+
+ y_coordinates = np.arange(scan_y_max, scan_y_min, -step)
+
+ # Compute collision counts using NumPy histogram (O(m + n))
+ y1_arr = np.array([b["y1"] for b in char_y_bounds], dtype=np.float32)
+ y2_arr = np.array([b["y2"] for b in char_y_bounds], dtype=np.float32)
+ collision_counts = self._compute_collision_counts_histogram(
+ y1_arr,
+ y2_arr,
+ scan_y_min,
+ scan_y_max,
+ step,
+ )
+
+ # 4. Find gaps (regions with low collision count) from the histogram.
+ gaps = []
+ in_gap = False
+ for i, count in enumerate(collision_counts):
+ if count < 1 and not in_gap:
+ in_gap = True
+ gap_start_index = i
+ elif count >= 1 and in_gap:
+ in_gap = False
+ gaps.append((gap_start_index, i - 1))
+ if in_gap:
+ gaps.append((gap_start_index, len(collision_counts) - 1))
+
+ # If no significant gaps are found, treat it as a single line.
+ if not gaps:
+ # all_chars.sort(key=lambda c: c.visual_bbox.box.x)
+ single_line_composition = self.create_line(all_chars)
+ paragraph.pdf_paragraph_composition = [
+ single_line_composition
+ ] + other_compositions
+ self.update_paragraph_data(paragraph)
+ return
+
+ # 5. Assign characters to lines based on the identified gaps.
+ # Calculate separator y-coordinates from the midpoints of the gaps.
+ separator_y_coords = sorted(
+ [y_coordinates[start_idx] for start_idx, end_idx in gaps],
+ reverse=True,
+ )
+
+ lines: list[list[PdfCharacter]] = [
+ [] for _ in range(len(separator_y_coords) + 1)
+ ]
+
+ for b in char_y_bounds:
+ char_y_center = (b["y1"] + b["y2"]) / 2
+ line_idx = 0
+ # Find which line bucket the character belongs to.
+ for sep_y in separator_y_coords:
+ if char_y_center > sep_y:
+ break
+ line_idx += 1
+ lines[line_idx].append(b["char"])
+
+ # 6. Rebuild the paragraph's composition list from the new lines.
+ new_line_compositions = []
+ for line_chars in lines:
+ if line_chars:
+ # Sort characters within each line by x-coordinate (left-to-right).
+ # line_chars.sort(key=lambda c: c.visual_bbox.box.x)
+ new_line_compositions.append(self.create_line(line_chars))
+
+ # The lines are already sorted vertically due to the scanning process.
+ paragraph.pdf_paragraph_composition = new_line_compositions + other_compositions
+ self.update_paragraph_data(paragraph)
+
+ def process_paragraph_spacing(self, paragraph: PdfParagraph):
+ if not paragraph.pdf_paragraph_composition:
+ return
+
+ # 处ç†è¡Œçº§åˆ«çš„ç©ºæ ¼
+ processed_lines = []
+ for composition in paragraph.pdf_paragraph_composition:
+ if not composition.pdf_line:
+ processed_lines.append(composition)
+ continue
+
+ line = composition.pdf_line
+ if not "".join(
+ x.char_unicode for x in line.pdf_character
+ ).strip(): # 跳过完全空白的行
+ continue
+
+ # 处ç†è¡Œå†…å—符的尾éšç©ºæ ¼
+ processed_chars = []
+ for char in line.pdf_character:
+ if not char.char_unicode.isspace():
+ processed_chars = processed_chars + [char]
+ elif processed_chars: # åªæœ‰åœ¨æœ‰éžç©ºæ ¼å—符åŽæ‰è€ƒè™‘ä¿ç•™ç©ºæ ¼
+ processed_chars.append(char)
+
+ # 移除尾éšç©ºæ ¼
+ while processed_chars and processed_chars[-1].char_unicode.isspace():
+ processed_chars.pop()
+
+ if processed_chars: # 如果行内还有å—符
+ line = self.create_line(processed_chars)
+ processed_lines.append(line)
+
+ paragraph.pdf_paragraph_composition = processed_lines
+ self.update_paragraph_data(paragraph)
+
+ def create_line(self, chars: list[PdfCharacter]) -> PdfParagraphComposition:
+ assert chars
+
+ line = PdfLine(pdf_character=chars)
+ self.update_line_data(line)
+ return PdfParagraphComposition(pdf_line=line)
+
+ def calculate_median_line_width(self, paragraphs: list[PdfParagraph]) -> float:
+ # 收集所有行的宽度
+ line_widths = []
+ for paragraph in paragraphs:
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_line:
+ line = composition.pdf_line
+ line_widths.append(line.box.x2 - line.box.x)
+
+ if not line_widths:
+ return 0.0
+
+ # 计算ä¸ä½æ•°
+ line_widths.sort()
+ mid = len(line_widths) // 2
+ if len(line_widths) % 2 == 0:
+ return (line_widths[mid - 1] + line_widths[mid]) / 2
+ return line_widths[mid]
+
+ def process_independent_paragraphs(
+ self,
+ paragraphs: list[PdfParagraph],
+ median_width: float,
+ ):
+ i = 0
+ while i < len(paragraphs):
+ paragraph = paragraphs[i]
+ if len(paragraph.pdf_paragraph_composition) <= 1: # 跳过åªæœ‰ä¸€è¡Œçš„段è½
+ i += 1
+ continue
+
+ j = 1
+ while j < len(paragraph.pdf_paragraph_composition):
+ prev_composition = paragraph.pdf_paragraph_composition[j - 1]
+ if not prev_composition.pdf_line:
+ j += 1
+ continue
+
+ prev_line = prev_composition.pdf_line
+ prev_width = prev_line.box.x2 - prev_line.box.x
+ prev_text = "".join([c.char_unicode for c in prev_line.pdf_character])
+
+ # 检查是å¦åŒ…å«è¿žç»çš„点（至少 20 个）
+ # å¦‚æžœæœ‰è‡³å°‘è¿žç» 20 个点，则代表这是目录æ¡ç›®
+ if re.search(r"\.{20,}", prev_text):
+ # 创建新的段è½
+ new_paragraph = PdfParagraph(
+ box=Box(0, 0, 0, 0), # 临时边界框
+ pdf_paragraph_composition=(
+ paragraph.pdf_paragraph_composition[j:]
+ ),
+ unicode="",
+ debug_id=generate_base58_id(),
+ layout_label=paragraph.layout_label,
+ layout_id=paragraph.layout_id,
+ )
+ # 更新原段è½
+ paragraph.pdf_paragraph_composition = (
+ paragraph.pdf_paragraph_composition[:j]
+ )
+
+ # 更新两个段è½çš„æ•°æ®
+ self.update_paragraph_data(paragraph)
+ self.update_paragraph_data(new_paragraph)
+
+ # 在原段è½åŽæ’入新段è½
+ paragraphs.insert(i + 1, new_paragraph)
+ break
+
+ # 如果å‰ä¸€è¡Œå®½åº¦å°äºŽä¸ä½æ•°çš„一åŠï¼Œå°†å½“å‰è¡ŒåŠåŽç»è¡Œåˆ†å‰²æˆæ–°æ®µè½
+ if (
+ self.translation_config.split_short_lines
+ and prev_width
+ < median_width * self.translation_config.short_line_split_factor
+ ) or (
+ paragraph.pdf_paragraph_composition
+ and (current_line := paragraph.pdf_paragraph_composition[j])
+ and (line := current_line.pdf_line)
+ and (chars := line.pdf_character)
+ and is_bullet_or_list_marker(chars)
+ ):
+ # 创建新的段è½
+ new_paragraph = PdfParagraph(
+ box=Box(0, 0, 0, 0), # 临时边界框
+ pdf_paragraph_composition=(
+ paragraph.pdf_paragraph_composition[j:]
+ ),
+ unicode="",
+ debug_id=generate_base58_id(),
+ layout_label=paragraph.layout_label,
+ layout_id=paragraph.layout_id,
+ )
+ # 更新原段è½
+ paragraph.pdf_paragraph_composition = (
+ paragraph.pdf_paragraph_composition[:j]
+ )
+
+ # 更新两个段è½çš„æ•°æ®
+ self.update_paragraph_data(paragraph)
+ self.update_paragraph_data(new_paragraph)
+
+ # 在原段è½åŽæ’入新段è½
+ paragraphs.insert(i + 1, new_paragraph)
+ break
+ j += 1
+ i += 1
+
+ @staticmethod
+ def is_bbox_contain_in_vertical(bbox1: Box, bbox2: Box) -> bool:
+ """Check if one bounding box is completely contained within the other."""
+ # Check if bbox1 is contained in bbox2
+ bbox1_in_bbox2 = bbox1.y >= bbox2.y and bbox1.y2 <= bbox2.y2
+ # Check if bbox2 is contained in bbox1
+ bbox2_in_bbox1 = bbox2.y >= bbox1.y and bbox2.y2 <= bbox1.y2
+ return bbox1_in_bbox2 or bbox2_in_bbox1
+
+ def fix_overlapping_paragraphs(self, page: Page):
+ """
+ Adjusts the bounding boxes of paragraphs on a page to resolve vertical overlaps.
+
+ Iteratively checks pairs of paragraphs and adjusts their vertical boundaries
+ (y and y2) if they overlap, aiming to place the boundary at the midpoint
+ of the vertical overlap.
+ """
+ paragraphs = page.pdf_paragraph
+ if not paragraphs or len(paragraphs) < 2:
+ return
+
+ max_iterations = len(paragraphs) * len(paragraphs) # Safety break
+ iterations = 0
+
+ while iterations < max_iterations:
+ iterations += 1
+ overlap_found_in_pass = False
+
+ for i in range(len(paragraphs)):
+ for j in range(i + 1, len(paragraphs)):
+ para1 = paragraphs[i]
+ para2 = paragraphs[j]
+
+ if para1.box is None or para2.box is None:
+ continue
+
+ if para1.xobj_id != para2.xobj_id:
+ continue
+
+ # Check for overlap using the existing method
+ if self.bbox_overlap(para1.box, para2.box):
+ if self.is_bbox_contain_in_vertical(para1.box, para2.box):
+ continue
+ # Calculate vertical overlap details
+ overlap_y_start = max(para1.box.y, para2.box.y)
+ overlap_y_end = min(para1.box.y2, para2.box.y2)
+ overlap_height = overlap_y_end - overlap_y_start
+
+ # Calculate horizontal overlap details
+ overlap_x_start = max(para1.box.x, para2.box.x)
+ overlap_x_end = min(para1.box.x2, para2.box.x2)
+ overlap_width = overlap_x_end - overlap_x_start
+
+ # Ensure there's a real 2D overlap, focusing on vertical adjustment
+ if overlap_height > 1e-6 and overlap_width > 1e-6:
+ overlap_found_in_pass = True
+
+ # Determine which paragraph is visually higher
+ if para1.box.y2 > para2.box.y and para1.box.y < para2.box.y:
+ lower_para = para1
+ higher_para = para2
+ # Handle cases where y values are identical (or very close)
+ # Prefer the one with smaller y2 as the higher one, or break tie arbitrarily
+ elif para1.box.y2 < para2.box.y2:
+ lower_para = para1
+ higher_para = para2
+ else:
+ lower_para = para2
+ higher_para = para1
+
+ # Calculate the midpoint of the vertical overlap
+ mid_y = overlap_y_start + overlap_height / 2
+
+ # Adjust boxes, ensuring they remain valid (y2 > y)
+ if mid_y > higher_para.box.y and mid_y < lower_para.box.y2:
+ higher_para.box.y = mid_y + 1
+ lower_para.box.y2 = mid_y - 1
+ else:
+ # This might happen if one box is fully contained vertically
+ # within another, or due to floating point issues.
+ # Log a warning and skip adjustment for this pair in this iteration.
+ # A more complex strategy might be needed for full containment.
+ logger.warning(
+ "Could not resolve overlap between paragraphs"
+ f" {higher_para.debug_id} and {lower_para.debug_id}"
+ " using simple midpoint strategy."
+ f" Midpoint: {mid_y},"
+ f" Higher Box: {higher_para.box},"
+ f" Lower Box: {lower_para.box}"
+ )
+
+ # If no overlaps were found and adjusted in this pass, we're done.
+ if not overlap_found_in_pass:
+ break
+
+ if iterations == max_iterations:
+ logger.warning(
+ f"Maximum iterations ({max_iterations}) reached in"
+ f" fix_overlapping_paragraphs for page {page.page_number}."
+ " Some overlaps might remain."
+ )
+
+ def _sort_characters_in_lines(self, page: Page):
+ """Sort characters in each line from left to right, top to bottom."""
+ for paragraph in page.pdf_paragraph:
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_line:
+ line = composition.pdf_line
+ line.pdf_character.sort(key=self._get_char_sort_key)
+
+ def _get_char_sort_key(self, char: PdfCharacter):
+ """Get sort key for character positioning (top to bottom, left to right)."""
+ visual_box = char.visual_bbox.box
+ pdf_box = char.box
+
+ # Use visual box if IoU with bbox is >= 0.1, otherwise use bbox
+ if calculate_iou_for_boxes(visual_box, pdf_box) >= 0.1:
+ box = visual_box
+ else:
+ box = pdf_box
+
+ # Sort by y coordinate first (top to bottom), then x coordinate (left to right)
+ # Note: In PDF coordinate system, y increases upward, so we negate y for top-to-bottom sorting
+ return (box.x, -box.y)
\ No newline at end of file
diff --git a/babeldoc/format/pdf/document_il/midend/remove_descent.py b/babeldoc/format/pdf/document_il/midend/remove_descent.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d68c603515f94cd69972a8fd75451e21cc608f5
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/remove_descent.py
@@ -0,0 +1,168 @@
+import logging
+from collections import Counter
+from functools import cache
+
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.translation_config import TranslationConfig
+
+logger = logging.getLogger(__name__)
+
+
+class RemoveDescent:
+ stage_name = "Remove Char Descent"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.translation_config = translation_config
+
+ def _remove_char_descent(
+ self,
+ char: il_version_1.PdfCharacter,
+ font: il_version_1.PdfFont,
+ ) -> float | None:
+ """Remove descent from a single character and return the descent value.
+
+ Args:
+ char: The character to process
+ font: The font used by this character
+
+ Returns:
+ The descent value if it was removed, None otherwise
+ """
+ if (
+ char.box
+ and char.box.y is not None
+ and char.box.y2 is not None
+ and font
+ and hasattr(font, "descent")
+ ):
+ descent = font.descent * char.pdf_style.font_size / 1000
+ if char.vertical:
+ # For vertical text, remove descent from x coordinates
+ char.box.x += descent
+ char.box.x2 += descent
+ else:
+ # For horizontal text, remove descent from y coordinates
+ char.box.y -= descent
+ char.box.y2 -= descent
+ return descent
+ return None
+
+ def process(self, document: il_version_1.Document):
+ """Process the document to remove descent adjustments from character boxes.
+
+ Args:
+ document: The document to process
+ """
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(document.page),
+ ) as pbar:
+ for page in document.page:
+ self.translation_config.raise_if_cancelled()
+ self.process_page(page)
+ pbar.advance()
+
+ def process_page(self, page: il_version_1.Page):
+ """Process a single page to remove descent adjustments.
+
+ Args:
+ page: The page to process
+ """
+ # Build font map including xobjects
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font}
+ page_fonts = {f.font_id: f for f in page.pdf_font}
+
+ # Add xobject fonts
+ for xobj in page.pdf_xobject:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ fonts[xobj.xobj_id][font.font_id] = font
+
+ @cache
+ def get_font(
+ font_id: str,
+ xobj_id: int | None = None,
+ ) -> il_version_1.PdfFont | None:
+ if xobj_id is not None and xobj_id in fonts:
+ font_map = fonts[xobj_id]
+ if isinstance(font_map, dict) and font_id in font_map:
+ return font_map[font_id]
+ return (
+ fonts.get(font_id)
+ if isinstance(fonts.get(font_id), il_version_1.PdfFont)
+ else None
+ )
+
+ # Process all standalone characters in the page
+ for char in page.pdf_character:
+ if font := get_font(char.pdf_style.font_id, char.xobj_id):
+ self._remove_char_descent(char, font)
+
+ # Process all paragraphs
+ for paragraph in page.pdf_paragraph:
+ descent_values = []
+ vertical_chars = []
+
+ # Process all characters in paragraph compositions
+ for comp in paragraph.pdf_paragraph_composition:
+ # Handle direct characters
+ if comp.pdf_character:
+ font = get_font(
+ comp.pdf_character.pdf_style.font_id,
+ comp.pdf_character.xobj_id,
+ )
+ if font:
+ descent = self._remove_char_descent(comp.pdf_character, font)
+ if descent is not None:
+ descent_values.append(descent)
+ vertical_chars.append(comp.pdf_character.vertical)
+
+ # Handle characters in PdfLine
+ elif comp.pdf_line:
+ for char in comp.pdf_line.pdf_character:
+ if font := get_font(char.pdf_style.font_id, char.xobj_id):
+ descent = self._remove_char_descent(char, font)
+ if descent is not None:
+ descent_values.append(descent)
+ vertical_chars.append(char.vertical)
+
+ # Handle characters in PdfFormula
+ elif comp.pdf_formula:
+ for char in comp.pdf_formula.pdf_character:
+ if font := get_font(char.pdf_style.font_id, char.xobj_id):
+ descent = self._remove_char_descent(char, font)
+ if descent is not None:
+ descent_values.append(descent)
+ vertical_chars.append(char.vertical)
+
+ # Handle characters in PdfSameStyleCharacters
+ elif comp.pdf_same_style_characters:
+ for char in comp.pdf_same_style_characters.pdf_character:
+ if font := get_font(char.pdf_style.font_id, char.xobj_id):
+ descent = self._remove_char_descent(char, font)
+ if descent is not None:
+ descent_values.append(descent)
+ vertical_chars.append(char.vertical)
+
+ # Adjust paragraph box based on most common descent value
+ if descent_values and paragraph.box:
+ # Calculate mode of descent values
+ descent_counter = Counter(descent_values)
+ most_common_descent = descent_counter.most_common(1)[0][0]
+
+ # Check if paragraph is vertical (all characters are vertical)
+ is_vertical = all(vertical_chars) if vertical_chars else False
+
+ # Adjust paragraph box
+ if paragraph.box.y is not None and paragraph.box.y2 is not None:
+ if is_vertical:
+ # For vertical paragraphs, adjust x coordinates
+ paragraph.box.x += most_common_descent
+ paragraph.box.x2 += most_common_descent
+ else:
+ # For horizontal paragraphs, adjust y coordinates
+ paragraph.box.y -= most_common_descent
+ paragraph.box.y2 -= most_common_descent
diff --git a/babeldoc/format/pdf/document_il/midend/styles_and_formulas.py b/babeldoc/format/pdf/document_il/midend/styles_and_formulas.py
new file mode 100644
index 0000000000000000000000000000000000000000..abbf07a64944b7ce7fc61eeb0d0831a966395fbb
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/styles_and_formulas.py
@@ -0,0 +1,1292 @@
+import math
+import re
+
+from babeldoc.format.pdf.document_il.il_version_1 import Box
+from babeldoc.format.pdf.document_il.il_version_1 import Document
+from babeldoc.format.pdf.document_il.il_version_1 import GraphicState
+from babeldoc.format.pdf.document_il.il_version_1 import Page
+from babeldoc.format.pdf.document_il.il_version_1 import PdfCharacter
+from babeldoc.format.pdf.document_il.il_version_1 import PdfFormula
+from babeldoc.format.pdf.document_il.il_version_1 import PdfLine
+from babeldoc.format.pdf.document_il.il_version_1 import PdfParagraphComposition
+from babeldoc.format.pdf.document_il.il_version_1 import PdfSameStyleCharacters
+from babeldoc.format.pdf.document_il.il_version_1 import PdfStyle
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.formular_helper import (
+ collect_page_formula_font_ids,
+)
+from babeldoc.format.pdf.document_il.utils.formular_helper import (
+ is_formulas_middle_char,
+)
+from babeldoc.format.pdf.document_il.utils.formular_helper import is_formulas_start_char
+from babeldoc.format.pdf.document_il.utils.formular_helper import update_formula_data
+from babeldoc.format.pdf.document_il.utils.layout_helper import LEFT_BRACKET
+from babeldoc.format.pdf.document_il.utils.layout_helper import RIGHT_BRACKET
+from babeldoc.format.pdf.document_il.utils.layout_helper import build_layout_index
+from babeldoc.format.pdf.document_il.utils.layout_helper import calculate_iou_for_boxes
+from babeldoc.format.pdf.document_il.utils.layout_helper import (
+ calculate_y_true_iou_for_boxes,
+)
+from babeldoc.format.pdf.document_il.utils.layout_helper import is_bullet_point
+from babeldoc.format.pdf.document_il.utils.layout_helper import (
+ is_curve_in_figure_table_layout,
+)
+from babeldoc.format.pdf.document_il.utils.layout_helper import (
+ is_curve_overlapping_with_paragraphs,
+)
+from babeldoc.format.pdf.document_il.utils.layout_helper import is_same_style
+from babeldoc.format.pdf.document_il.utils.spatial_analyzer import (
+ is_element_contained_in_formula,
+)
+from babeldoc.format.pdf.translation_config import TranslationConfig
+
+
+class StylesAndFormulas:
+ stage_name = "Parse Formulas and Styles"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.detailed_logger = None
+ self.translation_config = translation_config
+ self.font_mapper = FontMapper(translation_config)
+
+ def update_formula_data(self, formula: PdfFormula):
+ update_formula_data(formula)
+
+ def process(self, document: Document):
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Formula and Style Detection Started")
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(document.page),
+ ) as pbar:
+ for page in document.page:
+ self.translation_config.raise_if_cancelled()
+ self.process_page(page)
+ pbar.advance()
+
+ if self.detailed_logger:
+ formula_count = sum(
+ sum(1 for comp in para.pdf_paragraph_composition if hasattr(comp, 'pdf_formula') and comp.pdf_formula)
+ for page in document.page
+ for para in page.pdf_paragraph
+ if hasattr(page, 'pdf_paragraph')
+ )
+
+ self.detailed_logger.log_step(
+ "Formula and Style Detection Complete",
+ f"Total formulas detected: {formula_count}"
+ )
+
+ def update_all_formula_data(self, page: Page):
+ for para in page.pdf_paragraph:
+ for comp in para.pdf_paragraph_composition:
+ if comp.pdf_formula:
+ self.update_formula_data(comp.pdf_formula)
+
+ def _calculate_element_formula_iou(
+ self, element_box: Box, formula_box: Box, tolerance: float = 2.0
+ ) -> float:
+ """Calculate precise IoU between an element and a formula with tolerance.
+
+ Args:
+ element_box: Bounding box of the element (curve/form)
+ formula_box: Bounding box of the formula
+ tolerance: Tolerance to expand formula box for containment check
+
+ Returns:
+ IoU value between element and expanded formula box
+ """
+ if element_box is None or formula_box is None:
+ return 0.0
+
+ # Expand formula box by tolerance for more lenient containment check
+ expanded_formula_box = Box(
+ x=formula_box.x - tolerance,
+ y=formula_box.y - tolerance,
+ x2=formula_box.x2 + tolerance,
+ y2=formula_box.y2 + tolerance,
+ )
+
+ return calculate_iou_for_boxes(element_box, expanded_formula_box)
+
+ def _is_element_contained_exact(
+ self,
+ element_box: Box,
+ formula_box: Box,
+ containment_threshold: float = 0.95,
+ ) -> bool:
+ """Check if an element is contained within a formula with zero tolerance.
+
+ Args:
+ element_box: Bounding box of the element (curve/form)
+ formula_box: Bounding box of the formula
+ containment_threshold: Minimum IoU ratio to consider as contained
+
+ Returns:
+ True if the element is contained within the formula (exact match)
+ """
+ if element_box is None or formula_box is None:
+ return False
+
+ # Use formula box without any tolerance expansion
+ iou = calculate_iou_for_boxes(element_box, formula_box)
+ return iou >= containment_threshold
+
+ def _calculate_element_formula_distance(
+ self, element_box: Box, formula_box: Box
+ ) -> float:
+ """Calculate the shortest distance between an element and a formula.
+
+ Args:
+ element_box: Bounding box of the element (curve/form)
+ formula_box: Bounding box of the formula
+
+ Returns:
+ Shortest distance between the element and formula boxes
+ """
+ if element_box is None or formula_box is None:
+ return float("inf")
+
+ # Calculate horizontal distance
+ if element_box.x2 < formula_box.x:
+ # Element is to the left of formula
+ dx = formula_box.x - element_box.x2
+ elif element_box.x > formula_box.x2:
+ # Element is to the right of formula
+ dx = element_box.x - formula_box.x2
+ else:
+ # Horizontal overlap
+ dx = 0.0
+
+ # Calculate vertical distance
+ if element_box.y2 < formula_box.y:
+ # Element is above formula
+ dy = formula_box.y - element_box.y2
+ elif element_box.y > formula_box.y2:
+ # Element is below formula
+ dy = element_box.y - formula_box.y2
+ else:
+ # Vertical overlap
+ dy = 0.0
+
+ # Return Euclidean distance
+ return (dx * dx + dy * dy) ** 0.5
+
+ def _collect_element_formula_candidates(
+ self, page: Page
+ ) -> tuple[list, dict, dict]:
+ """Collect all potential assignments of elements to formulas.
+
+ Uses two-level IoU matching strategy:
+ 1. Exact IoU matching (zero tolerance) - highest priority
+ 2. Tolerant IoU matching (2.0 tolerance, distance-sorted) - second priority
+
+ Returns:
+ Tuple of (all_formulas, curve_candidates, form_candidates) where:
+ - all_formulas: list of (formula, paragraph_xobj_id) tuples
+ - curve_candidates: dict mapping curve index to (curve, candidates) tuples
+ - form_candidates: dict mapping form index to (form, candidates) tuples
+ where candidates is a list of (formula_index, score, match_type) tuples
+ """
+ curve_candidates = {}
+ form_candidates = {}
+
+ # Configuration parameters
+ max_tolerant_distance = 100.0 # Maximum distance for tolerant matching scoring
+
+ if not page.pdf_paragraph:
+ return [], curve_candidates, form_candidates
+
+ # Collect all formulas from all paragraphs with their index
+ all_formulas = []
+ for paragraph in page.pdf_paragraph:
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_formula:
+ all_formulas.append((composition.pdf_formula, paragraph.xobj_id))
+
+ # Check each curve against all formulas
+ for curve_idx, curve in enumerate(page.pdf_curve):
+ if not curve.box:
+ continue
+
+ candidates = []
+ for formula_idx, (formula, paragraph_xobj_id) in enumerate(all_formulas):
+ if not formula.box:
+ continue
+
+ # Check xobj_id compatibility
+ if paragraph_xobj_id is not None and curve.xobj_id != paragraph_xobj_id:
+ continue
+
+ # Level 1: Exact IoU matching (zero tolerance) - highest priority
+ if self._is_element_contained_exact(curve.box, formula.box):
+ iou = calculate_iou_for_boxes(curve.box, formula.box)
+ candidates.append((formula_idx, iou, "iou_exact"))
+ # Level 2: Tolerant IoU matching (with tolerance) - distance sorted
+ elif is_element_contained_in_formula(curve.box, formula.box):
+ distance = self._calculate_element_formula_distance(
+ curve.box, formula.box
+ )
+ # Convert distance to score (closer = higher score)
+ # Score range: 0.5-0.9 to ensure lower than exact IoU
+ distance_factor = max(0.0, 1.0 - distance / max_tolerant_distance)
+ score = 0.5 + 0.4 * distance_factor
+ candidates.append((formula_idx, score, "iou_tolerant"))
+
+ if candidates:
+ curve_candidates[curve_idx] = (curve, candidates)
+
+ # Check each form against all formulas
+ for form_idx, form in enumerate(page.pdf_form):
+ if not form.box:
+ continue
+
+ candidates = []
+ for formula_idx, (formula, paragraph_xobj_id) in enumerate(all_formulas):
+ if not formula.box:
+ continue
+
+ # Check xobj_id compatibility
+ if paragraph_xobj_id is not None and form.xobj_id != paragraph_xobj_id:
+ continue
+
+ # Level 1: Exact IoU matching (zero tolerance) - highest priority
+ if self._is_element_contained_exact(form.box, formula.box):
+ iou = calculate_iou_for_boxes(form.box, formula.box)
+ candidates.append((formula_idx, iou, "iou_exact"))
+ # Level 2: Tolerant IoU matching (with tolerance) - distance sorted
+ elif is_element_contained_in_formula(form.box, formula.box):
+ distance = self._calculate_element_formula_distance(
+ form.box, formula.box
+ )
+ # Convert distance to score (closer = higher score)
+ # Score range: 0.5-0.9 to ensure lower than exact IoU
+ distance_factor = max(0.0, 1.0 - distance / max_tolerant_distance)
+ score = 0.5 + 0.4 * distance_factor
+ candidates.append((formula_idx, score, "iou_tolerant"))
+
+ if candidates:
+ form_candidates[form_idx] = (form, candidates)
+
+ return all_formulas, curve_candidates, form_candidates
+
+ def _resolve_assignment_conflicts(
+ self, curve_candidates: dict, form_candidates: dict
+ ) -> tuple[dict, list, list]:
+ """Resolve assignment conflicts using prioritized matching strategy.
+
+ Args:
+ curve_candidates: dict mapping curve index to (curve, candidates) tuples
+ form_candidates: dict mapping form index to (form, candidates) tuples
+ where candidates is a list of (formula_index, score, match_type) tuples
+
+ Returns:
+ Tuple of (formula_assignments, curves_to_remove, forms_to_remove) where:
+ - formula_assignments: dict mapping formula_index to (curves, forms) tuples
+ - curves_to_remove: list of curves to remove from page level
+ - forms_to_remove: list of forms to remove from page level
+ """
+ formula_assignments = {}
+ curves_to_remove = []
+ forms_to_remove = []
+
+ def _get_best_candidate(candidates):
+ """Get the best candidate using priority: Exact IoU > Tolerant IoU, then by score."""
+ if not candidates:
+ return None
+
+ # Sort by match_type priority and then by score (descending)
+ def sort_key(candidate):
+ formula_idx, score, match_type = candidate
+ # Exact IoU matches get priority 1, tolerant IoU matches get priority 2
+ priority = 1 if match_type == "iou_exact" else 2
+ # Return tuple for sorting: (priority, -score) for descending score within priority
+ return (priority, -score)
+
+ sorted_candidates = sorted(candidates, key=sort_key)
+ return sorted_candidates[0]
+
+ # Resolve curve assignments
+ for _curve_idx, (curve, candidates) in curve_candidates.items():
+ if not candidates:
+ continue
+
+ best_candidate = _get_best_candidate(candidates)
+ if best_candidate:
+ best_formula_idx, best_score, match_type = best_candidate
+
+ # Add to assignments
+ if best_formula_idx not in formula_assignments:
+ formula_assignments[best_formula_idx] = ([], [])
+ formula_assignments[best_formula_idx][0].append(curve)
+ curves_to_remove.append(curve)
+
+ # Resolve form assignments
+ for _form_idx, (form, candidates) in form_candidates.items():
+ if not candidates:
+ continue
+
+ best_candidate = _get_best_candidate(candidates)
+ if best_candidate:
+ best_formula_idx, best_score, match_type = best_candidate
+
+ # Add to assignments
+ if best_formula_idx not in formula_assignments:
+ formula_assignments[best_formula_idx] = ([], [])
+ formula_assignments[best_formula_idx][1].append(form)
+ forms_to_remove.append(form)
+
+ return formula_assignments, curves_to_remove, forms_to_remove
+
+ def collect_contained_elements(self, page: Page):
+ """Collect curves and forms that are contained within formulas.
+
+ Uses two-phase assignment strategy to ensure each element is assigned
+ to only one formula based on highest IoU value.
+ """
+ if not page.pdf_paragraph:
+ return
+
+ # Phase 1: Collect all potential element-formula assignments
+ all_formulas, curve_candidates, form_candidates = (
+ self._collect_element_formula_candidates(page)
+ )
+
+ # Phase 2: Resolve conflicts using IoU maximization
+ formula_assignments, curves_to_remove, forms_to_remove = (
+ self._resolve_assignment_conflicts(curve_candidates, form_candidates)
+ )
+
+ # Apply the resolved assignments using formula indices
+ for formula_idx, (
+ assigned_curves,
+ assigned_forms,
+ ) in formula_assignments.items():
+ formula = all_formulas[formula_idx][0] # Extract formula from tuple
+ formula.pdf_curve.extend(assigned_curves)
+ formula.pdf_form.extend(assigned_forms)
+
+ # Remove assigned elements from page level
+ for curve in curves_to_remove:
+ if curve in page.pdf_curve:
+ page.pdf_curve.remove(curve)
+
+ for form in forms_to_remove:
+ if form in page.pdf_form:
+ page.pdf_form.remove(form)
+
+ def process_page(self, page: Page):
+ """处理页面，包括公式识别和偏移量计算"""
+ self.process_page_formulas(page)
+ # self.process_page_offsets(page)
+ self.process_comma_formulas(page)
+ self.merge_overlapping_formulas(page)
+ if not self.translation_config.skip_formula_offset_calculation:
+ self.process_page_offsets(page)
+ self.process_translatable_formulas(page)
+ self.update_all_formula_data(page)
+ if not self.translation_config.ocr_workaround:
+ self.collect_contained_elements(page)
+
+ # Process remaining non-formula lines after formula assignment is complete
+ if self.translation_config.remove_non_formula_lines:
+ self.remove_non_formula_lines_from_paragraphs(page)
+
+ if not self.translation_config.skip_formula_offset_calculation:
+ self.process_page_offsets(page)
+ self.update_all_formula_data(page)
+ self.process_page_styles(page)
+
+ def update_line_data(self, line: PdfLine):
+ min_x = min(char.visual_bbox.box.x for char in line.pdf_character)
+ min_y = min(char.visual_bbox.box.y for char in line.pdf_character)
+ max_x = max(char.visual_bbox.box.x2 for char in line.pdf_character)
+ max_y = max(char.visual_bbox.box.y2 for char in line.pdf_character)
+ line.box = Box(min_x, min_y, max_x, max_y)
+
+ def _classify_characters_in_composition(
+ self,
+ composition: PdfParagraphComposition,
+ formula_font_ids: set[int],
+ first_is_bullet_so_far: bool,
+ line_index: int,
+ ) -> tuple[list[tuple[PdfCharacter, bool]], bool]:
+ """
+ Phase 1: Classify every character in a composition as either formula or text.
+ This preserves the original logic, including the sticky `first_is_bullet` flag.
+ """
+ tagged_chars = []
+ is_formula_tags = []
+
+ line = composition.pdf_line
+ if not line or not line.pdf_character:
+ return [], first_is_bullet_so_far
+
+ first_is_bullet = first_is_bullet_so_far
+ in_formula_state = False
+ in_corner_mark_state = False
+ corner_mark_info = []
+
+ # Determine the `is_formula` tag for each character
+ for i, char in enumerate(line.pdf_character):
+ # The original logic for `first_is_bullet`: it is set if any segment starts with a bullet.
+ # A "segment" started when `current_chars` was empty.
+ # We determine the start of a segment by looking at the previous char's tag.
+ is_start_of_segment = i == 0 or (
+ len(is_formula_tags) > 0 and is_formula_tags[-1] != in_formula_state
+ )
+ if not first_is_bullet and is_start_of_segment and is_bullet_point(char):
+ first_is_bullet = True
+
+ is_formula = (
+ ( # 区分公式开头的字符&公式中间的字符。主要是逗号不能在公式开头，但是可以在中间。
+ char.formula_layout_id
+ or (
+ is_formulas_start_char(
+ char.char_unicode,
+ self.font_mapper,
+ self.translation_config,
+ )
+ and not in_formula_state
+ )
+ or (
+ is_formulas_middle_char(
+ char.char_unicode,
+ self.font_mapper,
+ self.translation_config,
+ )
+ and in_formula_state
+ )
+ ) # 公式字符
+ or char.pdf_style.font_id in formula_font_ids # 公式字体
+ or char.vertical # 垂直字体
+ or (
+ # 如果是程序添加的 dummy 空格
+ char.char_unicode is None and in_formula_state
+ )
+ or (
+ # 如果字符的视觉框和实际框不一致，则认为是公式字符
+ char.box.x > char.visual_bbox.box.x2
+ or char.box.x2 < char.visual_bbox.box.x
+ or char.box.y > char.visual_bbox.box.y2
+ or char.box.y2 < char.visual_bbox.box.y
+ )
+ )
+
+ previous_char = line.pdf_character[i - 1] if i > 0 else None
+ next_char = (
+ line.pdf_character[i + 1] if i < len(line.pdf_character) - 1 else None
+ )
+ isspace = char.char_unicode.isspace() if char.char_unicode else False
+ prev_is_space = (
+ previous_char.char_unicode.isspace()
+ if previous_char and previous_char.char_unicode
+ else False
+ )
+
+ is_corner_mark = (
+ (
+ previous_char is not None
+ and not isspace
+ and not prev_is_space
+ and not first_is_bullet
+ # 角标字体，有 0.76 的角标和 0.799 的大写，这里用 0.79 取中，同时考虑首字母放大的情况
+ and char.pdf_style.font_size
+ < previous_char.pdf_style.font_size * 0.79
+ and not in_corner_mark_state
+ )
+ or (
+ previous_char is not None
+ and not isspace
+ and not prev_is_space
+ and not first_is_bullet
+ # 角标字体，有 0.76 的角标和 0.799 的大写，这里用 0.79 取中，同时考虑首字母放大的情况
+ and char.pdf_style.font_size
+ < previous_char.pdf_style.font_size * 1.1
+ and in_corner_mark_state
+ )
+ or (
+ # 检查段落开始的角标：当没有前一个字符时，通过下一个字符判断
+ previous_char is None
+ and next_char is not None
+ and not isspace
+ and not prev_is_space
+ and not first_is_bullet
+ # 当前字符字体大小明显小于下一个字符，判定为角标
+ and char.pdf_style.font_size < next_char.pdf_style.font_size * 0.79
+ and not in_corner_mark_state
+ )
+ )
+
+ is_formula = is_formula or is_corner_mark
+
+ if char.char_unicode == " ":
+ is_formula = in_formula_state
+
+ # This simulates the state change for the next iteration
+ if is_formula != in_formula_state:
+ in_formula_state = is_formula
+
+ in_corner_mark_state = is_corner_mark
+ is_formula_tags.append(is_formula)
+ corner_mark_info.append(is_corner_mark)
+
+ for char, is_formula, is_corner_mark in zip(
+ line.pdf_character, is_formula_tags, corner_mark_info, strict=False
+ ):
+ tagged_chars.append((char, is_formula, is_corner_mark))
+
+ return tagged_chars, first_is_bullet
+
+ def _group_classified_characters(
+ self,
+ tagged_chars: list[tuple[PdfCharacter, bool, bool]],
+ line_index: int,
+ ) -> list[PdfParagraphComposition]:
+ """
+ Phase 2: Group consecutive characters with the same tag into new compositions.
+ """
+ if not tagged_chars:
+ return []
+
+ new_compositions = []
+ current_chars = []
+ current_tag = tagged_chars[0][1]
+ current_corner_mark_flags = []
+
+ for char, is_formula_tag, is_corner_mark in tagged_chars:
+ if is_formula_tag == current_tag:
+ current_chars.append(char)
+ current_corner_mark_flags.append(is_corner_mark)
+ else:
+ # Check if any character in current group is a corner mark
+ has_corner_mark = any(current_corner_mark_flags)
+ new_compositions.append(
+ self.create_composition(
+ current_chars, current_tag, line_index, has_corner_mark
+ ),
+ )
+ current_chars = [char]
+ current_tag = is_formula_tag
+ current_corner_mark_flags = [is_corner_mark]
+
+ if current_chars:
+ # Check if any character in final group is a corner mark
+ has_corner_mark = any(current_corner_mark_flags)
+ new_compositions.append(
+ self.create_composition(
+ current_chars, current_tag, line_index, has_corner_mark
+ ),
+ )
+
+ return new_compositions
+
+ def process_page_formulas(self, page: Page):
+ if not page.pdf_paragraph:
+ return
+
+ page_level_formula_font_ids, xobj_specific_formula_font_ids = (
+ collect_page_formula_font_ids(
+ page, self.translation_config.formular_font_pattern
+ )
+ )
+
+ for paragraph in page.pdf_paragraph:
+ if not paragraph.pdf_paragraph_composition:
+ continue
+
+ current_formula_font_ids: set[int]
+ if (
+ paragraph.xobj_id
+ and paragraph.xobj_id in xobj_specific_formula_font_ids
+ ):
+ current_formula_font_ids = xobj_specific_formula_font_ids[
+ paragraph.xobj_id
+ ]
+ else:
+ current_formula_font_ids = page_level_formula_font_ids
+
+ new_paragraph_compositions = []
+ # This flag is carried through all compositions in a paragraph, as in the original implementation.
+ first_is_bullet = False
+
+ for line_index, composition in enumerate(
+ paragraph.pdf_paragraph_composition
+ ):
+ (
+ tagged_chars,
+ first_is_bullet,
+ ) = self._classify_characters_in_composition(
+ composition,
+ current_formula_font_ids,
+ first_is_bullet,
+ line_index,
+ )
+
+ if not tagged_chars:
+ new_paragraph_compositions.append(composition)
+ continue
+
+ grouped_compositions = self._group_classified_characters(
+ tagged_chars, line_index
+ )
+ new_paragraph_compositions.extend(grouped_compositions)
+
+ paragraph.pdf_paragraph_composition = new_paragraph_compositions
+
+ def process_translatable_formulas(self, page: Page):
+ """将需要正常翻译的公式（如纯数字、数字加逗号等）转换为普通文本行"""
+ if not page.pdf_paragraph:
+ return
+
+ for paragraph in page.pdf_paragraph:
+ if not paragraph.pdf_paragraph_composition:
+ continue
+
+ new_compositions = []
+ for composition in paragraph.pdf_paragraph_composition:
+ if (
+ composition.pdf_formula is not None
+ and not composition.pdf_formula.is_corner_mark
+ and self.is_translatable_formula(
+ composition.pdf_formula,
+ )
+ ):
+ # 将可翻译公式转换为普通文本行
+ new_line = PdfLine(
+ pdf_character=composition.pdf_formula.pdf_character,
+ )
+ self.update_line_data(new_line)
+ new_compositions.append(PdfParagraphComposition(pdf_line=new_line))
+ else:
+ new_compositions.append(composition)
+
+ paragraph.pdf_paragraph_composition = new_compositions
+
+ def process_page_styles(self, page: Page):
+ """处理页面中的文本样式，识别相同样式的文本"""
+ if not page.pdf_paragraph:
+ return
+
+ for paragraph in page.pdf_paragraph:
+ if not paragraph.pdf_paragraph_composition:
+ continue
+
+ # 计算基准样式（除公式外所有文字样式的交集）
+ base_style = self._calculate_base_style(paragraph)
+ paragraph.pdf_style = base_style
+
+ # 重新组织段落中的文本，将相同样式的文本组合在一起
+ new_compositions = []
+ current_chars = []
+ current_style = None
+
+ for comp in paragraph.pdf_paragraph_composition:
+ if comp.pdf_formula is not None:
+ if current_chars:
+ new_comp = self._create_same_style_composition(
+ current_chars,
+ current_style,
+ )
+ new_compositions.append(new_comp)
+ current_chars = []
+ new_compositions.append(comp)
+ continue
+
+ if not comp.pdf_line:
+ new_compositions.append(comp)
+ continue
+
+ for char in comp.pdf_line.pdf_character:
+ char_style = char.pdf_style
+ if current_style is None:
+ current_style = char_style
+ current_chars.append(char)
+ elif is_same_style(char_style, current_style):
+ current_chars.append(char)
+ else:
+ if current_chars:
+ new_comp = self._create_same_style_composition(
+ current_chars,
+ current_style,
+ )
+ new_compositions.append(new_comp)
+ current_chars = [char]
+ current_style = char_style
+
+ if current_chars:
+ new_comp = self._create_same_style_composition(
+ current_chars,
+ current_style,
+ )
+ new_compositions.append(new_comp)
+
+ paragraph.pdf_paragraph_composition = new_compositions
+
+ def _calculate_base_style(self, paragraph) -> PdfStyle:
+ """计算段落的基准样式（除公式外所有文字样式的交集）"""
+ styles = []
+ for comp in paragraph.pdf_paragraph_composition:
+ if isinstance(comp, PdfFormula):
+ continue
+ if not comp.pdf_line:
+ continue
+ for char in comp.pdf_line.pdf_character:
+ styles.append(char.pdf_style)
+
+ if not styles:
+ return None
+
+ # 返回所有样式的交集
+ base_style = styles[0]
+ for style in styles[1:]:
+ # 更新基准样式为所有样式的交集
+ base_style = self._merge_styles(base_style, style)
+
+ # 如果 font_id 或 font_size 为 None，则使用众数
+ if base_style.font_id is None:
+ base_style.font_id = self._get_mode_value([s.font_id for s in styles])
+ if base_style.font_size is None:
+ base_style.font_size = self._get_mode_value([s.font_size for s in styles])
+
+ return base_style
+
+ def _get_mode_value(self, values):
+ """计算列表中的众数"""
+ if not values:
+ return None
+ from collections import Counter
+
+ counter = Counter(values)
+ return counter.most_common(1)[0][0]
+
+ def _merge_styles(self, style1, style2):
+ """合并两个样式，返回它们的交集"""
+ if style1 is None or style1.font_size is None:
+ return style2
+ if style2 is None or style2.font_size is None:
+ return style1
+
+ return PdfStyle(
+ font_id=style1.font_id if style1.font_id == style2.font_id else None,
+ font_size=(
+ style1.font_size
+ if math.fabs(style1.font_size - style2.font_size) < 0.02
+ else None
+ ),
+ graphic_state=self._merge_graphic_states(
+ style1.graphic_state,
+ style2.graphic_state,
+ ),
+ )
+
+ def _merge_graphic_states(self, state1, state2):
+ """合并两个 GraphicState，返回它们的交集"""
+ if state1 is None:
+ return state2
+ if state2 is None:
+ return state1
+
+ return GraphicState(
+ passthrough_per_char_instruction=(
+ state1.passthrough_per_char_instruction
+ if state1.passthrough_per_char_instruction
+ == state2.passthrough_per_char_instruction
+ else None
+ ),
+ )
+
+ def _create_same_style_composition(
+ self,
+ chars: list[PdfCharacter],
+ style,
+ ) -> PdfParagraphComposition:
+ """创建具有相同样式的文本组合"""
+ if not chars:
+ return None
+
+ # 计算边界框
+ min_x = min(char.visual_bbox.box.x for char in chars)
+ min_y = min(char.visual_bbox.box.y for char in chars)
+ max_x = max(char.visual_bbox.box.x2 for char in chars)
+ max_y = max(char.visual_bbox.box.y2 for char in chars)
+ box = Box(min_x, min_y, max_x, max_y)
+
+ return PdfParagraphComposition(
+ pdf_same_style_characters=PdfSameStyleCharacters(
+ box=box,
+ pdf_style=style,
+ pdf_character=chars,
+ ),
+ )
+
+ def process_page_offsets(self, page: Page):
+ """计算公式的 x 和 y 偏移量"""
+ if not page.pdf_paragraph:
+ return
+
+ for paragraph in page.pdf_paragraph:
+ if paragraph.debug_id is None:
+ continue
+ if not paragraph.pdf_paragraph_composition:
+ continue
+
+ # 计算该段落的行间距，用其 80% 作为容差
+ # line_spacing = self.calculate_line_spacing(paragraph)
+ # y_tolerance = line_spacing * 0.8
+
+ for i, composition in enumerate(paragraph.pdf_paragraph_composition):
+ if not composition.pdf_formula:
+ continue
+
+ formula = composition.pdf_formula
+ left_char = None
+ right_char = None
+
+ left_iou = 0
+ right_iou = 0
+
+ # 查找左边最近的同一行的文本
+ for j in range(i - 1, -1, -1):
+ comp = paragraph.pdf_paragraph_composition[j]
+ if comp.pdf_line:
+ for char in reversed(comp.pdf_line.pdf_character):
+ if not char.pdf_character_id:
+ continue
+ # 检查 y 坐标是否接近，判断是否在同一行
+ left_iou = calculate_y_true_iou_for_boxes(
+ formula.box, char.box
+ )
+ if left_iou > 0.6:
+ left_char = char
+ break
+ break
+
+ # 查找右边最近的同一行的文本
+ for j in range(i + 1, len(paragraph.pdf_paragraph_composition)):
+ comp = paragraph.pdf_paragraph_composition[j]
+ if comp.pdf_line:
+ for char in comp.pdf_line.pdf_character:
+ if not char.pdf_character_id:
+ continue
+ # 检查 y 坐标是否接近，判断是否在同一行
+ right_iou = calculate_y_true_iou_for_boxes(
+ formula.box, char.box
+ )
+ if right_iou > 0.6:
+ right_char = char
+ break
+ break
+
+ # If both text segments exist, keep the one with higher IOU
+ if left_char and right_char:
+ if left_iou < right_iou:
+ left_char = None
+ elif right_iou < left_iou:
+ right_char = None
+ # If IOUs are equal, keep both
+
+ # 计算 x 偏移量（相对于左边文本）
+ if left_char:
+ formula.x_offset = formula.box.x - left_char.box.x2
+ else:
+ formula.x_offset = 0 # 如果左边没有文字，x_offset 应该为 0
+ if abs(formula.x_offset) < 0.1:
+ formula.x_offset = 0
+ if formula.x_offset > 10:
+ formula.x_offset = 0
+ # if formula.x_offset > 0:
+ # formula.x_offset = 0
+ if formula.x_offset < -5:
+ formula.x_offset = 0
+
+ # 计算 y 偏移量
+ if left_char:
+ # 使用底部坐标计算偏移量
+ formula.y_offset = formula.box.y - left_char.box.y
+ elif right_char:
+ formula.y_offset = formula.box.y - right_char.box.y
+ else:
+ formula.y_offset = 0
+
+ if abs(formula.y_offset) < 0.1:
+ formula.y_offset = 0
+
+ if max(abs(formula.y_offset), abs(formula.x_offset)) > 10:
+ pass
+ # logging.debug(
+ # f"公式 {formula.box} 的偏移量过大：{formula.x_offset}, {formula.y_offset}"
+ # )
+
+ def calculate_line_spacing(self, paragraph) -> float:
+ """计算段落中的平均行间距"""
+ if not paragraph.pdf_paragraph_composition:
+ return 0.0
+
+ # 收集所有文本行的 y 坐标
+ line_y_positions = []
+ for comp in paragraph.pdf_paragraph_composition:
+ if comp.pdf_line:
+ line_y_positions.append(comp.pdf_line.box.y)
+
+ if len(line_y_positions) < 2:
+ return 10.0 # 如果只有一行或没有行，返回一个默认值
+
+ # 计算相邻行之间的 y 差值
+ line_spacings = []
+ for i in range(len(line_y_positions) - 1):
+ spacing = abs(line_y_positions[i] - line_y_positions[i + 1])
+ if spacing > 0: # 忽略重叠的行
+ line_spacings.append(spacing)
+
+ if not line_spacings:
+ return 10.0 # 如果没有有效的行间距，返回默认值
+
+ # 使用中位数来避免异常值的影响
+ median_spacing = sorted(line_spacings)[len(line_spacings) // 2]
+ return median_spacing
+
+ def create_composition(
+ self,
+ chars: list[PdfCharacter],
+ is_formula: bool,
+ line_index: int,
+ is_corner_mark: bool = False,
+ ) -> PdfParagraphComposition:
+ if is_formula:
+ formula = PdfFormula(pdf_character=chars, line_id=line_index)
+ formula.is_corner_mark = is_corner_mark
+ self.update_formula_data(formula)
+ return PdfParagraphComposition(pdf_formula=formula)
+ else:
+ new_line = PdfLine(pdf_character=chars)
+ self.update_line_data(new_line)
+ return PdfParagraphComposition(pdf_line=new_line)
+
+ def is_translatable_formula(self, formula: PdfFormula) -> bool:
+ """判断公式是否只包含需要正常翻译的字符（数字、空格和英文逗号）"""
+ if all(char.formula_layout_id for char in formula.pdf_character):
+ return False
+
+ text = "".join(char.char_unicode for char in formula.pdf_character)
+ if formula.y_offset > 0.1:
+ return False
+ return bool(re.match(r"^[0-9, .]+$", text))
+
+ def should_split_formula(self, formula: PdfFormula) -> bool:
+ """判断公式是否需要按逗号拆分（包含逗号且有其他特殊符号）"""
+
+ if all(x.formula_layout_id for x in formula.pdf_character):
+ return False
+
+ text = "".join(char.char_unicode for char in formula.pdf_character)
+ # 必须包含逗号
+ if "," not in text:
+ return False
+ # 检查是否包含除了数字和 [] 之外的其他符号
+ text_without_basic = re.sub(r"[0-9\[\],\s]", "", text)
+ return bool(text_without_basic)
+
+ def split_formula_by_comma(
+ self,
+ formula: PdfFormula,
+ ) -> list[tuple[list[PdfCharacter], PdfCharacter]]:
+ """按逗号拆分公式字符，返回 (字符组，逗号字符) 的列表，最后一组的逗号字符为 None。
+ 只有不在括号内的逗号才会被用作分隔符。支持的括号对包括：
+ - (cid:8) 和 (cid:9)
+ - ( 和 )
+ - (cid:16) 和 (cid:17)
+ """
+ result = []
+ current_chars = []
+ bracket_level = 0 # 跟踪括号的层数
+
+ for char in formula.pdf_character:
+ # 检查是否是左括号
+ if char.char_unicode in LEFT_BRACKET:
+ bracket_level += 1
+ current_chars.append(char)
+ # 检查是否是右括号
+ elif char.char_unicode in RIGHT_BRACKET:
+ bracket_level = max(0, bracket_level - 1) # 防止括号不匹配的情况
+ current_chars.append(char)
+ # 检查是否是逗号，且不在括号内
+ elif char.char_unicode == "," and bracket_level == 0:
+ if current_chars:
+ result.append((current_chars, char))
+ current_chars = []
+ else:
+ current_chars.append(char)
+
+ if current_chars:
+ result.append((current_chars, None)) # 最后一组没有逗号
+
+ return result
+
+ def merge_formulas(self, formula1: PdfFormula, formula2: PdfFormula) -> PdfFormula:
+ """合并两个公式，保持字符的相对位置"""
+ # 合并所有字符
+ all_chars = formula1.pdf_character + formula2.pdf_character
+ # 按 y 坐标和 x 坐标排序，确保字符顺序正确
+ # sorted_chars = sorted(
+ # all_chars, key=lambda c: (c.visual_bbox.box.y, c.visual_bbox.box.x))
+
+ # 继承第一个公式的行 ID
+ merged_formula = PdfFormula(pdf_character=all_chars, line_id=formula1.line_id)
+ self.update_formula_data(merged_formula)
+ return merged_formula
+
+ def is_x_axis_contained(self, box1: Box, box2: Box) -> bool:
+ """判断 box1 的 x 轴是否完全包含在 box2 的 x 轴内，或反之"""
+ return (box1.x >= box2.x and box1.x2 <= box2.x2) or (
+ box2.x >= box1.x and box2.x2 <= box1.x2
+ )
+
+ def has_y_intersection(self, box1: Box, box2: Box) -> bool:
+ """判断两个 box 的 y 轴是否有交集"""
+ tolerance = 1.0
+ return not (box1.y2 < box2.y - tolerance or box2.y2 < box1.y - tolerance)
+
+ def is_x_axis_adjacent(self, box1: Box, box2: Box, tolerance: float = 2.0) -> bool:
+ """判断两个 box 在 x 轴上是否相邻或有交集"""
+ # 检查是否有交集
+ has_intersection = not (box1.x2 < box2.x or box2.x2 < box1.x)
+
+ # 检查 box1 是否在 box2 左边且相邻
+ left_adjacent = abs(box1.x2 - box2.x) <= tolerance
+ # 检查 box2 是否在 box1 左边且相邻
+ right_adjacent = abs(box2.x2 - box1.x) <= tolerance
+
+ return has_intersection or left_adjacent or right_adjacent
+
+ def calculate_y_iou(self, box1: Box, box2: Box) -> float:
+ """计算两个 box 在 y 轴上的 IOU (Intersection over Union)"""
+ # 计算交集
+ intersection_start = max(box1.y, box2.y)
+ intersection_end = min(box1.y2, box2.y2)
+ intersection_length = max(0, intersection_end - intersection_start)
+
+ # 计算并集
+ box1_height = box1.y2 - box1.y
+ box2_height = box2.y2 - box2.y
+ union_length = box1_height + box2_height - intersection_length
+
+ # 避免除零错误
+ if union_length <= 0:
+ return 0.0
+
+ return intersection_length / union_length
+
+ def merge_overlapping_formulas(self, page: Page):
+ """
+ 合并符合以下条件的公式：
+ 1. x 轴重叠且 y 轴有交集的相邻公式，或者
+ 2. x 轴相邻且 y 轴 IOU > 0.5 的相邻公式，或者
+ 3. 所有字符的 layout id 都相同的相邻公式，或者
+ 4. 任意两个公式的 IOU > 0.8
+ 角标可能会被识别成单独的公式，需要合并
+ """
+ if not page.pdf_paragraph:
+ return
+
+ for paragraph in page.pdf_paragraph:
+ if not paragraph.pdf_paragraph_composition:
+ continue
+
+ # 重复执行合并过程，直到没有更多可以合并的公式
+ merged = True
+ while merged:
+ merged = False
+ for i in range(len(paragraph.pdf_paragraph_composition)):
+ if merged:
+ break
+ comp1 = paragraph.pdf_paragraph_composition[i]
+ if comp1.pdf_formula is None:
+ continue
+
+ for j in range(i + 1, len(paragraph.pdf_paragraph_composition)):
+ comp2 = paragraph.pdf_paragraph_composition[j]
+ if comp2.pdf_formula is None:
+ continue
+
+ formula1 = comp1.pdf_formula
+ formula2 = comp2.pdf_formula
+
+ # 检查合并条件：
+ # 0. 必须在同一行（line_id 相同），以及
+ # 1. x 轴重叠且 y 轴有交集，或者
+ # 2. x 轴相邻且 y 轴 IOU > 0.5，或者
+ # 3. 所有字符的 layout id 都相同，或者
+ # 4. 任意两个公式的 IOU > 0.8
+
+ # 检查是否在同一行
+ same_line = formula1.line_id == formula2.line_id
+
+ should_merge = same_line and (
+ (
+ j == i + 1
+ and (
+ (
+ self.is_x_axis_contained(
+ formula1.box, formula2.box
+ )
+ and self.has_y_intersection(
+ formula1.box, formula2.box
+ )
+ )
+ or (
+ self.is_x_axis_adjacent(
+ formula1.box, formula2.box
+ )
+ and self.calculate_y_iou(
+ formula1.box, formula2.box
+ )
+ > 0.5
+ )
+ )
+ )
+ or (self._have_same_layout_ids(formula1, formula2, page))
+ or (
+ calculate_iou_for_boxes(formula1.box, formula2.box)
+ > 0.8
+ )
+ or (
+ calculate_iou_for_boxes(formula2.box, formula1.box)
+ > 0.8
+ )
+ )
+
+ if should_merge:
+ # 合并公式
+ merged_formula = self.merge_formulas(formula1, formula2)
+ paragraph.pdf_paragraph_composition[i] = (
+ PdfParagraphComposition(
+ pdf_formula=merged_formula,
+ )
+ )
+ # 删除第二个公式
+ del paragraph.pdf_paragraph_composition[j]
+ merged = True
+ break
+
+ def _have_same_layout_ids(
+ self, formula1: PdfFormula, formula2: PdfFormula, page: Page
+ ) -> bool:
+ """检查两个公式的所有字符是否具有相同的 layout id"""
+ # 获取 formula1 中所有字符的 layout id
+ formula1_layout_ids = set()
+ for char in formula1.pdf_character:
+ if char.char_unicode == " ":
+ continue
+ layout = char.formula_layout_id
+ if layout:
+ formula1_layout_ids.add(layout)
+
+ # 获取 formula2 中所有字符的 layout id
+ formula2_layout_ids = set()
+ for char in formula2.pdf_character:
+ if char.char_unicode == " ":
+ continue
+ layout = char.formula_layout_id
+ if layout:
+ formula2_layout_ids.add(layout)
+
+ # 如果任一公式没有有效的 layout id，则不合并
+ if not (len(formula1_layout_ids) == len(formula2_layout_ids) == 1):
+ return False
+
+ # 检查两个公式的 layout id 集合是否相同
+ return formula1_layout_ids == formula2_layout_ids
+
+ def process_comma_formulas(self, page: Page):
+ """处理包含逗号的复杂公式，将其按逗号拆分"""
+ if not page.pdf_paragraph:
+ return
+
+ for paragraph in page.pdf_paragraph:
+ if not paragraph.pdf_paragraph_composition:
+ continue
+
+ new_compositions = []
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_formula is not None and self.should_split_formula(
+ composition.pdf_formula,
+ ):
+ # 按逗号拆分公式
+ char_groups = self.split_formula_by_comma(composition.pdf_formula)
+ for chars, comma in char_groups:
+ if chars: # 忽略空组（连续的逗号）
+ # 继承原公式的行 ID
+ formula = PdfFormula(
+ pdf_character=chars,
+ line_id=composition.pdf_formula.line_id,
+ )
+ self.update_formula_data(formula)
+ new_compositions.append(
+ PdfParagraphComposition(pdf_formula=formula),
+ )
+
+ # 如果有逗号，添加为文本行
+ if comma:
+ comma_line = PdfLine(pdf_character=[comma])
+ self.update_line_data(comma_line)
+ new_compositions.append(
+ PdfParagraphComposition(pdf_line=comma_line),
+ )
+ else:
+ new_compositions.append(composition)
+
+ paragraph.pdf_paragraph_composition = new_compositions
+
+ def remove_non_formula_lines_from_paragraphs(self, page: Page):
+ """Remove non-formula lines from paragraphs.
+
+ This method processes curves that remain in page.pdf_curve after
+ collect_contained_elements() has assigned formula-related curves to formulas.
+ All remaining curves are non-formula lines, but we need to be careful
+ not to remove lines from figure/table areas.
+
+ Args:
+ page: The page to process
+ """
+ if not page.pdf_curve:
+ return
+
+ # Build layout index for efficient spatial queries
+ layout_index, layout_map = build_layout_index(page)
+
+ curves_to_remove = []
+
+ # Get configuration thresholds
+ protection_threshold = getattr(
+ self.translation_config, "figure_table_protection_threshold", 0.9
+ )
+ overlap_threshold = getattr(
+ self.translation_config, "non_formula_line_iou_threshold", 0.9
+ )
+
+ for curve in page.pdf_curve:
+ # Skip if curve is in figure/table layout areas
+ if is_curve_in_figure_table_layout(
+ curve, layout_index, layout_map, protection_threshold
+ ):
+ continue
+
+ # Only remove if curve overlaps with text paragraph areas
+ if is_curve_overlapping_with_paragraphs(
+ curve, page.pdf_paragraph, overlap_threshold
+ ):
+ curves_to_remove.append(curve)
+
+ # Remove identified curves
+ removed_count = 0
+ for curve in curves_to_remove:
+ if curve in page.pdf_curve:
+ page.pdf_curve.remove(curve)
+ removed_count += 1
+
+ if removed_count > 0:
+ import logging
+
+ logger = logging.getLogger(__name__)
+ logger.debug(f"Removed {removed_count} non-formula lines from paragraphs")
diff --git a/babeldoc/format/pdf/document_il/midend/t_v5.py b/babeldoc/format/pdf/document_il/midend/t_v5.py
new file mode 100644
index 0000000000000000000000000000000000000000..040f3132058a4a168af1677b2e80abc93fd7f457
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/t_v5.py
@@ -0,0 +1,2383 @@
+from __future__ import annotations
+
+import copy
+import logging
+import re
+import statistics
+import unicodedata
+from functools import cache
+
+import pymupdf
+import regex
+from rtree import index
+
+from babeldoc.const import WATERMARK_VERSION
+from babeldoc.format.pdf.document_il import Box
+from babeldoc.format.pdf.document_il import PdfCharacter
+from babeldoc.format.pdf.document_il import PdfCurve
+from babeldoc.format.pdf.document_il import PdfForm
+from babeldoc.format.pdf.document_il import PdfFormula
+from babeldoc.format.pdf.document_il import PdfParagraphComposition
+from babeldoc.format.pdf.document_il import PdfStyle
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.formular_helper import update_formula_data
+from babeldoc.format.pdf.document_il.utils.layout_helper import box_to_tuple
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from babeldoc.format.pdf.translation_config import WatermarkOutputMode
+from arabic_reshaper import reshape
+from bidi.algorithm import get_display
+
+
+logger = logging.getLogger(__name__)
+
+LINE_BREAK_REGEX = regex.compile(
+ r"^["
+ r"a-z"
+ r"A-Z"
+ r"0-9"
+ r"\u00C0-\u00FF" # Latin-1 Supplement
+ r"\u0100-\u017F" # Latin Extended A
+ r"\u0180-\u024F" # Latin Extended B
+ r"\u1E00-\u1EFF" # Latin Extended Additional
+ r"\u2C60-\u2C7F" # Latin Extended C
+ r"\uA720-\uA7FF" # Latin Extended D
+ r"\uAB30-\uAB6F" # Latin Extended E
+ r"\u0250-\u02A0" # IPA Extensions
+ r"\u0400-\u04FF" # Cyrillic
+ r"\u0300-\u036F" # Combining Diacritical Marks
+ r"\u0500-\u052F" # Cyrillic Supplement
+ r"\u0370-\u03FF" # Greek and Coptic
+ r"\u2DE0-\u2DFF" # Cyrillic Extended-A
+ r"\uA650-\uA69F" # Cyrillic Extended-B
+ r"\u1200-\u137F" # Ethiopic
+ r"\u1380-\u139F" # Ethiopic Supplement
+ r"\u2D80-\u2DDF" # Ethiopic Extended
+ r"\uAB00-\uAB2F" # Ethiopic Extended-A
+ r"\U0001E7E0-\U0001E7FF" # Ethiopic Extended-B
+ r"\u0E80-\u0EFF" # Lao
+ r"\u0D00-\u0D7F" # Malayalam
+ r"\u0A80-\u0AFF" # Gujarati
+ r"\u0E00-\u0E7F" # Thai
+ r"\u1000-\u109F" # Myanmar
+ r"\uAA60-\uAA7F" # Myanmar Extended-A
+ r"\uA9E0-\uA9FF" # Myanmar Extended-B
+ r"\U000116D0-\U000116FF" # Myanmar Extended-C
+ r"\u0B80-\u0BFF" # Tamil
+ r"\u0C00-\u0C7F" # Telugu
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0530-\u058F" # Armenian
+ r"\u10A0-\u10FF" # Georgian
+ r"\u1C90-\u1CBF" # Georgian Extended
+ r"\u2D00-\u2D2F" # Georgian Supplement
+ r"\u1780-\u17FF" # Khmer
+ r"\u19E0-\u19FF" # Khmer Symbols
+ r"\U00010B00-\U00010B3F" # Avestan
+ r"\u1D00-\u1D7F" # Phonetic Extensions
+ r"\u1400-\u167F" # Unified Canadian Aboriginal Syllabics
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0780-\u07BF" # Thaana
+ r"\U0001E900-\U0001E95F" # Adlam
+ r"\u1C80-\u1C8F" # Cyrillic Extended-C
+ r"\U0001E030-\U0001E08F" # Cyrillic Extended-D
+ r"\uA000-\uA48F" # Yi Syllables
+ r"\uA490-\uA4CF" # Yi Radicals
+ r"'"
+ r"-" # Hyphen
+ r"·" # Middle Dot (U+00B7) For CatalÃÂÂ
+ r"Ê»" # Spacing Modifier Letters U+02BB
+ r"]+$"
+)
+
+
+class TypesettingUnit:
+ def __str__(self):
+ return self.try_get_unicode() or ""
+
+ def __init__(
+ self,
+ char: PdfCharacter | None = None,
+ formular: PdfFormula | None = None,
+ unicode: str | None = None,
+ font: pymupdf.Font | None = None,
+ original_font: il_version_1.PdfFont | None = None,
+ font_size: float | None = None,
+ style: PdfStyle | None = None,
+ xobj_id: int | None = None,
+ debug_info: bool = False,
+ ):
+ assert (char is not None) + (formular is not None) + (
+ unicode is not None
+ ) == 1, "Only one of chars and formular can be not None"
+ self.char = char
+ self.formular = formular
+ self.unicode = unicode
+ self.x = None
+ self.y = None
+ self.scale = None
+ self.debug_info = debug_info
+
+ # Cache variables
+ self.box_cache: Box | None = None
+ self.can_break_line_cache: bool | None = None
+ self.is_cjk_char_cache: bool | None = None
+ self.mixed_character_blacklist_cache: bool | None = None
+ self.is_space_cache: bool | None = None
+ self.is_hung_punctuation_cache: bool | None = None
+ self.is_cannot_appear_in_line_end_punctuation_cache: bool | None = None
+ self.can_passthrough_cache: bool | None = None
+ self.width_cache: float | None = None
+ self.height_cache: float | None = None
+
+ self.font_size: float | None = None
+
+ if unicode:
+ assert font_size, "Font size must be provided when unicode is provided"
+ assert style, "Style must be provided when unicode is provided"
+ assert len(unicode) == 1, "Unicode must be a single character"
+ assert xobj_id is not None, (
+ "Xobj id must be provided when unicode is provided"
+ )
+
+ self.font = font
+ if font is not None and hasattr(font, "font_id"):
+ self.font_id = font.font_id
+ else:
+ self.font_id = "base"
+ if original_font:
+ self.original_font = original_font
+ else:
+ self.original_font = None
+
+ self.font_size = font_size
+ self.style = style
+ self.xobj_id = xobj_id
+
+ def try_resue_cache(self, old_tu: TypesettingUnit):
+ if old_tu.is_cjk_char_cache is not None:
+ self.is_cjk_char_cache = old_tu.is_cjk_char_cache
+
+ if old_tu.can_break_line_cache is not None:
+ self.can_break_line_cache = old_tu.can_break_line_cache
+
+ if old_tu.is_space_cache is not None:
+ self.is_space_cache = old_tu.is_space_cache
+
+ if old_tu.is_hung_punctuation_cache is not None:
+ self.is_hung_punctuation_cache = old_tu.is_hung_punctuation_cache
+
+ if old_tu.is_cannot_appear_in_line_end_punctuation_cache is not None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ old_tu.is_cannot_appear_in_line_end_punctuation_cache
+ )
+
+ if old_tu.can_passthrough_cache is not None:
+ self.can_passthrough_cache = old_tu.can_passthrough_cache
+
+ if old_tu.mixed_character_blacklist_cache is not None:
+ self.mixed_character_blacklist_cache = (
+ old_tu.mixed_character_blacklist_cache
+ )
+
+
+ def try_get_unicode(self) -> str | None:
+ if self.char:
+ return self.char.char_unicode
+ elif self.formular:
+ return None
+ elif self.unicode:
+ return self.unicode
+
+ @property
+ def mixed_character_blacklist(self):
+ if self.mixed_character_blacklist_cache is None:
+ self.mixed_character_blacklist_cache = self.calc_mixed_character_blacklist()
+
+ return self.mixed_character_blacklist_cache
+
+ def calc_mixed_character_blacklist(self):
+ unicode = self.try_get_unicode()
+ if unicode:
+ return unicode in [
+ "。",
+ "，",
+ "：",
+ "？",
+ "ï¼ÂÂ",
+ ]
+ return False
+
+ @property
+ def can_break_line(self):
+ if self.can_break_line_cache is None:
+ self.can_break_line_cache = self.calc_can_break_line()
+
+ return self.can_break_line_cache
+
+ def calc_can_break_line(self):
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return True
+ if LINE_BREAK_REGEX.match(unicode):
+ return False
+ return True
+
+ @property
+ def is_cjk_char(self):
+ if self.is_cjk_char_cache is None:
+ self.is_cjk_char_cache = self.calc_is_cjk_char()
+
+ return self.is_cjk_char_cache
+
+ def calc_is_cjk_char(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ if "(cid" in unicode:
+ return False
+ if len(unicode) > 1:
+ return False
+ assert len(unicode) == 1, "Unicode must be a single character"
+ if unicode in [
+ "（",
+ "）",
+ "ã€ÂÂ",
+ "】",
+ "《",
+ "》",
+ "ã€â€Â",
+ "〕",
+ "〈",
+ "〉",
+ "〖",
+ "ã€â€â€",
+ "「",
+ "ã€ÂÂ",
+ "『",
+ "ã€ÂÂ",
+ "ã€ÂÂ",
+ "。",
+ "：",
+ "？",
+ "ï¼ÂÂ",
+ "，",
+ ]:
+ return True
+ if unicode:
+ if re.match(
+ r"^["
+ r"\u3000-\u303f" # CJK Symbols and Punctuation
+ r"\u3040-\u309f" # Hiragana
+ r"\u30a0-\u30ff" # Katakana
+ r"\u3100-\u312f" # Bopomofo
+ r"\uac00-\ud7af" # Hangul Syllables
+ r"\u1100-\u11ff" # Hangul Jamo
+ r"\u3130-\u318f" # Hangul Compatibility Jamo
+ r"\ua960-\ua97f" # Hangul Jamo Extended-A
+ r"\ud7b0-\ud7ff" # Hangul Jamo Extended-B
+ r"\u3190-\u319f" # Kanbun
+ r"\u3200-\u32ff" # Enclosed CJK Letters and Months
+ r"\u3300-\u33ff" # CJK Compatibility
+ r"\ufe30-\ufe4f" # CJK Compatibility Forms
+ r"\u4e00-\u9fff" # CJK Unified Ideographs
+ r"\u2e80-\u2eff" # CJK Radicals Supplement
+ r"\u31c0-\u31ef" # CJK Strokes
+ r"\u2f00-\u2fdf" # Kangxi Radicals
+ r"\ufe10-\ufe1f" # Vertical Forms
+ r"]+$",
+ unicode,
+ ):
+ return True
+ try:
+ unicodedata_name = unicodedata.name(unicode)
+ return (
+ "CJK UNIFIED IDEOGRAPH" in unicodedata_name
+ or "FULLWIDTH" in unicodedata_name
+ )
+ except ValueError:
+ return False
+ return False
+
+ @property
+ def is_space(self):
+ if self.is_space_cache is None:
+ self.is_space_cache = self.calc_is_space()
+
+ return self.is_space_cache
+
+ def calc_is_space(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ return unicode == " "
+
+ @property
+ def is_hung_punctuation(self):
+ if self.is_hung_punctuation_cache is None:
+ self.is_hung_punctuation_cache = self.calc_is_hung_punctuation()
+
+ return self.is_hung_punctuation_cache
+
+ def calc_is_hung_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+
+ if unicode:
+ return unicode in [
+ # 英文标点
+ ",",
+ ".",
+ ":",
+ ";",
+ "?",
+ "!",
+ # ä¸ÂÂ文点å·
+ "，", # é€â€â€ÃƒÂ¥Ã‚·
+ "。", # Ã¥ÂÂ¥å·
+ "．", # 全角åÂÂ¥å·
+ "ã€ÂÂ", # é¡¿å·
+ "：", # 冒å·
+ "ï¼›", # 分å·
+ "ï¼ÂÂ", # å¹å·
+ "‼", # Ã¥ÂÂŒå¹å·
+ "？", # éâ€â€Ã‚®Ã¥Â·
+ "â‡", # Ã¥ÂÂΎâ€â€Ã‚®Ã¥Â·
+ # 结æŸ引å·
+ "â€ÂÂ", # å³åÂŒ引å·
+ "’", # å³å•引å·
+ "ã€ÂÂ", # å³直角å•引å·
+ "ã€ÂÂ", # å³直角åÂŒ引å·
+ # 结æŸ括å·
+ ")", # å³圆括å·
+ "]", # å³方括å·
+ "}", # å³花括å·
+ "）", # å³圆括å·
+ "〕", # å³龟çâ€Â²æ‹¬å·
+ "〉", # å³å•书åÂÂÂÂå·
+ "】", # å³黑色方头括å·
+ "ã€â€â€", # å³空白方头括å·
+ "ï¼½", # 全角å³方括å·
+ "ï½ÂÂ", # 全角å³花括å·
+ # 结æŸåÂŒ书åÂÂÂÂå·
+ "》", # å³åÂŒ书åÂÂÂÂå·
+ # 连接å·
+ "～", # 全角波浪å·
+ "-", # è¿žåÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥â€¡ÂÂå·
+ "–", # çŸÂÂ破折å· (EN DASH)
+ "â€â€Â", # 长破折å· (EM DASH)
+ # éâ€â€Ã‚´Ã©Å¡â€Âå·
+ "·", # ä¸ÂÂéâ€â€Ã‚´Ã§â€šÂ¹
+ "・", # 片å‡åÂÂÂÂä¸ÂÂéâ€â€Ã‚´Ã§â€šÂ¹
+ "‧", # è¿žåÂÂâ€â€ÃƒÂ§Ã¢â‚¬Å¡Ã‚¹
+ # 分éšâ€Âå·
+ "/", # æ–œæÂÂÂÂ
+ "ï¼ÂÂ", # 全角斜æÂÂÂÂ
+ "â„", # 分数斜æÂÂÂÂ
+ ]
+ return False
+
+ @property
+ def is_cannot_appear_in_line_end_punctuation(self):
+ if self.is_cannot_appear_in_line_end_punctuation_cache is None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ self.calc_is_cannot_appear_in_line_end_punctuation()
+ )
+
+ return self.is_cannot_appear_in_line_end_punctuation_cache
+
+ def calc_is_cannot_appear_in_line_end_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ return unicode in [
+ # 开始引å·
+ "“", # å·¦åÂŒ引å·
+ "‘", # å·¦å•引å·
+ "「", # 左直角å•引å·
+ "『", # 左直角åÂŒ引å·
+ # 开始括å·
+ "(", # 左圆括å·
+ "[", # 左方括å·
+ "{", # 左花括å·
+ "（", # 左圆括å·
+ "ã€â€Â", # 左龟çâ€Â²æ‹¬å·
+ "〈", # å·¦å•书åÂÂÂÂå·
+ "《", # å·¦åÂŒ书åÂÂÂÂå·
+ # 开始å•åÂŒ书åÂÂÂÂå·
+ "〖", # 左空白方头括å·
+ "〘", # 左黑色方头括å·
+ "〚", # å·¦å•书åÂÂÂÂå·
+ ]
+
+ def passthrough(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ if self.char:
+ return [self.char], [], []
+ elif self.formular:
+ return (
+ self.formular.pdf_character,
+ self.formular.pdf_curve,
+ self.formular.pdf_form,
+ )
+ elif self.unicode:
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ return [], [], []
+
+ @property
+ def can_passthrough(self):
+ if self.can_passthrough_cache is None:
+ self.can_passthrough_cache = self.calc_can_passthrough()
+
+ return self.can_passthrough_cache
+
+ def calc_can_passthrough(self):
+ return self.unicode is None
+
+ def calculate_box(self):
+ if self.char:
+ box = copy.deepcopy(self.char.box)
+ if self.char.visual_bbox and self.char.visual_bbox.box:
+ box.y = self.char.visual_bbox.box.y
+ box.y2 = self.char.visual_bbox.box.y2
+ # return self.char.visual_bbox.box
+
+ return box
+ elif self.formular:
+ return self.formular.box
+ # if self.formular.x_offset <= 0.5:
+ # return self.formular.box
+ # formular_box = copy.copy(self.formular.box)
+ # formular_box.x2 += self.formular.x_advance
+ # return formular_box
+ elif self.unicode:
+ char_width = self.font.char_lengths(self.unicode, self.font_size)[0]
+ if self.x is None or self.y is None or self.scale is None:
+ return Box(0, 0, char_width, self.font_size)
+ return Box(self.x, self.y, self.x + char_width, self.y + self.font_size)
+
+ @property
+ def box(self):
+ if not self.box_cache:
+ self.box_cache = self.calculate_box()
+
+ return self.box_cache
+
+ @property
+ def width(self):
+ if self.width_cache is None:
+ self.width_cache = self.calc_width()
+
+ return self.width_cache
+
+ def calc_width(self):
+ box = self.box
+ return box.x2 - box.x
+
+ @property
+ def height(self):
+ if self.height_cache is None:
+ self.height_cache = self.calc_height()
+
+ return self.height_cache
+
+ def calc_height(self):
+ box = self.box
+ return box.y2 - box.y
+
+ def relocate(
+ self,
+ x: float,
+ y: float,
+ scale: float,
+ ) -> TypesettingUnit:
+ """é‡ÂÂ定ä½ÂÂ并缩æâ€Â¾æŽ’版å•元
+
+ Args:
+ x: æ–°çš„ x Ã¥ÂÂÂÂæ ‡
+ y: æ–°çš„ y Ã¥ÂÂÂÂæ ‡
+ scale: 缩æâ€Â¾å› åÂÂÂÂ
+
+ Returns:
+ 新的排版å•元
+ """
+ if self.char:
+ # 创建新的åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â¯Â¹Ã¨Â±Â¡
+ new_char = PdfCharacter(
+ pdf_character_id=self.char.pdf_character_id,
+ char_unicode=self.char.char_unicode,
+ box=Box(
+ x=x,
+ y=y,
+ x2=x + self.width * scale,
+ y2=y + self.height * scale,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.char.pdf_style.font_id,
+ font_size=self.char.pdf_style.font_size * scale,
+ graphic_state=self.char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=self.char.vertical,
+ advance=self.char.advance * scale if self.char.advance else None,
+ debug_info=self.debug_info,
+ xobj_id=self.char.xobj_id,
+ )
+ new_tu = TypesettingUnit(char=new_char)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.formular:
+ # 创建新的公å¼ÂÂ对象，ä¿ÂÂæŒÂÂ内部åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã§Å¡â€žÃ§â€ºÂ¸Ã¥Â¯Â¹Ã¤Â½ÂÂç½®
+ new_chars = []
+ min_x = self.formular.box.x
+ min_y = self.formular.box.y
+
+ for char in self.formular.pdf_character:
+ # 计ç®â€â€ÃƒÂ§Ã¢â‚¬ÂºÃ‚¸Ã¥Â¯Â¹Ã¤Â½ÂÂç½®
+ rel_x = char.box.x - min_x
+ rel_y = char.box.y - min_y
+
+ visual_rel_x = char.visual_bbox.box.x - min_x
+ visual_rel_y = char.visual_bbox.box.y - min_y
+
+ # 创建新的åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â¯Â¹Ã¨Â±Â¡
+ new_char = PdfCharacter(
+ pdf_character_id=char.pdf_character_id,
+ char_unicode=char.char_unicode,
+ box=Box(
+ x=x + (rel_x + self.formular.x_offset) * scale,
+ y=y + (rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (rel_x + (char.box.x2 - char.box.x) + self.formular.x_offset)
+ * scale,
+ y2=y
+ + (rel_y + (char.box.y2 - char.box.y) + self.formular.y_offset)
+ * scale,
+ ),
+ visual_bbox=il_version_1.VisualBbox(
+ box=Box(
+ x=x + (visual_rel_x + self.formular.x_offset) * scale,
+ y=y + (visual_rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (
+ visual_rel_x
+ + (char.visual_bbox.box.x2 - char.visual_bbox.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=y
+ + (
+ visual_rel_y
+ + (char.visual_bbox.box.y2 - char.visual_bbox.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ ),
+ ),
+ pdf_style=PdfStyle(
+ font_id=char.pdf_style.font_id,
+ font_size=char.pdf_style.font_size * scale,
+ graphic_state=char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=char.vertical,
+ advance=char.advance * scale if char.advance else None,
+ xobj_id=char.xobj_id,
+ )
+ new_chars.append(new_char)
+
+ # Calculate bounding box from new_chars
+ min_x = min(char.visual_bbox.box.x for char in new_chars)
+ min_y = min(char.visual_bbox.box.y for char in new_chars)
+ max_x = max(char.visual_bbox.box.x2 for char in new_chars)
+ max_y = max(char.visual_bbox.box.y2 for char in new_chars)
+
+ new_formula = PdfFormula(
+ box=Box(
+ x=min_x,
+ y=min_y,
+ x2=max_x,
+ y2=max_y,
+ ),
+ pdf_character=new_chars,
+ x_offset=self.formular.x_offset * scale,
+ y_offset=self.formular.y_offset * scale,
+ x_advance=self.formular.x_advance * scale,
+ )
+
+ # Handle contained curves
+ new_curves = []
+ for curve in self.formular.pdf_curve:
+ new_curve = self._transform_curve_for_relocation(
+ curve,
+ self.formular.box.x,
+ self.formular.box.y,
+ x,
+ y,
+ scale,
+ )
+ new_curves.append(new_curve)
+ new_formula.pdf_curve = new_curves
+
+ # Handle contained forms
+ new_forms = []
+ for form in self.formular.pdf_form:
+ new_form = self._transform_form_for_relocation(
+ form, self.formular.box.x, self.formular.box.y, x, y, scale
+ )
+ new_forms.append(new_form)
+ new_formula.pdf_form = new_forms
+
+ update_formula_data(new_formula)
+
+ new_tu = TypesettingUnit(formular=new_formula)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.unicode:
+ # 对于 Unicode Ã¥ÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¯Â¼Å’我们å˜储新的ä½ÂÂ置信æ¯
+ new_unit = TypesettingUnit(
+ unicode=self.unicode,
+ font=self.font,
+ original_font=self.original_font,
+ font_size=self.font_size * scale,
+ style=self.style,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ new_unit.x = x
+ new_unit.y = y
+ new_unit.scale = scale
+ new_unit.try_resue_cache(self)
+ return new_unit
+
+ def _transform_curve_for_relocation(
+ self,
+ curve,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a curve for formula relocation."""
+ import copy
+
+ new_curve = copy.deepcopy(curve)
+
+ if new_curve.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_curve.box.x - original_formula_x
+ rel_y = new_curve.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_curve.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (
+ rel_x
+ + (new_curve.box.x2 - new_curve.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=new_y
+ + (
+ rel_y
+ + (new_curve.box.y2 - new_curve.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original CTM
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_curve.relocation_transform = list(relocation_matrix)
+
+ return new_curve
+
+ def _transform_form_for_relocation(
+ self,
+ form,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a form for formula relocation."""
+ import copy
+
+ new_form = copy.deepcopy(form)
+
+ if new_form.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_form.box.x - original_formula_x
+ rel_y = new_form.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_form.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (rel_x + (new_form.box.x2 - new_form.box.x) + self.formular.x_offset)
+ * scale,
+ y2=new_y
+ + (rel_y + (new_form.box.y2 - new_form.box.y) + self.formular.y_offset)
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original matrices
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_form.relocation_transform = list(relocation_matrix)
+
+ return new_form
+
+ def render(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ """渲染排版å•元为 PdfCharacter åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+
+ Returns:
+ PdfCharacter åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ """
+ if self.can_passthrough:
+ return self.passthrough()
+ elif self.unicode:
+ assert self.x is not None, (
+ "x position must be set, should be set by `relocate`"
+ )
+ assert self.y is not None, (
+ "y position must be set, should be set by `relocate`"
+ )
+ assert self.scale is not None, (
+ "scale must be set, should be set by `relocate`"
+ )
+ x = self.x
+ y = self.y
+ # if self.original_font and self.font and hasattr(self.original_font, "descent") and hasattr(self.font, "descent_fontmap"):
+ # original_descent = self.original_font.descent
+ # new_descent = self.font.descent_fontmap
+ # y -= (original_descent - new_descent) * self.font_size / 1000
+
+ # 计ç®â€â€ÃƒÂ¥Ã‚Ââ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â®Â½Ã¥ÂºÂ¦
+ char_width = self.width
+
+ # Handle case when font is None (no suitable font found for this character)
+ if self.font is None:
+ logger.warning(
+ f"No font available for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using font_id='{self.font_id}' with glyph_id=0"
+ )
+ glyph_id = 0 # Use glyph 0 as fallback (usually .notdef)
+ else:
+ glyph_id = self.font.has_glyph(ord(self.unicode))
+ if glyph_id == 0 or glyph_id is None:
+ logger.warning(
+ f"Font '{self.font_id}' doesn't have glyph for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using glyph_id=0"
+ )
+ glyph_id = 0
+
+ new_char = PdfCharacter(
+ pdf_character_id=glyph_id,
+ char_unicode=self.unicode,
+ box=Box(
+ x=x, # 使çâ€Â¨å˜储的ä½ÂÂç½®
+ y=y,
+ x2=x + char_width,
+ y2=y + self.font_size,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.font_id,
+ font_size=self.font_size,
+ graphic_state=self.style.graphic_state,
+ ),
+ scale=self.scale,
+ vertical=False,
+ advance=char_width,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ return [new_char], [], []
+ else:
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ return [], [], []
+
+
+class Typesetting:
+ stage_name = "Typesetting"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.font_mapper = FontMapper(translation_config)
+ self.translation_config = translation_config
+ self.lang_code = self.translation_config.lang_out.upper()
+ # Ensure detailed_logger attribute exists to avoid attribute access errors
+ self.detailed_logger = None
+ self.is_cjk = (
+ # Why zh-CN/zh-HK/zh-TW here but not zh-Hans and so on?
+ # See https://funstory-ai.github.io/BabelDOC/supported_languages/
+ ("ZH" in self.lang_code) # C
+ or ("JA" in self.lang_code)
+ or ("JP" in self.lang_code) # J
+ or ("KR" in self.lang_code) # K
+ or ("CN" in self.lang_code)
+ or ("HK" in self.lang_code)
+ or ("TW" in self.lang_code)
+ )
+
+ def preprocess_document(self, document: il_version_1.Document, pbar):
+ """预处ç†文档，获å–æ¯ÂÂ个段è½的最优缩æâ€Â¾å› åÂÂÂÂ，ä¸ÂÂ执行实际排版"""
+ all_scales: list[float] = []
+ all_paragraphs: list[il_version_1.PdfParagraph] = []
+
+ for page in document.page:
+ pbar.advance()
+ # 准备åÂÂâ€â€ÃƒÂ¤Ã‚½â€œÃ¤Â¿Â¡Ã¦Â¯（å¤ÂÂ制自 render_page 的逻辑）
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if (
+ xobj.xobj_id in fonts
+ and isinstance(fonts[xobj.xobj_id], dict)
+ and font.font_id
+ ):
+ fonts[xobj.xobj_id][font.font_id] = font
+
+ # 处ç†æ¯ÂÂ个段è½
+ for paragraph in page.pdf_paragraph:
+ all_paragraphs.append(paragraph)
+ unit_count = 0
+ try:
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ unit_count = len(typesetting_units)
+ for unit in typesetting_units:
+ if unit.formular:
+ unit_count += len(unit.formular.pdf_character) - 1
+
+ # 如果所有å•元都å¯以直接传递，则 scale = 1.0
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.optimal_scale = 1.0
+ else:
+ # 获å–最优缩æâ€Â¾å› åÂÂÂÂ
+ optimal_scale = self._get_optimal_scale(
+ paragraph, page, typesetting_units
+ )
+ paragraph.optimal_scale = optimal_scale
+ except Exception as e:
+ # 如果预处ç†出éâ€Â™ï¼Œé»˜è®¤ä½¿çâ€Â¨ 1.0 缩æâ€Â¾å› åÂÂÂÂ
+ logger.warning(f"预处ç†段è½æâ€â€Ã‚¶Ã¥â€¡ÂºÃ©â€Â™ï¼š{e}")
+ paragraph.optimal_scale = 1.0
+
+ if paragraph.optimal_scale is not None:
+ all_scales.extend([paragraph.optimal_scale] * unit_count)
+
+ # 获å–缩æâ€Â¾å› åÂÂÂÂçš„ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ if all_scales:
+ try:
+ modes = statistics.multimode(all_scales)
+ mode_scale = min(modes)
+ except statistics.StatisticsError:
+ logger.warning(
+ "Could not find a mode for paragraph scales. Falling back to median."
+ )
+ mode_scale = statistics.median(all_scales)
+ # 将所有大于ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã§Å¡â€žÃ¥â‚¬Â¼Ã¤Â¿Â®Ã¦â€Â¹ä¸ºä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ for paragraph in all_paragraphs:
+ if (
+ paragraph.optimal_scale is not None
+ and paragraph.optimal_scale > mode_scale
+ ):
+ paragraph.optimal_scale = mode_scale
+ else:
+ logger.error(
+ "document_scales is empty, there seems no paragraph in this PDF"
+ )
+
+ def shape_arabic_text(self, text: str) -> str:
+ """Shape and reorder Arabic text if output language is Arabic.
+
+ Args:
+ text: Input text to shape
+
+ Returns:
+ Shaped and reordered text if language is Arabic, original text otherwise
+ """
+ if not text:
+ return text
+
+ # Robust Arabic output detection: accept explicit 'ar', 'ara', 'arabic'
+ # or formats containing '-ar', '->ar', or '/ar' as a target marker (e.g. 'en-ar', 'en->ar')
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = False
+ if lang_out in ("en-ar, ar", "ara", "arabic"):
+ is_arabic = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic = True
+
+ if is_arabic:
+ logger.debug("Shaping Arabic text")
+ # Flip parentheses and brackets for RTL display
+ # text = text.replace("(", "\x00")
+ # text = text.replace(")", "(")
+ # text = text.replace("\x00", ")")
+ # text = text.replace("[", "\x01")
+ # text = text.replace("]", "[")
+ # text = text.replace("\x01", "]")
+ # text = text.replace("{", "\x02")
+ # text = text.replace("}", "{")
+ # text = text.replace("\x02", "}")
+ try:
+ if not re.search(r'[\uFB50-\uFDFF\uFE70-\uFEFF]', text):
+ # Extract inline tags before shaping to prevent corruption
+ tag_pattern = r'<[^>]+>'
+ tags = []
+ tag_positions = []
+ for match in re.finditer(tag_pattern, text):
+ tags.append(match.group(0))
+ tag_positions.append((match.start(), match.end()))
+
+ if tags:
+ text_without_tags = text
+ placeholder_map = {}
+ for i in range(len(tags) - 1, -1, -1):
+ start, end = tag_positions[i]
+ placeholder = f"\u200D{i}\u200D"
+ placeholder_map[placeholder] = tags[i]
+ text_without_tags = text_without_tags[:start] + placeholder + text_without_tags[end:]
+
+ # Reshape Arabic text for proper character joining
+ from arabic_reshaper import ArabicReshaper
+ configuration = {
+ 'delete_harakat': False, # Keep diacritical marks
+ 'support_ligatures': True, # Support Arabic ligatures
+ 'RIAL SIGN': True,
+ 'ARABIC COMMA': True,
+ 'ARABIC SEMICOLON': True,
+ 'ARABIC QUESTION MARK': True,
+ 'ZWNJ': True, # Zero Width Non-Joiner
+ }
+
+ reshaper = ArabicReshaper(configuration=configuration)
+ reshaped_text = reshaper.reshape(text_without_tags)
+ display_text = get_display(reshaped_text, base_dir='R')
+
+ # Restore tags
+ # for placeholder, tag in placeholder_map.items():
+ # display_text = display_text.replace(placeholder, tag)
+ return display_text
+ else:
+ # No tags, process normally
+ # Reshape Arabic text for proper character joining
+ from arabic_reshaper import ArabicReshaper
+ configuration = {
+ 'delete_harakat': False, # Keep diacritical marks
+ 'support_ligatures': True, # Support Arabic ligatures
+ 'RIAL SIGN': True,
+ 'ARABIC COMMA': True,
+ 'ARABIC SEMICOLON': True,
+ 'ARABIC QUESTION MARK': True,
+ 'ZWNJ': True, # Zero Width Non-Joiner
+ }
+
+ reshaper = ArabicReshaper(configuration=configuration)
+ reshaped_text = reshaper.reshape(text)
+ display_text = get_display(reshaped_text, base_dir='R')
+ return display_text
+ else:
+ display_text = text
+ return display_text
+ except Exception as e:
+ logger.warning(f"Failed to shape Arabic text: {e}")
+ return text
+
+ return text
+
+ # # Robust Arabic output detection: accept explicit 'ar', 'ara', 'arabic'
+ # # or formats containing '-ar', '->ar', or '/ar' as a target marker (e.g. 'en-ar', 'en->ar')
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar, ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # if is_arabic:
+ # logger.debug("Shaping Arabic text")
+ # # Flip parentheses and brackets for RTL display
+ # # text = text.replace("(", "\x00")
+ # # text = text.replace(")", "(")
+ # # text = text.replace("\x00", ")")
+ # # text = text.replace("[", "\x01")
+ # # text = text.replace("]", "[")
+ # # text = text.replace("\x01", "]")
+ # # text = text.replace("{", "\x02")
+ # # text = text.replace("}", "{")
+ # # text = text.replace("\x02", "}")
+ # try:
+ # if not re.search(r'[\uFB50-\uFDFF\uFE70-\uFEFF]', text):
+ # # Reshape Arabic text for proper character joining
+ # from arabic_reshaper import ArabicReshaper
+ # configuration = {
+ # 'delete_harakat': False, # Keep diacritical marks
+ # 'support_ligatures': True, # Support Arabic ligatures
+ # 'RIAL SIGN': True,
+ # 'ARABIC COMMA': True,
+ # 'ARABIC SEMICOLON': True,
+ # 'ARABIC QUESTION MARK': True,
+ # 'ZWNJ': True, # Zero Width Non-Joiner
+ # }
+
+ # reshaper = ArabicReshaper(configuration=configuration)
+ # reshaped_text = reshaper.reshape(text)
+ # display_text = get_display(reshaped_text, base_dir='R')
+ # else:
+ # display_text = text
+ # return display_text
+ # except Exception as e:
+ # logger.warning(f"Failed to shape Arabic text: {e}")
+ # return text
+
+ # return text
+
+ def _find_optimal_scale_and_layout(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ initial_scale: float = 1.0,
+ use_english_line_break: bool = True,
+ apply_layout: bool = False,
+ ) -> tuple[float, list[TypesettingUnit] | None]:
+ """查找最优缩æâ€Â¾å› åÂÂÂÂ并å¯选择性地执行布局
+
+ Args:
+ paragraph: 段è½对象
+ page: 页é¢对象
+ typesetting_units: 排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ initial_scale: åˆÂÂ始缩æâ€Â¾å› åÂÂÂÂ
+ use_english_line_break: 是å¦使çâ€Â¨è‹±æ–‡æ¢行规则
+ apply_layout: 是å¦åºâ€Âçâ€Â¨å¸ƒå±€åˆ° paragraph（True æâ€â€Ã‚¶Ã¦â€°Â§Ã¨Â¡Å’实际排版）
+
+ Returns:
+ tuple[float, list[TypesettingUnit] | None]: (最终缩æâ€Â¾å› åÂÂÂÂ，排版åÂÂŽçš„å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨Ã¦Ë†â€“ None)
+ """
+ if not paragraph.box:
+ return initial_scale, None
+
+ box = paragraph.box
+ scale = initial_scale
+ line_skip = 1.50 if self.is_cjk else 1.3
+ min_scale = 0.1
+ expand_space_flag = 0
+ final_typeset_units = None
+
+ while scale >= min_scale:
+ try:
+ # Check if Arabic to disable English line breaking
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic_layout = False
+ if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ is_arabic_layout = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic_layout = True
+
+ # For Arabic, disable English line breaking to prevent premature breaks
+ effective_line_break = use_english_line_break and not is_arabic_layout
+
+ # å°ÂÂ试布局排版å•元
+ typeset_units, all_units_fit = self._layout_typesetting_units(
+ typesetting_units,
+ box,
+ scale,
+ line_skip,
+ paragraph,
+ effective_line_break,
+ )
+
+ # 如果所有å•元都æâ€Â¾å¾â€â€ÃƒÂ¤Ã‚¸â€¹
+ if all_units_fit:
+ # Apply RTL margin mirroring for Arabic documents
+ # Apply RTL margin mirroring for Arabic documents
+ # The _mirror_margins_for_rtl method now checks paragraph attributes itself
+ typeset_units = self._mirror_margins_for_rtl(
+ typeset_units,
+ box,
+ paragraph
+ )
+
+ if apply_layout:
+ # 实际åºâ€Âçâ€Â¨æŽ’版结果
+ paragraph.scale = scale
+ paragraph.pdf_paragraph_composition = []
+ for unit in typeset_units:
+ chars, curves, forms = unit.render()
+ for char in chars:
+ paragraph.pdf_paragraph_composition.append(
+ PdfParagraphComposition(pdf_character=char),
+ )
+ for curve in curves:
+ page.pdf_curve.append(curve)
+ for form in forms:
+ page.pdf_form.append(form)
+ final_typeset_units = typeset_units
+ return scale, final_typeset_units
+ except Exception:
+ # 如果布局检查出éâ€Â™ï¼Œç»§ç»ÂÂå°ÂÂ试下一个缩æâ€Â¾å› åÂÂÂÂ
+ pass
+
+ # 添加与原 retypeset 一致的逻辑检查
+ if not hasattr(paragraph, "debug_id") or not paragraph.debug_id:
+ return scale, final_typeset_units
+
+ # å‡ÂÂå°ÂÂ缩æâ€Â¾å› åÂÂÂÂ
+ if scale > 0.6:
+ scale -= 0.05
+ else:
+ scale -= 0.1
+
+ if scale < 0.7:
+ space_expanded = False # 标记是å¦æˆÂÂ功扩展了空éâ€â€Ã‚´
+
+ if expand_space_flag == 0:
+ # å°ÂÂ试å‘下扩展
+ try:
+ min_y = self.get_max_bottom_space(box, page) + 2
+ if min_y < box.y:
+ expanded_box = Box(x=box.x, y=min_y, x2=box.x2, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # 更新段è½的边界æ¡â€Â
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 1
+
+ # åª有æˆÂÂ功扩展空éâ€â€Ã‚´Ã¦â€â€Ã‚¶Ã¦â€°Â continue，å¦则继ç»ÂÂå‡ÂÂå° scale
+ if space_expanded:
+ continue
+
+ elif expand_space_flag == 1:
+ # å°ÂÂ试å‘å³扩展
+ try:
+ max_x = self.get_max_right_space(box, page) - 5
+ if max_x > box.x2:
+ expanded_box = Box(x=box.x, y=box.y, x2=max_x, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # 更新段è½的边界æ¡â€Â
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 2
+
+ # åª有æˆÂÂ功扩展空éâ€â€Ã‚´Ã¦â€â€Ã‚¶Ã¦â€°Â continue，å¦则继ç»ÂÂå‡ÂÂå° scale
+ if space_expanded:
+ continue
+
+ # åª有在扩展å°ÂÂ试阶段 (expand_space_flag < 2) ä¸â€Â扩展失败æâ€â€Ã‚¶Ã¦â€°ÂÂé‡ÂÂç½® scale
+ # 当 expand_space_flag >= 2 æâ€â€Ã‚¶Ã¯Â¼Å’说明已ç»ÂÂå°ÂÂ试过所有扩展，åºâ€Â该继ç»ÂÂæ£常的 scale å‡ÂÂå°ÂÂ
+ if expand_space_flag < 2:
+ # 如果æâ€â€Ã‚ Ã¦Â³â€¢Ã¦â€°Â©Ã¥Â±â€¢Ã§Â©ÂºÃ©â€â€Ã‚´Ã¯Â¼Å’é‡ÂÂç½® scale 并继ç»ÂÂ循环
+ scale = 1.0
+
+ # 如果ä»ÂÂ然æâ€Â¾ä¸ÂÂ下，å°ÂÂ试去除英文æ¢行é™ÂÂ制
+ if use_english_line_break:
+ return self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ initial_scale,
+ use_english_line_break=False,
+ apply_layout=apply_layout,
+ )
+
+ # 最åÂÂŽè¿â€Â回最å°ÂÂ缩æâ€Â¾å› åÂÂÂÂ
+ return min_scale, final_typeset_units
+
+ def _get_optimal_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ use_english_line_break: bool = True,
+ ) -> float:
+ """获å–段è½的最优缩æâ€Â¾å› åÂÂÂÂ，ä¸ÂÂ执行实际排版"""
+ scale, _ = self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ 1.0,
+ use_english_line_break,
+ apply_layout=False,
+ )
+ return scale
+
+ def retypeset_with_precomputed_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ precomputed_scale: float,
+ use_english_line_break: bool = True,
+ ):
+ """使çâ€Â¨é¢„计ç®â€â€ÃƒÂ§Ã…¡â€žÃ§Â¼Â©Ã¦â€Â¾å› åÂÂÂÂ进行排版"""
+ if not paragraph.box:
+ return
+
+ # 使çâ€Â¨é€šçâ€Â¨æ–¹æ³•è¿›è¡ŒæŽ’版，传入预计ç®â€â€ÃƒÂ§Ã…¡â€žÃ§Â¼Â©Ã¦â€Â¾å› åÂÂÂÂ作为åˆÂÂ始值
+ self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ precomputed_scale,
+ use_english_line_break,
+ apply_layout=True,
+ )
+
+ def typesetting_document(self, document: il_version_1.Document):
+ # Add detailed logging at the start
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Started")
+
+ # 原有的æŽ'版逻è¾'
+ if self.translation_config.progress_monitor:
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(document.page) * 2,
+ ) as pbar:
+ # 预处ç†ï¼šèŽ·å–所有段è½çš„最优缩æ"¾å› Ã¥ÂÂÂ
+ self.preprocess_document(document, pbar)
+
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+ pbar.advance()
+ else:
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+
+ # Add detailed logging at the end
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Complete")
+
+ def render_page(self, page: il_version_1.Page):
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if font.font_id:
+ fonts[xobj.xobj_id][font.font_id] = font
+ if (
+ page.page_number == 0
+ and self.translation_config.watermark_output_mode
+ == WatermarkOutputMode.Watermarked
+ ):
+ self.add_watermark(page)
+ try:
+ para_index = index.Index()
+ para_map = {}
+ #
+ valid_paras = [
+ p
+ for p in page.pdf_paragraph
+ if p.box
+ and all(c is not None for c in [p.box.x, p.box.y, p.box.x2, p.box.y2])
+ ]
+
+ for i, para in enumerate(valid_paras):
+ para_map[i] = para
+ para_index.insert(i, box_to_tuple(para.box))
+
+ for i, p_upper in para_map.items():
+ if not (p_upper.box and p_upper.box.y is not None):
+ continue
+
+ # Calculate paragraph height and set required gap accordingly
+ para_height = p_upper.box.y2 - p_upper.box.y
+ required_gap = 0.5 if para_height < 36 else 3
+
+ check_area = il_version_1.Box(
+ x=p_upper.box.x,
+ y=p_upper.box.y - required_gap,
+ x2=p_upper.box.x2,
+ y2=p_upper.box.y,
+ )
+
+ candidate_ids = list(para_index.intersection(box_to_tuple(check_area)))
+
+ conflicting_paras = []
+ for para_id in candidate_ids:
+ if para_id == i:
+ continue
+ p_lower = para_map[para_id]
+ if not (
+ p_lower.box
+ and p_upper.box
+ and p_lower.box.x2 < p_upper.box.x
+ or p_lower.box.x > p_upper.box.x2
+ ):
+ conflicting_paras.append(p_lower)
+
+ if conflicting_paras:
+ max_y2 = max(
+ p.box.y2
+ for p in conflicting_paras
+ if p.box and p.box.y2 is not None
+ )
+
+ new_y = max_y2 + required_gap
+ if p_upper.box and new_y < p_upper.box.y2:
+ p_upper.box.y = new_y
+ except Exception as e:
+ logger.warning(
+ f"Failed to adjust paragraph positions on page {page.page_number}: {e}"
+ )
+ # 开始实际的渲染过程
+ for paragraph in page.pdf_paragraph:
+ self.render_paragraph(paragraph, page, fonts)
+
+ def add_watermark(self, page: il_version_1.Page):
+ page_width = page.cropbox.box.x2 - page.cropbox.box.x
+ page_height = page.cropbox.box.y2 - page.cropbox.box.y
+ style = il_version_1.PdfStyle(
+ font_id="base",
+ font_size=6,
+ graphic_state=il_version_1.GraphicState(),
+ )
+ text = f"本文档çâ€Â± funstory.ai 的开溠PDF 翻译库 BabelDOC {WATERMARK_VERSION} (http://yadt.io) 翻译，本仓库æ£在积æžÂÂ的建设当ä¸ÂÂ，欢迎 star 和关注。"
+ if self.translation_config.debug:
+ text += "\n 当å‰ÂÂ为 DEBUG 模å¼ÂÂ，将显示更多辅助信æ¯。请注æ„ÂÂ，部分框的ä½ÂÂ置对åºâ€Â原文，但在译文ä¸ÂÂå¯能ä¸ÂÂæ£确。"
+ page.pdf_paragraph.append(
+ il_version_1.PdfParagraph(
+ first_line_indent=False,
+ box=il_version_1.Box(
+ x=page.cropbox.box.x + page_width * 0.05,
+ y=page.cropbox.box.y,
+ x2=page.cropbox.box.x2,
+ y2=page.cropbox.box.y2 - page_height * 0.05,
+ ),
+ vertical=False,
+ pdf_style=style,
+ pdf_paragraph_composition=[
+ il_version_1.PdfParagraphComposition(
+ pdf_same_style_unicode_characters=il_version_1.PdfSameStyleUnicodeCharacters(
+ unicode=text,
+ pdf_style=style,
+ ),
+ ),
+ ],
+ xobj_id=-1,
+ ),
+ )
+
+ def render_paragraph(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ],
+ ):
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ # 如果所有å•元都å¯以直接传递，则直接传递
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.scale = 1.0
+ paragraph.pdf_paragraph_composition = self.create_passthrough_composition(
+ typesetting_units,
+ )
+ else:
+ # 使çâ€Â¨é¢„计ç®â€â€ÃƒÂ§Ã…¡â€žÃ§Â¼Â©Ã¦â€Â¾å› åÂÂÂÂ进行é‡ÂÂ排ç‰Ëâ€
+ precomputed_scale = (
+ paragraph.optimal_scale if paragraph.optimal_scale is not None else 1.0
+ )
+
+ # 如果有å•元æâ€â€Ã‚ Ã¦Â³â€¢Ã§â€ºÂ´Ã¦Å½Â¥Ã¤Â¼Â Ã©â‚¬â€™Ã¯Â¼Å’则进行é‡ÂÂ排ç‰Ëâ€
+ paragraph.pdf_paragraph_composition = []
+ self.retypeset_with_precomputed_scale(
+ paragraph, page, typesetting_units, precomputed_scale
+ )
+
+ # é‡ÂÂ排版åÂŽ，é‡ÂÂ新设置段è½å„åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã§Å¡â€ž render order
+ self._update_paragraph_render_order(paragraph)
+ # Log the typeset text block with coordinates
+ if hasattr(self, 'detailed_logger') and self.detailed_logger:
+ try:
+ # Extract the complete text from the paragraph
+ paragraph_text = ""
+ if hasattr(paragraph, 'unicode') and paragraph.unicode:
+ paragraph_text = paragraph.unicode
+ elif hasattr(paragraph, 'pdf_paragraph_composition'):
+ text_parts = []
+ for comp in paragraph.pdf_paragraph_composition:
+ if comp.pdf_character and hasattr(comp.pdf_character, 'char_unicode'):
+ if comp.pdf_character.char_unicode:
+ text_parts.append(comp.pdf_character.char_unicode)
+ elif comp.pdf_line and hasattr(comp.pdf_line, 'pdf_character'):
+ for char in comp.pdf_line.pdf_character:
+ if hasattr(char, 'char_unicode') and char.char_unicode:
+ text_parts.append(char.char_unicode)
+ elif comp.pdf_same_style_unicode_characters:
+ if comp.pdf_same_style_unicode_characters.unicode:
+ text_parts.append(comp.pdf_same_style_unicode_characters.unicode)
+ paragraph_text = "".join(text_parts)
+
+ # Determine paragraph type based on layout
+ paragraph_type = "paragraph" # default
+ if hasattr(paragraph, 'layout') and paragraph.layout:
+ layout_name = paragraph.layout.class_name if hasattr(paragraph.layout, 'class_name') else str(paragraph.layout)
+ if 'title' in layout_name.lower() or 'heading' in layout_name.lower():
+ paragraph_type = "heading"
+ elif 'list' in layout_name.lower():
+ paragraph_type = "list_item"
+ # Check if text starts with bullet point
+ if paragraph_text and len(paragraph_text) > 0:
+ first_char = paragraph_text[0]
+ if first_char in ['•', '◦', '▪', '▫', '●', '○', '■', '□', '▶', '▷', '-', '·']:
+ paragraph_type = "bullet_point"
+
+ # Get box coordinates
+ if hasattr(paragraph, 'box') and paragraph.box:
+ box_coords = {
+ 'x': paragraph.box.x,
+ 'y': paragraph.box.y,
+ 'x2': paragraph.box.x2,
+ 'y2': paragraph.box.y2
+ }
+
+ # Get page number
+ page_num = page.page_number if hasattr(page, 'page_number') else 0
+
+ # Get scale
+ scale = paragraph.scale if hasattr(paragraph, 'scale') else None
+
+ # Log the typeset text block
+ self.detailed_logger.log_typeset_text_block(
+ page_num=page_num,
+ paragraph_type=paragraph_type,
+ text=paragraph_text,
+ box_coords=box_coords,
+ scale=scale
+ )
+ except Exception as e:
+ # Silently fail if logging has issues
+ pass
+
+ def _get_width_before_next_break_point(
+ self, typesetting_units: list[TypesettingUnit], scale: float
+ ) -> float:
+ if not typesetting_units:
+ return 0
+ if typesetting_units[0].can_break_line:
+ return 0
+
+ total_width = 0
+ for unit in typesetting_units:
+ if unit.can_break_line:
+ return total_width * scale
+ total_width += unit.width
+ return total_width * scale
+
+ def _layout_typesetting_units(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ box: Box,
+ scale: float,
+ line_skip: float,
+ paragraph: il_version_1.PdfParagraph,
+ use_english_line_break: bool = True,
+ ) -> tuple[list[TypesettingUnit], bool]:
+ """布局排版å•元。
+
+ Args:
+ typesetting_units: è¦ÂÂ布局的排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ box: 布局边界æ¡â€Â
+ scale: 缩æâ€Â¾å› åÂÂÂÂ
+
+ Returns:
+ tuple[list[TypesettingUnit], bool]: (已布局的排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨Ã¯Â¼Å’是å¦所有å•元都æâ€Â¾å¾â€â€ÃƒÂ¤Ã‚¸â€¹)
+ """
+ # 计ç®â€â€ÃƒÂ¥Ã‚Ââ€â€ÃƒÂ¥Ã‚·ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ font_sizes = []
+ for unit in typesetting_units:
+ if unit.font_size:
+ font_sizes.append(unit.font_size)
+ if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ font_sizes.append(unit.char.pdf_style.font_size)
+ font_sizes.sort()
+ font_size = statistics.mode(font_sizes)
+
+ space_width = (
+ self.font_mapper.base_font.char_lengths("ä½ ", font_size * scale)[0] * 0.5
+ )
+
+ # 计ç®â€â€ÃƒÂ¨Ã‚¡Å’高（使çâ€Â¨ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã¯Â¼â€°
+ unit_heights = (
+ [unit.height for unit in typesetting_units] if typesetting_units else []
+ )
+ if not unit_heights:
+ avg_height = 0
+ elif len(unit_heights) == 1:
+ avg_height = unit_heights[0] * scale
+ else:
+ try:
+ avg_height = statistics.mode(unit_heights) * scale
+ except statistics.StatisticsError:
+ # 如果没有ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã¯Â¼Ë†Ã¦â€°â‚¬Ã¦Å“‰å€¼éƒ½å‡ºçŽ°ç›¸åÂŒ次数），则使çâ€Â¨å¹³å‡值
+ avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # Check if output language is Arabic for RTL layout
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = False
+ if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ is_arabic = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic = True
+
+ # Initialize position - for Arabic (RTL), start from right; for LTR, start from left
+ if is_arabic:
+ # For RTL: start from right edge and work left
+ current_x = box.x2
+ current_y = box.y2 - avg_height
+ else:
+ # For LTR: start from left edge and work right
+ current_x = box.x
+ current_y = box.y2 - avg_height
+
+ box = copy.deepcopy(box)
+ # box.y -= avg_height * (line_spacing - 1.01) # line_spacing 已被替æ¢为 line_skip
+ line_height = 0
+ current_line_heights = [] # Ã¥ÂÂËÅ"储å½â€Å"å‰ÂÂ行所有元素的é«ËÅ"度
+
+ # Ã¥ÂÂËÅ"储已排版的å•元
+ typeset_units = []
+ all_units_fit = True
+ last_unit: TypesettingUnit | None = None
+ line_ys = [current_y]
+ is_first_line = True
+ prev_x = None
+ if paragraph.first_line_indent:
+ if is_arabic:
+ # For RTL: apply indent from right side
+ current_x -= space_width * 4
+ else:
+ # For LTR: apply indent from left side
+ current_x += space_width * 4
+ # For Arabic (RTL), process units in reverse order; for LTR, process normally
+ units_to_process = list(reversed(typesetting_units)) if is_arabic else typesetting_units
+
+ # éÂÂÂÂ历所有排版å•元
+ for i, unit in enumerate(units_to_process):
+ # Get original index for width calculation
+ orig_idx = len(typesetting_units) - 1 - i if is_arabic else i
+
+ # 计ç®â€â€ÃƒÂ¥Ã‚½â€Å"å‰ÂÂå•元在å½â€Å"å‰ÂÂ缩æâ€Â¾ä¸‹çš„尺寸
+ unit_width = unit.width * scale
+ unit_height = unit.height * scale
+
+ # 跳过行首的空格
+ if is_arabic:
+ # For RTL: skip leading spaces at right edge
+ if current_x == box.x2 and unit.is_space:
+ continue
+ else:
+ # For LTR: skip leading spaces at left edge
+ if current_x == box.x and unit.is_space:
+ continue
+
+ # Apply spacing between CJK and non-CJK characters (only for LTR)
+ if not is_arabic and (
+ last_unit # 有上一个å•元
+ and last_unit.is_cjk_char ^ unit.is_cjk_char # ä¸ÂÂ英文交界处
+ and (
+ last_unit.box
+ and last_unit.box.y
+ and current_y - 0.1
+ <= last_unit.box.y2
+ <= current_y + line_height + 0.1
+ ) # 在åÂŒ一行，ä¸â€Â有垂直é‡ÂÂÃ¥ÂÂÂÂ
+ and not last_unit.mixed_character_blacklist # ä¸ÂÂæËÅ"¯æ··æŽ’空格黑åÂÂÂÂå•åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦
+ and not unit.mixed_character_blacklist # Ã¥ÂÂŒä¸ÅÂ
+ and current_x > box.x # ä¸ÂÂæËÅ"¯è¡Œé¦–
+ and unit.try_get_unicode() != " " # ä¸ÂÂæËÅ"¯ç©ºæ ¼
+ and last_unit.try_get_unicode() != " " # ä¸ÂÂæËÅ"¯ç©ºæ ¼
+ and last_unit.try_get_unicode()
+ not in [
+ "。",
+ "ï¼ÂÂ",
+ "？",
+ "ï¼›",
+ "：",
+ "，",
+ ]
+ ):
+ current_x += space_width * 0.5
+ # Calculate width before next break point (for LTR only)
+ if use_english_line_break and not is_arabic:
+ width_before_next_break_point = self._get_width_before_next_break_point(
+ typesetting_units[orig_idx:], scale
+ )
+ else:
+ width_before_next_break_point = 0
+
+ # Check if we need to break line - different logic for RTL vs LTR
+ need_line_break = False
+ if not unit.is_hung_punctuation:
+ if is_arabic:
+ # For RTL: check if we've gone past the left boundary
+ # Position unit so its left edge is at current_x - unit_width
+ if (current_x - unit_width < box.x):
+ need_line_break = True
+ elif (
+ unit.is_cannot_appear_in_line_end_punctuation
+ and current_x - unit_width * 2 < box.x
+ ):
+ need_line_break = True
+ else:
+ # For LTR: check if we've gone past the right boundary
+ if (current_x + unit_width > box.x2):
+ need_line_break = True
+ elif (
+ use_english_line_break
+ and current_x + unit_width + width_before_next_break_point > box.x2
+ ):
+ need_line_break = True
+ elif (
+ unit.is_cannot_appear_in_line_end_punctuation
+ and current_x + unit_width * 2 > box.x2
+ ):
+ need_line_break = True
+
+ if need_line_break:
+ # æ¢行
+ if is_arabic:
+ current_x = box.x2
+ else:
+ current_x = box.x
+
+ if not current_line_heights:
+ return [], False
+ max_height = max(current_line_heights)
+ mode_height = statistics.mode(current_line_heights)
+
+ current_y -= max(mode_height * line_skip, max_height * 1.05)
+ line_ys.append(current_y)
+ line_height = 0.0
+ current_line_heights = [] # 清空å½â€Å"å‰ÂÂè¡Œé«ËÅ"度åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ is_first_line = False
+
+ # 检查æËÅ"¯å¦超出底部边界
+ # if current_y - unit_height < box.y:
+ if current_y < box.y:
+ all_units_fit = False
+ # 这里ä¸ÂÂ覠break，继ç»ÂÂ排版剩余内容
+
+ if unit.is_space:
+ line_height = max(line_height, unit_height)
+ continue
+
+ # Position unit - for RTL, place from right to left; for LTR, place from left to right
+ if is_arabic:
+ # For RTL: position unit so its right edge is at current_x
+ # The unit's x position will be current_x - unit_width
+ unit_x = current_x - unit_width
+ relocated_unit = unit.relocate(unit_x, current_y, scale)
+ # Update current_x to the left edge of the unit (for next unit)
+ current_x = unit_x
+ else:
+ # For LTR: position unit at current_x
+ relocated_unit = unit.relocate(current_x, current_y, scale)
+ # Update current_x to the right edge of the unit (for next unit)
+ current_x = relocated_unit.box.x2
+
+ typeset_units.append(relocated_unit)
+
+ # 添加å½â€Å"å‰ÂÂå•元的é«ËÅ"度到å½â€Å"å‰ÂÂè¡Œé«ËÅ"度åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ if not unit.is_space:
+ current_line_heights.append(unit_height)
+
+ if is_arabic and prev_x is not None and current_x > prev_x:
+ logger.warning(f"RTL position error: current_x ({current_x}) > prev_x ({prev_x})")
+
+ last_unit = relocated_unit
+ prev_x = current_x
+
+ # For Arabic, reverse the units order since we processed them in reverse
+ # This ensures the final order matches the logical text order
+ if is_arabic and typeset_units:
+ typeset_units = list(reversed(typeset_units))
+
+ return typeset_units, all_units_fit
+
+ def _mirror_margins_for_rtl(
+ self,
+ typeset_units: list[TypesettingUnit],
+ box: Box,
+ paragraph: il_version_1.PdfParagraph,
+ ) -> list[TypesettingUnit]:
+ """
+ Mirror left margins to right margins for RTL languages (Arabic).
+ This function preserves all original formatting and styling while adjusting
+ margins and indentation to follow RTL conventions.
+
+ Enhanced to:
+ - Check text_direction and text_align attributes
+ - Handle first-line indent reversal
+ - Properly align all lines to the right
+
+ Args:
+ typeset_units: Already laid out typesetting units
+ box: The paragraph's bounding box
+ paragraph: The paragraph object containing metadata
+
+ Returns:
+ list[TypesettingUnit]: Units with mirrored margins
+ """
+ if not typeset_units or not box:
+ return typeset_units
+
+ # Check if this paragraph should be RTL
+ is_rtl = False
+
+ # Primary check: use text_direction attribute if available
+ if hasattr(paragraph, 'text_direction') and paragraph.text_direction == 'rtl':
+ is_rtl = True
+ logger.debug(f"RTL detected via text_direction attribute for paragraph {paragraph.debug_id}")
+ # Secondary check: use text_align attribute
+ elif hasattr(paragraph, 'text_align') and paragraph.text_align == 'right':
+ is_rtl = True
+ logger.debug(f"RTL detected via text_align attribute for paragraph {paragraph.debug_id}")
+ # Fallback: check language configuration
+ elif not hasattr(paragraph, 'text_direction'):
+ lang_out = (self.translation_config.lang_out or "").lower()
+ if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ is_rtl = True
+ logger.debug(f"RTL detected via language config: {lang_out}")
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_rtl = True
+ logger.debug(f"RTL detected via language pattern: {lang_out}")
+
+ if not is_rtl:
+ logger.debug(f"Not RTL paragraph, skipping margin mirroring")
+ return typeset_units
+
+ # Check if this is a table paragraph (tables have their own layout)
+ is_table_paragraph = False
+ if hasattr(paragraph, 'pdf_paragraph_composition'):
+ for comp in paragraph.pdf_paragraph_composition:
+ if hasattr(comp, 'pdf_table') and comp.pdf_table:
+ is_table_paragraph = True
+ break
+
+ # Don't adjust table content
+ if is_table_paragraph:
+ logger.debug(f"Skipping RTL adjustment for table paragraph")
+ return typeset_units
+
+ logger.info(f"Applying RTL margin mirroring for paragraph {paragraph.debug_id}")
+
+ # Group units by line (Y coordinate)
+ lines_dict = {}
+ for unit in typeset_units:
+ if unit.box and unit.box.y is not None:
+ line_y = round(unit.box.y, 1)
+ if line_y not in lines_dict:
+ lines_dict[line_y] = []
+ lines_dict[line_y].append(unit)
+
+ # Sort lines by Y coordinate (top to bottom)
+ sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # Process each line to mirror margins
+ for line_index, line_y in enumerate(sorted_line_ys):
+ line_units = lines_dict[line_y]
+ if not line_units:
+ continue
+
+ # Find the leftmost position in this line (original left margin)
+ leftmost_x = min(u.box.x for u in line_units if u.box and u.box.x is not None)
+
+ # Calculate the left margin from the box's left edge
+ left_margin = leftmost_x - box.x
+
+ # For RTL, we want the same margin amount on the right side
+ # So the rightmost position should be: box.x2 - left_margin
+ target_rightmost_x = box.x2 - left_margin
+
+ # Find the current rightmost position
+ rightmost_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+
+ # Calculate the shift amount to align the rightmost position
+ shift_x = target_rightmost_x - rightmost_x
+
+ logger.debug(
+ f"Line {line_index} (y={line_y}): "
+ f"leftmost={leftmost_x:.2f}, left_margin={left_margin:.2f}, "
+ f"target_rightmost={target_rightmost_x:.2f}, current_rightmost={rightmost_x:.2f}, "
+ f"shift={shift_x:.2f}"
+ )
+
+ # Apply the shift to all units in this line
+ for unit in line_units:
+ if unit.box:
+ unit.box.x += shift_x
+ unit.box.x2 += shift_x
+ if unit.x is not None:
+ unit.x += shift_x
+
+ # Update character box if present
+ if unit.char:
+ if unit.char.box:
+ unit.char.box.x += shift_x
+ unit.char.box.x2 += shift_x
+ if hasattr(unit.char, 'visual_bbox') and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ unit.char.visual_bbox.box.x += shift_x
+ unit.char.visual_bbox.box.x2 += shift_x
+
+ logger.info(f"RTL margin mirroring completed for paragraph {paragraph.debug_id}")
+ return typeset_units
+
+# CORRECT FIX FOR ARABIC TEXT LAYOUT
+# Replace the _layout_typesetting_units function in typesetting.py (lines 1346-1502)
+
+ # def _layout_typesetting_units(
+ # self,
+ # typesetting_units: list[TypesettingUnit],
+ # box: Box,
+ # scale: float,
+ # line_skip: float,
+ # paragraph: il_version_1.PdfParagraph,
+ # use_english_line_break: bool = True,
+ # ) -> tuple[list[TypesettingUnit], bool]:
+ # """布局排版å•元。
+
+ # Args:
+ # typesetting_units: è¦ÂÂ布局的排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ # box: 布局边界æ¡â€Â
+ # scale: 缩æâ€Â¾å› åÂÂÂÂ
+
+ # Returns:
+ # tuple[list[TypesettingUnit], bool]: (已布局的排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨Ã¯Â¼Å’是å¦所有å•元都æâ€Â¾å¾â€â€ÃƒÂ¤Ã‚¸â€¹)
+ # """
+ # # 计ç®â€â€ÃƒÂ¥Ã‚Ââ€â€ÃƒÂ¥Ã‚·ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ # font_sizes = []
+ # for unit in typesetting_units:
+ # if unit.font_size:
+ # font_sizes.append(unit.font_size)
+ # if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ # font_sizes.append(unit.char.pdf_style.font_size)
+ # font_sizes.sort()
+ # font_size = statistics.mode(font_sizes)
+
+ # space_width = (
+ # self.font_mapper.base_font.char_lengths("ä½ ", font_size * scale)[0] * 0.5
+ # )
+
+ # # 计ç®â€â€ÃƒÂ¨Ã‚¡Å’高（使çâ€Â¨ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã¯Â¼â€°
+ # unit_heights = (
+ # [unit.height for unit in typesetting_units] if typesetting_units else []
+ # )
+ # if not unit_heights:
+ # avg_height = 0
+ # elif len(unit_heights) == 1:
+ # avg_height = unit_heights[0] * scale
+ # else:
+ # try:
+ # avg_height = statistics.mode(unit_heights) * scale
+ # except statistics.StatisticsError:
+ # # 如果没有ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã¯Â¼Ë†Ã¦â€°â‚¬Ã¦Å“‰å€¼éƒ½å‡ºçŽ°ç›¸åÂŒ次数），则使çâ€Â¨å¹³å‡值
+ # avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # # *** NEW: Detect Arabic language ***
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # # åˆÂÂ始化ä½ÂÂ置为å³上角，并å‡ÂÂ去一个平å‡行高
+ # # *** CHANGED: For Arabic, calculate total line width first and start from right ***
+ # current_x = box.x
+ # current_y = box.y2 - avg_height
+ # box = copy.deepcopy(box)
+ # line_height = 0
+ # current_line_heights = [] # å˜储当å‰ÂÂ行所有元素的高度
+
+ # # å˜储已排版的å•元
+ # typeset_units = []
+ # all_units_fit = True
+ # last_unit: TypesettingUnit | None = None
+ # line_ys = [current_y]
+ # if paragraph.first_line_indent:
+ # current_x += space_width * 4
+ # # éÂÂÂÂ历所有排版å•元
+ # for i, unit in enumerate(typesetting_units):
+ # # 计ç®â€â€ÃƒÂ¥Ã‚½â€œÃ¥â€°ÂÂå•元在当å‰ÂÂ缩æâ€Â¾ä¸‹çš„尺寸
+ # unit_width = unit.width * scale
+ # unit_height = unit.height * scale
+
+ # # 跳过行首的空格
+ # if current_x == box.x and unit.is_space:
+ # continue
+
+ # if (
+ # last_unit # 有上一个å•元
+ # and last_unit.is_cjk_char ^ unit.is_cjk_char # ä¸ÂÂ英文交界处
+ # and (
+ # last_unit.box
+ # and last_unit.box.y
+ # and current_y - 0.1
+ # <= last_unit.box.y2
+ # <= current_y + line_height + 0.1
+ # ) # 在åÂŒ一行，ä¸â€Â有垂直é‡ÂÂÃ¥ÂÂÂÂ
+ # and not last_unit.mixed_character_blacklist # ä¸ÂÂ是混排空格黑åÂÂÂÂå•åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦
+ # and not unit.mixed_character_blacklist # Ã¥ÂÂŒä¸ÅÂ
+ # and current_x > box.x # ä¸ÂÂ是行首
+ # and unit.try_get_unicode() != " " # ä¸ÂÂ是空格
+ # and last_unit.try_get_unicode() != " " # ä¸ÂÂ是空格
+ # and last_unit.try_get_unicode()
+ # not in [
+ # "。",
+ # "ï¼ÂÂ",
+ # "？",
+ # "ï¼›",
+ # "：",
+ # "，",
+ # ]
+ # ):
+ # current_x += space_width * 0.5
+ # if use_english_line_break:
+ # width_before_next_break_point = self._get_width_before_next_break_point(
+ # typesetting_units[i:], scale
+ # )
+ # else:
+ # width_before_next_break_point = 0
+
+ # # 如果当å‰ÂÂè¡Œæâ€Â¾ä¸ÂÂ下这个元素，æ¢行
+ # if not unit.is_hung_punctuation and (
+ # (current_x + unit_width > box.x2)
+ # or (
+ # use_english_line_break
+ # and current_x + unit_width + width_before_next_break_point > box.x2
+ # )
+ # or (
+ # unit.is_cannot_appear_in_line_end_punctuation
+ # and current_x + unit_width * 2 > box.x2
+ # )
+ # ):
+ # # æ¢行
+ # current_x = box.x
+ # if not current_line_heights:
+ # return [], False
+ # max_height = max(current_line_heights)
+ # mode_height = statistics.mode(current_line_heights)
+
+ # current_y -= max(mode_height * line_skip, max_height * 1.05)
+ # line_ys.append(current_y)
+ # line_height = 0.0
+ # current_line_heights = [] # 清空当å‰ÂÂ行高度åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+
+ # # 检查是å¦超出底部边界
+ # # if current_y - unit_height < box.y:
+ # if current_y < box.y:
+ # all_units_fit = False
+ # # 这里ä¸ÂÂ覠break，继ç»ÂÂ排版剩余内容
+
+ # if unit.is_space:
+ # line_height = max(line_height, unit_height)
+ # continue
+
+ # # æâ€Â¾ç½®å½“å‰ÂÂå•元
+ # relocated_unit = unit.relocate(current_x, current_y, scale)
+ # typeset_units.append(relocated_unit)
+
+ # # 添加当å‰ÂÂå•元的高度到当å‰ÂÂ行高度åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ # if not unit.is_space:
+ # current_line_heights.append(unit_height)
+
+ # prev_x = current_x
+ # # æ›´æ–° x Ã¥ÂÂÂÂæ ‡
+ # current_x = relocated_unit.box.x2
+ # if prev_x > current_x:
+ # logger.warning(f"Ã¥ÂÂÂÂ标回绕ï¼ÂÂï¼ÂÂï¼ÂÂTypesettingUnit: {unit.box}, ")
+
+ # last_unit = relocated_unit
+
+ # # *** NEW: For Arabic, right-align each line ***
+ # if is_arabic and typeset_units:
+ # # Group units by line (Y coordinate)
+ # lines = {}
+ # for unit in typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # line_y = round(unit.box.y, 1)
+ # if line_y not in lines:
+ # lines[line_y] = []
+ # lines[line_y].append(unit)
+
+ # # Right-align each line
+ # for line_y, line_units in lines.items():
+ # if not line_units:
+ # continue
+
+ # # Find the rightmost position of this line
+ # line_max_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+
+ # # Calculate how much to shift right
+ # shift_x = box.x2 - line_max_x
+
+ # # Shift all units in this line to the right
+ # for unit in line_units:
+ # if unit.box:
+ # unit.box.x += shift_x
+ # unit.box.x2 += shift_x
+ # if unit.x is not None:
+ # unit.x += shift_x
+ # # Update character box if present
+ # if unit.char and unit.char.box:
+ # unit.char.box.x += shift_x
+ # unit.char.box.x2 += shift_x
+ # if unit.char and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ # unit.char.visual_bbox.box.x += shift_x
+ # unit.char.visual_bbox.box.x2 += shift_x
+ # # Check if output language is Arabic
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # # If Arabic, reverse the line order
+ # if is_arabic and typeset_units:
+ # # Group units by line (using Y coordinates)
+ # lines_dict = {}
+ # for unit in typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # # Round Y coordinate to group units on the same line
+ # line_y = round(unit.box.y, 1)
+ # if line_y not in lines_dict:
+ # lines_dict[line_y] = []
+ # lines_dict[line_y].append(unit)
+
+ # # Sort lines by Y coordinate (top to bottom) and reverse
+ # sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # # Rebuild typeset_units with reversed line order
+ # reversed_typeset_units = []
+ # for line_y in reversed(sorted_line_ys):
+ # reversed_typeset_units.extend(lines_dict[line_y])
+
+ # # Now reposition all units to swap their Y coordinates
+ # # Map old Y positions to new Y positions
+ # y_mapping = {}
+ # for i, old_y in enumerate(sorted_line_ys):
+ # new_y = sorted_line_ys[len(sorted_line_ys) - 1 - i]
+ # y_mapping[old_y] = new_y
+
+ # # Update Y coordinates for all units
+ # for unit in reversed_typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # old_y = round(unit.box.y, 1)
+ # if old_y in y_mapping:
+ # new_y = y_mapping[old_y]
+ # y_diff = new_y - old_y
+ # # Update the unit's Y position
+ # if unit.y is not None:
+ # unit.y += y_diff
+ # if unit.box:
+ # unit.box.y += y_diff
+ # unit.box.y2 += y_diff
+
+ # typeset_units = reversed_typeset_units
+
+ # return typeset_units, all_units_fit
+
+ def create_typesetting_units(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ fonts: dict[str, il_version_1.PdfFont],
+ ) -> list[TypesettingUnit]:
+ if not paragraph.pdf_paragraph_composition:
+ return []
+ result = []
+
+ @cache
+ def get_font(font_id: str, xobj_id: int | None):
+ if xobj_id in fonts:
+ font = fonts[xobj_id][font_id]
+ else:
+ font = fonts[font_id]
+ return font
+
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition is None:
+ continue
+ if composition.pdf_line:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_line.pdf_character
+ ],
+ )
+ elif composition.pdf_character:
+ result.append(
+ TypesettingUnit(
+ char=composition.pdf_character,
+ debug_info=paragraph.debug_info,
+ ),
+ )
+ elif composition.pdf_same_style_characters:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_same_style_characters.pdf_character
+ ],
+ )
+ elif composition.pdf_same_style_unicode_characters:
+ style = composition.pdf_same_style_unicode_characters.pdf_style
+ if style is None:
+ logger.warning(
+ f"Style is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font_id = style.font_id
+ if font_id is None:
+ logger.warning(
+ f"Font ID is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font = get_font(font_id, paragraph.xobj_id)
+ if composition.pdf_same_style_unicode_characters.unicode:
+ unicode_text = composition.pdf_same_style_unicode_characters.unicode
+ shaped_text = self.shape_arabic_text(unicode_text)
+ result.extend(
+ [
+ TypesettingUnit(
+ unicode=char_unicode,
+ font=self.font_mapper.map(
+ font,
+ char_unicode,
+ ),
+ original_font=font,
+ font_size=style.font_size,
+ style=style,
+ xobj_id=paragraph.xobj_id,
+ debug_info=composition.pdf_same_style_unicode_characters.debug_info
+ or False,
+ )
+ for char_unicode in shaped_text # Use shaped_text instead of original
+ if char_unicode not in ("\n",)
+ ],
+ )
+ elif composition.pdf_formula:
+ result.extend([TypesettingUnit(formular=composition.pdf_formula)])
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ result = list(
+ filter(
+ lambda x: x.unicode is None or x.font is not None,
+ result,
+ ),
+ )
+
+ if any(x.width < 0 for x in result):
+ logger.warning("有排版å•元宽度å°ÂÂ于 0，请检查åÂÂâ€â€ÃƒÂ¤Ã‚½â€œÃ¦ËœÂ Ã¥Â°â€žÃ¦ËœÂ¯Ã¥Â¦æ£确。")
+ return result
+
+ def create_passthrough_composition(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ ) -> list[PdfParagraphComposition]:
+ """从排版å•元创建直接传递的段è½组åˆ。
+
+ Args:
+ typesetting_units: 排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+
+ Returns:
+ 段è½组åˆåˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ """
+ composition = []
+ for unit in typesetting_units:
+ if unit.formular:
+ # 对于公å¼ÂÂå•元，直接创建包å«完整公å¼ÂÂ的组åÂÂËâ€
+ composition.append(PdfParagraphComposition(pdf_formula=unit.formular))
+ else:
+ # 对于åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â•元，使çâ€Â¨åŽŸæœ‰é€»è¾‘
+ chars, curves, forms = unit.passthrough()
+ composition.extend(
+ [PdfParagraphComposition(pdf_character=char) for char in chars],
+ )
+ return composition
+
+ def get_max_right_space(self, current_box: Box, page) -> float:
+ """获å–段è½å³侧最大å¯çâ€Â¨ç©ºéâ€â€Ã‚´
+
+ Args:
+ current_box: 当å‰ÂÂ段è½的边界æ¡â€Â
+ page: 当å‰ÂÂ页é¢
+
+ Returns:
+ å¯以扩展到的最大 x Ã¥ÂÂÂÂæ ‡
+ """
+ # 获å–页é¢的è£ÂÂ剪框作为åˆÂÂ始最大é™ÂÂ制
+ max_x = page.cropbox.box.x2 * 0.9
+
+ # 检查所有å¯能的阻挡元ç´ÂÂ
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # 跳过当å‰ÂÂ段è½
+ continue
+ # åª考虑在当å‰ÂÂ段è½å³侧ä¸â€Â有垂直é‡ÂÂå 的元ç´ÂÂ
+ if para.box.x > current_box.x and not (
+ para.box.y >= current_box.y2 or para.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, para.box.x)
+ for char in page.pdf_character:
+ if char.box.x > current_box.x and not (
+ char.box.y >= current_box.y2 or char.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, char.box.x)
+ # 检查图形
+ for figure in page.pdf_figure:
+ if figure.box.x > current_box.x and not (
+ figure.box.y >= current_box.y2 or figure.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, figure.box.x)
+
+ return max_x
+
+ def get_max_bottom_space(self, current_box: Box, page: il_version_1.Page) -> float:
+ """获å–段è½下方最大å¯çâ€Â¨ç©ºéâ€â€Ã‚´
+
+ Args:
+ current_box: 当å‰ÂÂ段è½的边界æ¡â€Â
+ page: 当å‰ÂÂ页é¢
+
+ Returns:
+ å¯以扩展到的最尠y Ã¥ÂÂÂÂæ ‡
+ """
+ # 获å–页é¢的è£ÂÂ剪框作为åˆÂÂ始最å°ÂÂé™ÂÂ制
+ min_y = page.cropbox.box.y * 1.1
+
+ # 检查所有å¯能的阻挡元ç´ÂÂ
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # 跳过当å‰ÂÂ段è½
+ continue
+ # åª考虑在当å‰ÂÂ段è½下方ä¸â€Â有水平é‡ÂÂå 的元ç´ÂÂ
+ if para.box.y2 < current_box.y and not (
+ para.box.x >= current_box.x2 or para.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, para.box.y2)
+ for char in page.pdf_character:
+ if char.box.y2 < current_box.y and not (
+ char.box.x >= current_box.x2 or char.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, char.box.y2)
+ # 检查图形
+ for figure in page.pdf_figure:
+ if figure.box.y2 < current_box.y and not (
+ figure.box.x >= current_box.x2 or figure.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, figure.box.y2)
+
+ return min_y
+
+ def _update_paragraph_render_order(self, paragraph: il_version_1.PdfParagraph):
+ """
+ é‡ÂÂ新设置段è½å„åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã§Å¡â€ž render order
+ 主 render order ç‰于 paragraph çš„ renderorder，sub render order 从 1 开始自增
+ """
+ if not hasattr(paragraph, "render_order") or paragraph.render_order is None:
+ return
+
+ main_render_order = paragraph.render_order
+ sub_render_order = 1
+
+ # éÂÂÂÂ历段è½的所有组æˆÂÂ部åˆâ€Â
+ for composition in paragraph.pdf_paragraph_composition:
+ # 检查å•个åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦
+ if composition.pdf_character:
+ char = composition.pdf_character
+ char.render_order = main_render_order
+ char.sub_render_order = sub_render_order
+ sub_render_order += 1
\ No newline at end of file
diff --git a/babeldoc/format/pdf/document_il/midend/table_parser.py b/babeldoc/format/pdf/document_il/midend/table_parser.py
new file mode 100644
index 0000000000000000000000000000000000000000..13a8351c7efe9906097c2d35b8d2e36b216c8709
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/table_parser.py
@@ -0,0 +1,166 @@
+import logging
+from pathlib import Path
+
+import cv2
+import numpy as np
+from pymupdf import Document
+
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.utils.mupdf_helper import get_no_rotation_img
+from babeldoc.format.pdf.document_il.utils.style_helper import GREEN
+from babeldoc.format.pdf.translation_config import TranslationConfig
+
+logger = logging.getLogger(__name__)
+
+
+class TableParser:
+ stage_name = "Parse Table"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.translation_config = translation_config
+ self.model = translation_config.table_model
+
+ def _save_debug_image(self, image: np.ndarray, layouts, page_number: int):
+ """Save debug image with drawn boxes if debug mode is enabled."""
+ if not self.translation_config.debug:
+ return
+
+ if not isinstance(layouts, list):
+ layouts = [layouts]
+ debug_dir = Path(
+ self.translation_config.get_working_file_path("table-ocr-box-image")
+ )
+ debug_dir.mkdir(parents=True, exist_ok=True)
+
+ # Draw boxes on the image
+ debug_image = image.copy()
+ for layout in layouts:
+ for box in layout.boxes:
+ x0, y0, x1, y1 = box.xyxy
+ cv2.rectangle(
+ debug_image,
+ (int(x0), int(y0)),
+ (int(x1), int(y1)),
+ (0, 255, 0),
+ 2,
+ )
+ # Add text label
+ cv2.putText(
+ debug_image,
+ layout.names[box.cls],
+ (int(x0), int(y0) - 5),
+ cv2.FONT_HERSHEY_SIMPLEX,
+ 0.5,
+ (0, 255, 0),
+ 1,
+ )
+
+ # Save the image
+ output_path = debug_dir / f"{page_number}.jpg"
+ cv2.imwrite(str(output_path), debug_image)
+
+ def _save_debug_box_to_page(self, page: il_version_1.Page):
+ """Save debug boxes and text labels to the PDF page."""
+ if not self.translation_config.debug:
+ return
+
+ color = GREEN
+
+ for layout in page.page_layout:
+ # Create a rectangle box
+ rect = il_version_1.PdfRectangle(
+ box=il_version_1.Box(
+ x=layout.box.x,
+ y=layout.box.y,
+ x2=layout.box.x2,
+ y2=layout.box.y2,
+ ),
+ graphic_state=color,
+ debug_info=True,
+ )
+ page.pdf_rectangle.append(rect)
+
+ # Create text label at top-left corner
+ # Note: PDF coordinates are from bottom-left,
+ # so we use y2 for top position
+ style = il_version_1.PdfStyle(
+ font_id="base",
+ font_size=4,
+ graphic_state=color,
+ )
+ page.pdf_paragraph.append(
+ il_version_1.PdfParagraph(
+ first_line_indent=False,
+ box=il_version_1.Box(
+ x=layout.box.x,
+ y=layout.box.y2,
+ x2=layout.box.x2,
+ y2=layout.box.y2 + 5,
+ ),
+ vertical=False,
+ pdf_style=style,
+ unicode=layout.class_name,
+ pdf_paragraph_composition=[
+ il_version_1.PdfParagraphComposition(
+ pdf_same_style_unicode_characters=il_version_1.PdfSameStyleUnicodeCharacters(
+ unicode=layout.class_name,
+ pdf_style=style,
+ debug_info=True,
+ ),
+ ),
+ ],
+ xobj_id=-1,
+ ),
+ )
+
+ def process(self, docs: il_version_1.Document, mupdf_doc: Document):
+ """Generate layouts for all pages that need to be translated."""
+ # Get pages that need to be translated
+ have_table_pages = {}
+ for page in docs.page:
+ for layout in page.page_layout:
+ if layout.class_name == "table":
+ have_table_pages[page.page_number] = page
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(have_table_pages),
+ ) as progress:
+ # Process predictions for each page
+ for page, layouts in self.model.handle_document(
+ have_table_pages.values(),
+ mupdf_doc,
+ self.translation_config,
+ self._save_debug_image,
+ ):
+ page_layouts = []
+ for layout in layouts.boxes:
+ # Convert coordinate system from picture to il
+ # system to the il coordinate system
+ x0, y0, x1, y1 = layout.xyxy
+ # pix = mupdf_doc[page.page_number].get_pixmap()
+ pix = get_no_rotation_img(mupdf_doc[page.page_number])
+ h, w = pix.height, pix.width
+ x0, y0, x1, y1 = (
+ np.clip(int(x0 - 1), 0, w - 1),
+ np.clip(int(h - y1 - 1), 0, h - 1),
+ np.clip(int(x1 + 1), 0, w - 1),
+ np.clip(int(h - y0 + 1), 0, h - 1),
+ )
+ page_layout = il_version_1.PageLayout(
+ id=len(page_layouts) + 1,
+ box=il_version_1.Box(
+ x0.item(),
+ y0.item(),
+ x1.item(),
+ y1.item(),
+ ),
+ conf=layout.conf.item(),
+ class_name=layouts.names[layout.cls],
+ )
+ page_layouts.append(page_layout)
+
+ page.page_layout.extend(page_layouts)
+ self._save_debug_box_to_page(page)
+ progress.advance(1)
+
+ return docs
diff --git a/babeldoc/format/pdf/document_il/midend/typesetting-v1.py b/babeldoc/format/pdf/document_il/midend/typesetting-v1.py
new file mode 100644
index 0000000000000000000000000000000000000000..75f61e9b1b5dbd7c61877b5154cfdf0aaaf40c40
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/typesetting-v1.py
@@ -0,0 +1,2134 @@
+from __future__ import annotations
+
+import copy
+import logging
+import re
+import statistics
+import unicodedata
+from functools import cache
+
+import pymupdf
+import regex
+from rtree import index
+
+from babeldoc.const import WATERMARK_VERSION
+from babeldoc.format.pdf.document_il import Box
+from babeldoc.format.pdf.document_il import PdfCharacter
+from babeldoc.format.pdf.document_il import PdfCurve
+from babeldoc.format.pdf.document_il import PdfForm
+from babeldoc.format.pdf.document_il import PdfFormula
+from babeldoc.format.pdf.document_il import PdfParagraphComposition
+from babeldoc.format.pdf.document_il import PdfStyle
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.formular_helper import update_formula_data
+from babeldoc.format.pdf.document_il.utils.layout_helper import box_to_tuple
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from babeldoc.format.pdf.translation_config import WatermarkOutputMode
+from arabic_reshaper import reshape
+from bidi.algorithm import get_display
+
+
+logger = logging.getLogger(__name__)
+
+LINE_BREAK_REGEX = regex.compile(
+ r"^["
+ r"a-z"
+ r"A-Z"
+ r"0-9"
+ r"\u00C0-\u00FF" # Latin-1 Supplement
+ r"\u0100-\u017F" # Latin Extended A
+ r"\u0180-\u024F" # Latin Extended B
+ r"\u1E00-\u1EFF" # Latin Extended Additional
+ r"\u2C60-\u2C7F" # Latin Extended C
+ r"\uA720-\uA7FF" # Latin Extended D
+ r"\uAB30-\uAB6F" # Latin Extended E
+ r"\u0250-\u02A0" # IPA Extensions
+ r"\u0400-\u04FF" # Cyrillic
+ r"\u0300-\u036F" # Combining Diacritical Marks
+ r"\u0500-\u052F" # Cyrillic Supplement
+ r"\u0370-\u03FF" # Greek and Coptic
+ r"\u2DE0-\u2DFF" # Cyrillic Extended-A
+ r"\uA650-\uA69F" # Cyrillic Extended-B
+ r"\u1200-\u137F" # Ethiopic
+ r"\u1380-\u139F" # Ethiopic Supplement
+ r"\u2D80-\u2DDF" # Ethiopic Extended
+ r"\uAB00-\uAB2F" # Ethiopic Extended-A
+ r"\U0001E7E0-\U0001E7FF" # Ethiopic Extended-B
+ r"\u0E80-\u0EFF" # Lao
+ r"\u0D00-\u0D7F" # Malayalam
+ r"\u0A80-\u0AFF" # Gujarati
+ r"\u0E00-\u0E7F" # Thai
+ r"\u1000-\u109F" # Myanmar
+ r"\uAA60-\uAA7F" # Myanmar Extended-A
+ r"\uA9E0-\uA9FF" # Myanmar Extended-B
+ r"\U000116D0-\U000116FF" # Myanmar Extended-C
+ r"\u0B80-\u0BFF" # Tamil
+ r"\u0C00-\u0C7F" # Telugu
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0530-\u058F" # Armenian
+ r"\u10A0-\u10FF" # Georgian
+ r"\u1C90-\u1CBF" # Georgian Extended
+ r"\u2D00-\u2D2F" # Georgian Supplement
+ r"\u1780-\u17FF" # Khmer
+ r"\u19E0-\u19FF" # Khmer Symbols
+ r"\U00010B00-\U00010B3F" # Avestan
+ r"\u1D00-\u1D7F" # Phonetic Extensions
+ r"\u1400-\u167F" # Unified Canadian Aboriginal Syllabics
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0780-\u07BF" # Thaana
+ r"\U0001E900-\U0001E95F" # Adlam
+ r"\u1C80-\u1C8F" # Cyrillic Extended-C
+ r"\U0001E030-\U0001E08F" # Cyrillic Extended-D
+ r"\uA000-\uA48F" # Yi Syllables
+ r"\uA490-\uA4CF" # Yi Radicals
+ r"'"
+ r"-" # Hyphen
+ r"·" # Middle Dot (U+00B7) For CatalÃÂÂÂ
+ r"Ê»" # Spacing Modifier Letters U+02BB
+ r"]+$"
+)
+
+
+class TypesettingUnit:
+ def __str__(self):
+ return self.try_get_unicode() or ""
+
+ def __init__(
+ self,
+ char: PdfCharacter | None = None,
+ formular: PdfFormula | None = None,
+ unicode: str | None = None,
+ font: pymupdf.Font | None = None,
+ original_font: il_version_1.PdfFont | None = None,
+ font_size: float | None = None,
+ style: PdfStyle | None = None,
+ xobj_id: int | None = None,
+ debug_info: bool = False,
+ ):
+ assert (char is not None) + (formular is not None) + (
+ unicode is not None
+ ) == 1, "Only one of chars and formular can be not None"
+ self.char = char
+ self.formular = formular
+ self.unicode = unicode
+ self.x = None
+ self.y = None
+ self.scale = None
+ self.debug_info = debug_info
+
+ # Cache variables
+ self.box_cache: Box | None = None
+ self.can_break_line_cache: bool | None = None
+ self.is_cjk_char_cache: bool | None = None
+ self.mixed_character_blacklist_cache: bool | None = None
+ self.is_space_cache: bool | None = None
+ self.is_hung_punctuation_cache: bool | None = None
+ self.is_cannot_appear_in_line_end_punctuation_cache: bool | None = None
+ self.can_passthrough_cache: bool | None = None
+ self.width_cache: float | None = None
+ self.height_cache: float | None = None
+
+ self.font_size: float | None = None
+
+ if unicode:
+ assert font_size, "Font size must be provided when unicode is provided"
+ assert style, "Style must be provided when unicode is provided"
+ assert len(unicode) == 1, "Unicode must be a single character"
+ assert xobj_id is not None, (
+ "Xobj id must be provided when unicode is provided"
+ )
+
+ self.font = font
+ if font is not None and hasattr(font, "font_id"):
+ self.font_id = font.font_id
+ else:
+ self.font_id = "base"
+ if original_font:
+ self.original_font = original_font
+ else:
+ self.original_font = None
+
+ self.font_size = font_size
+ self.style = style
+ self.xobj_id = xobj_id
+
+ def try_resue_cache(self, old_tu: TypesettingUnit):
+ if old_tu.is_cjk_char_cache is not None:
+ self.is_cjk_char_cache = old_tu.is_cjk_char_cache
+
+ if old_tu.can_break_line_cache is not None:
+ self.can_break_line_cache = old_tu.can_break_line_cache
+
+ if old_tu.is_space_cache is not None:
+ self.is_space_cache = old_tu.is_space_cache
+
+ if old_tu.is_hung_punctuation_cache is not None:
+ self.is_hung_punctuation_cache = old_tu.is_hung_punctuation_cache
+
+ if old_tu.is_cannot_appear_in_line_end_punctuation_cache is not None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ old_tu.is_cannot_appear_in_line_end_punctuation_cache
+ )
+
+ if old_tu.can_passthrough_cache is not None:
+ self.can_passthrough_cache = old_tu.can_passthrough_cache
+
+ if old_tu.mixed_character_blacklist_cache is not None:
+ self.mixed_character_blacklist_cache = (
+ old_tu.mixed_character_blacklist_cache
+ )
+
+
+ def try_get_unicode(self) -> str | None:
+ if self.char:
+ return self.char.char_unicode
+ elif self.formular:
+ return None
+ elif self.unicode:
+ return self.unicode
+
+ @property
+ def mixed_character_blacklist(self):
+ if self.mixed_character_blacklist_cache is None:
+ self.mixed_character_blacklist_cache = self.calc_mixed_character_blacklist()
+
+ return self.mixed_character_blacklist_cache
+
+ def calc_mixed_character_blacklist(self):
+ unicode = self.try_get_unicode()
+ if unicode:
+ return unicode in [
+ "。",
+ "，",
+ "：",
+ "？",
+ "ï¼ÂÂÂ",
+ ]
+ return False
+
+ @property
+ def can_break_line(self):
+ if self.can_break_line_cache is None:
+ self.can_break_line_cache = self.calc_can_break_line()
+
+ return self.can_break_line_cache
+
+ def calc_can_break_line(self):
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return True
+ if LINE_BREAK_REGEX.match(unicode):
+ return False
+ return True
+
+ @property
+ def is_cjk_char(self):
+ if self.is_cjk_char_cache is None:
+ self.is_cjk_char_cache = self.calc_is_cjk_char()
+
+ return self.is_cjk_char_cache
+
+ def calc_is_cjk_char(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ if "(cid" in unicode:
+ return False
+ if len(unicode) > 1:
+ return False
+ assert len(unicode) == 1, "Unicode must be a single character"
+ if unicode in [
+ "（",
+ "）",
+ "ã€ÂÂÂ",
+ "】",
+ "《",
+ "》",
+ "ã€â€ÂÂ",
+ "〕",
+ "〈",
+ "〉",
+ "〖",
+ "ã€â€â€Â",
+ "「",
+ "ã€ÂÂÂ",
+ "『",
+ "ã€ÂÂÂ",
+ "ã€ÂÂÂ",
+ "。",
+ "：",
+ "？",
+ "ï¼ÂÂÂ",
+ "，",
+ ]:
+ return True
+ if unicode:
+ if re.match(
+ r"^["
+ r"\u3000-\u303f" # CJK Symbols and Punctuation
+ r"\u3040-\u309f" # Hiragana
+ r"\u30a0-\u30ff" # Katakana
+ r"\u3100-\u312f" # Bopomofo
+ r"\uac00-\ud7af" # Hangul Syllables
+ r"\u1100-\u11ff" # Hangul Jamo
+ r"\u3130-\u318f" # Hangul Compatibility Jamo
+ r"\ua960-\ua97f" # Hangul Jamo Extended-A
+ r"\ud7b0-\ud7ff" # Hangul Jamo Extended-B
+ r"\u3190-\u319f" # Kanbun
+ r"\u3200-\u32ff" # Enclosed CJK Letters and Months
+ r"\u3300-\u33ff" # CJK Compatibility
+ r"\ufe30-\ufe4f" # CJK Compatibility Forms
+ r"\u4e00-\u9fff" # CJK Unified Ideographs
+ r"\u2e80-\u2eff" # CJK Radicals Supplement
+ r"\u31c0-\u31ef" # CJK Strokes
+ r"\u2f00-\u2fdf" # Kangxi Radicals
+ r"\ufe10-\ufe1f" # Vertical Forms
+ r"]+$",
+ unicode,
+ ):
+ return True
+ try:
+ unicodedata_name = unicodedata.name(unicode)
+ return (
+ "CJK UNIFIED IDEOGRAPH" in unicodedata_name
+ or "FULLWIDTH" in unicodedata_name
+ )
+ except ValueError:
+ return False
+ return False
+
+ @property
+ def is_space(self):
+ if self.is_space_cache is None:
+ self.is_space_cache = self.calc_is_space()
+
+ return self.is_space_cache
+
+ def calc_is_space(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ return unicode == " "
+
+ @property
+ def is_hung_punctuation(self):
+ if self.is_hung_punctuation_cache is None:
+ self.is_hung_punctuation_cache = self.calc_is_hung_punctuation()
+
+ return self.is_hung_punctuation_cache
+
+ def calc_is_hung_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+
+ if unicode:
+ return unicode in [
+ # 英文标点
+ ",",
+ ".",
+ ":",
+ ";",
+ "?",
+ "!",
+ # ä¸ÂÂÂ文点å·
+ "，", # é€â€â€Âå·
+ "。", # Ã¥ÂÂÂ¥å·
+ "．", # 全角åÂÂÂ¥å·
+ "ã€ÂÂÂ", # é¡¿å·
+ "：", # 冒å·
+ "ï¼›", # 分å·
+ "ï¼ÂÂÂ", # å¹å·
+ "‼", # Ã¥ÂÂÂŒå¹å·
+ "？", # éâ€â€Â®å·
+ "â‡", # Ã¥ÂÂÂΎâ€â€Â®å·
+ # 结æŸ引å·
+ "â€ÂÂÂ", # å³åÂÂŒ引å·
+ "’", # å³å•引å·
+ "ã€ÂÂÂ", # å³直角å•引å·
+ "ã€ÂÂÂ", # å³直角åÂÂŒ引å·
+ # 结æŸ括å·
+ ")", # å³圆括å·
+ "]", # å³方括å·
+ "}", # å³花括å·
+ "）", # å³圆括å·
+ "〕", # å³龟çâ€Â²括å·
+ "〉", # å³å•书åÂÂÂÂÂÂå·
+ "】", # å³黑色方头括å·
+ "ã€â€â€Â", # å³空白方头括å·
+ "ï¼½", # 全角å³方括å·
+ "ï½ÂÂÂ", # 全角å³花括å·
+ # 结æŸåÂÂŒ书åÂÂÂÂÂÂå·
+ "》", # å³åÂÂŒ书åÂÂÂÂÂÂå·
+ # 连接å·
+ "～", # 全角波浪å·
+ "-", # è¿žåÂÂÂâ€â€Â符å‡ÂÂÂå·
+ "–", # çŸÂÂÂ破折å· (EN DASH)
+ "â€â€ÂÂ", # 长破折å· (EM DASH)
+ # éâ€â€Â´éšâ€ÂÂå·
+ "·", # ä¸ÂÂÂéâ€â€Â´ç‚¹
+ "・", # 片å‡åÂÂÂÂÂÂä¸ÂÂÂéâ€â€Â´ç‚¹
+ "‧", # è¿žåÂÂÂâ€â€Â点
+ # 分éšâ€ÂÂå·
+ "/", # æ–œæÂÂÂÂÂÂ
+ "ï¼ÂÂÂ", # 全角斜æÂÂÂÂÂÂ
+ "â„", # 分数斜æÂÂÂÂÂÂ
+ ]
+ return False
+
+ @property
+ def is_cannot_appear_in_line_end_punctuation(self):
+ if self.is_cannot_appear_in_line_end_punctuation_cache is None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ self.calc_is_cannot_appear_in_line_end_punctuation()
+ )
+
+ return self.is_cannot_appear_in_line_end_punctuation_cache
+
+ def calc_is_cannot_appear_in_line_end_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ return unicode in [
+ # 开始引å·
+ "“", # å·¦åÂÂŒ引å·
+ "‘", # å·¦å•引å·
+ "「", # 左直角å•引å·
+ "『", # 左直角åÂÂŒ引å·
+ # 开始括å·
+ "(", # 左圆括å·
+ "[", # 左方括å·
+ "{", # 左花括å·
+ "（", # 左圆括å·
+ "ã€â€ÂÂ", # 左龟çâ€Â²括å·
+ "〈", # å·¦å•书åÂÂÂÂÂÂå·
+ "《", # å·¦åÂÂŒ书åÂÂÂÂÂÂå·
+ # 开始å•åÂÂŒ书åÂÂÂÂÂÂå·
+ "〖", # 左空白方头括å·
+ "〘", # 左黑色方头括å·
+ "〚", # å·¦å•书åÂÂÂÂÂÂå·
+ ]
+
+ def passthrough(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ if self.char:
+ return [self.char], [], []
+ elif self.formular:
+ return (
+ self.formular.pdf_character,
+ self.formular.pdf_curve,
+ self.formular.pdf_form,
+ )
+ elif self.unicode:
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ return [], [], []
+
+ @property
+ def can_passthrough(self):
+ if self.can_passthrough_cache is None:
+ self.can_passthrough_cache = self.calc_can_passthrough()
+
+ return self.can_passthrough_cache
+
+ def calc_can_passthrough(self):
+ return self.unicode is None
+
+ def calculate_box(self):
+ if self.char:
+ box = copy.deepcopy(self.char.box)
+ if self.char.visual_bbox and self.char.visual_bbox.box:
+ box.y = self.char.visual_bbox.box.y
+ box.y2 = self.char.visual_bbox.box.y2
+ # return self.char.visual_bbox.box
+
+ return box
+ elif self.formular:
+ return self.formular.box
+ # if self.formular.x_offset <= 0.5:
+ # return self.formular.box
+ # formular_box = copy.copy(self.formular.box)
+ # formular_box.x2 += self.formular.x_advance
+ # return formular_box
+ elif self.unicode:
+ char_width = self.font.char_lengths(self.unicode, self.font_size)[0]
+ if self.x is None or self.y is None or self.scale is None:
+ return Box(0, 0, char_width, self.font_size)
+ return Box(self.x, self.y, self.x + char_width, self.y + self.font_size)
+
+ @property
+ def box(self):
+ if not self.box_cache:
+ self.box_cache = self.calculate_box()
+
+ return self.box_cache
+
+ @property
+ def width(self):
+ if self.width_cache is None:
+ self.width_cache = self.calc_width()
+
+ return self.width_cache
+
+ def calc_width(self):
+ box = self.box
+ return box.x2 - box.x
+
+ @property
+ def height(self):
+ if self.height_cache is None:
+ self.height_cache = self.calc_height()
+
+ return self.height_cache
+
+ def calc_height(self):
+ box = self.box
+ return box.y2 - box.y
+
+ def relocate(
+ self,
+ x: float,
+ y: float,
+ scale: float,
+ ) -> TypesettingUnit:
+ """é‡ÂÂÂ定ä½ÂÂÂ并缩æâ€Â¾排版å•元
+
+ Args:
+ x: æ–°çš„ x Ã¥ÂÂÂÂÂÂæ ‡
+ y: æ–°çš„ y Ã¥ÂÂÂÂÂÂæ ‡
+ scale: 缩æâ€Â¾因åÂÂÂÂÂÂ
+
+ Returns:
+ 新的排版å•元
+ """
+ if self.char:
+ # 创建新的åÂÂÂâ€â€Â符对象
+ new_char = PdfCharacter(
+ pdf_character_id=self.char.pdf_character_id,
+ char_unicode=self.char.char_unicode,
+ box=Box(
+ x=x,
+ y=y,
+ x2=x + self.width * scale,
+ y2=y + self.height * scale,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.char.pdf_style.font_id,
+ font_size=self.char.pdf_style.font_size * scale,
+ graphic_state=self.char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=self.char.vertical,
+ advance=self.char.advance * scale if self.char.advance else None,
+ debug_info=self.debug_info,
+ xobj_id=self.char.xobj_id,
+ )
+ new_tu = TypesettingUnit(char=new_char)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.formular:
+ # 创建新的公å¼ÂÂÂ对象，ä¿ÂÂÂæŒÂÂÂ内部åÂÂÂâ€â€Â符的相对ä½ÂÂÂç½®
+ new_chars = []
+ min_x = self.formular.box.x
+ min_y = self.formular.box.y
+
+ for char in self.formular.pdf_character:
+ # 计ç®â€â€Â相对ä½ÂÂÂç½®
+ rel_x = char.box.x - min_x
+ rel_y = char.box.y - min_y
+
+ visual_rel_x = char.visual_bbox.box.x - min_x
+ visual_rel_y = char.visual_bbox.box.y - min_y
+
+ # 创建新的åÂÂÂâ€â€Â符对象
+ new_char = PdfCharacter(
+ pdf_character_id=char.pdf_character_id,
+ char_unicode=char.char_unicode,
+ box=Box(
+ x=x + (rel_x + self.formular.x_offset) * scale,
+ y=y + (rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (rel_x + (char.box.x2 - char.box.x) + self.formular.x_offset)
+ * scale,
+ y2=y
+ + (rel_y + (char.box.y2 - char.box.y) + self.formular.y_offset)
+ * scale,
+ ),
+ visual_bbox=il_version_1.VisualBbox(
+ box=Box(
+ x=x + (visual_rel_x + self.formular.x_offset) * scale,
+ y=y + (visual_rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (
+ visual_rel_x
+ + (char.visual_bbox.box.x2 - char.visual_bbox.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=y
+ + (
+ visual_rel_y
+ + (char.visual_bbox.box.y2 - char.visual_bbox.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ ),
+ ),
+ pdf_style=PdfStyle(
+ font_id=char.pdf_style.font_id,
+ font_size=char.pdf_style.font_size * scale,
+ graphic_state=char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=char.vertical,
+ advance=char.advance * scale if char.advance else None,
+ xobj_id=char.xobj_id,
+ )
+ new_chars.append(new_char)
+
+ # Calculate bounding box from new_chars
+ min_x = min(char.visual_bbox.box.x for char in new_chars)
+ min_y = min(char.visual_bbox.box.y for char in new_chars)
+ max_x = max(char.visual_bbox.box.x2 for char in new_chars)
+ max_y = max(char.visual_bbox.box.y2 for char in new_chars)
+
+ new_formula = PdfFormula(
+ box=Box(
+ x=min_x,
+ y=min_y,
+ x2=max_x,
+ y2=max_y,
+ ),
+ pdf_character=new_chars,
+ x_offset=self.formular.x_offset * scale,
+ y_offset=self.formular.y_offset * scale,
+ x_advance=self.formular.x_advance * scale,
+ )
+
+ # Handle contained curves
+ new_curves = []
+ for curve in self.formular.pdf_curve:
+ new_curve = self._transform_curve_for_relocation(
+ curve,
+ self.formular.box.x,
+ self.formular.box.y,
+ x,
+ y,
+ scale,
+ )
+ new_curves.append(new_curve)
+ new_formula.pdf_curve = new_curves
+
+ # Handle contained forms
+ new_forms = []
+ for form in self.formular.pdf_form:
+ new_form = self._transform_form_for_relocation(
+ form, self.formular.box.x, self.formular.box.y, x, y, scale
+ )
+ new_forms.append(new_form)
+ new_formula.pdf_form = new_forms
+
+ update_formula_data(new_formula)
+
+ new_tu = TypesettingUnit(formular=new_formula)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.unicode:
+ # 对于 Unicode Ã¥ÂÂÂâ€â€Â符，我们å˜储新的ä½ÂÂÂ置信æ¯
+ new_unit = TypesettingUnit(
+ unicode=self.unicode,
+ font=self.font,
+ original_font=self.original_font,
+ font_size=self.font_size * scale,
+ style=self.style,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ new_unit.x = x
+ new_unit.y = y
+ new_unit.scale = scale
+ new_unit.try_resue_cache(self)
+ return new_unit
+
+ def _transform_curve_for_relocation(
+ self,
+ curve,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a curve for formula relocation."""
+ import copy
+
+ new_curve = copy.deepcopy(curve)
+
+ if new_curve.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_curve.box.x - original_formula_x
+ rel_y = new_curve.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_curve.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (
+ rel_x
+ + (new_curve.box.x2 - new_curve.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=new_y
+ + (
+ rel_y
+ + (new_curve.box.y2 - new_curve.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original CTM
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_curve.relocation_transform = list(relocation_matrix)
+
+ return new_curve
+
+ def _transform_form_for_relocation(
+ self,
+ form,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a form for formula relocation."""
+ import copy
+
+ new_form = copy.deepcopy(form)
+
+ if new_form.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_form.box.x - original_formula_x
+ rel_y = new_form.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_form.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (rel_x + (new_form.box.x2 - new_form.box.x) + self.formular.x_offset)
+ * scale,
+ y2=new_y
+ + (rel_y + (new_form.box.y2 - new_form.box.y) + self.formular.y_offset)
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original matrices
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_form.relocation_transform = list(relocation_matrix)
+
+ return new_form
+
+ def render(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ """渲染排版å•元为 PdfCharacter åˆâ€â€Â表
+
+ Returns:
+ PdfCharacter åˆâ€â€Â表
+ """
+ if self.can_passthrough:
+ return self.passthrough()
+ elif self.unicode:
+ assert self.x is not None, (
+ "x position must be set, should be set by `relocate`"
+ )
+ assert self.y is not None, (
+ "y position must be set, should be set by `relocate`"
+ )
+ assert self.scale is not None, (
+ "scale must be set, should be set by `relocate`"
+ )
+ x = self.x
+ y = self.y
+ # if self.original_font and self.font and hasattr(self.original_font, "descent") and hasattr(self.font, "descent_fontmap"):
+ # original_descent = self.original_font.descent
+ # new_descent = self.font.descent_fontmap
+ # y -= (original_descent - new_descent) * self.font_size / 1000
+
+ # 计ç®â€â€ÂÃ¥ÂÂÂâ€â€Â符宽度
+ char_width = self.width
+
+ # Handle case when font is None (no suitable font found for this character)
+ if self.font is None:
+ logger.warning(
+ f"No font available for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using font_id='{self.font_id}' with glyph_id=0"
+ )
+ glyph_id = 0 # Use glyph 0 as fallback (usually .notdef)
+ else:
+ glyph_id = self.font.has_glyph(ord(self.unicode))
+ if glyph_id == 0 or glyph_id is None:
+ logger.warning(
+ f"Font '{self.font_id}' doesn't have glyph for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using glyph_id=0"
+ )
+ glyph_id = 0
+
+ new_char = PdfCharacter(
+ pdf_character_id=glyph_id,
+ char_unicode=self.unicode,
+ box=Box(
+ x=x, # 使çâ€Â¨å˜储的ä½ÂÂÂç½®
+ y=y,
+ x2=x + char_width,
+ y2=y + self.font_size,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.font_id,
+ font_size=self.font_size,
+ graphic_state=self.style.graphic_state,
+ ),
+ scale=self.scale,
+ vertical=False,
+ advance=char_width,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ return [new_char], [], []
+ else:
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ return [], [], []
+
+
+class Typesetting:
+ stage_name = "Typesetting"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.font_mapper = FontMapper(translation_config)
+ self.translation_config = translation_config
+ self.lang_code = self.translation_config.lang_out.upper()
+ # Ensure detailed_logger attribute exists to avoid attribute access errors
+ self.detailed_logger = None
+ self.is_cjk = (
+ # Why zh-CN/zh-HK/zh-TW here but not zh-Hans and so on?
+ # See https://funstory-ai.github.io/BabelDOC/supported_languages/
+ ("ZH" in self.lang_code) # C
+ or ("JA" in self.lang_code)
+ or ("JP" in self.lang_code) # J
+ or ("KR" in self.lang_code) # K
+ or ("CN" in self.lang_code)
+ or ("HK" in self.lang_code)
+ or ("TW" in self.lang_code)
+ )
+
+ def preprocess_document(self, document: il_version_1.Document, pbar):
+ """预处ç†文档，获å–æ¯ÂÂÂ个段è½的最优缩æâ€Â¾因åÂÂÂÂÂÂ，ä¸ÂÂÂ执行实际排版"""
+ all_scales: list[float] = []
+ all_paragraphs: list[il_version_1.PdfParagraph] = []
+
+ for page in document.page:
+ pbar.advance()
+ # 准备åÂÂÂâ€â€Â体信æ¯（å¤ÂÂÂ制自 render_page 的逻辑）
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if (
+ xobj.xobj_id in fonts
+ and isinstance(fonts[xobj.xobj_id], dict)
+ and font.font_id
+ ):
+ fonts[xobj.xobj_id][font.font_id] = font
+
+ # 处ç†æ¯ÂÂÂ个段è½
+ for paragraph in page.pdf_paragraph:
+ all_paragraphs.append(paragraph)
+ unit_count = 0
+ try:
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ unit_count = len(typesetting_units)
+ for unit in typesetting_units:
+ if unit.formular:
+ unit_count += len(unit.formular.pdf_character) - 1
+
+ # 如果所有å•元都å¯以直接传递，则 scale = 1.0
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.optimal_scale = 1.0
+ else:
+ # 获å–最优缩æâ€Â¾因åÂÂÂÂÂÂ
+ optimal_scale = self._get_optimal_scale(
+ paragraph, page, typesetting_units
+ )
+ paragraph.optimal_scale = optimal_scale
+ except Exception as e:
+ # 如果预处ç†出éâ€Â™，默认使çâ€Â¨ 1.0 缩æâ€Â¾因åÂÂÂÂÂÂ
+ logger.warning(f"预处ç†段è½æâ€â€Â¶å‡ºéâ€Â™：{e}")
+ paragraph.optimal_scale = 1.0
+
+ if paragraph.optimal_scale is not None:
+ all_scales.extend([paragraph.optimal_scale] * unit_count)
+
+ # 获å–缩æâ€Â¾因åÂÂÂÂÂÂçš„ä¼â€â€Âæ•°
+ if all_scales:
+ try:
+ modes = statistics.multimode(all_scales)
+ mode_scale = min(modes)
+ except statistics.StatisticsError:
+ logger.warning(
+ "Could not find a mode for paragraph scales. Falling back to median."
+ )
+ mode_scale = statistics.median(all_scales)
+ # 将所有大于ä¼â€â€Â数的值修æâ€Â¹为ä¼â€â€Âæ•°
+ for paragraph in all_paragraphs:
+ if (
+ paragraph.optimal_scale is not None
+ and paragraph.optimal_scale > mode_scale
+ ):
+ paragraph.optimal_scale = mode_scale
+ else:
+ logger.error(
+ "document_scales is empty, there seems no paragraph in this PDF"
+ )
+
+ def shape_arabic_text(self, text: str) -> str:
+ """Shape and reorder Arabic text if output language is Arabic.
+
+ Args:
+ text: Input text to shape
+
+ Returns:
+ Shaped and reordered text if language is Arabic, original text otherwise
+ """
+ if not text:
+ return text
+
+ # Robust Arabic output detection: accept explicit 'ar', 'ara', 'arabic'
+ # or formats containing '-ar', '->ar', or '/ar' as a target marker (e.g. 'en-ar', 'en->ar')
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = False
+ if lang_out in ("en-ar, ar", "ara", "arabic"):
+ is_arabic = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic = True
+
+ if is_arabic:
+ logger.debug("Shaping Arabic text")
+ # Flip parentheses and brackets for RTL display
+ # text = text.replace("(", "\x00")
+ # text = text.replace(")", "(")
+ # text = text.replace("\x00", ")")
+ # text = text.replace("[", "\x01")
+ # text = text.replace("]", "[")
+ # text = text.replace("\x01", "]")
+ # text = text.replace("{", "\x02")
+ # text = text.replace("}", "{")
+ # text = text.replace("\x02", "}")
+ try:
+ if not re.search(r'[\uFB50-\uFDFF\uFE70-\uFEFF]', text):
+ # Reshape Arabic text for proper character joining
+ from arabic_reshaper import ArabicReshaper
+ configuration = {
+ 'delete_harakat': False, # Keep diacritical marks
+ 'support_ligatures': True, # Support Arabic ligatures
+ 'RIAL SIGN': True,
+ 'ARABIC COMMA': True,
+ 'ARABIC SEMICOLON': True,
+ 'ARABIC QUESTION MARK': True,
+ 'ZWNJ': True, # Zero Width Non-Joiner
+ }
+
+ reshaper = ArabicReshaper(configuration=configuration)
+ reshaped_text = reshaper.reshape(text)
+ display_text = get_display(reshaped_text, base_dir='R')
+ else:
+ display_text = text
+ return display_text
+ except Exception as e:
+ logger.warning(f"Failed to shape Arabic text: {e}")
+ return text
+
+ return text
+
+ def _find_optimal_scale_and_layout(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ initial_scale: float = 1.0,
+ use_english_line_break: bool = True,
+ apply_layout: bool = False,
+ ) -> tuple[float, list[TypesettingUnit] | None]:
+ """查找最优缩æâ€Â¾因åÂÂÂÂÂÂ并å¯选择性地执行布局
+
+ Args:
+ paragraph: 段è½对象
+ page: 页é¢对象
+ typesetting_units: 排版å•元åˆâ€â€Â表
+ initial_scale: åˆÂÂÂ始缩æâ€Â¾因åÂÂÂÂÂÂ
+ use_english_line_break: 是å¦使çâ€Â¨英文æ¢行规则
+ apply_layout: 是å¦åºâ€ÂÂçâ€Â¨布局到 paragraph（True æâ€â€Â¶æ‰§è¡Œå®žé™…排版）
+
+ Returns:
+ tuple[float, list[TypesettingUnit] | None]: (最终缩æâ€Â¾因åÂÂÂÂÂÂ，排版åÂÂÂŽçš„å•元åˆâ€â€Â表或 None)
+ """
+ if not paragraph.box:
+ return initial_scale, None
+
+ box = paragraph.box
+ scale = initial_scale
+ line_skip = 1.50 if self.is_cjk else 1.3
+ min_scale = 0.1
+ expand_space_flag = 0
+ final_typeset_units = None
+
+ while scale >= min_scale:
+ try:
+ # å°ÂÂÂ试布局排版å•元
+ typeset_units, all_units_fit = self._layout_typesetting_units(
+ typesetting_units,
+ box,
+ scale,
+ line_skip,
+ paragraph,
+ use_english_line_break,
+ )
+
+ # 如果所有å•元都æâ€Â¾å¾â€â€Â下
+ if all_units_fit:
+ if apply_layout:
+ # 实际åºâ€ÂÂçâ€Â¨排版结果
+ paragraph.scale = scale
+ paragraph.pdf_paragraph_composition = []
+ for unit in typeset_units:
+ chars, curves, forms = unit.render()
+ for char in chars:
+ paragraph.pdf_paragraph_composition.append(
+ PdfParagraphComposition(pdf_character=char),
+ )
+ for curve in curves:
+ page.pdf_curve.append(curve)
+ for form in forms:
+ page.pdf_form.append(form)
+ final_typeset_units = typeset_units
+ return scale, final_typeset_units
+ except Exception:
+ # 如果布局检查出éâ€Â™，继ç»ÂÂÂå°ÂÂÂ试下一个缩æâ€Â¾因åÂÂÂÂÂÂ
+ pass
+
+ # 添加与原 retypeset 一致的逻辑检查
+ if not hasattr(paragraph, "debug_id") or not paragraph.debug_id:
+ return scale, final_typeset_units
+
+ # å‡ÂÂÂå°ÂÂÂ缩æâ€Â¾因åÂÂÂÂÂÂ
+ if scale > 0.6:
+ scale -= 0.05
+ else:
+ scale -= 0.1
+
+ if scale < 0.7:
+ space_expanded = False # 标记是å¦æˆÂÂÂ功扩展了空éâ€â€Â´
+
+ if expand_space_flag == 0:
+ # å°ÂÂÂ试å‘下扩展
+ try:
+ min_y = self.get_max_bottom_space(box, page) + 2
+ if min_y < box.y:
+ expanded_box = Box(x=box.x, y=min_y, x2=box.x2, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # 更新段è½的边界æ¡â€ÂÂ
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 1
+
+ # åª有æˆÂÂÂ功扩展空éâ€â€Â´æâ€â€Â¶æ‰ continue，å¦则继ç»ÂÂÂå‡ÂÂÂå° scale
+ if space_expanded:
+ continue
+
+ elif expand_space_flag == 1:
+ # å°ÂÂÂ试å‘å³扩展
+ try:
+ max_x = self.get_max_right_space(box, page) - 5
+ if max_x > box.x2:
+ expanded_box = Box(x=box.x, y=box.y, x2=max_x, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # 更新段è½的边界æ¡â€ÂÂ
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 2
+
+ # åª有æˆÂÂÂ功扩展空éâ€â€Â´æâ€â€Â¶æ‰ continue，å¦则继ç»ÂÂÂå‡ÂÂÂå° scale
+ if space_expanded:
+ continue
+
+ # åª有在扩展å°ÂÂÂ试阶段 (expand_space_flag < 2) ä¸â€ÂÂ扩展失败æâ€â€Â¶æ‰ÂÂÂé‡ÂÂÂç½® scale
+ # 当 expand_space_flag >= 2 æâ€â€Â¶ï¼Œè¯´æ˜Žå·²ç»ÂÂÂå°ÂÂÂ试过所有扩展，åºâ€ÂÂ该继ç»ÂÂÂæ£常的 scale å‡ÂÂÂå°ÂÂÂ
+ if expand_space_flag < 2:
+ # 如果æâ€â€Â æ³•æ‰©å±•ç©ºéâ€â€Â´ï¼Œé‡ÂÂÂç½® scale 并继ç»ÂÂÂ循环
+ scale = 1.0
+
+ # 如果ä»ÂÂÂ然æâ€Â¾ä¸ÂÂÂ下，å°ÂÂÂ试去除英文æ¢行é™ÂÂÂ制
+ if use_english_line_break:
+ return self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ initial_scale,
+ use_english_line_break=False,
+ apply_layout=apply_layout,
+ )
+
+ # 最åÂÂÂŽè¿â€ÂÂ回最å°ÂÂÂ缩æâ€Â¾因åÂÂÂÂÂÂ
+ return min_scale, final_typeset_units
+
+ def _get_optimal_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ use_english_line_break: bool = True,
+ ) -> float:
+ """获å–段è½的最优缩æâ€Â¾因åÂÂÂÂÂÂ，ä¸ÂÂÂ执行实际排版"""
+ scale, _ = self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ 1.0,
+ use_english_line_break,
+ apply_layout=False,
+ )
+ return scale
+
+ def retypeset_with_precomputed_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ precomputed_scale: float,
+ use_english_line_break: bool = True,
+ ):
+ """使çâ€Â¨预计ç®â€â€Â的缩æâ€Â¾因åÂÂÂÂÂÂ进行排版"""
+ if not paragraph.box:
+ return
+
+ # 使çâ€Â¨通çâ€Â¨方法进行排版，传入预计ç®â€â€Â的缩æâ€Â¾因åÂÂÂÂÂÂ作为åˆÂÂÂ始值
+ self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ precomputed_scale,
+ use_english_line_break,
+ apply_layout=True,
+ )
+
+ def typesetting_document(self, document: il_version_1.Document):
+ # Add detailed logging at the start
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Started")
+
+ # 原有的æŽ'版逻è¾'
+ if self.translation_config.progress_monitor:
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(document.page) * 2,
+ ) as pbar:
+ # 预处ç†ï¼šèŽ·å–所有段è½çš„最优缩æ"¾å› Ã¥ÂÂÂÂ
+ self.preprocess_document(document, pbar)
+
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+ pbar.advance()
+ else:
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+
+ # Add detailed logging at the end
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Complete")
+
+ def render_page(self, page: il_version_1.Page):
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if font.font_id:
+ fonts[xobj.xobj_id][font.font_id] = font
+ if (
+ page.page_number == 0
+ and self.translation_config.watermark_output_mode
+ == WatermarkOutputMode.Watermarked
+ ):
+ self.add_watermark(page)
+ try:
+ para_index = index.Index()
+ para_map = {}
+ #
+ valid_paras = [
+ p
+ for p in page.pdf_paragraph
+ if p.box
+ and all(c is not None for c in [p.box.x, p.box.y, p.box.x2, p.box.y2])
+ ]
+
+ for i, para in enumerate(valid_paras):
+ para_map[i] = para
+ para_index.insert(i, box_to_tuple(para.box))
+
+ for i, p_upper in para_map.items():
+ if not (p_upper.box and p_upper.box.y is not None):
+ continue
+
+ # Calculate paragraph height and set required gap accordingly
+ para_height = p_upper.box.y2 - p_upper.box.y
+ required_gap = 0.5 if para_height < 36 else 3
+
+ check_area = il_version_1.Box(
+ x=p_upper.box.x,
+ y=p_upper.box.y - required_gap,
+ x2=p_upper.box.x2,
+ y2=p_upper.box.y,
+ )
+
+ candidate_ids = list(para_index.intersection(box_to_tuple(check_area)))
+
+ conflicting_paras = []
+ for para_id in candidate_ids:
+ if para_id == i:
+ continue
+ p_lower = para_map[para_id]
+ if not (
+ p_lower.box
+ and p_upper.box
+ and p_lower.box.x2 < p_upper.box.x
+ or p_lower.box.x > p_upper.box.x2
+ ):
+ conflicting_paras.append(p_lower)
+
+ if conflicting_paras:
+ max_y2 = max(
+ p.box.y2
+ for p in conflicting_paras
+ if p.box and p.box.y2 is not None
+ )
+
+ new_y = max_y2 + required_gap
+ if p_upper.box and new_y < p_upper.box.y2:
+ p_upper.box.y = new_y
+ except Exception as e:
+ logger.warning(
+ f"Failed to adjust paragraph positions on page {page.page_number}: {e}"
+ )
+ # 开始实际的渲染过程
+ for paragraph in page.pdf_paragraph:
+ self.render_paragraph(paragraph, page, fonts)
+
+ def add_watermark(self, page: il_version_1.Page):
+ page_width = page.cropbox.box.x2 - page.cropbox.box.x
+ page_height = page.cropbox.box.y2 - page.cropbox.box.y
+ style = il_version_1.PdfStyle(
+ font_id="base",
+ font_size=6,
+ graphic_state=il_version_1.GraphicState(),
+ )
+ text = f"本文档çâ€Â± funstory.ai 的开溠PDF 翻译库 BabelDOC {WATERMARK_VERSION} (http://yadt.io) 翻译，本仓库æ£在积æžÂÂÂ的建设当ä¸ÂÂÂ，欢迎 star 和关注。"
+ if self.translation_config.debug:
+ text += "\n 当å‰ÂÂÂ为 DEBUG 模å¼ÂÂÂ，将显示更多辅助信æ¯。请注æ„ÂÂÂ，部分框的ä½ÂÂÂ置对åºâ€ÂÂ原文，但在译文ä¸ÂÂÂå¯能ä¸ÂÂÂæ£确。"
+ page.pdf_paragraph.append(
+ il_version_1.PdfParagraph(
+ first_line_indent=False,
+ box=il_version_1.Box(
+ x=page.cropbox.box.x + page_width * 0.05,
+ y=page.cropbox.box.y,
+ x2=page.cropbox.box.x2,
+ y2=page.cropbox.box.y2 - page_height * 0.05,
+ ),
+ vertical=False,
+ pdf_style=style,
+ pdf_paragraph_composition=[
+ il_version_1.PdfParagraphComposition(
+ pdf_same_style_unicode_characters=il_version_1.PdfSameStyleUnicodeCharacters(
+ unicode=text,
+ pdf_style=style,
+ ),
+ ),
+ ],
+ xobj_id=-1,
+ ),
+ )
+
+ def render_paragraph(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ],
+ ):
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ # 如果所有å•元都å¯以直接传递，则直接传递
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.scale = 1.0
+ paragraph.pdf_paragraph_composition = self.create_passthrough_composition(
+ typesetting_units,
+ )
+ else:
+ # 使çâ€Â¨预计ç®â€â€Â的缩æâ€Â¾因åÂÂÂÂÂÂ进行é‡ÂÂÂ排ç‰Ëâ€Â
+ precomputed_scale = (
+ paragraph.optimal_scale if paragraph.optimal_scale is not None else 1.0
+ )
+
+ # 如果有å•元æâ€â€Â æ³•ç›´æŽ¥ä¼ é€’，则进行é‡ÂÂÂ排ç‰Ëâ€Â
+ paragraph.pdf_paragraph_composition = []
+ self.retypeset_with_precomputed_scale(
+ paragraph, page, typesetting_units, precomputed_scale
+ )
+
+ # é‡ÂÂÂ排版åÂÂŽ，é‡ÂÂÂ新设置段è½å„åÂÂÂâ€â€Â符的 render order
+ self._update_paragraph_render_order(paragraph)
+
+ def _is_arabic_char(self, char: str) -> bool:
+ """Check if character is Arabic - OPTIMIZED"""
+ if not char:
+ return False
+ try:
+ code_point = ord(char[0])
+ return (0x0600 <= code_point <= 0x06FF) or (0xFB50 <= code_point <= 0xFDFF) or (0xFE70 <= code_point <= 0xFEFF)
+ except:
+ return False
+
+ def _layout_typesetting_units(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ box: Box,
+ scale: float,
+ line_skip: float,
+ paragraph: il_version_1.PdfParagraph,
+ use_english_line_break: bool = True,
+ ) -> tuple[list[TypesettingUnit], bool]:
+ """布局排版å•元 - OPTIMIZED FOR ARABIC RTL"""
+
+ # Detect Arabic FIRST
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = any(marker in lang_out for marker in ["ar", "arabic", "ara"])
+
+ # 计算å—体大å°Â
+ font_sizes = []
+ for unit in typesetting_units:
+ if unit.font_size:
+ font_sizes.append(unit.font_size)
+ if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ font_sizes.append(unit.char.pdf_style.font_size)
+ if not font_sizes:
+ font_sizes = [12]
+ font_sizes.sort()
+ font_size = statistics.mode(font_sizes)
+
+ space_width = (
+ self.font_mapper.base_font.char_lengths("ä½ ", font_size * scale)[0] * 0.5
+ )
+
+ # 计算行高
+ unit_heights = [unit.height for unit in typesetting_units] if typesetting_units else []
+ if not unit_heights:
+ avg_height = 0
+ elif len(unit_heights) == 1:
+ avg_height = unit_heights[0] * scale
+ else:
+ try:
+ avg_height = statistics.mode(unit_heights) * scale
+ except statistics.StatisticsError:
+ avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # åˆÂ始化
+ current_x = box.x
+ current_y = box.y2 - avg_height
+ box = copy.deepcopy(box)
+ line_height = 0
+ current_line_heights = []
+ typeset_units = []
+ all_units_fit = True
+ last_unit: TypesettingUnit | None = None
+ line_ys = [current_y]
+
+ if paragraph.first_line_indent:
+ current_x += space_width * 4
+
+ # OPTIMIZED ARABIC WORD-LEVEL PROCESSING
+ if is_arabic:
+ # CRITICAL: Capture original English left margin BEFORE typesetting
+ # This preserves the margin hierarchy for titles vs paragraphs
+ original_left_margin = 0
+ if typesetting_units and hasattr(typesetting_units[0], 'x') and typesetting_units[0].x is not None:
+ # Find the minimum X position from the original English layout
+ original_min_x = min(u.x for u in typesetting_units if hasattr(u, 'x') and u.x is not None)
+ original_left_margin = original_min_x - box.x
+
+ i = 0
+ safety_counter = 0
+ max_iterations = len(typesetting_units) * 2 # Safety limit
+
+ while i < len(typesetting_units) and safety_counter < max_iterations:
+ safety_counter += 1
+
+ # Collect word (simple: until space or end)
+ word_units = []
+ while i < len(typesetting_units):
+ unit = typesetting_units[i]
+ if unit.is_space:
+ if word_units:
+ i += 1
+ break
+ word_units.append(unit)
+ i += 1
+ if len(word_units) > 100: # Safety: max word length
+ break
+
+ if not word_units:
+ continue
+
+ # Calculate word width
+ word_width = sum(u.width * scale for u in word_units)
+
+ # Skip leading spaces
+ if current_x == box.x and word_units and word_units[0].is_space:
+ continue
+
+ # Check if needs new line
+ if current_x + word_width > box.x2 and current_x > box.x:
+ current_x = box.x
+ if current_line_heights:
+ max_height = max(current_line_heights)
+ mode_height = statistics.mode(current_line_heights) if len(current_line_heights) > 1 else max_height
+ current_y -= max(mode_height * line_skip, max_height * 1.05)
+ line_ys.append(current_y)
+ current_line_heights = []
+
+ if current_y < box.y:
+ all_units_fit = False
+
+ # Place word units
+ for unit in word_units:
+ if unit.is_space and current_x == box.x:
+ continue
+
+ unit_width = unit.width * scale
+ unit_height = unit.height * scale
+
+ # CJK spacing
+ if (last_unit and last_unit.is_cjk_char ^ unit.is_cjk_char
+ and not unit.is_space and current_x > box.x):
+ current_x += space_width * 0.5
+
+ relocated_unit = unit.relocate(current_x, current_y, scale)
+ typeset_units.append(relocated_unit)
+
+ if not unit.is_space:
+ current_line_heights.append(unit_height)
+
+ current_x = relocated_unit.box.x2
+ last_unit = relocated_unit
+
+ # Right-align Arabic lines (but NOT table content)
+ # Check if this paragraph is inside a table by examining layout_label
+ is_table_content = False
+ if paragraph.layout_label:
+ layout_label_lower = paragraph.layout_label.lower()
+ # Exclude ONLY actual table cell content from right-alignment
+ # NOTE: "table_title", "table_caption" are headings, NOT table content!
+ # We only want to exclude: table_cell, table_text, wired_table_cell, wireless_table_cell
+ if any(table_marker in layout_label_lower for table_marker in [
+ 'table_cell', 'table_text', 'wired_table_cell', 'wireless_table_cell'
+ ]):
+ is_table_content = True
+
+ # Only apply right-alignment if NOT table content
+ if typeset_units and not is_table_content:
+ lines_dict = {}
+ for unit in typeset_units:
+ if unit.box and unit.box.y is not None:
+ line_y = round(unit.box.y, 1)
+ if line_y not in lines_dict:
+ lines_dict[line_y] = []
+ lines_dict[line_y].append(unit)
+
+ # CRITICAL FIX: Use the original English left margin as the right margin
+ # This directly mirrors the English layout hierarchy in Arabic RTL
+ # Titles with small English left margin → small Arabic right margin (flush right)
+ # Paragraphs with large English left margin → large Arabic right margin (indented from right)
+
+ # The original_left_margin was captured BEFORE typesetting from the English positions
+ right_margin = original_left_margin
+
+ for line_y, line_units in lines_dict.items():
+ if line_units:
+ # Calculate shift to position line from the right with the mirrored margin
+ line_max_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+ target_right_position = box.x2 - right_margin
+ shift_x = target_right_position - line_max_x
+
+ for unit in line_units:
+ if unit.box:
+ unit.box.x += shift_x
+ unit.box.x2 += shift_x
+ if unit.x is not None:
+ unit.x += shift_x
+ if unit.char and unit.char.box:
+ unit.char.box.x += shift_x
+ unit.char.box.x2 += shift_x
+ if unit.char and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ unit.char.visual_bbox.box.x += shift_x
+ unit.char.visual_bbox.box.x2 += shift_x
+ else:
+ # ORIGINAL NON-ARABIC LOGIC (UNCHANGED)
+ for i, unit in enumerate(typesetting_units):
+ unit_width = unit.width * scale
+ unit_height = unit.height * scale
+
+ if current_x == box.x and unit.is_space:
+ continue
+
+ if (last_unit and last_unit.is_cjk_char ^ unit.is_cjk_char
+ and last_unit.box and last_unit.box.y
+ and current_y - 0.1 <= last_unit.box.y2 <= current_y + line_height + 0.1
+ and not last_unit.mixed_character_blacklist and not unit.mixed_character_blacklist
+ and current_x > box.x and unit.try_get_unicode() != " "
+ and last_unit.try_get_unicode() != " "
+ and last_unit.try_get_unicode() not in ["。", "，", "ã€Â", "ï¼›", "ï¼Â", "？"]):
+ current_x += space_width * 0.5
+
+ if use_english_line_break:
+ width_before_next_break_point = self._get_width_before_next_break_point(typesetting_units[i:], scale)
+ else:
+ width_before_next_break_point = 0
+
+ if not unit.is_hung_punctuation and (
+ (current_x + unit_width > box.x2) or
+ (use_english_line_break and current_x + unit_width + width_before_next_break_point > box.x2) or
+ (unit.is_cannot_appear_in_line_end_punctuation and current_x + unit_width * 2 > box.x2)):
+
+ current_x = box.x
+ if not current_line_heights:
+ return [], False
+ max_height = max(current_line_heights)
+ mode_height = statistics.mode(current_line_heights)
+ current_y -= max(mode_height * line_skip, max_height * 1.05)
+ line_ys.append(current_y)
+ line_height = 0.0
+ current_line_heights = []
+
+ if current_y < box.y:
+ all_units_fit = False
+
+ if unit.is_space:
+ line_height = max(line_height, unit_height)
+ continue
+
+ relocated_unit = unit.relocate(current_x, current_y, scale)
+ typeset_units.append(relocated_unit)
+
+ if not unit.is_space:
+ current_line_heights.append(unit_height)
+
+ prev_x = current_x
+ current_x = relocated_unit.box.x2
+ if prev_x > current_x:
+ logger.warning(f"Ã¥ÂÂ标回退ï¼Âï¼Âï¼ÂTypesettingUnit: {unit.box}, ")
+
+ last_unit = relocated_unit
+ # If Arabic, reverse the line order
+ if is_arabic and typeset_units:
+ # Group units by line (using Y coordinates)
+ lines_dict = {}
+ for unit in typeset_units:
+ if unit.box and unit.box.y is not None:
+ # Round Y coordinate to group units on the same line
+ line_y = round(unit.box.y, 1)
+ if line_y not in lines_dict:
+ lines_dict[line_y] = []
+ lines_dict[line_y].append(unit)
+
+ # Sort lines by Y coordinate (top to bottom) and reverse
+ sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # Rebuild typeset_units with reversed line order
+ reversed_typeset_units = []
+ for line_y in reversed(sorted_line_ys):
+ reversed_typeset_units.extend(lines_dict[line_y])
+
+ # Now reposition all units to swap their Y coordinates
+ # Map old Y positions to new Y positions
+ y_mapping = {}
+ for i, old_y in enumerate(sorted_line_ys):
+ new_y = sorted_line_ys[len(sorted_line_ys) - 1 - i]
+ y_mapping[old_y] = new_y
+
+ # Update Y coordinates for all units
+ for unit in reversed_typeset_units:
+ if unit.box and unit.box.y is not None:
+ old_y = round(unit.box.y, 1)
+ if old_y in y_mapping:
+ new_y = y_mapping[old_y]
+ y_diff = new_y - old_y
+ # Update the unit's Y position
+ if unit.y is not None:
+ unit.y += y_diff
+ if unit.box:
+ unit.box.y += y_diff
+ unit.box.y2 += y_diff
+
+ typeset_units = reversed_typeset_units
+
+ return typeset_units, all_units_fit
+
+# CORRECT FIX FOR ARABIC TEXT LAYOUT
+# Replace the _layout_typesetting_units function in typesetting.py (lines 1346-1502)
+
+ # def _layout_typesetting_units(
+ # self,
+ # typesetting_units: list[TypesettingUnit],
+ # box: Box,
+ # scale: float,
+ # line_skip: float,
+ # paragraph: il_version_1.PdfParagraph,
+ # use_english_line_break: bool = True,
+ # ) -> tuple[list[TypesettingUnit], bool]:
+ # """布局排版å•元。
+
+ # Args:
+ # typesetting_units: è¦ÂÂÂ布局的排版å•元åˆâ€â€Â表
+ # box: 布局边界æ¡â€ÂÂ
+ # scale: 缩æâ€Â¾因åÂÂÂÂÂÂ
+
+ # Returns:
+ # tuple[list[TypesettingUnit], bool]: (已布局的排版å•元åˆâ€â€Â表，是å¦所有å•元都æâ€Â¾å¾â€â€Â下)
+ # """
+ # # 计ç®â€â€ÂÃ¥ÂÂÂâ€â€Âå·ä¼â€â€Âæ•°
+ # font_sizes = []
+ # for unit in typesetting_units:
+ # if unit.font_size:
+ # font_sizes.append(unit.font_size)
+ # if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ # font_sizes.append(unit.char.pdf_style.font_size)
+ # font_sizes.sort()
+ # font_size = statistics.mode(font_sizes)
+
+ # space_width = (
+ # self.font_mapper.base_font.char_lengths("ä½ ", font_size * scale)[0] * 0.5
+ # )
+
+ # # 计ç®â€â€Â行高（使çâ€Â¨ä¼â€â€Â数）
+ # unit_heights = (
+ # [unit.height for unit in typesetting_units] if typesetting_units else []
+ # )
+ # if not unit_heights:
+ # avg_height = 0
+ # elif len(unit_heights) == 1:
+ # avg_height = unit_heights[0] * scale
+ # else:
+ # try:
+ # avg_height = statistics.mode(unit_heights) * scale
+ # except statistics.StatisticsError:
+ # # 如果没有ä¼â€â€Â数（所有值都出现相åÂÂŒ次数），则使çâ€Â¨平å‡值
+ # avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # # *** NEW: Detect Arabic language ***
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # # åˆÂÂÂ始化ä½ÂÂÂ置为å³上角，并å‡ÂÂÂ去一个平å‡行高
+ # # *** CHANGED: For Arabic, calculate total line width first and start from right ***
+ # current_x = box.x
+ # current_y = box.y2 - avg_height
+ # box = copy.deepcopy(box)
+ # line_height = 0
+ # current_line_heights = [] # å˜储当å‰ÂÂÂ行所有元素的高度
+
+ # # å˜储已排版的å•元
+ # typeset_units = []
+ # all_units_fit = True
+ # last_unit: TypesettingUnit | None = None
+ # line_ys = [current_y]
+ # if paragraph.first_line_indent:
+ # current_x += space_width * 4
+ # # éÂÂÂÂÂÂ历所有排版å•元
+ # for i, unit in enumerate(typesetting_units):
+ # # 计ç®â€â€Â当å‰ÂÂÂå•元在当å‰ÂÂÂ缩æâ€Â¾下的尺寸
+ # unit_width = unit.width * scale
+ # unit_height = unit.height * scale
+
+ # # 跳过行首的空格
+ # if current_x == box.x and unit.is_space:
+ # continue
+
+ # if (
+ # last_unit # 有上一个å•元
+ # and last_unit.is_cjk_char ^ unit.is_cjk_char # ä¸ÂÂÂ英文交界处
+ # and (
+ # last_unit.box
+ # and last_unit.box.y
+ # and current_y - 0.1
+ # <= last_unit.box.y2
+ # <= current_y + line_height + 0.1
+ # ) # 在åÂÂŒ一行，ä¸â€ÂÂ有垂直é‡ÂÂÂÃ¥ÂÂÂÂÂÂ
+ # and not last_unit.mixed_character_blacklist # ä¸ÂÂÂ是混排空格黑åÂÂÂÂÂÂå•åÂÂÂâ€â€Â符
+ # and not unit.mixed_character_blacklist # Ã¥ÂÂÂŒä¸ÅÂÂ
+ # and current_x > box.x # ä¸ÂÂÂ是行首
+ # and unit.try_get_unicode() != " " # ä¸ÂÂÂ是空格
+ # and last_unit.try_get_unicode() != " " # ä¸ÂÂÂ是空格
+ # and last_unit.try_get_unicode()
+ # not in [
+ # "。",
+ # "ï¼ÂÂÂ",
+ # "？",
+ # "ï¼›",
+ # "：",
+ # "，",
+ # ]
+ # ):
+ # current_x += space_width * 0.5
+ # if use_english_line_break:
+ # width_before_next_break_point = self._get_width_before_next_break_point(
+ # typesetting_units[i:], scale
+ # )
+ # else:
+ # width_before_next_break_point = 0
+
+ # # 如果当å‰ÂÂÂè¡Œæâ€Â¾ä¸ÂÂÂ下这个元素，æ¢行
+ # if not unit.is_hung_punctuation and (
+ # (current_x + unit_width > box.x2)
+ # or (
+ # use_english_line_break
+ # and current_x + unit_width + width_before_next_break_point > box.x2
+ # )
+ # or (
+ # unit.is_cannot_appear_in_line_end_punctuation
+ # and current_x + unit_width * 2 > box.x2
+ # )
+ # ):
+ # # æ¢行
+ # current_x = box.x
+ # if not current_line_heights:
+ # return [], False
+ # max_height = max(current_line_heights)
+ # mode_height = statistics.mode(current_line_heights)
+
+ # current_y -= max(mode_height * line_skip, max_height * 1.05)
+ # line_ys.append(current_y)
+ # line_height = 0.0
+ # current_line_heights = [] # 清空当å‰ÂÂÂ行高度åˆâ€â€Â表
+
+ # # 检查是å¦超出底部边界
+ # # if current_y - unit_height < box.y:
+ # if current_y < box.y:
+ # all_units_fit = False
+ # # 这里ä¸ÂÂÂ覠break，继ç»ÂÂÂ排版剩余内容
+
+ # if unit.is_space:
+ # line_height = max(line_height, unit_height)
+ # continue
+
+ # # æâ€Â¾置当å‰ÂÂÂå•元
+ # relocated_unit = unit.relocate(current_x, current_y, scale)
+ # typeset_units.append(relocated_unit)
+
+ # # 添加当å‰ÂÂÂå•元的高度到当å‰ÂÂÂ行高度åˆâ€â€Â表
+ # if not unit.is_space:
+ # current_line_heights.append(unit_height)
+
+ # prev_x = current_x
+ # # æ›´æ–° x Ã¥ÂÂÂÂÂÂæ ‡
+ # current_x = relocated_unit.box.x2
+ # if prev_x > current_x:
+ # logger.warning(f"Ã¥ÂÂÂÂÂÂ标回绕ï¼ÂÂÂï¼ÂÂÂï¼ÂÂÂTypesettingUnit: {unit.box}, ")
+
+ # last_unit = relocated_unit
+
+ # # *** NEW: For Arabic, right-align each line ***
+ # if is_arabic and typeset_units:
+ # # Group units by line (Y coordinate)
+ # lines = {}
+ # for unit in typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # line_y = round(unit.box.y, 1)
+ # if line_y not in lines:
+ # lines[line_y] = []
+ # lines[line_y].append(unit)
+
+ # # Right-align each line
+ # for line_y, line_units in lines.items():
+ # if not line_units:
+ # continue
+
+ # # Find the rightmost position of this line
+ # line_max_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+
+ # # Calculate how much to shift right
+ # shift_x = box.x2 - line_max_x
+
+ # # Shift all units in this line to the right
+ # for unit in line_units:
+ # if unit.box:
+ # unit.box.x += shift_x
+ # unit.box.x2 += shift_x
+ # if unit.x is not None:
+ # unit.x += shift_x
+ # # Update character box if present
+ # if unit.char and unit.char.box:
+ # unit.char.box.x += shift_x
+ # unit.char.box.x2 += shift_x
+ # if unit.char and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ # unit.char.visual_bbox.box.x += shift_x
+ # unit.char.visual_bbox.box.x2 += shift_x
+ # # Check if output language is Arabic
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # # If Arabic, reverse the line order
+ # if is_arabic and typeset_units:
+ # # Group units by line (using Y coordinates)
+ # lines_dict = {}
+ # for unit in typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # # Round Y coordinate to group units on the same line
+ # line_y = round(unit.box.y, 1)
+ # if line_y not in lines_dict:
+ # lines_dict[line_y] = []
+ # lines_dict[line_y].append(unit)
+
+ # # Sort lines by Y coordinate (top to bottom) and reverse
+ # sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # # Rebuild typeset_units with reversed line order
+ # reversed_typeset_units = []
+ # for line_y in reversed(sorted_line_ys):
+ # reversed_typeset_units.extend(lines_dict[line_y])
+
+ # # Now reposition all units to swap their Y coordinates
+ # # Map old Y positions to new Y positions
+ # y_mapping = {}
+ # for i, old_y in enumerate(sorted_line_ys):
+ # new_y = sorted_line_ys[len(sorted_line_ys) - 1 - i]
+ # y_mapping[old_y] = new_y
+
+ # # Update Y coordinates for all units
+ # for unit in reversed_typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # old_y = round(unit.box.y, 1)
+ # if old_y in y_mapping:
+ # new_y = y_mapping[old_y]
+ # y_diff = new_y - old_y
+ # # Update the unit's Y position
+ # if unit.y is not None:
+ # unit.y += y_diff
+ # if unit.box:
+ # unit.box.y += y_diff
+ # unit.box.y2 += y_diff
+
+ # typeset_units = reversed_typeset_units
+
+ # return typeset_units, all_units_fit
+
+ def create_typesetting_units(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ fonts: dict[str, il_version_1.PdfFont],
+ ) -> list[TypesettingUnit]:
+ if not paragraph.pdf_paragraph_composition:
+ return []
+ result = []
+
+ @cache
+ def get_font(font_id: str, xobj_id: int | None):
+ if xobj_id in fonts:
+ font = fonts[xobj_id][font_id]
+ else:
+ font = fonts[font_id]
+ return font
+
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition is None:
+ continue
+ if composition.pdf_line:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_line.pdf_character
+ ],
+ )
+ elif composition.pdf_character:
+ result.append(
+ TypesettingUnit(
+ char=composition.pdf_character,
+ debug_info=paragraph.debug_info,
+ ),
+ )
+ elif composition.pdf_same_style_characters:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_same_style_characters.pdf_character
+ ],
+ )
+ elif composition.pdf_same_style_unicode_characters:
+ style = composition.pdf_same_style_unicode_characters.pdf_style
+ if style is None:
+ logger.warning(
+ f"Style is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font_id = style.font_id
+ if font_id is None:
+ logger.warning(
+ f"Font ID is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font = get_font(font_id, paragraph.xobj_id)
+ if composition.pdf_same_style_unicode_characters.unicode:
+ unicode_text = composition.pdf_same_style_unicode_characters.unicode
+ shaped_text = self.shape_arabic_text(unicode_text)
+ result.extend(
+ [
+ TypesettingUnit(
+ unicode=char_unicode,
+ font=self.font_mapper.map(
+ font,
+ char_unicode,
+ ),
+ original_font=font,
+ font_size=style.font_size,
+ style=style,
+ xobj_id=paragraph.xobj_id,
+ debug_info=composition.pdf_same_style_unicode_characters.debug_info
+ or False,
+ )
+ for char_unicode in shaped_text # Use shaped_text instead of original
+ if char_unicode not in ("\n",)
+ ],
+ )
+ elif composition.pdf_formula:
+ result.extend([TypesettingUnit(formular=composition.pdf_formula)])
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ result = list(
+ filter(
+ lambda x: x.unicode is None or x.font is not None,
+ result,
+ ),
+ )
+
+ if any(x.width < 0 for x in result):
+ logger.warning("有排版å•元宽度å°ÂÂÂ于 0，请检查åÂÂÂâ€â€Â体映射是å¦æ£确。")
+ return result
+
+ def create_passthrough_composition(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ ) -> list[PdfParagraphComposition]:
+ """从排版å•元创建直接传递的段è½组åˆ。
+
+ Args:
+ typesetting_units: 排版å•元åˆâ€â€Â表
+
+ Returns:
+ 段è½组åˆåˆâ€â€Â表
+ """
+ composition = []
+ for unit in typesetting_units:
+ if unit.formular:
+ # 对于公å¼ÂÂÂå•元，直接创建包å«完整公å¼ÂÂÂ的组åÂÂÂËâ€Â
+ composition.append(PdfParagraphComposition(pdf_formula=unit.formular))
+ else:
+ # 对于åÂÂÂâ€â€Â符å•元，使çâ€Â¨原有逻辑
+ chars, curves, forms = unit.passthrough()
+ composition.extend(
+ [PdfParagraphComposition(pdf_character=char) for char in chars],
+ )
+ return composition
+
+ def get_max_right_space(self, current_box: Box, page) -> float:
+ """获å–段è½å³侧最大å¯çâ€Â¨空éâ€â€Â´
+
+ Args:
+ current_box: 当å‰ÂÂÂ段è½的边界æ¡â€ÂÂ
+ page: 当å‰ÂÂÂ页é¢
+
+ Returns:
+ å¯以扩展到的最大 x Ã¥ÂÂÂÂÂÂæ ‡
+ """
+ # 获å–页é¢的è£ÂÂÂ剪框作为åˆÂÂÂ始最大é™ÂÂÂ制
+ max_x = page.cropbox.box.x2 * 0.9
+
+ # 检查所有å¯能的阻挡元ç´ÂÂÂ
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # 跳过当å‰ÂÂÂ段è½
+ continue
+ # åª考虑在当å‰ÂÂÂ段è½å³侧ä¸â€ÂÂ有垂直é‡ÂÂÂå 的元ç´ÂÂÂ
+ if para.box.x > current_box.x and not (
+ para.box.y >= current_box.y2 or para.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, para.box.x)
+ for char in page.pdf_character:
+ if char.box.x > current_box.x and not (
+ char.box.y >= current_box.y2 or char.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, char.box.x)
+ # 检查图形
+ for figure in page.pdf_figure:
+ if figure.box.x > current_box.x and not (
+ figure.box.y >= current_box.y2 or figure.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, figure.box.x)
+
+ return max_x
+
+ def get_max_bottom_space(self, current_box: Box, page: il_version_1.Page) -> float:
+ """获å–段è½下方最大å¯çâ€Â¨空éâ€â€Â´
+
+ Args:
+ current_box: 当å‰ÂÂÂ段è½的边界æ¡â€ÂÂ
+ page: 当å‰ÂÂÂ页é¢
+
+ Returns:
+ å¯以扩展到的最尠y Ã¥ÂÂÂÂÂÂæ ‡
+ """
+ # 获å–页é¢的è£ÂÂÂ剪框作为åˆÂÂÂ始最å°ÂÂÂé™ÂÂÂ制
+ min_y = page.cropbox.box.y * 1.1
+
+ # 检查所有å¯能的阻挡元ç´ÂÂÂ
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # 跳过当å‰ÂÂÂ段è½
+ continue
+ # åª考虑在当å‰ÂÂÂ段è½下方ä¸â€ÂÂ有水平é‡ÂÂÂå 的元ç´ÂÂÂ
+ if para.box.y2 < current_box.y and not (
+ para.box.x >= current_box.x2 or para.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, para.box.y2)
+ for char in page.pdf_character:
+ if char.box.y2 < current_box.y and not (
+ char.box.x >= current_box.x2 or char.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, char.box.y2)
+ # 检查图形
+ for figure in page.pdf_figure:
+ if figure.box.y2 < current_box.y and not (
+ figure.box.x >= current_box.x2 or figure.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, figure.box.y2)
+
+ return min_y
+
+ def _update_paragraph_render_order(self, paragraph: il_version_1.PdfParagraph):
+ """
+ é‡ÂÂÂ新设置段è½å„åÂÂÂâ€â€Â符的 render order
+ 主 render order ç‰于 paragraph çš„ renderorder，sub render order 从 1 开始自增
+ """
+ if not hasattr(paragraph, "render_order") or paragraph.render_order is None:
+ return
+
+ main_render_order = paragraph.render_order
+ sub_render_order = 1
+
+ # éÂÂÂÂÂÂ历段è½的所有组æˆÂÂÂ部åˆâ€ÂÂ
+ for composition in paragraph.pdf_paragraph_composition:
+ # 检查å•个åÂÂÂâ€â€Â符
+ if composition.pdf_character:
+ char = composition.pdf_character
+ char.render_order = main_render_order
+ char.sub_render_order = sub_render_order
+ sub_render_order += 1
\ No newline at end of file
diff --git a/babeldoc/format/pdf/document_il/midend/typesetting.py b/babeldoc/format/pdf/document_il/midend/typesetting.py
new file mode 100644
index 0000000000000000000000000000000000000000..8ad5f7411f7ec7dd8258c4f7674a8e7a88790fdb
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/typesetting.py
@@ -0,0 +1,1857 @@
+from __future__ import annotations
+
+import copy
+import logging
+import re
+import statistics
+import unicodedata
+from functools import cache
+
+import pymupdf
+import regex
+from rtree import index
+
+from babeldoc.const import WATERMARK_VERSION
+from babeldoc.format.pdf.document_il import Box
+from babeldoc.format.pdf.document_il import PdfCharacter
+from babeldoc.format.pdf.document_il import PdfCurve
+from babeldoc.format.pdf.document_il import PdfForm
+from babeldoc.format.pdf.document_il import PdfFormula
+from babeldoc.format.pdf.document_il import PdfParagraphComposition
+from babeldoc.format.pdf.document_il import PdfStyle
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.formular_helper import update_formula_data
+from babeldoc.format.pdf.document_il.utils.layout_helper import box_to_tuple
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from babeldoc.format.pdf.translation_config import WatermarkOutputMode
+from arabic_reshaper import reshape
+from bidi.algorithm import get_display
+
+
+logger = logging.getLogger(__name__)
+
+LINE_BREAK_REGEX = regex.compile(
+ r"^["
+ r"a-z"
+ r"A-Z"
+ r"0-9"
+ r"\u00C0-\u00FF" # Latin-1 Supplement
+ r"\u0100-\u017F" # Latin Extended A
+ r"\u0180-\u024F" # Latin Extended B
+ r"\u1E00-\u1EFF" # Latin Extended Additional
+ r"\u2C60-\u2C7F" # Latin Extended C
+ r"\uA720-\uA7FF" # Latin Extended D
+ r"\uAB30-\uAB6F" # Latin Extended E
+ r"\u0250-\u02A0" # IPA Extensions
+ r"\u0400-\u04FF" # Cyrillic
+ r"\u0300-\u036F" # Combining Diacritical Marks
+ r"\u0500-\u052F" # Cyrillic Supplement
+ r"\u0370-\u03FF" # Greek and Coptic
+ r"\u2DE0-\u2DFF" # Cyrillic Extended-A
+ r"\uA650-\uA69F" # Cyrillic Extended-B
+ r"\u1200-\u137F" # Ethiopic
+ r"\u1380-\u139F" # Ethiopic Supplement
+ r"\u2D80-\u2DDF" # Ethiopic Extended
+ r"\uAB00-\uAB2F" # Ethiopic Extended-A
+ r"\U0001E7E0-\U0001E7FF" # Ethiopic Extended-B
+ r"\u0E80-\u0EFF" # Lao
+ r"\u0D00-\u0D7F" # Malayalam
+ r"\u0A80-\u0AFF" # Gujarati
+ r"\u0E00-\u0E7F" # Thai
+ r"\u1000-\u109F" # Myanmar
+ r"\uAA60-\uAA7F" # Myanmar Extended-A
+ r"\uA9E0-\uA9FF" # Myanmar Extended-B
+ r"\U000116D0-\U000116FF" # Myanmar Extended-C
+ r"\u0B80-\u0BFF" # Tamil
+ r"\u0C00-\u0C7F" # Telugu
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0530-\u058F" # Armenian
+ r"\u10A0-\u10FF" # Georgian
+ r"\u1C90-\u1CBF" # Georgian Extended
+ r"\u2D00-\u2D2F" # Georgian Supplement
+ r"\u1780-\u17FF" # Khmer
+ r"\u19E0-\u19FF" # Khmer Symbols
+ r"\U00010B00-\U00010B3F" # Avestan
+ r"\u1D00-\u1D7F" # Phonetic Extensions
+ r"\u1400-\u167F" # Unified Canadian Aboriginal Syllabics
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0780-\u07BF" # Thaana
+ r"\U0001E900-\U0001E95F" # Adlam
+ r"\u1C80-\u1C8F" # Cyrillic Extended-C
+ r"\U0001E030-\U0001E08F" # Cyrillic Extended-D
+ r"\uA000-\uA48F" # Yi Syllables
+ r"\uA490-\uA4CF" # Yi Radicals
+ r"'"
+ r"-" # Hyphen
+ r"\u00B7" # Middle Dot
+ r"\u02BB" # Spacing Modifier Letters
+ r"]+ $"
+)
+
+
+class TypesettingUnit:
+ def __str__(self):
+ return self.try_get_unicode() or ""
+
+ def __init__(
+ self,
+ char: PdfCharacter | None = None,
+ formular: PdfFormula | None = None,
+ unicode: str | None = None,
+ font: pymupdf.Font | None = None,
+ original_font: il_version_1.PdfFont | None = None,
+ font_size: float | None = None,
+ style: PdfStyle | None = None,
+ xobj_id: int | None = None,
+ debug_info: bool = False,
+ ):
+ assert (char is not None) + (formular is not None) + (
+ unicode is not None
+ ) == 1, "Only one of chars and formular can be not None"
+ self.char = char
+ self.formular = formular
+ self.unicode = unicode
+ self.x = None
+ self.y = None
+ self.scale = None
+ self.debug_info = debug_info
+
+ # Cache variables
+ self.box_cache: Box | None = None
+ self.can_break_line_cache: bool | None = None
+ self.is_cjk_char_cache: bool | None = None
+ self.mixed_character_blacklist_cache: bool | None = None
+ self.is_space_cache: bool | None = None
+ self.is_hung_punctuation_cache: bool | None = None
+ self.is_cannot_appear_in_line_end_punctuation_cache: bool | None = None
+ self.can_passthrough_cache: bool | None = None
+ self.width_cache: float | None = None
+ self.height_cache: float | None = None
+
+ self.font_size: float | None = None
+
+ if unicode:
+ assert font_size, "Font size must be provided when unicode is provided"
+ assert style, "Style must be provided when unicode is provided"
+ assert len(unicode) == 1, "Unicode must be a single character"
+ assert xobj_id is not None, (
+ "Xobj id must be provided when unicode is provided"
+ )
+
+ self.font = font
+ if font is not None and hasattr(font, "font_id"):
+ self.font_id = font.font_id
+ else:
+ self.font_id = "base"
+ if original_font:
+ self.original_font = original_font
+ else:
+ self.original_font = None
+
+ self.font_size = font_size
+ self.style = style
+ self.xobj_id = xobj_id
+
+ def try_resue_cache(self, old_tu: TypesettingUnit):
+ if old_tu.is_cjk_char_cache is not None:
+ self.is_cjk_char_cache = old_tu.is_cjk_char_cache
+
+ if old_tu.can_break_line_cache is not None:
+ self.can_break_line_cache = old_tu.can_break_line_cache
+
+ if old_tu.is_space_cache is not None:
+ self.is_space_cache = old_tu.is_space_cache
+
+ if old_tu.is_hung_punctuation_cache is not None:
+ self.is_hung_punctuation_cache = old_tu.is_hung_punctuation_cache
+
+ if old_tu.is_cannot_appear_in_line_end_punctuation_cache is not None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ old_tu.is_cannot_appear_in_line_end_punctuation_cache
+ )
+
+ if old_tu.can_passthrough_cache is not None:
+ self.can_passthrough_cache = old_tu.can_passthrough_cache
+
+ if old_tu.mixed_character_blacklist_cache is not None:
+ self.mixed_character_blacklist_cache = (
+ old_tu.mixed_character_blacklist_cache
+ )
+
+
+ def try_get_unicode(self) -> str | None:
+ if self.char:
+ return self.char.char_unicode
+ elif self.formular:
+ return None
+ elif self.unicode:
+ return self.unicode
+
+ @property
+ def mixed_character_blacklist(self):
+ if self.mixed_character_blacklist_cache is None:
+ self.mixed_character_blacklist_cache = self.calc_mixed_character_blacklist()
+
+ return self.mixed_character_blacklist_cache
+
+ def calc_mixed_character_blacklist(self):
+ unicode = self.try_get_unicode()
+ if unicode:
+ return unicode in [
+ "£",
+ "，",
+ "！",
+ "：",
+ "）",
+ ]
+ return False
+
+ @property
+ def can_break_line(self):
+ if self.can_break_line_cache is None:
+ self.can_break_line_cache = self.calc_can_break_line()
+
+ return self.can_break_line_cache
+
+ def calc_can_break_line(self):
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return True
+ if LINE_BREAK_REGEX.match(unicode):
+ return False
+ return True
+
+ @property
+ def is_cjk_char(self):
+ if self.is_cjk_char_cache is None:
+ self.is_cjk_char_cache = self.calc_is_cjk_char()
+
+ return self.is_cjk_char_cache
+
+ def calc_is_cjk_char(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ if "(cid" in unicode:
+ return False
+ if len(unicode) > 1:
+ return False
+ assert len(unicode) == 1, "Unicode must be a single character"
+ if unicode in [
+ "（",
+ "）",
+ "，",
+ "。",
+ "、",
+ "；",
+ "：",
+ "？",
+ "！",
+ "）",
+ "，",
+ "！",
+ "：",
+ "）",
+ ]:
+ return True
+ if unicode:
+ if re.match(
+ r"^["
+ r"\u3000-\u303f" # CJK Symbols and Punctuation
+ r"\u3040-\u309f" # Hiragana
+ r"\u30a0-\u30ff" # Katakana
+ r"\u3100-\u312f" # Bopomofo
+ r"\uac00-\ud7af" # Hangul Syllables
+ r"\u1100-\u11ff" # Hangul Jamo
+ r"\u3130-\u318f" # Hangul Compatibility Jamo
+ r"\ua960-\ua97f" # Hangul Jamo Extended-A
+ r"\ud7b0-\ud7ff" # Hangul Jamo Extended-B
+ r"\u3190-\u319f" # Kanbun
+ r"\u3200-\u32ff" # Enclosed CJK Letters and Months
+ r"\u3300-\u33ff" # CJK Compatibility
+ r"\ufe30-\ufe4f" # CJK Compatibility Forms
+ r"\u4e00-\u9fff" # CJK Unified Ideographs
+ r"\u2e80-\u2eff" # CJK Radicals Supplement
+ r"\u31c0-\u31ef" # CJK Strokes
+ r"\u2f00-\u2fdf" # Kangxi Radicals
+ r"\ufe10-\ufe1f" # Vertical Forms
+ r"]+$",
+ unicode,
+ ):
+ return True
+ try:
+ unicodedata_name = unicodedata.name(unicode)
+ return (
+ "CJK UNIFIED IDEOGRAPH" in unicodedata_name
+ or "FULLWIDTH" in unicodedata_name
+ )
+ except ValueError:
+ return False
+ return False
+
+ @property
+ def is_space(self):
+ if self.is_space_cache is None:
+ self.is_space_cache = self.calc_is_space()
+
+ return self.is_space_cache
+
+ def calc_is_space(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ return unicode == " "
+
+ @property
+ def is_hung_punctuation(self):
+ if self.is_hung_punctuation_cache is None:
+ self.is_hung_punctuation_cache = self.calc_is_hung_punctuation()
+
+ return self.is_hung_punctuation_cache
+
+ def calc_is_hung_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+
+ if unicode:
+ return unicode in [
+ # English punctuation
+ ",",
+ ".",
+ ":",
+ ";",
+ "?",
+ "!",
+ # Chinese punctuation
+ "，", # Comma
+ "。", # Period
+ "：", # Colon
+ "；", # Semicolon
+ "？", # Question mark
+ "！", # Exclamation mark
+ "、", # Enumeration comma
+ # Closing brackets
+ ")", # Right parenthesis
+ "]", # Right square bracket
+ "}", # Right curly bracket
+ "）", # Right parenthesis
+ "】", # Right square bracket
+ "》", # Right double angle bracket
+ "』", # Right single quotation mark
+ "」", # Right corner bracket
+ # Connected line symbols
+ "–", # EN DASH
+ "—", # EM DASH
+ # Special punctuation
+ "·", # Middle dot
+ "…", # Ellipsis
+ "°", # Degree symbol
+ # Slash
+ "/", # Slash
+ "／", # Fullwidth solidus
+ "‰", # Per mille sign
+ ]
+ return False
+
+ @property
+ def is_cannot_appear_in_line_end_punctuation(self):
+ if self.is_cannot_appear_in_line_end_punctuation_cache is None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ self.calc_is_cannot_appear_in_line_end_punctuation()
+ )
+
+ return self.is_cannot_appear_in_line_end_punctuation_cache
+
+ def calc_is_cannot_appear_in_line_end_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ return unicode in [
+ # Opening brackets
+ """, # Left double quotation mark
+ "'", # Left single quotation mark
+ "《", # Left double angle bracket
+ "『", # Left single quotation mark
+ # Opening brackets
+ "(", # Left parenthesis
+ "[", # Left square bracket
+ "{", # Left curly bracket
+ "（", # Left parenthesis
+ "【", # Left square bracket
+ "《", # Left double angle bracket
+ "『", # Left single quotation mark
+ # Cannot appear at end of line - combined with closing brackets
+ """, # Right double quotation mark
+ "'", # Right single quotation mark
+ "》", # Right double angle bracket
+ "』", # Right single quotation mark
+ ]
+
+ def passthrough(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ if self.char:
+ return [self.char], [], []
+ elif self.formular:
+ return (
+ self.formular.pdf_character,
+ self.formular.pdf_curve,
+ self.formular.pdf_form,
+ )
+ elif self.unicode:
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ return [], [], []
+
+ @property
+ def can_passthrough(self):
+ if self.can_passthrough_cache is None:
+ self.can_passthrough_cache = self.calc_can_passthrough()
+
+ return self.can_passthrough_cache
+
+ def calc_can_passthrough(self):
+ return self.unicode is None
+
+ def calculate_box(self):
+ if self.char:
+ box = copy.deepcopy(self.char.box)
+ if self.char.visual_bbox and self.char.visual_bbox.box:
+ box.y = self.char.visual_bbox.box.y
+ box.y2 = self.char.visual_bbox.box.y2
+
+ return box
+ elif self.formular:
+ return self.formular.box
+ elif self.unicode:
+ char_width = self.font.char_lengths(self.unicode, self.font_size)[0]
+ if self.x is None or self.y is None or self.scale is None:
+ return Box(0, 0, char_width, self.font_size)
+ return Box(self.x, self.y, self.x + char_width, self.y + self.font_size)
+
+ @property
+ def box(self):
+ if not self.box_cache:
+ self.box_cache = self.calculate_box()
+
+ return self.box_cache
+
+ @property
+ def width(self):
+ if self.width_cache is None:
+ self.width_cache = self.calc_width()
+
+ return self.width_cache
+
+ def calc_width(self):
+ box = self.box
+ return box.x2 - box.x
+
+ @property
+ def height(self):
+ if self.height_cache is None:
+ self.height_cache = self.calc_height()
+
+ return self.height_cache
+
+ def calc_height(self):
+ box = self.box
+ return box.y2 - box.y
+
+ def relocate(
+ self,
+ x: float,
+ y: float,
+ scale: float,
+ ) -> TypesettingUnit:
+ """Relocate and scale the typesetting unit
+
+ Args:
+ x: New x position
+ y: New y position
+ scale: Scale factor
+
+ Returns:
+ New relocated and scaled typesetting unit
+ """
+ if self.char:
+ # Create new character object
+ new_char = PdfCharacter(
+ pdf_character_id=self.char.pdf_character_id,
+ char_unicode=self.char.char_unicode,
+ box=Box(
+ x=x,
+ y=y,
+ x2=x + self.width * scale,
+ y2=y + self.height * scale,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.char.pdf_style.font_id,
+ font_size=self.char.pdf_style.font_size * scale,
+ graphic_state=self.char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=self.char.vertical,
+ advance=self.char.advance * scale if self.char.advance else None,
+ debug_info=self.debug_info,
+ xobj_id=self.char.xobj_id,
+ )
+ new_tu = TypesettingUnit(char=new_char)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.formular:
+ # Create new formula object and recursively relocate child characters
+ new_chars = []
+ min_x = self.formular.box.x
+ min_y = self.formular.box.y
+
+ for char in self.formular.pdf_character:
+ # Calculate relative position
+ rel_x = char.box.x - min_x
+ rel_y = char.box.y - min_y
+
+ visual_rel_x = char.visual_bbox.box.x - min_x
+ visual_rel_y = char.visual_bbox.box.y - min_y
+
+ # Create new character object
+ new_char = PdfCharacter(
+ pdf_character_id=char.pdf_character_id,
+ char_unicode=char.char_unicode,
+ box=Box(
+ x=x + (rel_x + self.formular.x_offset) * scale,
+ y=y + (rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (rel_x + (char.box.x2 - char.box.x) + self.formular.x_offset)
+ * scale,
+ y2=y
+ + (rel_y + (char.box.y2 - char.box.y) + self.formular.y_offset)
+ * scale,
+ ),
+ visual_bbox=il_version_1.VisualBbox(
+ box=Box(
+ x=x + (visual_rel_x + self.formular.x_offset) * scale,
+ y=y + (visual_rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (
+ visual_rel_x
+ + (char.visual_bbox.box.x2 - char.visual_bbox.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=y
+ + (
+ visual_rel_y
+ + (char.visual_bbox.box.y2 - char.visual_bbox.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ ),
+ ),
+ pdf_style=PdfStyle(
+ font_id=char.pdf_style.font_id,
+ font_size=char.pdf_style.font_size * scale,
+ graphic_state=char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=char.vertical,
+ advance=char.advance * scale if char.advance else None,
+ xobj_id=char.xobj_id,
+ )
+ new_chars.append(new_char)
+
+ # Calculate bounding box from new_chars
+ min_x = min(char.visual_bbox.box.x for char in new_chars)
+ min_y = min(char.visual_bbox.box.y for char in new_chars)
+ max_x = max(char.visual_bbox.box.x2 for char in new_chars)
+ max_y = max(char.visual_bbox.box.y2 for char in new_chars)
+
+ new_formula = PdfFormula(
+ box=Box(
+ x=min_x,
+ y=min_y,
+ x2=max_x,
+ y2=max_y,
+ ),
+ pdf_character=new_chars,
+ x_offset=self.formular.x_offset * scale,
+ y_offset=self.formular.y_offset * scale,
+ x_advance=self.formular.x_advance * scale,
+ )
+
+ # Handle contained curves
+ new_curves = []
+ for curve in self.formular.pdf_curve:
+ new_curve = self._transform_curve_for_relocation(
+ curve,
+ self.formular.box.x,
+ self.formular.box.y,
+ x,
+ y,
+ scale,
+ )
+ new_curves.append(new_curve)
+ new_formula.pdf_curve = new_curves
+
+ # Handle contained forms
+ new_forms = []
+ for form in self.formular.pdf_form:
+ new_form = self._transform_form_for_relocation(
+ form, self.formular.box.x, self.formular.box.y, x, y, scale
+ )
+ new_forms.append(new_form)
+ new_formula.pdf_form = new_forms
+
+ update_formula_data(new_formula)
+
+ new_tu = TypesettingUnit(formular=new_formula)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.unicode:
+ # For Unicode, store position info and create new TypesettingUnit
+ new_unit = TypesettingUnit(
+ unicode=self.unicode,
+ font=self.font,
+ original_font=self.original_font,
+ font_size=self.font_size * scale,
+ style=self.style,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ new_unit.x = x
+ new_unit.y = y
+ new_unit.scale = scale
+ new_unit.try_resue_cache(self)
+ return new_unit
+
+ def _transform_curve_for_relocation(
+ self,
+ curve,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a curve for formula relocation."""
+ import copy
+
+ new_curve = copy.deepcopy(curve)
+
+ if new_curve.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_curve.box.x - original_formula_x
+ rel_y = new_curve.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_curve.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (
+ rel_x
+ + (new_curve.box.x2 - new_curve.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=new_y
+ + (
+ rel_y
+ + (new_curve.box.y2 - new_curve.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original CTM
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_curve.relocation_transform = list(relocation_matrix)
+
+ return new_curve
+
+ def _transform_form_for_relocation(
+ self,
+ form,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a form for formula relocation."""
+ import copy
+
+ new_form = copy.deepcopy(form)
+
+ if new_form.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_form.box.x - original_formula_x
+ rel_y = new_form.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_form.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (rel_x + (new_form.box.x2 - new_form.box.x) + self.formular.x_offset)
+ * scale,
+ y2=new_y
+ + (rel_y + (new_form.box.y2 - new_form.box.y) + self.formular.y_offset)
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original matrices
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_form.relocation_transform = list(relocation_matrix)
+
+ return new_form
+
+ def render(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ """Render the typesetting unit to PdfCharacter list
+
+ Returns:
+ PdfCharacter list
+ """
+ if self.can_passthrough:
+ return self.passthrough()
+ elif self.unicode:
+ assert self.x is not None, (
+ "x position must be set, should be set by `relocate`"
+ )
+ assert self.y is not None, (
+ "y position must be set, should be set by `relocate`"
+ )
+ assert self.scale is not None, (
+ "scale must be set, should be set by `relocate`"
+ )
+ x = self.x
+ y = self.y
+
+ # Calculate character width
+ char_width = self.width
+
+ # Handle case when font is None (no suitable font found for this character)
+ if self.font is None:
+ logger.warning(
+ f"No font available for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using font_id='{self.font_id}' with glyph_id=0"
+ )
+ glyph_id = 0 # Use glyph 0 as fallback (usually .notdef)
+ else:
+ glyph_id = self.font.has_glyph(ord(self.unicode))
+ if glyph_id == 0 or glyph_id is None:
+ logger.warning(
+ f"Font '{self.font_id}' doesn't have glyph for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using glyph_id=0"
+ )
+ glyph_id = 0
+
+ new_char = PdfCharacter(
+ pdf_character_id=glyph_id,
+ char_unicode=self.unicode,
+ box=Box(
+ x=x, # Use stored x position
+ y=y,
+ x2=x + char_width,
+ y2=y + self.font_size,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.font_id,
+ font_size=self.font_size,
+ graphic_state=self.style.graphic_state,
+ ),
+ scale=self.scale,
+ vertical=False,
+ advance=char_width,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ return [new_char], [], []
+ else:
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ return [], [], []
+
+
+class Typesetting:
+ stage_name = "Typesetting"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.font_mapper = FontMapper(translation_config)
+ self.translation_config = translation_config
+ self.lang_code = self.translation_config.lang_out.upper()
+ # Ensure detailed_logger attribute exists to avoid attribute access errors
+ self.detailed_logger = None
+ self.is_cjk = (
+ # Why zh-CN/zh-HK/zh-TW here but not zh-Hans and so on?
+ # See https://funstory-ai.github.io/BabelDOC/supported_languages/
+ ("ZH" in self.lang_code) # C
+ or ("JA" in self.lang_code)
+ or ("JP" in self.lang_code) # J
+ or ("KR" in self.lang_code) # K
+ or ("CN" in self.lang_code)
+ or ("HK" in self.lang_code)
+ or ("TW" in self.lang_code)
+ )
+
+ def preprocess_document(self, document: il_version_1.Document, pbar):
+ """Preprocess document - calculate optimal scale for each paragraph and cache the result"""
+ all_scales: list[float] = []
+ all_paragraphs: list[il_version_1.PdfParagraph] = []
+
+ for page in document.page:
+ pbar.advance()
+ # Build font dictionary for current page rendering logic
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if (
+ xobj.xobj_id in fonts
+ and isinstance(fonts[xobj.xobj_id], dict)
+ and font.font_id
+ ):
+ fonts[xobj.xobj_id][font.font_id] = font
+
+ # Preprocess each paragraph
+ for paragraph in page.pdf_paragraph:
+ all_paragraphs.append(paragraph)
+ unit_count = 0
+ try:
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ unit_count = len(typesetting_units)
+ for unit in typesetting_units:
+ if unit.formular:
+ unit_count += len(unit.formular.pdf_character) - 1
+
+ # Get optimal scale value (if all can passthrough, scale = 1.0)
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.optimal_scale = 1.0
+ else:
+ # Get optimal scale factor
+ optimal_scale = self._get_optimal_scale(
+ paragraph, page, typesetting_units
+ )
+ paragraph.optimal_scale = optimal_scale
+ except Exception as e:
+ # If preprocessing paragraph fails, default scale is 1.0
+ logger.warning(f"Preprocessing paragraph failed: {e}")
+ paragraph.optimal_scale = 1.0
+
+ if paragraph.optimal_scale is not None:
+ all_scales.extend([paragraph.optimal_scale] * unit_count)
+
+ # Get optimal scale factor, estimate the mode of the distribution as starting point for scale selection
+ if all_scales:
+ try:
+ modes = statistics.multimode(all_scales)
+ mode_scale = min(modes)
+ except statistics.StatisticsError:
+ logger.warning(
+ "Could not find a mode for paragraph scales. Falling back to median."
+ )
+ mode_scale = statistics.median(all_scales)
+ # Too many times using a smaller scale will affect readability, so use the mode as the upper limit
+ for paragraph in all_paragraphs:
+ if (
+ paragraph.optimal_scale is not None
+ and paragraph.optimal_scale > mode_scale
+ ):
+ paragraph.optimal_scale = mode_scale
+ else:
+ logger.error(
+ "document_scales is empty, there seems no paragraph in this PDF"
+ )
+
+ def shape_arabic_text(self, text: str) -> str:
+ """Shape and reorder Arabic text if output language is Arabic.
+
+ Args:
+ text: Input text to shape
+
+ Returns:
+ Shaped and reordered text if language is Arabic, original text otherwise
+ """
+ if not text:
+ return text
+
+ # Robust Arabic output detection: accept explicit 'ar', 'ara', 'arabic'
+ # or formats containing '-ar', '->ar', or '/ar' as a target marker (e.g. 'en-ar', 'en->ar')
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = False
+ if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ is_arabic = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic = True
+
+ if is_arabic:
+ logger.debug("Shaping Arabic text")
+ try:
+ if not re.search(r'[\uFB50-\uFDFF\uFE70-\uFEFF]', text):
+ # Reshape Arabic text for proper character joining
+ from arabic_reshaper import ArabicReshaper
+ configuration = {
+ 'delete_harakat': False, # Keep diacritical marks
+ 'support_ligatures': True, # Support Arabic ligatures
+ 'RIAL SIGN': True,
+ 'ARABIC COMMA': True,
+ 'ARABIC SEMICOLON': True,
+ 'ARABIC QUESTION MARK': True,
+ 'ZWNJ': True, # Zero Width Non-Joiner
+ }
+
+ reshaper = ArabicReshaper(configuration=configuration)
+ reshaped_text = reshaper.reshape(text)
+ display_text = get_display(reshaped_text, base_dir='R')
+ else:
+ display_text = text
+ return display_text
+ except Exception as e:
+ logger.warning(f"Failed to shape Arabic text: {e}")
+ return text
+
+ return text
+
+ def _find_optimal_scale_and_layout(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ initial_scale: float = 1.0,
+ use_english_line_break: bool = True,
+ apply_layout: bool = False,
+ ) -> tuple[float, list[TypesettingUnit] | None]:
+ """Find the optimal scale factor and apply layout if needed
+
+ Args:
+ paragraph: Paragraph object
+ page: Page object
+ typesetting_units: List of typesetting units
+ initial_scale: Starting scale factor
+ use_english_line_break: Whether to use English line breaking rules
+ apply_layout: Whether to apply layout and update the paragraph
+
+ Returns:
+ tuple[float, list[TypesettingUnit] | None]: (optimal scale factor, laid out typesetting units or None)
+ """
+ if not paragraph.box:
+ return initial_scale, None
+
+ box = paragraph.box
+ scale = initial_scale
+ line_skip = 1.50 if self.is_cjk else 1.3
+ min_scale = 0.1
+ expand_space_flag = 0
+ final_typeset_units = None
+
+ while scale >= min_scale:
+ try:
+ # Try to layout typesetting units
+ typeset_units, all_units_fit = self._layout_typesetting_units(
+ typesetting_units,
+ box,
+ scale,
+ line_skip,
+ paragraph,
+ use_english_line_break,
+ )
+
+ # If all typesetting units fit within the box
+ if all_units_fit:
+ if apply_layout:
+ # Apply layout and write to paragraph
+ paragraph.scale = scale
+ paragraph.pdf_paragraph_composition = []
+ for unit in typeset_units:
+ chars, curves, forms = unit.render()
+ for char in chars:
+ paragraph.pdf_paragraph_composition.append(
+ PdfParagraphComposition(pdf_character=char),
+ )
+ for curve in curves:
+ page.pdf_curve.append(curve)
+ for form in forms:
+ page.pdf_form.append(form)
+ final_typeset_units = typeset_units
+ return scale, final_typeset_units
+ except Exception:
+ # If layout fails, check for overflow and try a smaller scale
+ pass
+
+ # Add retypeset logic if needed
+ if not hasattr(paragraph, "debug_id") or not paragraph.debug_id:
+ return scale, final_typeset_units
+
+ # Reduce scale factor
+ if scale > 0.6:
+ scale -= 0.05
+ else:
+ scale -= 0.1
+
+ if scale < 0.7:
+ space_expanded = False # Track whether space has been added
+
+ if expand_space_flag == 0:
+ # Try expanding bottom space
+ try:
+ min_y = self.get_max_bottom_space(box, page) + 2
+ if min_y < box.y:
+ expanded_box = Box(x=box.x, y=min_y, x2=box.x2, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # Update paragraph box boundary
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 1
+
+ # If space expansion is possible, continue to try new scale
+ if space_expanded:
+ continue
+
+ elif expand_space_flag == 1:
+ # Try expanding right space
+ try:
+ max_x = self.get_max_right_space(box, page) - 5
+ if max_x > box.x2:
+ expanded_box = Box(x=box.x, y=box.y, x2=max_x, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # Update paragraph box boundary
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 2
+
+ # If space expansion is possible, continue to try new scale
+ if space_expanded:
+ continue
+
+ # If no space can be expanded (expand_space_flag < 2), reset scale
+ # When expand_space_flag >= 2, the space has been exhausted and normal scale selection continues
+ if expand_space_flag < 2:
+ # Reset if there was no space expansion, retry scale loop from 1.0
+ scale = 1.0
+
+ # If English line break fails, try fallback with no line break
+ if use_english_line_break:
+ return self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ initial_scale,
+ use_english_line_break=False,
+ apply_layout=apply_layout,
+ )
+
+ # Return the smallest scale factor
+ return min_scale, final_typeset_units
+
+ def _get_optimal_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ use_english_line_break: bool = True,
+ ) -> float:
+ """Get optimal scale factor for paragraph, without applying layout"""
+ scale, _ = self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ 1.0,
+ use_english_line_break,
+ apply_layout=False,
+ )
+ return scale
+
+ def retypeset_with_precomputed_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ precomputed_scale: float,
+ use_english_line_break: bool = True,
+ ):
+ """Use precomputed scale factor to layout typesetting units"""
+ if not paragraph.box:
+ return
+
+ # Using the precomputed scale factor to layout typesetting units
+ self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ precomputed_scale,
+ use_english_line_break,
+ apply_layout=True,
+ )
+
+ def typesetting_document(self, document: il_version_1.Document):
+ # Add detailed logging at the start
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Started")
+
+ # Batch preprocess document - calculate optimal scale for each paragraph
+ if self.translation_config.progress_monitor:
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(document.page) * 2,
+ ) as pbar:
+ # Preprocess - calculate optimal scale factor for each paragraph
+ self.preprocess_document(document, pbar)
+
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+ pbar.advance()
+ else:
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+
+ # Add detailed logging at the end
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Complete")
+
+ def render_page(self, page: il_version_1.Page):
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if font.font_id:
+ fonts[xobj.xobj_id][font.font_id] = font
+ if (
+ page.page_number == 0
+ and self.translation_config.watermark_output_mode
+ == WatermarkOutputMode.Watermarked
+ ):
+ self.add_watermark(page)
+ try:
+ para_index = index.Index()
+ para_map = {}
+
+ valid_paras = [
+ p
+ for p in page.pdf_paragraph
+ if p.box
+ and all(c is not None for c in [p.box.x, p.box.y, p.box.x2, p.box.y2])
+ ]
+
+ for i, para in enumerate(valid_paras):
+ para_map[i] = para
+ para_index.insert(i, box_to_tuple(para.box))
+
+ for i, p_upper in para_map.items():
+ if not (p_upper.box and p_upper.box.y is not None):
+ continue
+
+ # Calculate paragraph height and set required gap accordingly
+ para_height = p_upper.box.y2 - p_upper.box.y
+ required_gap = 0.5 if para_height < 36 else 3
+
+ check_area = il_version_1.Box(
+ x=p_upper.box.x,
+ y=p_upper.box.y - required_gap,
+ x2=p_upper.box.x2,
+ y2=p_upper.box.y,
+ )
+
+ candidate_ids = list(para_index.intersection(box_to_tuple(check_area)))
+
+ conflicting_paras = []
+ for para_id in candidate_ids:
+ if para_id == i:
+ continue
+ p_lower = para_map[para_id]
+ if not (
+ p_lower.box
+ and p_upper.box
+ and p_lower.box.x2 < p_upper.box.x
+ or p_lower.box.x > p_upper.box.x2
+ ):
+ conflicting_paras.append(p_lower)
+
+ if conflicting_paras:
+ max_y2 = max(
+ p.box.y2
+ for p in conflicting_paras
+ if p.box and p.box.y2 is not None
+ )
+
+ new_y = max_y2 + required_gap
+ if p_upper.box and new_y < p_upper.box.y2:
+ p_upper.box.y = new_y
+ except Exception as e:
+ logger.warning(
+ f"Failed to adjust paragraph positions on page {page.page_number}: {e}"
+ )
+ # Start typesetting layout rendering
+ for paragraph in page.pdf_paragraph:
+ self.render_paragraph(paragraph, page, fonts)
+
+ def add_watermark(self, page: il_version_1.Page):
+ page_width = page.cropbox.box.x2 - page.cropbox.box.x
+ page_height = page.cropbox.box.y2 - page.cropbox.box.y
+ style = il_version_1.PdfStyle(
+ font_id="base",
+ font_size=6,
+ graphic_state=il_version_1.GraphicState(),
+ )
+ text = f"This document was translated by funstory.ai using open-source PDF translation software BabelDOC {WATERMARK_VERSION} (http://yadt.io). For commercial use, please contact us for a custom version. We welcome feedback and contributions to the open-source project. Please star on GitHub."
+ if self.translation_config.debug:
+ text += "\n This is DEBUG mode. Do not share or use this document for production. Please contact us if you have questions."
+ page.pdf_paragraph.append(
+ il_version_1.PdfParagraph(
+ first_line_indent=False,
+ box=il_version_1.Box(
+ x=page.cropbox.box.x + page_width * 0.05,
+ y=page.cropbox.box.y,
+ x2=page.cropbox.box.x2,
+ y2=page.cropbox.box.y2 - page_height * 0.05,
+ ),
+ vertical=False,
+ pdf_style=style,
+ pdf_paragraph_composition=[
+ il_version_1.PdfParagraphComposition(
+ pdf_same_style_unicode_characters=il_version_1.PdfSameStyleUnicodeCharacters(
+ unicode=text,
+ pdf_style=style,
+ ),
+ ),
+ ],
+ xobj_id=-1,
+ ),
+ )
+
+ def render_paragraph(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ],
+ ):
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ # If all typesetting units can be passed through directly, no need to layout
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.scale = 1.0
+ paragraph.pdf_paragraph_composition = self.create_passthrough_composition(
+ typesetting_units,
+ )
+ else:
+ # Use precomputed scale factor to layout typesetting units
+ precomputed_scale = (
+ paragraph.optimal_scale if paragraph.optimal_scale is not None else 1.0
+ )
+
+ # If precomputed scale is available, use precomputed scale for layout
+ paragraph.pdf_paragraph_composition = []
+ self.retypeset_with_precomputed_scale(
+ paragraph, page, typesetting_units, precomputed_scale
+ )
+
+ # Update paragraph render order for child characters
+ self._update_paragraph_render_order(paragraph)
+
+ def _is_arabic_char(self, char: str) -> bool:
+ """Check if character is Arabic - OPTIMIZED"""
+ if not char:
+ return False
+ try:
+ code_point = ord(char[0])
+ return (0x0600 <= code_point <= 0x06FF) or (0xFB50 <= code_point <= 0xFDFF) or (0xFE70 <= code_point <= 0xFEFF)
+ except:
+ return False
+
+ def _layout_typesetting_units(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ box: Box,
+ scale: float,
+ line_skip: float,
+ paragraph: il_version_1.PdfParagraph,
+ use_english_line_break: bool = True,
+ ) -> tuple[list[TypesettingUnit], bool]:
+ """Layout typesetting units - OPTIMIZED FOR ARABIC RTL"""
+
+ # Detect Arabic FIRST
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = any(marker in lang_out for marker in ["ar", "arabic", "ara"])
+
+ # Calculate font size
+ font_sizes = []
+ for unit in typesetting_units:
+ if unit.font_size:
+ font_sizes.append(unit.font_size)
+ if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ font_sizes.append(unit.char.pdf_style.font_size)
+ if not font_sizes:
+ font_sizes = [12]
+ font_sizes.sort()
+ font_size = statistics.mode(font_sizes)
+
+ space_width = (
+ self.font_mapper.base_font.char_lengths(" ", font_size * scale)[0] * 0.5
+ )
+
+ # Calculate line height
+ unit_heights = [unit.height for unit in typesetting_units] if typesetting_units else []
+ if not unit_heights:
+ avg_height = 0
+ elif len(unit_heights) == 1:
+ avg_height = unit_heights[0] * scale
+ else:
+ try:
+ avg_height = statistics.mode(unit_heights) * scale
+ except statistics.StatisticsError:
+ avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # Initialize position
+ current_x = box.x
+ current_y = box.y2 - avg_height
+ box = copy.deepcopy(box)
+ line_height = 0
+ current_line_heights = []
+ typeset_units = []
+ all_units_fit = True
+ last_unit: TypesettingUnit | None = None
+ line_ys = [current_y]
+
+ if paragraph.first_line_indent:
+ current_x += space_width * 4
+
+ # OPTIMIZED ARABIC WORD-LEVEL PROCESSING
+ if is_arabic:
+ # CRITICAL: Capture original English left margin BEFORE typesetting
+ # This preserves the margin hierarchy for titles vs paragraphs
+ original_left_margin = 0
+ if typesetting_units and hasattr(typesetting_units[0], 'x') and typesetting_units[0].x is not None:
+ # Find the minimum X position from the original English layout
+ original_min_x = min(u.x for u in typesetting_units if hasattr(u, 'x') and u.x is not None)
+ original_left_margin = original_min_x - box.x
+
+ i = 0
+ safety_counter = 0
+ max_iterations = len(typesetting_units) * 2 # Safety limit
+
+ while i < len(typesetting_units) and safety_counter < max_iterations:
+ safety_counter += 1
+
+ # Collect word (simple: until space or end)
+ word_units = []
+ while i < len(typesetting_units):
+ unit = typesetting_units[i]
+ if unit.is_space:
+ if word_units:
+ i += 1
+ break
+ word_units.append(unit)
+ i += 1
+ if len(word_units) > 100: # Safety: max word length
+ break
+
+ if not word_units:
+ continue
+
+ # Calculate word width
+ word_width = sum(u.width * scale for u in word_units)
+
+ # Skip leading spaces
+ if current_x == box.x and word_units and word_units[0].is_space:
+ continue
+
+ # Check if needs new line
+ if current_x + word_width > box.x2 and current_x > box.x:
+ current_x = box.x
+ if current_line_heights:
+ max_height = max(current_line_heights)
+ mode_height = statistics.mode(current_line_heights) if len(current_line_heights) > 1 else max_height
+ current_y -= max(mode_height * line_skip, max_height * 1.05)
+ line_ys.append(current_y)
+ current_line_heights = []
+
+ if current_y < box.y:
+ all_units_fit = False
+
+ # Place word units
+ for unit in word_units:
+ if unit.is_space and current_x == box.x:
+ continue
+
+ unit_width = unit.width * scale
+ unit_height = unit.height * scale
+
+ # CJK spacing
+ if (last_unit and last_unit.is_cjk_char ^ unit.is_cjk_char
+ and not unit.is_space and current_x > box.x):
+ current_x += space_width * 0.5
+
+ relocated_unit = unit.relocate(current_x, current_y, scale)
+ typeset_units.append(relocated_unit)
+
+ if not unit.is_space:
+ current_line_heights.append(unit_height)
+
+ current_x = relocated_unit.box.x2
+ last_unit = relocated_unit
+
+ # Right-align Arabic lines (but NOT table content)
+ # Check if this paragraph is inside a table by examining layout_label
+ is_table_content = False
+ if paragraph.layout_label:
+ layout_label_lower = paragraph.layout_label.lower()
+ # Exclude ONLY actual table cell content from right-alignment
+ # NOTE: "table_title", "table_caption" are headings, NOT table content!
+ # We only want to exclude: table_cell, table_text, wired_table_cell, wireless_table_cell
+ if any(table_marker in layout_label_lower for table_marker in [
+ 'table_cell', 'table_text', 'wired_table_cell', 'wireless_table_cell'
+ ]):
+ is_table_content = True
+
+ # Only apply right-alignment if NOT table content
+ if typeset_units and not is_table_content:
+ lines_dict = {}
+ for unit in typeset_units:
+ if unit.box and unit.box.y is not None:
+ line_y = round(unit.box.y, 1)
+ if line_y not in lines_dict:
+ lines_dict[line_y] = []
+ lines_dict[line_y].append(unit)
+
+ # CRITICAL FIX: Use the original English left margin as the right margin
+ # This directly mirrors the English layout hierarchy in Arabic RTL
+ # Titles with small English left margin -> small Arabic right margin (flush right)
+ # Paragraphs with large English left margin -> large Arabic right margin (indented from right)
+
+ # The original_left_margin was captured BEFORE typesetting from the English positions
+ right_margin = original_left_margin
+
+ for line_y, line_units in lines_dict.items():
+ if line_units:
+ # Calculate shift to position line from the right with the mirrored margin
+ line_max_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+ target_right_position = box.x2 - right_margin
+ shift_x = target_right_position - line_max_x
+
+ for unit in line_units:
+ if unit.box:
+ unit.box.x += shift_x
+ unit.box.x2 += shift_x
+ if unit.x is not None:
+ unit.x += shift_x
+ if unit.char and unit.char.box:
+ unit.char.box.x += shift_x
+ unit.char.box.x2 += shift_x
+ if unit.char and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ unit.char.visual_bbox.box.x += shift_x
+ unit.char.visual_bbox.box.x2 += shift_x
+ else:
+ # ORIGINAL NON-ARABIC LOGIC (UNCHANGED)
+ for i, unit in enumerate(typesetting_units):
+ unit_width = unit.width * scale
+ unit_height = unit.height * scale
+
+ if current_x == box.x and unit.is_space:
+ continue
+
+ if (last_unit and last_unit.is_cjk_char ^ unit.is_cjk_char
+ and last_unit.box and last_unit.box.y
+ and current_y - 0.1 <= last_unit.box.y2 <= current_y + line_height + 0.1
+ and not last_unit.mixed_character_blacklist and not unit.mixed_character_blacklist
+ and current_x > box.x and unit.try_get_unicode() != " "
+ and last_unit.try_get_unicode() != " "
+ and last_unit.try_get_unicode() not in ["、", "，", "。", "：", "！", "？"]):
+ current_x += space_width * 0.5
+
+ if use_english_line_break:
+ width_before_next_break_point = self._get_width_before_next_break_point(typesetting_units[i:], scale)
+ else:
+ width_before_next_break_point = 0
+
+ if not unit.is_hung_punctuation and (
+ (current_x + unit_width > box.x2) or
+ (use_english_line_break and current_x + unit_width + width_before_next_break_point > box.x2) or
+ (unit.is_cannot_appear_in_line_end_punctuation and current_x + unit_width * 2 > box.x2)):
+
+ current_x = box.x
+ if not current_line_heights:
+ return [], False
+ max_height = max(current_line_heights)
+ mode_height = statistics.mode(current_line_heights)
+ current_y -= max(mode_height * line_skip, max_height * 1.05)
+ line_ys.append(current_y)
+ line_height = 0.0
+ current_line_heights = []
+
+ if current_y < box.y:
+ all_units_fit = False
+
+ if unit.is_space:
+ line_height = max(line_height, unit_height)
+ continue
+
+ relocated_unit = unit.relocate(current_x, current_y, scale)
+ typeset_units.append(relocated_unit)
+
+ if not unit.is_space:
+ current_line_heights.append(unit_height)
+
+ prev_x = current_x
+ current_x = relocated_unit.box.x2
+ if prev_x > current_x:
+ logger.warning(f"Position regression occurred, TypesettingUnit: {unit.box}, ")
+
+ last_unit = relocated_unit
+
+ # Check if output language is Arabic
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = False
+ if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ is_arabic = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic = True
+
+ # If Arabic, reverse the line order
+ if is_arabic and typeset_units:
+ # Group units by line (using Y coordinates)
+ lines_dict = {}
+ for unit in typeset_units:
+ if unit.box and unit.box.y is not None:
+ # Round Y coordinate to group units on the same line
+ line_y = round(unit.box.y, 1)
+ if line_y not in lines_dict:
+ lines_dict[line_y] = []
+ lines_dict[line_y].append(unit)
+
+ # Sort lines by Y coordinate (top to bottom) and reverse
+ sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # Rebuild typeset_units with reversed line order
+ reversed_typeset_units = []
+ for line_y in reversed(sorted_line_ys):
+ reversed_typeset_units.extend(lines_dict[line_y])
+
+ # Now reposition all units to swap their Y coordinates
+ # Map old Y positions to new Y positions
+ y_mapping = {}
+ for i, old_y in enumerate(sorted_line_ys):
+ new_y = sorted_line_ys[len(sorted_line_ys) - 1 - i]
+ y_mapping[old_y] = new_y
+
+ # Update Y coordinates for all units
+ for unit in reversed_typeset_units:
+ if unit.box and unit.box.y is not None:
+ old_y = round(unit.box.y, 1)
+ if old_y in y_mapping:
+ new_y = y_mapping[old_y]
+ y_diff = new_y - old_y
+ # Update the unit's Y position
+ if unit.y is not None:
+ unit.y += y_diff
+ if unit.box:
+ unit.box.y += y_diff
+ unit.box.y2 += y_diff
+
+ typeset_units = reversed_typeset_units
+
+ return typeset_units, all_units_fit
+
+ def _get_width_before_next_break_point(
+ self, units: list[TypesettingUnit], scale: float
+ ) -> float:
+ """Calculate the width before the next line break point"""
+ width = 0.0
+ for unit in units:
+ if unit.can_break_line:
+ break
+ width += unit.width * scale
+ return width
+
+ def create_typesetting_units(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ fonts: dict[str, il_version_1.PdfFont],
+ ) -> list[TypesettingUnit]:
+ if not paragraph.pdf_paragraph_composition:
+ return []
+ result = []
+
+ @cache
+ def get_font(font_id: str, xobj_id: int | None):
+ if xobj_id in fonts:
+ font = fonts[xobj_id][font_id]
+ else:
+ font = fonts[font_id]
+ return font
+
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition is None:
+ continue
+ if composition.pdf_line:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_line.pdf_character
+ ],
+ )
+ elif composition.pdf_character:
+ result.append(
+ TypesettingUnit(
+ char=composition.pdf_character,
+ debug_info=paragraph.debug_info,
+ ),
+ )
+ elif composition.pdf_same_style_characters:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_same_style_characters.pdf_character
+ ],
+ )
+ elif composition.pdf_same_style_unicode_characters:
+ style = composition.pdf_same_style_unicode_characters.pdf_style
+ if style is None:
+ logger.warning(
+ f"Style is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font_id = style.font_id
+ if font_id is None:
+ logger.warning(
+ f"Font ID is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font = get_font(font_id, paragraph.xobj_id)
+ if composition.pdf_same_style_unicode_characters.unicode:
+ unicode_text = composition.pdf_same_style_unicode_characters.unicode
+ shaped_text = self.shape_arabic_text(unicode_text)
+ result.extend(
+ [
+ TypesettingUnit(
+ unicode=char_unicode,
+ font=self.font_mapper.map(
+ font,
+ char_unicode,
+ ),
+ original_font=font,
+ font_size=style.font_size,
+ style=style,
+ xobj_id=paragraph.xobj_id,
+ debug_info=composition.pdf_same_style_unicode_characters.debug_info
+ or False,
+ )
+ for char_unicode in shaped_text # Use shaped_text instead of original
+ if char_unicode not in ("\n",)
+ ],
+ )
+ elif composition.pdf_formula:
+ result.extend([TypesettingUnit(formular=composition.pdf_formula)])
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ result = list(
+ filter(
+ lambda x: x.unicode is None or x.font is not None,
+ result,
+ ),
+ )
+
+ if any(x.width < 0 for x in result):
+ logger.warning("Typesetting unit width is less than 0, please check if positioning is incorrect or if text is being drawn in reverse")
+ return result
+
+ def create_passthrough_composition(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ ) -> list[PdfParagraphComposition]:
+ """Create passthrough composition from typesetting units - used when all units can be directly passed through
+
+ Args:
+ typesetting_units: List of typesetting units
+
+ Returns:
+ Paragraph composition list
+ """
+ composition = []
+ for unit in typesetting_units:
+ if unit.formular:
+ # For formula units, directly create PdfParagraphComposition containing the formula object
+ composition.append(PdfParagraphComposition(pdf_formula=unit.formular))
+ else:
+ # For character units, use existing logic to passthrough
+ chars, curves, forms = unit.passthrough()
+ composition.extend(
+ [PdfParagraphComposition(pdf_character=char) for char in chars],
+ )
+ return composition
+
+ def get_max_right_space(self, current_box: Box, page) -> float:
+ """Get the maximum right space available next to the current paragraph
+
+ Args:
+ current_box: Current paragraph bounding box
+ page: Current page
+
+ Returns:
+ Maximum available right edge x position
+ """
+ # Get page's right margin as the upper limit
+ max_x = page.cropbox.box.x2 * 0.9
+
+ # Check for content on the right side that may interfere
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # Skip current paragraph
+ continue
+ # If the paragraph is on the right side of current paragraph and their vertical ranges overlap
+ if para.box.x > current_box.x and not (
+ para.box.y >= current_box.y2 or para.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, para.box.x)
+ for char in page.pdf_character:
+ if char.box.x > current_box.x and not (
+ char.box.y >= current_box.y2 or char.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, char.box.x)
+ # Check figures
+ for figure in page.pdf_figure:
+ if figure.box.x > current_box.x and not (
+ figure.box.y >= current_box.y2 or figure.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, figure.box.x)
+
+ return max_x
+
+ def get_max_bottom_space(self, current_box: Box, page: il_version_1.Page) -> float:
+ """Get the maximum bottom space available below the current paragraph
+
+ Args:
+ current_box: Current paragraph bounding box
+ page: Current page
+
+ Returns:
+ Maximum available bottom edge y position
+ """
+ # Get page's bottom margin as the lower limit
+ min_y = page.cropbox.box.y * 1.1
+
+ # Check for content below that may interfere
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # Skip current paragraph
+ continue
+ # If the paragraph is below current paragraph and their horizontal ranges overlap
+ if para.box.y2 < current_box.y and not (
+ para.box.x >= current_box.x2 or para.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, para.box.y2)
+ for char in page.pdf_character:
+ if char.box.y2 < current_box.y and not (
+ char.box.x >= current_box.x2 or char.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, char.box.y2)
+ # Check figures
+ for figure in page.pdf_figure:
+ if figure.box.y2 < current_box.y and not (
+ figure.box.x >= current_box.x2 or figure.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, figure.box.y2)
+
+ return min_y
+
+ def _update_paragraph_render_order(self, paragraph: il_version_1.PdfParagraph):
+ """
+ Update paragraph render order for child characters.
+ From render order = paragraph's render order, sub render order starts from 1
+ """
+ if not hasattr(paragraph, "render_order") or paragraph.render_order is None:
+ return
+
+ main_render_order = paragraph.render_order
+ sub_render_order = 1
+
+ # Iterate through paragraph composition list
+ for composition in paragraph.pdf_paragraph_composition:
+ # Check for character and assign render order
+ if composition.pdf_character:
+ char = composition.pdf_character
+ char.render_order = main_render_order
+ char.sub_render_order = sub_render_order
+ sub_render_order += 1
diff --git a/babeldoc/format/pdf/document_il/midend/typesetting_v2.py b/babeldoc/format/pdf/document_il/midend/typesetting_v2.py
new file mode 100644
index 0000000000000000000000000000000000000000..c56c4fc9b3d080032c76752e8a9bb6c29d554ac9
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/typesetting_v2.py
@@ -0,0 +1,2116 @@
+from __future__ import annotations
+
+import copy
+import logging
+import re
+import statistics
+import unicodedata
+from functools import cache
+
+import pymupdf
+import regex
+from rtree import index
+
+from babeldoc.const import WATERMARK_VERSION
+from babeldoc.format.pdf.document_il import Box
+from babeldoc.format.pdf.document_il import PdfCharacter
+from babeldoc.format.pdf.document_il import PdfCurve
+from babeldoc.format.pdf.document_il import PdfForm
+from babeldoc.format.pdf.document_il import PdfFormula
+from babeldoc.format.pdf.document_il import PdfParagraphComposition
+from babeldoc.format.pdf.document_il import PdfStyle
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.formular_helper import update_formula_data
+from babeldoc.format.pdf.document_il.utils.layout_helper import box_to_tuple
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from babeldoc.format.pdf.translation_config import WatermarkOutputMode
+from arabic_reshaper import reshape
+from bidi.algorithm import get_display
+
+
+logger = logging.getLogger(__name__)
+
+LINE_BREAK_REGEX = regex.compile(
+ r"^["
+ r"a-z"
+ r"A-Z"
+ r"0-9"
+ r"\u00C0-\u00FF" # Latin-1 Supplement
+ r"\u0100-\u017F" # Latin Extended A
+ r"\u0180-\u024F" # Latin Extended B
+ r"\u1E00-\u1EFF" # Latin Extended Additional
+ r"\u2C60-\u2C7F" # Latin Extended C
+ r"\uA720-\uA7FF" # Latin Extended D
+ r"\uAB30-\uAB6F" # Latin Extended E
+ r"\u0250-\u02A0" # IPA Extensions
+ r"\u0400-\u04FF" # Cyrillic
+ r"\u0300-\u036F" # Combining Diacritical Marks
+ r"\u0500-\u052F" # Cyrillic Supplement
+ r"\u0370-\u03FF" # Greek and Coptic
+ r"\u2DE0-\u2DFF" # Cyrillic Extended-A
+ r"\uA650-\uA69F" # Cyrillic Extended-B
+ r"\u1200-\u137F" # Ethiopic
+ r"\u1380-\u139F" # Ethiopic Supplement
+ r"\u2D80-\u2DDF" # Ethiopic Extended
+ r"\uAB00-\uAB2F" # Ethiopic Extended-A
+ r"\U0001E7E0-\U0001E7FF" # Ethiopic Extended-B
+ r"\u0E80-\u0EFF" # Lao
+ r"\u0D00-\u0D7F" # Malayalam
+ r"\u0A80-\u0AFF" # Gujarati
+ r"\u0E00-\u0E7F" # Thai
+ r"\u1000-\u109F" # Myanmar
+ r"\uAA60-\uAA7F" # Myanmar Extended-A
+ r"\uA9E0-\uA9FF" # Myanmar Extended-B
+ r"\U000116D0-\U000116FF" # Myanmar Extended-C
+ r"\u0B80-\u0BFF" # Tamil
+ r"\u0C00-\u0C7F" # Telugu
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0530-\u058F" # Armenian
+ r"\u10A0-\u10FF" # Georgian
+ r"\u1C90-\u1CBF" # Georgian Extended
+ r"\u2D00-\u2D2F" # Georgian Supplement
+ r"\u1780-\u17FF" # Khmer
+ r"\u19E0-\u19FF" # Khmer Symbols
+ r"\U00010B00-\U00010B3F" # Avestan
+ r"\u1D00-\u1D7F" # Phonetic Extensions
+ r"\u1400-\u167F" # Unified Canadian Aboriginal Syllabics
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0780-\u07BF" # Thaana
+ r"\U0001E900-\U0001E95F" # Adlam
+ r"\u1C80-\u1C8F" # Cyrillic Extended-C
+ r"\U0001E030-\U0001E08F" # Cyrillic Extended-D
+ r"\uA000-\uA48F" # Yi Syllables
+ r"\uA490-\uA4CF" # Yi Radicals
+ r"'"
+ r"-" # Hyphen
+ r"·" # Middle Dot (U+00B7) For CatalÃÂÂ
+ r"Ê»" # Spacing Modifier Letters U+02BB
+ r"]+$"
+)
+
+
+class TypesettingUnit:
+ def __str__(self):
+ return self.try_get_unicode() or ""
+
+ def __init__(
+ self,
+ char: PdfCharacter | None = None,
+ formular: PdfFormula | None = None,
+ unicode: str | None = None,
+ font: pymupdf.Font | None = None,
+ original_font: il_version_1.PdfFont | None = None,
+ font_size: float | None = None,
+ style: PdfStyle | None = None,
+ xobj_id: int | None = None,
+ debug_info: bool = False,
+ ):
+ assert (char is not None) + (formular is not None) + (
+ unicode is not None
+ ) == 1, "Only one of chars and formular can be not None"
+ self.char = char
+ self.formular = formular
+ self.unicode = unicode
+ self.x = None
+ self.y = None
+ self.scale = None
+ self.debug_info = debug_info
+
+ # Cache variables
+ self.box_cache: Box | None = None
+ self.can_break_line_cache: bool | None = None
+ self.is_cjk_char_cache: bool | None = None
+ self.mixed_character_blacklist_cache: bool | None = None
+ self.is_space_cache: bool | None = None
+ self.is_hung_punctuation_cache: bool | None = None
+ self.is_cannot_appear_in_line_end_punctuation_cache: bool | None = None
+ self.can_passthrough_cache: bool | None = None
+ self.width_cache: float | None = None
+ self.height_cache: float | None = None
+
+ self.font_size: float | None = None
+
+ if unicode:
+ assert font_size, "Font size must be provided when unicode is provided"
+ assert style, "Style must be provided when unicode is provided"
+ assert len(unicode) == 1, "Unicode must be a single character"
+ assert xobj_id is not None, (
+ "Xobj id must be provided when unicode is provided"
+ )
+
+ self.font = font
+ if font is not None and hasattr(font, "font_id"):
+ self.font_id = font.font_id
+ else:
+ self.font_id = "base"
+ if original_font:
+ self.original_font = original_font
+ else:
+ self.original_font = None
+
+ self.font_size = font_size
+ self.style = style
+ self.xobj_id = xobj_id
+
+ def try_resue_cache(self, old_tu: TypesettingUnit):
+ if old_tu.is_cjk_char_cache is not None:
+ self.is_cjk_char_cache = old_tu.is_cjk_char_cache
+
+ if old_tu.can_break_line_cache is not None:
+ self.can_break_line_cache = old_tu.can_break_line_cache
+
+ if old_tu.is_space_cache is not None:
+ self.is_space_cache = old_tu.is_space_cache
+
+ if old_tu.is_hung_punctuation_cache is not None:
+ self.is_hung_punctuation_cache = old_tu.is_hung_punctuation_cache
+
+ if old_tu.is_cannot_appear_in_line_end_punctuation_cache is not None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ old_tu.is_cannot_appear_in_line_end_punctuation_cache
+ )
+
+ if old_tu.can_passthrough_cache is not None:
+ self.can_passthrough_cache = old_tu.can_passthrough_cache
+
+ if old_tu.mixed_character_blacklist_cache is not None:
+ self.mixed_character_blacklist_cache = (
+ old_tu.mixed_character_blacklist_cache
+ )
+
+
+ def try_get_unicode(self) -> str | None:
+ if self.char:
+ return self.char.char_unicode
+ elif self.formular:
+ return None
+ elif self.unicode:
+ return self.unicode
+
+ @property
+ def mixed_character_blacklist(self):
+ if self.mixed_character_blacklist_cache is None:
+ self.mixed_character_blacklist_cache = self.calc_mixed_character_blacklist()
+
+ return self.mixed_character_blacklist_cache
+
+ def calc_mixed_character_blacklist(self):
+ unicode = self.try_get_unicode()
+ if unicode:
+ return unicode in [
+ "。",
+ "，",
+ "：",
+ "？",
+ "ï¼ÂÂ",
+ ]
+ return False
+
+ @property
+ def can_break_line(self):
+ if self.can_break_line_cache is None:
+ self.can_break_line_cache = self.calc_can_break_line()
+
+ return self.can_break_line_cache
+
+ def calc_can_break_line(self):
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return True
+ if LINE_BREAK_REGEX.match(unicode):
+ return False
+ return True
+
+ @property
+ def is_cjk_char(self):
+ if self.is_cjk_char_cache is None:
+ self.is_cjk_char_cache = self.calc_is_cjk_char()
+
+ return self.is_cjk_char_cache
+
+ def calc_is_cjk_char(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ if "(cid" in unicode:
+ return False
+ if len(unicode) > 1:
+ return False
+ assert len(unicode) == 1, "Unicode must be a single character"
+ if unicode in [
+ "（",
+ "）",
+ "ã€ÂÂ",
+ "】",
+ "《",
+ "》",
+ "ã€â€Â",
+ "〕",
+ "〈",
+ "〉",
+ "〖",
+ "ã€â€â€",
+ "「",
+ "ã€ÂÂ",
+ "『",
+ "ã€ÂÂ",
+ "ã€ÂÂ",
+ "。",
+ "：",
+ "？",
+ "ï¼ÂÂ",
+ "，",
+ ]:
+ return True
+ if unicode:
+ if re.match(
+ r"^["
+ r"\u3000-\u303f" # CJK Symbols and Punctuation
+ r"\u3040-\u309f" # Hiragana
+ r"\u30a0-\u30ff" # Katakana
+ r"\u3100-\u312f" # Bopomofo
+ r"\uac00-\ud7af" # Hangul Syllables
+ r"\u1100-\u11ff" # Hangul Jamo
+ r"\u3130-\u318f" # Hangul Compatibility Jamo
+ r"\ua960-\ua97f" # Hangul Jamo Extended-A
+ r"\ud7b0-\ud7ff" # Hangul Jamo Extended-B
+ r"\u3190-\u319f" # Kanbun
+ r"\u3200-\u32ff" # Enclosed CJK Letters and Months
+ r"\u3300-\u33ff" # CJK Compatibility
+ r"\ufe30-\ufe4f" # CJK Compatibility Forms
+ r"\u4e00-\u9fff" # CJK Unified Ideographs
+ r"\u2e80-\u2eff" # CJK Radicals Supplement
+ r"\u31c0-\u31ef" # CJK Strokes
+ r"\u2f00-\u2fdf" # Kangxi Radicals
+ r"\ufe10-\ufe1f" # Vertical Forms
+ r"]+$",
+ unicode,
+ ):
+ return True
+ try:
+ unicodedata_name = unicodedata.name(unicode)
+ return (
+ "CJK UNIFIED IDEOGRAPH" in unicodedata_name
+ or "FULLWIDTH" in unicodedata_name
+ )
+ except ValueError:
+ return False
+ return False
+
+ @property
+ def is_space(self):
+ if self.is_space_cache is None:
+ self.is_space_cache = self.calc_is_space()
+
+ return self.is_space_cache
+
+ def calc_is_space(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ return unicode == " "
+
+ @property
+ def is_hung_punctuation(self):
+ if self.is_hung_punctuation_cache is None:
+ self.is_hung_punctuation_cache = self.calc_is_hung_punctuation()
+
+ return self.is_hung_punctuation_cache
+
+ def calc_is_hung_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+
+ if unicode:
+ return unicode in [
+ # 英文标点
+ ",",
+ ".",
+ ":",
+ ";",
+ "?",
+ "!",
+ # ä¸ÂÂ文点å·
+ "，", # é€â€â€ÃƒÂ¥Ã‚·
+ "。", # Ã¥ÂÂ¥å·
+ "．", # 全角åÂÂ¥å·
+ "ã€ÂÂ", # é¡¿å·
+ "：", # 冒å·
+ "ï¼›", # 分å·
+ "ï¼ÂÂ", # å¹å·
+ "‼", # Ã¥ÂÂŒå¹å·
+ "？", # éâ€â€Ã‚®Ã¥Â·
+ "â‡", # Ã¥ÂÂΎâ€â€Ã‚®Ã¥Â·
+ # 结æŸ引å·
+ "â€ÂÂ", # å³åÂŒ引å·
+ "’", # å³å•引å·
+ "ã€ÂÂ", # å³直角å•引å·
+ "ã€ÂÂ", # å³直角åÂŒ引å·
+ # 结æŸ括å·
+ ")", # å³圆括å·
+ "]", # å³方括å·
+ "}", # å³花括å·
+ "）", # å³圆括å·
+ "〕", # å³龟çâ€Â²æ‹¬å·
+ "〉", # å³å•书åÂÂÂÂå·
+ "】", # å³黑色方头括å·
+ "ã€â€â€", # å³空白方头括å·
+ "ï¼½", # 全角å³方括å·
+ "ï½ÂÂ", # 全角å³花括å·
+ # 结æŸåÂŒ书åÂÂÂÂå·
+ "》", # å³åÂŒ书åÂÂÂÂå·
+ # 连接å·
+ "～", # 全角波浪å·
+ "-", # è¿žåÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥â€¡ÂÂå·
+ "–", # çŸÂÂ破折å· (EN DASH)
+ "â€â€Â", # 长破折å· (EM DASH)
+ # éâ€â€Ã‚´Ã©Å¡â€Âå·
+ "·", # ä¸ÂÂéâ€â€Ã‚´Ã§â€šÂ¹
+ "・", # 片å‡åÂÂÂÂä¸ÂÂéâ€â€Ã‚´Ã§â€šÂ¹
+ "‧", # è¿žåÂÂâ€â€ÃƒÂ§Ã¢â‚¬Å¡Ã‚¹
+ # 分éšâ€Âå·
+ "/", # æ–œæÂÂÂÂ
+ "ï¼ÂÂ", # 全角斜æÂÂÂÂ
+ "â„", # 分数斜æÂÂÂÂ
+ ]
+ return False
+
+ @property
+ def is_cannot_appear_in_line_end_punctuation(self):
+ if self.is_cannot_appear_in_line_end_punctuation_cache is None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ self.calc_is_cannot_appear_in_line_end_punctuation()
+ )
+
+ return self.is_cannot_appear_in_line_end_punctuation_cache
+
+ def calc_is_cannot_appear_in_line_end_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ return unicode in [
+ # 开始引å·
+ "“", # å·¦åÂŒ引å·
+ "‘", # å·¦å•引å·
+ "「", # 左直角å•引å·
+ "『", # 左直角åÂŒ引å·
+ # 开始括å·
+ "(", # 左圆括å·
+ "[", # 左方括å·
+ "{", # 左花括å·
+ "（", # 左圆括å·
+ "ã€â€Â", # 左龟çâ€Â²æ‹¬å·
+ "〈", # å·¦å•书åÂÂÂÂå·
+ "《", # å·¦åÂŒ书åÂÂÂÂå·
+ # 开始å•åÂŒ书åÂÂÂÂå·
+ "〖", # 左空白方头括å·
+ "〘", # 左黑色方头括å·
+ "〚", # å·¦å•书åÂÂÂÂå·
+ ]
+
+ def passthrough(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ if self.char:
+ return [self.char], [], []
+ elif self.formular:
+ return (
+ self.formular.pdf_character,
+ self.formular.pdf_curve,
+ self.formular.pdf_form,
+ )
+ elif self.unicode:
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ return [], [], []
+
+ @property
+ def can_passthrough(self):
+ if self.can_passthrough_cache is None:
+ self.can_passthrough_cache = self.calc_can_passthrough()
+
+ return self.can_passthrough_cache
+
+ def calc_can_passthrough(self):
+ return self.unicode is None
+
+ def calculate_box(self):
+ if self.char:
+ box = copy.deepcopy(self.char.box)
+ if self.char.visual_bbox and self.char.visual_bbox.box:
+ box.y = self.char.visual_bbox.box.y
+ box.y2 = self.char.visual_bbox.box.y2
+ # return self.char.visual_bbox.box
+
+ return box
+ elif self.formular:
+ return self.formular.box
+ # if self.formular.x_offset <= 0.5:
+ # return self.formular.box
+ # formular_box = copy.copy(self.formular.box)
+ # formular_box.x2 += self.formular.x_advance
+ # return formular_box
+ elif self.unicode:
+ char_width = self.font.char_lengths(self.unicode, self.font_size)[0]
+ if self.x is None or self.y is None or self.scale is None:
+ return Box(0, 0, char_width, self.font_size)
+ return Box(self.x, self.y, self.x + char_width, self.y + self.font_size)
+
+ @property
+ def box(self):
+ if not self.box_cache:
+ self.box_cache = self.calculate_box()
+
+ return self.box_cache
+
+ @property
+ def width(self):
+ if self.width_cache is None:
+ self.width_cache = self.calc_width()
+
+ return self.width_cache
+
+ def calc_width(self):
+ box = self.box
+ return box.x2 - box.x
+
+ @property
+ def height(self):
+ if self.height_cache is None:
+ self.height_cache = self.calc_height()
+
+ return self.height_cache
+
+ def calc_height(self):
+ box = self.box
+ return box.y2 - box.y
+
+ def relocate(
+ self,
+ x: float,
+ y: float,
+ scale: float,
+ ) -> TypesettingUnit:
+ """é‡ÂÂ定ä½ÂÂ并缩æâ€Â¾æŽ’版å•元
+
+ Args:
+ x: æ–°çš„ x Ã¥ÂÂÂÂæ ‡
+ y: æ–°çš„ y Ã¥ÂÂÂÂæ ‡
+ scale: 缩æâ€Â¾å› åÂÂÂÂ
+
+ Returns:
+ 新的排版å•元
+ """
+ if self.char:
+ # 创建新的åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â¯Â¹Ã¨Â±Â¡
+ new_char = PdfCharacter(
+ pdf_character_id=self.char.pdf_character_id,
+ char_unicode=self.char.char_unicode,
+ box=Box(
+ x=x,
+ y=y,
+ x2=x + self.width * scale,
+ y2=y + self.height * scale,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.char.pdf_style.font_id,
+ font_size=self.char.pdf_style.font_size * scale,
+ graphic_state=self.char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=self.char.vertical,
+ advance=self.char.advance * scale if self.char.advance else None,
+ debug_info=self.debug_info,
+ xobj_id=self.char.xobj_id,
+ )
+ new_tu = TypesettingUnit(char=new_char)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.formular:
+ # 创建新的公å¼ÂÂ对象，ä¿ÂÂæŒÂÂ内部åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã§Å¡â€žÃ§â€ºÂ¸Ã¥Â¯Â¹Ã¤Â½ÂÂç½®
+ new_chars = []
+ min_x = self.formular.box.x
+ min_y = self.formular.box.y
+
+ for char in self.formular.pdf_character:
+ # 计ç®â€â€ÃƒÂ§Ã¢â‚¬ÂºÃ‚¸Ã¥Â¯Â¹Ã¤Â½ÂÂç½®
+ rel_x = char.box.x - min_x
+ rel_y = char.box.y - min_y
+
+ visual_rel_x = char.visual_bbox.box.x - min_x
+ visual_rel_y = char.visual_bbox.box.y - min_y
+
+ # 创建新的åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â¯Â¹Ã¨Â±Â¡
+ new_char = PdfCharacter(
+ pdf_character_id=char.pdf_character_id,
+ char_unicode=char.char_unicode,
+ box=Box(
+ x=x + (rel_x + self.formular.x_offset) * scale,
+ y=y + (rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (rel_x + (char.box.x2 - char.box.x) + self.formular.x_offset)
+ * scale,
+ y2=y
+ + (rel_y + (char.box.y2 - char.box.y) + self.formular.y_offset)
+ * scale,
+ ),
+ visual_bbox=il_version_1.VisualBbox(
+ box=Box(
+ x=x + (visual_rel_x + self.formular.x_offset) * scale,
+ y=y + (visual_rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (
+ visual_rel_x
+ + (char.visual_bbox.box.x2 - char.visual_bbox.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=y
+ + (
+ visual_rel_y
+ + (char.visual_bbox.box.y2 - char.visual_bbox.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ ),
+ ),
+ pdf_style=PdfStyle(
+ font_id=char.pdf_style.font_id,
+ font_size=char.pdf_style.font_size * scale,
+ graphic_state=char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=char.vertical,
+ advance=char.advance * scale if char.advance else None,
+ xobj_id=char.xobj_id,
+ )
+ new_chars.append(new_char)
+
+ # Calculate bounding box from new_chars
+ min_x = min(char.visual_bbox.box.x for char in new_chars)
+ min_y = min(char.visual_bbox.box.y for char in new_chars)
+ max_x = max(char.visual_bbox.box.x2 for char in new_chars)
+ max_y = max(char.visual_bbox.box.y2 for char in new_chars)
+
+ new_formula = PdfFormula(
+ box=Box(
+ x=min_x,
+ y=min_y,
+ x2=max_x,
+ y2=max_y,
+ ),
+ pdf_character=new_chars,
+ x_offset=self.formular.x_offset * scale,
+ y_offset=self.formular.y_offset * scale,
+ x_advance=self.formular.x_advance * scale,
+ )
+
+ # Handle contained curves
+ new_curves = []
+ for curve in self.formular.pdf_curve:
+ new_curve = self._transform_curve_for_relocation(
+ curve,
+ self.formular.box.x,
+ self.formular.box.y,
+ x,
+ y,
+ scale,
+ )
+ new_curves.append(new_curve)
+ new_formula.pdf_curve = new_curves
+
+ # Handle contained forms
+ new_forms = []
+ for form in self.formular.pdf_form:
+ new_form = self._transform_form_for_relocation(
+ form, self.formular.box.x, self.formular.box.y, x, y, scale
+ )
+ new_forms.append(new_form)
+ new_formula.pdf_form = new_forms
+
+ update_formula_data(new_formula)
+
+ new_tu = TypesettingUnit(formular=new_formula)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.unicode:
+ # 对于 Unicode Ã¥ÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¯Â¼Å’我们å˜储新的ä½ÂÂ置信æ¯
+ new_unit = TypesettingUnit(
+ unicode=self.unicode,
+ font=self.font,
+ original_font=self.original_font,
+ font_size=self.font_size * scale,
+ style=self.style,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ new_unit.x = x
+ new_unit.y = y
+ new_unit.scale = scale
+ new_unit.try_resue_cache(self)
+ return new_unit
+
+ def _transform_curve_for_relocation(
+ self,
+ curve,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a curve for formula relocation."""
+ import copy
+
+ new_curve = copy.deepcopy(curve)
+
+ if new_curve.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_curve.box.x - original_formula_x
+ rel_y = new_curve.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_curve.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (
+ rel_x
+ + (new_curve.box.x2 - new_curve.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=new_y
+ + (
+ rel_y
+ + (new_curve.box.y2 - new_curve.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original CTM
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_curve.relocation_transform = list(relocation_matrix)
+
+ return new_curve
+
+ def _transform_form_for_relocation(
+ self,
+ form,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a form for formula relocation."""
+ import copy
+
+ new_form = copy.deepcopy(form)
+
+ if new_form.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_form.box.x - original_formula_x
+ rel_y = new_form.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_form.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (rel_x + (new_form.box.x2 - new_form.box.x) + self.formular.x_offset)
+ * scale,
+ y2=new_y
+ + (rel_y + (new_form.box.y2 - new_form.box.y) + self.formular.y_offset)
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original matrices
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_form.relocation_transform = list(relocation_matrix)
+
+ return new_form
+
+ def render(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ """渲染排版å•元为 PdfCharacter åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+
+ Returns:
+ PdfCharacter åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ """
+ if self.can_passthrough:
+ return self.passthrough()
+ elif self.unicode:
+ assert self.x is not None, (
+ "x position must be set, should be set by `relocate`"
+ )
+ assert self.y is not None, (
+ "y position must be set, should be set by `relocate`"
+ )
+ assert self.scale is not None, (
+ "scale must be set, should be set by `relocate`"
+ )
+ x = self.x
+ y = self.y
+ # if self.original_font and self.font and hasattr(self.original_font, "descent") and hasattr(self.font, "descent_fontmap"):
+ # original_descent = self.original_font.descent
+ # new_descent = self.font.descent_fontmap
+ # y -= (original_descent - new_descent) * self.font_size / 1000
+
+ # 计ç®â€â€ÃƒÂ¥Ã‚Ââ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â®Â½Ã¥ÂºÂ¦
+ char_width = self.width
+
+ # Handle case when font is None (no suitable font found for this character)
+ if self.font is None:
+ logger.warning(
+ f"No font available for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using font_id='{self.font_id}' with glyph_id=0"
+ )
+ glyph_id = 0 # Use glyph 0 as fallback (usually .notdef)
+ else:
+ glyph_id = self.font.has_glyph(ord(self.unicode))
+ if glyph_id == 0 or glyph_id is None:
+ logger.warning(
+ f"Font '{self.font_id}' doesn't have glyph for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using glyph_id=0"
+ )
+ glyph_id = 0
+
+ new_char = PdfCharacter(
+ pdf_character_id=glyph_id,
+ char_unicode=self.unicode,
+ box=Box(
+ x=x, # 使çâ€Â¨å˜储的ä½ÂÂç½®
+ y=y,
+ x2=x + char_width,
+ y2=y + self.font_size,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.font_id,
+ font_size=self.font_size,
+ graphic_state=self.style.graphic_state,
+ ),
+ scale=self.scale,
+ vertical=False,
+ advance=char_width,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ return [new_char], [], []
+ else:
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ return [], [], []
+
+
+class Typesetting:
+ stage_name = "Typesetting"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.font_mapper = FontMapper(translation_config)
+ self.translation_config = translation_config
+ self.lang_code = self.translation_config.lang_out.upper()
+ # Ensure detailed_logger attribute exists to avoid attribute access errors
+ self.detailed_logger = None
+ self.is_cjk = (
+ # Why zh-CN/zh-HK/zh-TW here but not zh-Hans and so on?
+ # See https://funstory-ai.github.io/BabelDOC/supported_languages/
+ ("ZH" in self.lang_code) # C
+ or ("JA" in self.lang_code)
+ or ("JP" in self.lang_code) # J
+ or ("KR" in self.lang_code) # K
+ or ("CN" in self.lang_code)
+ or ("HK" in self.lang_code)
+ or ("TW" in self.lang_code)
+ )
+
+ def preprocess_document(self, document: il_version_1.Document, pbar):
+ """预处ç†文档，获å–æ¯ÂÂ个段è½的最优缩æâ€Â¾å› åÂÂÂÂ，ä¸ÂÂ执行实际排版"""
+ all_scales: list[float] = []
+ all_paragraphs: list[il_version_1.PdfParagraph] = []
+
+ for page in document.page:
+ pbar.advance()
+ # 准备åÂÂâ€â€ÃƒÂ¤Ã‚½â€œÃ¤Â¿Â¡Ã¦Â¯（å¤ÂÂ制自 render_page 的逻辑）
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if (
+ xobj.xobj_id in fonts
+ and isinstance(fonts[xobj.xobj_id], dict)
+ and font.font_id
+ ):
+ fonts[xobj.xobj_id][font.font_id] = font
+
+ # 处ç†æ¯ÂÂ个段è½
+ for paragraph in page.pdf_paragraph:
+ all_paragraphs.append(paragraph)
+ unit_count = 0
+ try:
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ unit_count = len(typesetting_units)
+ for unit in typesetting_units:
+ if unit.formular:
+ unit_count += len(unit.formular.pdf_character) - 1
+
+ # 如果所有å•元都å¯以直接传递，则 scale = 1.0
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.optimal_scale = 1.0
+ else:
+ # 获å–最优缩æâ€Â¾å› åÂÂÂÂ
+ optimal_scale = self._get_optimal_scale(
+ paragraph, page, typesetting_units
+ )
+ paragraph.optimal_scale = optimal_scale
+ except Exception as e:
+ # 如果预处ç†出éâ€Â™ï¼Œé»˜è®¤ä½¿çâ€Â¨ 1.0 缩æâ€Â¾å› åÂÂÂÂ
+ logger.warning(f"预处ç†段è½æâ€â€Ã‚¶Ã¥â€¡ÂºÃ©â€Â™ï¼š{e}")
+ paragraph.optimal_scale = 1.0
+
+ if paragraph.optimal_scale is not None:
+ all_scales.extend([paragraph.optimal_scale] * unit_count)
+
+ # 获å–缩æâ€Â¾å› åÂÂÂÂçš„ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ if all_scales:
+ try:
+ modes = statistics.multimode(all_scales)
+ mode_scale = min(modes)
+ except statistics.StatisticsError:
+ logger.warning(
+ "Could not find a mode for paragraph scales. Falling back to median."
+ )
+ mode_scale = statistics.median(all_scales)
+ # 将所有大于ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã§Å¡â€žÃ¥â‚¬Â¼Ã¤Â¿Â®Ã¦â€Â¹ä¸ºä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ for paragraph in all_paragraphs:
+ if (
+ paragraph.optimal_scale is not None
+ and paragraph.optimal_scale > mode_scale
+ ):
+ paragraph.optimal_scale = mode_scale
+ else:
+ logger.error(
+ "document_scales is empty, there seems no paragraph in this PDF"
+ )
+
+ def shape_arabic_text(self, text: str) -> str:
+ """Shape and reorder Arabic text if output language is Arabic.
+
+ Args:
+ text: Input text to shape
+
+ Returns:
+ Shaped and reordered text if language is Arabic, original text otherwise
+ """
+ if not text:
+ return text
+
+ # Robust Arabic output detection: accept explicit 'ar', 'ara', 'arabic'
+ # or formats containing '-ar', '->ar', or '/ar' as a target marker (e.g. 'en-ar', 'en->ar')
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = False
+ if lang_out in ("en-ar, ar", "ara", "arabic"):
+ is_arabic = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic = True
+
+ if is_arabic:
+ logger.debug("Shaping Arabic text")
+ # Flip parentheses and brackets for RTL display
+ # text = text.replace("(", "\x00")
+ # text = text.replace(")", "(")
+ # text = text.replace("\x00", ")")
+ # text = text.replace("[", "\x01")
+ # text = text.replace("]", "[")
+ # text = text.replace("\x01", "]")
+ # text = text.replace("{", "\x02")
+ # text = text.replace("}", "{")
+ # text = text.replace("\x02", "}")
+ try:
+ if not re.search(r'[\uFB50-\uFDFF\uFE70-\uFEFF]', text):
+ # Reshape Arabic text for proper character joining
+ from arabic_reshaper import ArabicReshaper
+ configuration = {
+ 'delete_harakat': False, # Keep diacritical marks
+ 'support_ligatures': True, # Support Arabic ligatures
+ 'RIAL SIGN': True,
+ 'ARABIC COMMA': True,
+ 'ARABIC SEMICOLON': True,
+ 'ARABIC QUESTION MARK': True,
+ 'ZWNJ': True, # Zero Width Non-Joiner
+ }
+
+ reshaper = ArabicReshaper(configuration=configuration)
+ reshaped_text = reshaper.reshape(text)
+ display_text = get_display(reshaped_text, base_dir='R')
+ else:
+ display_text = text
+ return display_text
+ except Exception as e:
+ logger.warning(f"Failed to shape Arabic text: {e}")
+ return text
+
+ return text
+
+ def _find_optimal_scale_and_layout(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ initial_scale: float = 1.0,
+ use_english_line_break: bool = True,
+ apply_layout: bool = False,
+ ) -> tuple[float, list[TypesettingUnit] | None]:
+ """查找最优缩æâ€Â¾å› åÂÂÂÂ并å¯选择性地执行布局
+
+ Args:
+ paragraph: 段è½对象
+ page: 页é¢对象
+ typesetting_units: 排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ initial_scale: åˆÂÂ始缩æâ€Â¾å› åÂÂÂÂ
+ use_english_line_break: 是å¦使çâ€Â¨è‹±æ–‡æ¢行规则
+ apply_layout: 是å¦åºâ€Âçâ€Â¨å¸ƒå±€åˆ° paragraph（True æâ€â€Ã‚¶Ã¦â€°Â§Ã¨Â¡Å’实际排版）
+
+ Returns:
+ tuple[float, list[TypesettingUnit] | None]: (最终缩æâ€Â¾å› åÂÂÂÂ，排版åÂÂŽçš„å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨Ã¦Ë†â€“ None)
+ """
+ if not paragraph.box:
+ return initial_scale, None
+
+ box = paragraph.box
+ scale = initial_scale
+ line_skip = 1.50 if self.is_cjk else 1.3
+ min_scale = 0.1
+ expand_space_flag = 0
+ final_typeset_units = None
+
+ while scale >= min_scale:
+ try:
+ # å°ÂÂ试布局排版å•元
+ typeset_units, all_units_fit = self._layout_typesetting_units(
+ typesetting_units,
+ box,
+ scale,
+ line_skip,
+ paragraph,
+ use_english_line_break,
+ )
+
+ # 如果所有å•元都æâ€Â¾å¾â€â€ÃƒÂ¤Ã‚¸â€¹
+ if all_units_fit:
+ if apply_layout:
+ # 实际åºâ€Âçâ€Â¨æŽ’版结果
+ paragraph.scale = scale
+ paragraph.pdf_paragraph_composition = []
+ for unit in typeset_units:
+ chars, curves, forms = unit.render()
+ for char in chars:
+ paragraph.pdf_paragraph_composition.append(
+ PdfParagraphComposition(pdf_character=char),
+ )
+ for curve in curves:
+ page.pdf_curve.append(curve)
+ for form in forms:
+ page.pdf_form.append(form)
+ final_typeset_units = typeset_units
+ return scale, final_typeset_units
+ except Exception:
+ # 如果布局检查出éâ€Â™ï¼Œç»§ç»ÂÂå°ÂÂ试下一个缩æâ€Â¾å› åÂÂÂÂ
+ pass
+
+ # 添加与原 retypeset 一致的逻辑检查
+ if not hasattr(paragraph, "debug_id") or not paragraph.debug_id:
+ return scale, final_typeset_units
+
+ # å‡ÂÂå°ÂÂ缩æâ€Â¾å› åÂÂÂÂ
+ if scale > 0.6:
+ scale -= 0.05
+ else:
+ scale -= 0.1
+
+ if scale < 0.7:
+ space_expanded = False # 标记是å¦æˆÂÂ功扩展了空éâ€â€Ã‚´
+
+ if expand_space_flag == 0:
+ # å°ÂÂ试å‘下扩展
+ try:
+ min_y = self.get_max_bottom_space(box, page) + 2
+ if min_y < box.y:
+ expanded_box = Box(x=box.x, y=min_y, x2=box.x2, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # 更新段è½的边界æ¡â€Â
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 1
+
+ # åª有æˆÂÂ功扩展空éâ€â€Ã‚´Ã¦â€â€Ã‚¶Ã¦â€°Â continue，å¦则继ç»ÂÂå‡ÂÂå° scale
+ if space_expanded:
+ continue
+
+ elif expand_space_flag == 1:
+ # å°ÂÂ试å‘å³扩展
+ try:
+ max_x = self.get_max_right_space(box, page) - 5
+ if max_x > box.x2:
+ expanded_box = Box(x=box.x, y=box.y, x2=max_x, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # 更新段è½的边界æ¡â€Â
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 2
+
+ # åª有æˆÂÂ功扩展空éâ€â€Ã‚´Ã¦â€â€Ã‚¶Ã¦â€°Â continue，å¦则继ç»ÂÂå‡ÂÂå° scale
+ if space_expanded:
+ continue
+
+ # åª有在扩展å°ÂÂ试阶段 (expand_space_flag < 2) ä¸â€Â扩展失败æâ€â€Ã‚¶Ã¦â€°ÂÂé‡ÂÂç½® scale
+ # 当 expand_space_flag >= 2 æâ€â€Ã‚¶Ã¯Â¼Å’说明已ç»ÂÂå°ÂÂ试过所有扩展，åºâ€Â该继ç»ÂÂæ£常的 scale å‡ÂÂå°ÂÂ
+ if expand_space_flag < 2:
+ # 如果æâ€â€Ã‚ Ã¦Â³â€¢Ã¦â€°Â©Ã¥Â±â€¢Ã§Â©ÂºÃ©â€â€Ã‚´Ã¯Â¼Å’é‡ÂÂç½® scale 并继ç»ÂÂ循环
+ scale = 1.0
+
+ # 如果ä»ÂÂ然æâ€Â¾ä¸ÂÂ下，å°ÂÂ试去除英文æ¢行é™ÂÂ制
+ if use_english_line_break:
+ return self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ initial_scale,
+ use_english_line_break=False,
+ apply_layout=apply_layout,
+ )
+
+ # 最åÂÂŽè¿â€Â回最å°ÂÂ缩æâ€Â¾å› åÂÂÂÂ
+ return min_scale, final_typeset_units
+
+ def _get_optimal_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ use_english_line_break: bool = True,
+ ) -> float:
+ """获å–段è½的最优缩æâ€Â¾å› åÂÂÂÂ，ä¸ÂÂ执行实际排版"""
+ scale, _ = self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ 1.0,
+ use_english_line_break,
+ apply_layout=False,
+ )
+ return scale
+
+ def retypeset_with_precomputed_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ precomputed_scale: float,
+ use_english_line_break: bool = True,
+ ):
+ """使çâ€Â¨é¢„计ç®â€â€ÃƒÂ§Ã…¡â€žÃ§Â¼Â©Ã¦â€Â¾å› åÂÂÂÂ进行排版"""
+ if not paragraph.box:
+ return
+
+ # 使çâ€Â¨é€šçâ€Â¨æ–¹æ³•è¿›è¡ŒæŽ’版，传入预计ç®â€â€ÃƒÂ§Ã…¡â€žÃ§Â¼Â©Ã¦â€Â¾å› åÂÂÂÂ作为åˆÂÂ始值
+ self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ precomputed_scale,
+ use_english_line_break,
+ apply_layout=True,
+ )
+
+ def typesetting_document(self, document: il_version_1.Document):
+ # Add detailed logging at the start
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Started")
+
+ # 原有的æŽ'版逻è¾'
+ if self.translation_config.progress_monitor:
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(document.page) * 2,
+ ) as pbar:
+ # 预处ç†ï¼šèŽ·å–所有段è½çš„最优缩æ"¾å› Ã¥ÂÂÂ
+ self.preprocess_document(document, pbar)
+
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+ pbar.advance()
+ else:
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+
+ # Add detailed logging at the end
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Complete")
+
+ def render_page(self, page: il_version_1.Page):
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if font.font_id:
+ fonts[xobj.xobj_id][font.font_id] = font
+ if (
+ page.page_number == 0
+ and self.translation_config.watermark_output_mode
+ == WatermarkOutputMode.Watermarked
+ ):
+ self.add_watermark(page)
+ try:
+ para_index = index.Index()
+ para_map = {}
+ #
+ valid_paras = [
+ p
+ for p in page.pdf_paragraph
+ if p.box
+ and all(c is not None for c in [p.box.x, p.box.y, p.box.x2, p.box.y2])
+ ]
+
+ for i, para in enumerate(valid_paras):
+ para_map[i] = para
+ para_index.insert(i, box_to_tuple(para.box))
+
+ for i, p_upper in para_map.items():
+ if not (p_upper.box and p_upper.box.y is not None):
+ continue
+
+ # Calculate paragraph height and set required gap accordingly
+ para_height = p_upper.box.y2 - p_upper.box.y
+ required_gap = 0.5 if para_height < 36 else 3
+
+ check_area = il_version_1.Box(
+ x=p_upper.box.x,
+ y=p_upper.box.y - required_gap,
+ x2=p_upper.box.x2,
+ y2=p_upper.box.y,
+ )
+
+ candidate_ids = list(para_index.intersection(box_to_tuple(check_area)))
+
+ conflicting_paras = []
+ for para_id in candidate_ids:
+ if para_id == i:
+ continue
+ p_lower = para_map[para_id]
+ if not (
+ p_lower.box
+ and p_upper.box
+ and p_lower.box.x2 < p_upper.box.x
+ or p_lower.box.x > p_upper.box.x2
+ ):
+ conflicting_paras.append(p_lower)
+
+ if conflicting_paras:
+ max_y2 = max(
+ p.box.y2
+ for p in conflicting_paras
+ if p.box and p.box.y2 is not None
+ )
+
+ new_y = max_y2 + required_gap
+ if p_upper.box and new_y < p_upper.box.y2:
+ p_upper.box.y = new_y
+ except Exception as e:
+ logger.warning(
+ f"Failed to adjust paragraph positions on page {page.page_number}: {e}"
+ )
+ # 开始实际的渲染过程
+ for paragraph in page.pdf_paragraph:
+ self.render_paragraph(paragraph, page, fonts)
+
+ def add_watermark(self, page: il_version_1.Page):
+ page_width = page.cropbox.box.x2 - page.cropbox.box.x
+ page_height = page.cropbox.box.y2 - page.cropbox.box.y
+ style = il_version_1.PdfStyle(
+ font_id="base",
+ font_size=6,
+ graphic_state=il_version_1.GraphicState(),
+ )
+ text = f"本文档çâ€Â± funstory.ai 的开溠PDF 翻译库 BabelDOC {WATERMARK_VERSION} (http://yadt.io) 翻译，本仓库æ£在积æžÂÂ的建设当ä¸ÂÂ，欢迎 star 和关注。"
+ if self.translation_config.debug:
+ text += "\n 当å‰ÂÂ为 DEBUG 模å¼ÂÂ，将显示更多辅助信æ¯。请注æ„ÂÂ，部分框的ä½ÂÂ置对åºâ€Â原文，但在译文ä¸ÂÂå¯能ä¸ÂÂæ£确。"
+ page.pdf_paragraph.append(
+ il_version_1.PdfParagraph(
+ first_line_indent=False,
+ box=il_version_1.Box(
+ x=page.cropbox.box.x + page_width * 0.05,
+ y=page.cropbox.box.y,
+ x2=page.cropbox.box.x2,
+ y2=page.cropbox.box.y2 - page_height * 0.05,
+ ),
+ vertical=False,
+ pdf_style=style,
+ pdf_paragraph_composition=[
+ il_version_1.PdfParagraphComposition(
+ pdf_same_style_unicode_characters=il_version_1.PdfSameStyleUnicodeCharacters(
+ unicode=text,
+ pdf_style=style,
+ ),
+ ),
+ ],
+ xobj_id=-1,
+ ),
+ )
+
+ def render_paragraph(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ],
+ ):
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ # 如果所有å•元都å¯以直接传递，则直接传递
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.scale = 1.0
+ paragraph.pdf_paragraph_composition = self.create_passthrough_composition(
+ typesetting_units,
+ )
+ else:
+ # 使çâ€Â¨é¢„计ç®â€â€ÃƒÂ§Ã…¡â€žÃ§Â¼Â©Ã¦â€Â¾å› åÂÂÂÂ进行é‡ÂÂ排ç‰Ëâ€
+ precomputed_scale = (
+ paragraph.optimal_scale if paragraph.optimal_scale is not None else 1.0
+ )
+
+ # 如果有å•元æâ€â€Ã‚ Ã¦Â³â€¢Ã§â€ºÂ´Ã¦Å½Â¥Ã¤Â¼Â Ã©â‚¬â€™Ã¯Â¼Å’则进行é‡ÂÂ排ç‰Ëâ€
+ paragraph.pdf_paragraph_composition = []
+ self.retypeset_with_precomputed_scale(
+ paragraph, page, typesetting_units, precomputed_scale
+ )
+
+ # é‡ÂÂ排版åÂŽ，é‡ÂÂ新设置段è½å„åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã§Å¡â€ž render order
+ self._update_paragraph_render_order(paragraph)
+
+ def _is_arabic_char(self, char: str) -> bool:
+ """Check if character is Arabic - OPTIMIZED"""
+ if not char:
+ return False
+ try:
+ code_point = ord(char[0])
+ return (0x0600 <= code_point <= 0x06FF) or (0xFB50 <= code_point <= 0xFDFF) or (0xFE70 <= code_point <= 0xFEFF)
+ except:
+ return False
+
+ def _layout_typesetting_units(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ box: Box,
+ scale: float,
+ line_skip: float,
+ paragraph: il_version_1.PdfParagraph,
+ use_english_line_break: bool = True,
+ ) -> tuple[list[TypesettingUnit], bool]:
+ """布局排版å•å…ƒ - OPTIMIZED FOR ARABIC RTL"""
+
+ # Detect Arabic FIRST
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = any(marker in lang_out for marker in ["ar", "arabic", "ara"])
+
+ # 计算å—体大å°
+ font_sizes = []
+ for unit in typesetting_units:
+ if unit.font_size:
+ font_sizes.append(unit.font_size)
+ if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ font_sizes.append(unit.char.pdf_style.font_size)
+ if not font_sizes:
+ font_sizes = [12]
+ font_sizes.sort()
+ font_size = statistics.mode(font_sizes)
+
+ space_width = (
+ self.font_mapper.base_font.char_lengths("ä½ ", font_size * scale)[0] * 0.5
+ )
+
+ # 计算行高
+ unit_heights = [unit.height for unit in typesetting_units] if typesetting_units else []
+ if not unit_heights:
+ avg_height = 0
+ elif len(unit_heights) == 1:
+ avg_height = unit_heights[0] * scale
+ else:
+ try:
+ avg_height = statistics.mode(unit_heights) * scale
+ except statistics.StatisticsError:
+ avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # åˆå§‹åŒ–
+ current_x = box.x
+ current_y = box.y2 - avg_height
+ box = copy.deepcopy(box)
+ line_height = 0
+ current_line_heights = []
+ typeset_units = []
+ all_units_fit = True
+ last_unit: TypesettingUnit | None = None
+ line_ys = [current_y]
+
+ if paragraph.first_line_indent:
+ current_x += space_width * 4
+
+ # OPTIMIZED ARABIC WORD-LEVEL PROCESSING
+ if is_arabic:
+ i = 0
+ safety_counter = 0
+ max_iterations = len(typesetting_units) * 2 # Safety limit
+
+ while i < len(typesetting_units) and safety_counter < max_iterations:
+ safety_counter += 1
+
+ # Collect word (simple: until space or end)
+ word_units = []
+ while i < len(typesetting_units):
+ unit = typesetting_units[i]
+ if unit.is_space:
+ if word_units:
+ i += 1
+ break
+ word_units.append(unit)
+ i += 1
+ if len(word_units) > 100: # Safety: max word length
+ break
+
+ if not word_units:
+ continue
+
+ # Calculate word width
+ word_width = sum(u.width * scale for u in word_units)
+
+ # Skip leading spaces
+ if current_x == box.x and word_units and word_units[0].is_space:
+ continue
+
+ # Check if needs new line
+ if current_x + word_width > box.x2 and current_x > box.x:
+ current_x = box.x
+ if current_line_heights:
+ max_height = max(current_line_heights)
+ mode_height = statistics.mode(current_line_heights) if len(current_line_heights) > 1 else max_height
+ current_y -= max(mode_height * line_skip, max_height * 1.05)
+ line_ys.append(current_y)
+ current_line_heights = []
+
+ if current_y < box.y:
+ all_units_fit = False
+
+ # Place word units
+ for unit in word_units:
+ if unit.is_space and current_x == box.x:
+ continue
+
+ unit_width = unit.width * scale
+ unit_height = unit.height * scale
+
+ # CJK spacing
+ if (last_unit and last_unit.is_cjk_char ^ unit.is_cjk_char
+ and not unit.is_space and current_x > box.x):
+ current_x += space_width * 0.5
+
+ relocated_unit = unit.relocate(current_x, current_y, scale)
+ typeset_units.append(relocated_unit)
+
+ if not unit.is_space:
+ current_line_heights.append(unit_height)
+
+ current_x = relocated_unit.box.x2
+ last_unit = relocated_unit
+
+ # Right-align Arabic lines (but NOT table content)
+ # Check if this paragraph is inside a table by examining layout_label
+ is_table_content = False
+ if paragraph.layout_label:
+ layout_label_lower = paragraph.layout_label.lower()
+ # Exclude ONLY actual table cell content from right-alignment
+ # NOTE: "table_title", "table_caption" are headings, NOT table content!
+ # We only want to exclude: table_cell, table_text, wired_table_cell, wireless_table_cell
+ if any(table_marker in layout_label_lower for table_marker in [
+ 'table_cell', 'table_text', 'wired_table_cell', 'wireless_table_cell'
+ ]):
+ is_table_content = True
+
+ # Only apply right-alignment if NOT table content
+ if typeset_units and not is_table_content:
+ lines_dict = {}
+ for unit in typeset_units:
+ if unit.box and unit.box.y is not None:
+ line_y = round(unit.box.y, 1)
+ if line_y not in lines_dict:
+ lines_dict[line_y] = []
+ lines_dict[line_y].append(unit)
+
+ for line_y, line_units in lines_dict.items():
+ if line_units:
+ line_max_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+ shift_x = box.x2 - line_max_x
+
+ for unit in line_units:
+ if unit.box:
+ unit.box.x += shift_x
+ unit.box.x2 += shift_x
+ if unit.x is not None:
+ unit.x += shift_x
+ if unit.char and unit.char.box:
+ unit.char.box.x += shift_x
+ unit.char.box.x2 += shift_x
+ if unit.char and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ unit.char.visual_bbox.box.x += shift_x
+ unit.char.visual_bbox.box.x2 += shift_x
+ else:
+ # ORIGINAL NON-ARABIC LOGIC (UNCHANGED)
+ for i, unit in enumerate(typesetting_units):
+ unit_width = unit.width * scale
+ unit_height = unit.height * scale
+
+ if current_x == box.x and unit.is_space:
+ continue
+
+ if (last_unit and last_unit.is_cjk_char ^ unit.is_cjk_char
+ and last_unit.box and last_unit.box.y
+ and current_y - 0.1 <= last_unit.box.y2 <= current_y + line_height + 0.1
+ and not last_unit.mixed_character_blacklist and not unit.mixed_character_blacklist
+ and current_x > box.x and unit.try_get_unicode() != " "
+ and last_unit.try_get_unicode() != " "
+ and last_unit.try_get_unicode() not in ["。", "，", "ã€", "ï¼›", "ï¼", "？"]):
+ current_x += space_width * 0.5
+
+ if use_english_line_break:
+ width_before_next_break_point = self._get_width_before_next_break_point(typesetting_units[i:], scale)
+ else:
+ width_before_next_break_point = 0
+
+ if not unit.is_hung_punctuation and (
+ (current_x + unit_width > box.x2) or
+ (use_english_line_break and current_x + unit_width + width_before_next_break_point > box.x2) or
+ (unit.is_cannot_appear_in_line_end_punctuation and current_x + unit_width * 2 > box.x2)):
+
+ current_x = box.x
+ if not current_line_heights:
+ return [], False
+ max_height = max(current_line_heights)
+ mode_height = statistics.mode(current_line_heights)
+ current_y -= max(mode_height * line_skip, max_height * 1.05)
+ line_ys.append(current_y)
+ line_height = 0.0
+ current_line_heights = []
+
+ if current_y < box.y:
+ all_units_fit = False
+
+ if unit.is_space:
+ line_height = max(line_height, unit_height)
+ continue
+
+ relocated_unit = unit.relocate(current_x, current_y, scale)
+ typeset_units.append(relocated_unit)
+
+ if not unit.is_space:
+ current_line_heights.append(unit_height)
+
+ prev_x = current_x
+ current_x = relocated_unit.box.x2
+ if prev_x > current_x:
+ logger.warning(f"åæ ‡å›žé€€ï¼ï¼ï¼TypesettingUnit: {unit.box}, ")
+
+ last_unit = relocated_unit
+ # If Arabic, reverse the line order
+ if is_arabic and typeset_units:
+ # Group units by line (using Y coordinates)
+ lines_dict = {}
+ for unit in typeset_units:
+ if unit.box and unit.box.y is not None:
+ # Round Y coordinate to group units on the same line
+ line_y = round(unit.box.y, 1)
+ if line_y not in lines_dict:
+ lines_dict[line_y] = []
+ lines_dict[line_y].append(unit)
+
+ # Sort lines by Y coordinate (top to bottom) and reverse
+ sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # Rebuild typeset_units with reversed line order
+ reversed_typeset_units = []
+ for line_y in reversed(sorted_line_ys):
+ reversed_typeset_units.extend(lines_dict[line_y])
+
+ # Now reposition all units to swap their Y coordinates
+ # Map old Y positions to new Y positions
+ y_mapping = {}
+ for i, old_y in enumerate(sorted_line_ys):
+ new_y = sorted_line_ys[len(sorted_line_ys) - 1 - i]
+ y_mapping[old_y] = new_y
+
+ # Update Y coordinates for all units
+ for unit in reversed_typeset_units:
+ if unit.box and unit.box.y is not None:
+ old_y = round(unit.box.y, 1)
+ if old_y in y_mapping:
+ new_y = y_mapping[old_y]
+ y_diff = new_y - old_y
+ # Update the unit's Y position
+ if unit.y is not None:
+ unit.y += y_diff
+ if unit.box:
+ unit.box.y += y_diff
+ unit.box.y2 += y_diff
+
+ typeset_units = reversed_typeset_units
+
+ return typeset_units, all_units_fit
+
+# CORRECT FIX FOR ARABIC TEXT LAYOUT
+# Replace the _layout_typesetting_units function in typesetting.py (lines 1346-1502)
+
+ # def _layout_typesetting_units(
+ # self,
+ # typesetting_units: list[TypesettingUnit],
+ # box: Box,
+ # scale: float,
+ # line_skip: float,
+ # paragraph: il_version_1.PdfParagraph,
+ # use_english_line_break: bool = True,
+ # ) -> tuple[list[TypesettingUnit], bool]:
+ # """布局排版å•元。
+
+ # Args:
+ # typesetting_units: è¦ÂÂ布局的排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ # box: 布局边界æ¡â€Â
+ # scale: 缩æâ€Â¾å› åÂÂÂÂ
+
+ # Returns:
+ # tuple[list[TypesettingUnit], bool]: (已布局的排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨Ã¯Â¼Å’是å¦所有å•元都æâ€Â¾å¾â€â€ÃƒÂ¤Ã‚¸â€¹)
+ # """
+ # # 计ç®â€â€ÃƒÂ¥Ã‚Ââ€â€ÃƒÂ¥Ã‚·ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ # font_sizes = []
+ # for unit in typesetting_units:
+ # if unit.font_size:
+ # font_sizes.append(unit.font_size)
+ # if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ # font_sizes.append(unit.char.pdf_style.font_size)
+ # font_sizes.sort()
+ # font_size = statistics.mode(font_sizes)
+
+ # space_width = (
+ # self.font_mapper.base_font.char_lengths("ä½ ", font_size * scale)[0] * 0.5
+ # )
+
+ # # 计ç®â€â€ÃƒÂ¨Ã‚¡Å’高（使çâ€Â¨ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã¯Â¼â€°
+ # unit_heights = (
+ # [unit.height for unit in typesetting_units] if typesetting_units else []
+ # )
+ # if not unit_heights:
+ # avg_height = 0
+ # elif len(unit_heights) == 1:
+ # avg_height = unit_heights[0] * scale
+ # else:
+ # try:
+ # avg_height = statistics.mode(unit_heights) * scale
+ # except statistics.StatisticsError:
+ # # 如果没有ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã¯Â¼Ë†Ã¦â€°â‚¬Ã¦Å“‰å€¼éƒ½å‡ºçŽ°ç›¸åÂŒ次数），则使çâ€Â¨å¹³å‡值
+ # avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # # *** NEW: Detect Arabic language ***
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # # åˆÂÂ始化ä½ÂÂ置为å³上角，并å‡ÂÂ去一个平å‡行高
+ # # *** CHANGED: For Arabic, calculate total line width first and start from right ***
+ # current_x = box.x
+ # current_y = box.y2 - avg_height
+ # box = copy.deepcopy(box)
+ # line_height = 0
+ # current_line_heights = [] # å˜储当å‰ÂÂ行所有元素的高度
+
+ # # å˜储已排版的å•元
+ # typeset_units = []
+ # all_units_fit = True
+ # last_unit: TypesettingUnit | None = None
+ # line_ys = [current_y]
+ # if paragraph.first_line_indent:
+ # current_x += space_width * 4
+ # # éÂÂÂÂ历所有排版å•元
+ # for i, unit in enumerate(typesetting_units):
+ # # 计ç®â€â€ÃƒÂ¥Ã‚½â€œÃ¥â€°ÂÂå•元在当å‰ÂÂ缩æâ€Â¾ä¸‹çš„尺寸
+ # unit_width = unit.width * scale
+ # unit_height = unit.height * scale
+
+ # # 跳过行首的空格
+ # if current_x == box.x and unit.is_space:
+ # continue
+
+ # if (
+ # last_unit # 有上一个å•元
+ # and last_unit.is_cjk_char ^ unit.is_cjk_char # ä¸ÂÂ英文交界处
+ # and (
+ # last_unit.box
+ # and last_unit.box.y
+ # and current_y - 0.1
+ # <= last_unit.box.y2
+ # <= current_y + line_height + 0.1
+ # ) # 在åÂŒ一行，ä¸â€Â有垂直é‡ÂÂÃ¥ÂÂÂÂ
+ # and not last_unit.mixed_character_blacklist # ä¸ÂÂ是混排空格黑åÂÂÂÂå•åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦
+ # and not unit.mixed_character_blacklist # Ã¥ÂÂŒä¸ÅÂ
+ # and current_x > box.x # ä¸ÂÂ是行首
+ # and unit.try_get_unicode() != " " # ä¸ÂÂ是空格
+ # and last_unit.try_get_unicode() != " " # ä¸ÂÂ是空格
+ # and last_unit.try_get_unicode()
+ # not in [
+ # "。",
+ # "ï¼ÂÂ",
+ # "？",
+ # "ï¼›",
+ # "：",
+ # "，",
+ # ]
+ # ):
+ # current_x += space_width * 0.5
+ # if use_english_line_break:
+ # width_before_next_break_point = self._get_width_before_next_break_point(
+ # typesetting_units[i:], scale
+ # )
+ # else:
+ # width_before_next_break_point = 0
+
+ # # 如果当å‰ÂÂè¡Œæâ€Â¾ä¸ÂÂ下这个元素，æ¢行
+ # if not unit.is_hung_punctuation and (
+ # (current_x + unit_width > box.x2)
+ # or (
+ # use_english_line_break
+ # and current_x + unit_width + width_before_next_break_point > box.x2
+ # )
+ # or (
+ # unit.is_cannot_appear_in_line_end_punctuation
+ # and current_x + unit_width * 2 > box.x2
+ # )
+ # ):
+ # # æ¢行
+ # current_x = box.x
+ # if not current_line_heights:
+ # return [], False
+ # max_height = max(current_line_heights)
+ # mode_height = statistics.mode(current_line_heights)
+
+ # current_y -= max(mode_height * line_skip, max_height * 1.05)
+ # line_ys.append(current_y)
+ # line_height = 0.0
+ # current_line_heights = [] # 清空当å‰ÂÂ行高度åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+
+ # # 检查是å¦超出底部边界
+ # # if current_y - unit_height < box.y:
+ # if current_y < box.y:
+ # all_units_fit = False
+ # # 这里ä¸ÂÂ覠break，继ç»ÂÂ排版剩余内容
+
+ # if unit.is_space:
+ # line_height = max(line_height, unit_height)
+ # continue
+
+ # # æâ€Â¾ç½®å½“å‰ÂÂå•元
+ # relocated_unit = unit.relocate(current_x, current_y, scale)
+ # typeset_units.append(relocated_unit)
+
+ # # 添加当å‰ÂÂå•元的高度到当å‰ÂÂ行高度åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ # if not unit.is_space:
+ # current_line_heights.append(unit_height)
+
+ # prev_x = current_x
+ # # æ›´æ–° x Ã¥ÂÂÂÂæ ‡
+ # current_x = relocated_unit.box.x2
+ # if prev_x > current_x:
+ # logger.warning(f"Ã¥ÂÂÂÂ标回绕ï¼ÂÂï¼ÂÂï¼ÂÂTypesettingUnit: {unit.box}, ")
+
+ # last_unit = relocated_unit
+
+ # # *** NEW: For Arabic, right-align each line ***
+ # if is_arabic and typeset_units:
+ # # Group units by line (Y coordinate)
+ # lines = {}
+ # for unit in typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # line_y = round(unit.box.y, 1)
+ # if line_y not in lines:
+ # lines[line_y] = []
+ # lines[line_y].append(unit)
+
+ # # Right-align each line
+ # for line_y, line_units in lines.items():
+ # if not line_units:
+ # continue
+
+ # # Find the rightmost position of this line
+ # line_max_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+
+ # # Calculate how much to shift right
+ # shift_x = box.x2 - line_max_x
+
+ # # Shift all units in this line to the right
+ # for unit in line_units:
+ # if unit.box:
+ # unit.box.x += shift_x
+ # unit.box.x2 += shift_x
+ # if unit.x is not None:
+ # unit.x += shift_x
+ # # Update character box if present
+ # if unit.char and unit.char.box:
+ # unit.char.box.x += shift_x
+ # unit.char.box.x2 += shift_x
+ # if unit.char and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ # unit.char.visual_bbox.box.x += shift_x
+ # unit.char.visual_bbox.box.x2 += shift_x
+ # # Check if output language is Arabic
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # # If Arabic, reverse the line order
+ # if is_arabic and typeset_units:
+ # # Group units by line (using Y coordinates)
+ # lines_dict = {}
+ # for unit in typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # # Round Y coordinate to group units on the same line
+ # line_y = round(unit.box.y, 1)
+ # if line_y not in lines_dict:
+ # lines_dict[line_y] = []
+ # lines_dict[line_y].append(unit)
+
+ # # Sort lines by Y coordinate (top to bottom) and reverse
+ # sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # # Rebuild typeset_units with reversed line order
+ # reversed_typeset_units = []
+ # for line_y in reversed(sorted_line_ys):
+ # reversed_typeset_units.extend(lines_dict[line_y])
+
+ # # Now reposition all units to swap their Y coordinates
+ # # Map old Y positions to new Y positions
+ # y_mapping = {}
+ # for i, old_y in enumerate(sorted_line_ys):
+ # new_y = sorted_line_ys[len(sorted_line_ys) - 1 - i]
+ # y_mapping[old_y] = new_y
+
+ # # Update Y coordinates for all units
+ # for unit in reversed_typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # old_y = round(unit.box.y, 1)
+ # if old_y in y_mapping:
+ # new_y = y_mapping[old_y]
+ # y_diff = new_y - old_y
+ # # Update the unit's Y position
+ # if unit.y is not None:
+ # unit.y += y_diff
+ # if unit.box:
+ # unit.box.y += y_diff
+ # unit.box.y2 += y_diff
+
+ # typeset_units = reversed_typeset_units
+
+ # return typeset_units, all_units_fit
+
+ def create_typesetting_units(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ fonts: dict[str, il_version_1.PdfFont],
+ ) -> list[TypesettingUnit]:
+ if not paragraph.pdf_paragraph_composition:
+ return []
+ result = []
+
+ @cache
+ def get_font(font_id: str, xobj_id: int | None):
+ if xobj_id in fonts:
+ font = fonts[xobj_id][font_id]
+ else:
+ font = fonts[font_id]
+ return font
+
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition is None:
+ continue
+ if composition.pdf_line:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_line.pdf_character
+ ],
+ )
+ elif composition.pdf_character:
+ result.append(
+ TypesettingUnit(
+ char=composition.pdf_character,
+ debug_info=paragraph.debug_info,
+ ),
+ )
+ elif composition.pdf_same_style_characters:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_same_style_characters.pdf_character
+ ],
+ )
+ elif composition.pdf_same_style_unicode_characters:
+ style = composition.pdf_same_style_unicode_characters.pdf_style
+ if style is None:
+ logger.warning(
+ f"Style is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font_id = style.font_id
+ if font_id is None:
+ logger.warning(
+ f"Font ID is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font = get_font(font_id, paragraph.xobj_id)
+ if composition.pdf_same_style_unicode_characters.unicode:
+ unicode_text = composition.pdf_same_style_unicode_characters.unicode
+ shaped_text = self.shape_arabic_text(unicode_text)
+ result.extend(
+ [
+ TypesettingUnit(
+ unicode=char_unicode,
+ font=self.font_mapper.map(
+ font,
+ char_unicode,
+ ),
+ original_font=font,
+ font_size=style.font_size,
+ style=style,
+ xobj_id=paragraph.xobj_id,
+ debug_info=composition.pdf_same_style_unicode_characters.debug_info
+ or False,
+ )
+ for char_unicode in shaped_text # Use shaped_text instead of original
+ if char_unicode not in ("\n",)
+ ],
+ )
+ elif composition.pdf_formula:
+ result.extend([TypesettingUnit(formular=composition.pdf_formula)])
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ result = list(
+ filter(
+ lambda x: x.unicode is None or x.font is not None,
+ result,
+ ),
+ )
+
+ if any(x.width < 0 for x in result):
+ logger.warning("有排版å•元宽度å°ÂÂ于 0，请检查åÂÂâ€â€ÃƒÂ¤Ã‚½â€œÃ¦ËœÂ Ã¥Â°â€žÃ¦ËœÂ¯Ã¥Â¦æ£确。")
+ return result
+
+ def create_passthrough_composition(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ ) -> list[PdfParagraphComposition]:
+ """从排版å•元创建直接传递的段è½组åˆ。
+
+ Args:
+ typesetting_units: 排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+
+ Returns:
+ 段è½组åˆåˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ """
+ composition = []
+ for unit in typesetting_units:
+ if unit.formular:
+ # 对于公å¼ÂÂå•元，直接创建包å«完整公å¼ÂÂ的组åÂÂËâ€
+ composition.append(PdfParagraphComposition(pdf_formula=unit.formular))
+ else:
+ # 对于åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â•元，使çâ€Â¨åŽŸæœ‰é€»è¾‘
+ chars, curves, forms = unit.passthrough()
+ composition.extend(
+ [PdfParagraphComposition(pdf_character=char) for char in chars],
+ )
+ return composition
+
+ def get_max_right_space(self, current_box: Box, page) -> float:
+ """获å–段è½å³侧最大å¯çâ€Â¨ç©ºéâ€â€Ã‚´
+
+ Args:
+ current_box: 当å‰ÂÂ段è½的边界æ¡â€Â
+ page: 当å‰ÂÂ页é¢
+
+ Returns:
+ å¯以扩展到的最大 x Ã¥ÂÂÂÂæ ‡
+ """
+ # 获å–页é¢的è£ÂÂ剪框作为åˆÂÂ始最大é™ÂÂ制
+ max_x = page.cropbox.box.x2 * 0.9
+
+ # 检查所有å¯能的阻挡元ç´ÂÂ
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # 跳过当å‰ÂÂ段è½
+ continue
+ # åª考虑在当å‰ÂÂ段è½å³侧ä¸â€Â有垂直é‡ÂÂå 的元ç´ÂÂ
+ if para.box.x > current_box.x and not (
+ para.box.y >= current_box.y2 or para.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, para.box.x)
+ for char in page.pdf_character:
+ if char.box.x > current_box.x and not (
+ char.box.y >= current_box.y2 or char.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, char.box.x)
+ # 检查图形
+ for figure in page.pdf_figure:
+ if figure.box.x > current_box.x and not (
+ figure.box.y >= current_box.y2 or figure.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, figure.box.x)
+
+ return max_x
+
+ def get_max_bottom_space(self, current_box: Box, page: il_version_1.Page) -> float:
+ """获å–段è½下方最大å¯çâ€Â¨ç©ºéâ€â€Ã‚´
+
+ Args:
+ current_box: 当å‰ÂÂ段è½的边界æ¡â€Â
+ page: 当å‰ÂÂ页é¢
+
+ Returns:
+ å¯以扩展到的最尠y Ã¥ÂÂÂÂæ ‡
+ """
+ # 获å–页é¢的è£ÂÂ剪框作为åˆÂÂ始最å°ÂÂé™ÂÂ制
+ min_y = page.cropbox.box.y * 1.1
+
+ # 检查所有å¯能的阻挡元ç´ÂÂ
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # 跳过当å‰ÂÂ段è½
+ continue
+ # åª考虑在当å‰ÂÂ段è½下方ä¸â€Â有水平é‡ÂÂå 的元ç´ÂÂ
+ if para.box.y2 < current_box.y and not (
+ para.box.x >= current_box.x2 or para.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, para.box.y2)
+ for char in page.pdf_character:
+ if char.box.y2 < current_box.y and not (
+ char.box.x >= current_box.x2 or char.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, char.box.y2)
+ # 检查图形
+ for figure in page.pdf_figure:
+ if figure.box.y2 < current_box.y and not (
+ figure.box.x >= current_box.x2 or figure.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, figure.box.y2)
+
+ return min_y
+
+ def _update_paragraph_render_order(self, paragraph: il_version_1.PdfParagraph):
+ """
+ é‡ÂÂ新设置段è½å„åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã§Å¡â€ž render order
+ 主 render order ç‰于 paragraph çš„ renderorder，sub render order 从 1 开始自增
+ """
+ if not hasattr(paragraph, "render_order") or paragraph.render_order is None:
+ return
+
+ main_render_order = paragraph.render_order
+ sub_render_order = 1
+
+ # éÂÂÂÂ历段è½的所有组æˆÂÂ部åˆâ€Â
+ for composition in paragraph.pdf_paragraph_composition:
+ # 检查å•个åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦
+ if composition.pdf_character:
+ char = composition.pdf_character
+ char.render_order = main_render_order
+ char.sub_render_order = sub_render_order
+ sub_render_order += 1
\ No newline at end of file
diff --git a/babeldoc/format/pdf/document_il/midend/typesetting_v3.py b/babeldoc/format/pdf/document_il/midend/typesetting_v3.py
new file mode 100644
index 0000000000000000000000000000000000000000..5278dfd1d567a2b0cb5f7e585972ed46dc7caa4d
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/typesetting_v3.py
@@ -0,0 +1,2103 @@
+from __future__ import annotations
+
+import copy
+import logging
+import re
+import statistics
+import unicodedata
+from functools import cache
+
+import pymupdf
+import regex
+from rtree import index
+
+from babeldoc.const import WATERMARK_VERSION
+from babeldoc.format.pdf.document_il import Box
+from babeldoc.format.pdf.document_il import PdfCharacter
+from babeldoc.format.pdf.document_il import PdfCurve
+from babeldoc.format.pdf.document_il import PdfForm
+from babeldoc.format.pdf.document_il import PdfFormula
+from babeldoc.format.pdf.document_il import PdfParagraphComposition
+from babeldoc.format.pdf.document_il import PdfStyle
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.formular_helper import update_formula_data
+from babeldoc.format.pdf.document_il.utils.layout_helper import box_to_tuple
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from babeldoc.format.pdf.translation_config import WatermarkOutputMode
+from arabic_reshaper import reshape
+from bidi.algorithm import get_display
+
+
+logger = logging.getLogger(__name__)
+
+LINE_BREAK_REGEX = regex.compile(
+ r"^["
+ r"a-z"
+ r"A-Z"
+ r"0-9"
+ r"\u00C0-\u00FF" # Latin-1 Supplement
+ r"\u0100-\u017F" # Latin Extended A
+ r"\u0180-\u024F" # Latin Extended B
+ r"\u1E00-\u1EFF" # Latin Extended Additional
+ r"\u2C60-\u2C7F" # Latin Extended C
+ r"\uA720-\uA7FF" # Latin Extended D
+ r"\uAB30-\uAB6F" # Latin Extended E
+ r"\u0250-\u02A0" # IPA Extensions
+ r"\u0400-\u04FF" # Cyrillic
+ r"\u0300-\u036F" # Combining Diacritical Marks
+ r"\u0500-\u052F" # Cyrillic Supplement
+ r"\u0370-\u03FF" # Greek and Coptic
+ r"\u2DE0-\u2DFF" # Cyrillic Extended-A
+ r"\uA650-\uA69F" # Cyrillic Extended-B
+ r"\u1200-\u137F" # Ethiopic
+ r"\u1380-\u139F" # Ethiopic Supplement
+ r"\u2D80-\u2DDF" # Ethiopic Extended
+ r"\uAB00-\uAB2F" # Ethiopic Extended-A
+ r"\U0001E7E0-\U0001E7FF" # Ethiopic Extended-B
+ r"\u0E80-\u0EFF" # Lao
+ r"\u0D00-\u0D7F" # Malayalam
+ r"\u0A80-\u0AFF" # Gujarati
+ r"\u0E00-\u0E7F" # Thai
+ r"\u1000-\u109F" # Myanmar
+ r"\uAA60-\uAA7F" # Myanmar Extended-A
+ r"\uA9E0-\uA9FF" # Myanmar Extended-B
+ r"\U000116D0-\U000116FF" # Myanmar Extended-C
+ r"\u0B80-\u0BFF" # Tamil
+ r"\u0C00-\u0C7F" # Telugu
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0530-\u058F" # Armenian
+ r"\u10A0-\u10FF" # Georgian
+ r"\u1C90-\u1CBF" # Georgian Extended
+ r"\u2D00-\u2D2F" # Georgian Supplement
+ r"\u1780-\u17FF" # Khmer
+ r"\u19E0-\u19FF" # Khmer Symbols
+ r"\U00010B00-\U00010B3F" # Avestan
+ r"\u1D00-\u1D7F" # Phonetic Extensions
+ r"\u1400-\u167F" # Unified Canadian Aboriginal Syllabics
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0780-\u07BF" # Thaana
+ r"\U0001E900-\U0001E95F" # Adlam
+ r"\u1C80-\u1C8F" # Cyrillic Extended-C
+ r"\U0001E030-\U0001E08F" # Cyrillic Extended-D
+ r"\uA000-\uA48F" # Yi Syllables
+ r"\uA490-\uA4CF" # Yi Radicals
+ r"'"
+ r"-" # Hyphen
+ r"·" # Middle Dot (U+00B7) For CatalÃÂ
+ r"Ê»" # Spacing Modifier Letters U+02BB
+ r"]+$"
+)
+
+
+class TypesettingUnit:
+ def __str__(self):
+ return self.try_get_unicode() or ""
+
+ def __init__(
+ self,
+ char: PdfCharacter | None = None,
+ formular: PdfFormula | None = None,
+ unicode: str | None = None,
+ font: pymupdf.Font | None = None,
+ original_font: il_version_1.PdfFont | None = None,
+ font_size: float | None = None,
+ style: PdfStyle | None = None,
+ xobj_id: int | None = None,
+ debug_info: bool = False,
+ ):
+ assert (char is not None) + (formular is not None) + (
+ unicode is not None
+ ) == 1, "Only one of chars and formular can be not None"
+ self.char = char
+ self.formular = formular
+ self.unicode = unicode
+ self.x = None
+ self.y = None
+ self.scale = None
+ self.debug_info = debug_info
+
+ # Cache variables
+ self.box_cache: Box | None = None
+ self.can_break_line_cache: bool | None = None
+ self.is_cjk_char_cache: bool | None = None
+ self.mixed_character_blacklist_cache: bool | None = None
+ self.is_space_cache: bool | None = None
+ self.is_hung_punctuation_cache: bool | None = None
+ self.is_cannot_appear_in_line_end_punctuation_cache: bool | None = None
+ self.can_passthrough_cache: bool | None = None
+ self.width_cache: float | None = None
+ self.height_cache: float | None = None
+
+ self.font_size: float | None = None
+
+ if unicode:
+ assert font_size, "Font size must be provided when unicode is provided"
+ assert style, "Style must be provided when unicode is provided"
+ assert len(unicode) == 1, "Unicode must be a single character"
+ assert xobj_id is not None, (
+ "Xobj id must be provided when unicode is provided"
+ )
+
+ self.font = font
+ if font is not None and hasattr(font, "font_id"):
+ self.font_id = font.font_id
+ else:
+ self.font_id = "base"
+ if original_font:
+ self.original_font = original_font
+ else:
+ self.original_font = None
+
+ self.font_size = font_size
+ self.style = style
+ self.xobj_id = xobj_id
+
+ def try_resue_cache(self, old_tu: TypesettingUnit):
+ if old_tu.is_cjk_char_cache is not None:
+ self.is_cjk_char_cache = old_tu.is_cjk_char_cache
+
+ if old_tu.can_break_line_cache is not None:
+ self.can_break_line_cache = old_tu.can_break_line_cache
+
+ if old_tu.is_space_cache is not None:
+ self.is_space_cache = old_tu.is_space_cache
+
+ if old_tu.is_hung_punctuation_cache is not None:
+ self.is_hung_punctuation_cache = old_tu.is_hung_punctuation_cache
+
+ if old_tu.is_cannot_appear_in_line_end_punctuation_cache is not None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ old_tu.is_cannot_appear_in_line_end_punctuation_cache
+ )
+
+ if old_tu.can_passthrough_cache is not None:
+ self.can_passthrough_cache = old_tu.can_passthrough_cache
+
+ if old_tu.mixed_character_blacklist_cache is not None:
+ self.mixed_character_blacklist_cache = (
+ old_tu.mixed_character_blacklist_cache
+ )
+
+
+ def try_get_unicode(self) -> str | None:
+ if self.char:
+ return self.char.char_unicode
+ elif self.formular:
+ return None
+ elif self.unicode:
+ return self.unicode
+
+ @property
+ def mixed_character_blacklist(self):
+ if self.mixed_character_blacklist_cache is None:
+ self.mixed_character_blacklist_cache = self.calc_mixed_character_blacklist()
+
+ return self.mixed_character_blacklist_cache
+
+ def calc_mixed_character_blacklist(self):
+ unicode = self.try_get_unicode()
+ if unicode:
+ return unicode in [
+ "。",
+ "，",
+ "：",
+ "？",
+ "ï¼Â",
+ ]
+ return False
+
+ @property
+ def can_break_line(self):
+ if self.can_break_line_cache is None:
+ self.can_break_line_cache = self.calc_can_break_line()
+
+ return self.can_break_line_cache
+
+ def calc_can_break_line(self):
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return True
+ if LINE_BREAK_REGEX.match(unicode):
+ return False
+ return True
+
+ @property
+ def is_cjk_char(self):
+ if self.is_cjk_char_cache is None:
+ self.is_cjk_char_cache = self.calc_is_cjk_char()
+
+ return self.is_cjk_char_cache
+
+ def calc_is_cjk_char(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ if "(cid" in unicode:
+ return False
+ if len(unicode) > 1:
+ return False
+ assert len(unicode) == 1, "Unicode must be a single character"
+ if unicode in [
+ "（",
+ "）",
+ "ã€Â",
+ "】",
+ "《",
+ "》",
+ "ã€â€",
+ "〕",
+ "〈",
+ "〉",
+ "〖",
+ "〗",
+ "「",
+ "ã€Â",
+ "『",
+ "ã€Â",
+ "ã€Â",
+ "。",
+ "：",
+ "？",
+ "ï¼Â",
+ "，",
+ ]:
+ return True
+ if unicode:
+ if re.match(
+ r"^["
+ r"\u3000-\u303f" # CJK Symbols and Punctuation
+ r"\u3040-\u309f" # Hiragana
+ r"\u30a0-\u30ff" # Katakana
+ r"\u3100-\u312f" # Bopomofo
+ r"\uac00-\ud7af" # Hangul Syllables
+ r"\u1100-\u11ff" # Hangul Jamo
+ r"\u3130-\u318f" # Hangul Compatibility Jamo
+ r"\ua960-\ua97f" # Hangul Jamo Extended-A
+ r"\ud7b0-\ud7ff" # Hangul Jamo Extended-B
+ r"\u3190-\u319f" # Kanbun
+ r"\u3200-\u32ff" # Enclosed CJK Letters and Months
+ r"\u3300-\u33ff" # CJK Compatibility
+ r"\ufe30-\ufe4f" # CJK Compatibility Forms
+ r"\u4e00-\u9fff" # CJK Unified Ideographs
+ r"\u2e80-\u2eff" # CJK Radicals Supplement
+ r"\u31c0-\u31ef" # CJK Strokes
+ r"\u2f00-\u2fdf" # Kangxi Radicals
+ r"\ufe10-\ufe1f" # Vertical Forms
+ r"]+$",
+ unicode,
+ ):
+ return True
+ try:
+ unicodedata_name = unicodedata.name(unicode)
+ return (
+ "CJK UNIFIED IDEOGRAPH" in unicodedata_name
+ or "FULLWIDTH" in unicodedata_name
+ )
+ except ValueError:
+ return False
+ return False
+
+ @property
+ def is_space(self):
+ if self.is_space_cache is None:
+ self.is_space_cache = self.calc_is_space()
+
+ return self.is_space_cache
+
+ def calc_is_space(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ return unicode == " "
+
+ @property
+ def is_hung_punctuation(self):
+ if self.is_hung_punctuation_cache is None:
+ self.is_hung_punctuation_cache = self.calc_is_hung_punctuation()
+
+ return self.is_hung_punctuation_cache
+
+ def calc_is_hung_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+
+ if unicode:
+ return unicode in [
+ # 英文标点
+ ",",
+ ".",
+ ":",
+ ";",
+ "?",
+ "!",
+ # ä¸Â文点å·
+ "，", # 逗å·
+ "。", # å¥å·
+ "．", # 全角åÂ¥å·
+ "ã€Â", # é¡¿å·
+ "：", # 冒å·
+ "ï¼›", # 分å·
+ "ï¼Â", # å¹å·
+ "‼", # åŒå¹å·
+ "？", # é—®å·
+ "â‡", # åŒ问å·
+ # 结æŸ引å·
+ "â€Â", # å³åŒ引å·
+ "’", # å³å•引å·
+ "ã€Â", # å³直角å•引å·
+ "ã€Â", # å³直角åŒ引å·
+ # 结æŸ括å·
+ ")", # å³圆括å·
+ "]", # å³方括å·
+ "}", # å³花括å·
+ "）", # å³圆括å·
+ "〕", # å³龟çâ€Â²Ã¦â€¹Â¬Ã¥Â·
+ "〉", # å³å•书åÂÂå·
+ "】", # å³黑色方头括å·
+ "〗", # å³空白方头括å·
+ "ï¼½", # 全角å³方括å·
+ "ï½Â", # 全角å³花括å·
+ # 结æŸåŒ书åÂÂå·
+ "》", # å³åŒ书åÂÂå·
+ # 连接å·
+ "～", # 全角波浪å·
+ "-", # è¿žå—符å‡Âå·
+ "–", # çŸÂ破折å· (EN DASH)
+ "â€â€", # 长破折å· (EM DASH)
+ # é—´éšâ€Ã¥Â·
+ "·", # ä¸Â间点
+ "・", # 片å‡åÂÂä¸Â间点
+ "‧", # è¿žå—点
+ # 分éšâ€Ã¥Â·
+ "/", # æ–œæÂÂ
+ "ï¼Â", # 全角斜æÂÂ
+ "â„", # 分数斜æÂÂ
+ ]
+ return False
+
+ @property
+ def is_cannot_appear_in_line_end_punctuation(self):
+ if self.is_cannot_appear_in_line_end_punctuation_cache is None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ self.calc_is_cannot_appear_in_line_end_punctuation()
+ )
+
+ return self.is_cannot_appear_in_line_end_punctuation_cache
+
+ def calc_is_cannot_appear_in_line_end_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ return unicode in [
+ # 开始引å·
+ "“", # å·¦åŒ引å·
+ "‘", # å·¦å•引å·
+ "「", # 左直角å•引å·
+ "『", # 左直角åŒ引å·
+ # 开始括å·
+ "(", # 左圆括å·
+ "[", # 左方括å·
+ "{", # 左花括å·
+ "（", # 左圆括å·
+ "ã€â€", # 左龟çâ€Â²Ã¦â€¹Â¬Ã¥Â·
+ "〈", # å·¦å•书åÂÂå·
+ "《", # å·¦åŒ书åÂÂå·
+ # 开始å•åŒ书åÂÂå·
+ "〖", # 左空白方头括å·
+ "〘", # 左黑色方头括å·
+ "〚", # å·¦å•书åÂÂå·
+ ]
+
+ def passthrough(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ if self.char:
+ return [self.char], [], []
+ elif self.formular:
+ return (
+ self.formular.pdf_character,
+ self.formular.pdf_curve,
+ self.formular.pdf_form,
+ )
+ elif self.unicode:
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ return [], [], []
+
+ @property
+ def can_passthrough(self):
+ if self.can_passthrough_cache is None:
+ self.can_passthrough_cache = self.calc_can_passthrough()
+
+ return self.can_passthrough_cache
+
+ def calc_can_passthrough(self):
+ return self.unicode is None
+
+ def calculate_box(self):
+ if self.char:
+ box = copy.deepcopy(self.char.box)
+ if self.char.visual_bbox and self.char.visual_bbox.box:
+ box.y = self.char.visual_bbox.box.y
+ box.y2 = self.char.visual_bbox.box.y2
+ # return self.char.visual_bbox.box
+
+ return box
+ elif self.formular:
+ return self.formular.box
+ # if self.formular.x_offset <= 0.5:
+ # return self.formular.box
+ # formular_box = copy.copy(self.formular.box)
+ # formular_box.x2 += self.formular.x_advance
+ # return formular_box
+ elif self.unicode:
+ char_width = self.font.char_lengths(self.unicode, self.font_size)[0]
+ if self.x is None or self.y is None or self.scale is None:
+ return Box(0, 0, char_width, self.font_size)
+ return Box(self.x, self.y, self.x + char_width, self.y + self.font_size)
+
+ @property
+ def box(self):
+ if not self.box_cache:
+ self.box_cache = self.calculate_box()
+
+ return self.box_cache
+
+ @property
+ def width(self):
+ if self.width_cache is None:
+ self.width_cache = self.calc_width()
+
+ return self.width_cache
+
+ def calc_width(self):
+ box = self.box
+ return box.x2 - box.x
+
+ @property
+ def height(self):
+ if self.height_cache is None:
+ self.height_cache = self.calc_height()
+
+ return self.height_cache
+
+ def calc_height(self):
+ box = self.box
+ return box.y2 - box.y
+
+ def relocate(
+ self,
+ x: float,
+ y: float,
+ scale: float,
+ ) -> TypesettingUnit:
+ """é‡Â定ä½Â并缩æâ€Â¾Ã¦Å½â€™Ã§â€°Ë†Ã¥Â•å…ƒ
+
+ Args:
+ x: æ–°çš„ x Ã¥ÂÂæ ‡
+ y: æ–°çš„ y Ã¥ÂÂæ ‡
+ scale: 缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ
+
+ Returns:
+ 新的排版å•元
+ """
+ if self.char:
+ # 创建新的å—符对象
+ new_char = PdfCharacter(
+ pdf_character_id=self.char.pdf_character_id,
+ char_unicode=self.char.char_unicode,
+ box=Box(
+ x=x,
+ y=y,
+ x2=x + self.width * scale,
+ y2=y + self.height * scale,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.char.pdf_style.font_id,
+ font_size=self.char.pdf_style.font_size * scale,
+ graphic_state=self.char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=self.char.vertical,
+ advance=self.char.advance * scale if self.char.advance else None,
+ debug_info=self.debug_info,
+ xobj_id=self.char.xobj_id,
+ )
+ new_tu = TypesettingUnit(char=new_char)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.formular:
+ # 创建新的公å¼Â对象，ä¿ÂæŒÂ内部å—符的相对ä½Âç½®
+ new_chars = []
+ min_x = self.formular.box.x
+ min_y = self.formular.box.y
+
+ for char in self.formular.pdf_character:
+ # 计算相对ä½Âç½®
+ rel_x = char.box.x - min_x
+ rel_y = char.box.y - min_y
+
+ visual_rel_x = char.visual_bbox.box.x - min_x
+ visual_rel_y = char.visual_bbox.box.y - min_y
+
+ # 创建新的å—符对象
+ new_char = PdfCharacter(
+ pdf_character_id=char.pdf_character_id,
+ char_unicode=char.char_unicode,
+ box=Box(
+ x=x + (rel_x + self.formular.x_offset) * scale,
+ y=y + (rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (rel_x + (char.box.x2 - char.box.x) + self.formular.x_offset)
+ * scale,
+ y2=y
+ + (rel_y + (char.box.y2 - char.box.y) + self.formular.y_offset)
+ * scale,
+ ),
+ visual_bbox=il_version_1.VisualBbox(
+ box=Box(
+ x=x + (visual_rel_x + self.formular.x_offset) * scale,
+ y=y + (visual_rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (
+ visual_rel_x
+ + (char.visual_bbox.box.x2 - char.visual_bbox.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=y
+ + (
+ visual_rel_y
+ + (char.visual_bbox.box.y2 - char.visual_bbox.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ ),
+ ),
+ pdf_style=PdfStyle(
+ font_id=char.pdf_style.font_id,
+ font_size=char.pdf_style.font_size * scale,
+ graphic_state=char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=char.vertical,
+ advance=char.advance * scale if char.advance else None,
+ xobj_id=char.xobj_id,
+ )
+ new_chars.append(new_char)
+
+ # Calculate bounding box from new_chars
+ min_x = min(char.visual_bbox.box.x for char in new_chars)
+ min_y = min(char.visual_bbox.box.y for char in new_chars)
+ max_x = max(char.visual_bbox.box.x2 for char in new_chars)
+ max_y = max(char.visual_bbox.box.y2 for char in new_chars)
+
+ new_formula = PdfFormula(
+ box=Box(
+ x=min_x,
+ y=min_y,
+ x2=max_x,
+ y2=max_y,
+ ),
+ pdf_character=new_chars,
+ x_offset=self.formular.x_offset * scale,
+ y_offset=self.formular.y_offset * scale,
+ x_advance=self.formular.x_advance * scale,
+ )
+
+ # Handle contained curves
+ new_curves = []
+ for curve in self.formular.pdf_curve:
+ new_curve = self._transform_curve_for_relocation(
+ curve,
+ self.formular.box.x,
+ self.formular.box.y,
+ x,
+ y,
+ scale,
+ )
+ new_curves.append(new_curve)
+ new_formula.pdf_curve = new_curves
+
+ # Handle contained forms
+ new_forms = []
+ for form in self.formular.pdf_form:
+ new_form = self._transform_form_for_relocation(
+ form, self.formular.box.x, self.formular.box.y, x, y, scale
+ )
+ new_forms.append(new_form)
+ new_formula.pdf_form = new_forms
+
+ update_formula_data(new_formula)
+
+ new_tu = TypesettingUnit(formular=new_formula)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.unicode:
+ # 对于 Unicode å—符，我们å˜储新的ä½Â置信æ¯
+ new_unit = TypesettingUnit(
+ unicode=self.unicode,
+ font=self.font,
+ original_font=self.original_font,
+ font_size=self.font_size * scale,
+ style=self.style,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ new_unit.x = x
+ new_unit.y = y
+ new_unit.scale = scale
+ new_unit.try_resue_cache(self)
+ return new_unit
+
+ def _transform_curve_for_relocation(
+ self,
+ curve,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a curve for formula relocation."""
+ import copy
+
+ new_curve = copy.deepcopy(curve)
+
+ if new_curve.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_curve.box.x - original_formula_x
+ rel_y = new_curve.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_curve.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (
+ rel_x
+ + (new_curve.box.x2 - new_curve.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=new_y
+ + (
+ rel_y
+ + (new_curve.box.y2 - new_curve.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original CTM
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_curve.relocation_transform = list(relocation_matrix)
+
+ return new_curve
+
+ def _transform_form_for_relocation(
+ self,
+ form,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a form for formula relocation."""
+ import copy
+
+ new_form = copy.deepcopy(form)
+
+ if new_form.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_form.box.x - original_formula_x
+ rel_y = new_form.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_form.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (rel_x + (new_form.box.x2 - new_form.box.x) + self.formular.x_offset)
+ * scale,
+ y2=new_y
+ + (rel_y + (new_form.box.y2 - new_form.box.y) + self.formular.y_offset)
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original matrices
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_form.relocation_transform = list(relocation_matrix)
+
+ return new_form
+
+ def render(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ """渲染排版å•元为 PdfCharacter 列表
+
+ Returns:
+ PdfCharacter 列表
+ """
+ if self.can_passthrough:
+ return self.passthrough()
+ elif self.unicode:
+ assert self.x is not None, (
+ "x position must be set, should be set by `relocate`"
+ )
+ assert self.y is not None, (
+ "y position must be set, should be set by `relocate`"
+ )
+ assert self.scale is not None, (
+ "scale must be set, should be set by `relocate`"
+ )
+ x = self.x
+ y = self.y
+ # if self.original_font and self.font and hasattr(self.original_font, "descent") and hasattr(self.font, "descent_fontmap"):
+ # original_descent = self.original_font.descent
+ # new_descent = self.font.descent_fontmap
+ # y -= (original_descent - new_descent) * self.font_size / 1000
+
+ # 计算å—符宽度
+ char_width = self.width
+
+ # Handle case when font is None (no suitable font found for this character)
+ if self.font is None:
+ logger.warning(
+ f"No font available for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using font_id='{self.font_id}' with glyph_id=0"
+ )
+ glyph_id = 0 # Use glyph 0 as fallback (usually .notdef)
+ else:
+ glyph_id = self.font.has_glyph(ord(self.unicode))
+ if glyph_id == 0 or glyph_id is None:
+ logger.warning(
+ f"Font '{self.font_id}' doesn't have glyph for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using glyph_id=0"
+ )
+ glyph_id = 0
+
+ new_char = PdfCharacter(
+ pdf_character_id=glyph_id,
+ char_unicode=self.unicode,
+ box=Box(
+ x=x, # 使çâ€Â¨Ã¥Â˜å‚¨çš„ä½Âç½®
+ y=y,
+ x2=x + char_width,
+ y2=y + self.font_size,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.font_id,
+ font_size=self.font_size,
+ graphic_state=self.style.graphic_state,
+ ),
+ scale=self.scale,
+ vertical=False,
+ advance=char_width,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ return [new_char], [], []
+ else:
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ return [], [], []
+
+
+class Typesetting:
+ stage_name = "Typesetting"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.font_mapper = FontMapper(translation_config)
+ self.translation_config = translation_config
+ self.lang_code = self.translation_config.lang_out.upper()
+ # Ensure detailed_logger attribute exists to avoid attribute access errors
+ self.detailed_logger = None
+ self.is_cjk = (
+ # Why zh-CN/zh-HK/zh-TW here but not zh-Hans and so on?
+ # See https://funstory-ai.github.io/BabelDOC/supported_languages/
+ ("ZH" in self.lang_code) # C
+ or ("JA" in self.lang_code)
+ or ("JP" in self.lang_code) # J
+ or ("KR" in self.lang_code) # K
+ or ("CN" in self.lang_code)
+ or ("HK" in self.lang_code)
+ or ("TW" in self.lang_code)
+ )
+
+ def preprocess_document(self, document: il_version_1.Document, pbar):
+ """预处ç†文档，获å–æ¯Â个段è½的最优缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ，ä¸Â执行实际排版"""
+ all_scales: list[float] = []
+ all_paragraphs: list[il_version_1.PdfParagraph] = []
+
+ for page in document.page:
+ pbar.advance()
+ # 准备å—体信æ¯（å¤Â制自 render_page 的逻辑）
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if (
+ xobj.xobj_id in fonts
+ and isinstance(fonts[xobj.xobj_id], dict)
+ and font.font_id
+ ):
+ fonts[xobj.xobj_id][font.font_id] = font
+
+ # 处ç†æ¯Â个段è½
+ for paragraph in page.pdf_paragraph:
+ all_paragraphs.append(paragraph)
+ unit_count = 0
+ try:
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ unit_count = len(typesetting_units)
+ for unit in typesetting_units:
+ if unit.formular:
+ unit_count += len(unit.formular.pdf_character) - 1
+
+ # 如果所有å•元都å¯以直接传递，则 scale = 1.0
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.optimal_scale = 1.0
+ else:
+ # 获å–最优缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ
+ optimal_scale = self._get_optimal_scale(
+ paragraph, page, typesetting_units
+ )
+ paragraph.optimal_scale = optimal_scale
+ except Exception as e:
+ # 如果预处ç†出éâ€â„¢Ã¯Â¼Å’默认使çâ€Â¨ 1.0 缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ
+ logger.warning(f"预处ç†段è½时出éâ€â„¢Ã¯Â¼Å¡{e}")
+ paragraph.optimal_scale = 1.0
+
+ if paragraph.optimal_scale is not None:
+ all_scales.extend([paragraph.optimal_scale] * unit_count)
+
+ # 获å–缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂçš„ä¼—æ•°
+ if all_scales:
+ try:
+ modes = statistics.multimode(all_scales)
+ mode_scale = min(modes)
+ except statistics.StatisticsError:
+ logger.warning(
+ "Could not find a mode for paragraph scales. Falling back to median."
+ )
+ mode_scale = statistics.median(all_scales)
+ # 将所有大于众数的值修æâ€Â¹Ã¤Â¸ÂºÃ¤Â¼â€”æ•°
+ for paragraph in all_paragraphs:
+ if (
+ paragraph.optimal_scale is not None
+ and paragraph.optimal_scale > mode_scale
+ ):
+ paragraph.optimal_scale = mode_scale
+ else:
+ logger.error(
+ "document_scales is empty, there seems no paragraph in this PDF"
+ )
+
+ def shape_arabic_text(self, text: str) -> str:
+ """Shape and reorder Arabic text if output language is Arabic.
+
+ Args:
+ text: Input text to shape
+
+ Returns:
+ Shaped and reordered text if language is Arabic, original text otherwise
+ """
+ if not text:
+ return text
+
+ # Robust Arabic output detection: accept explicit 'ar', 'ara', 'arabic'
+ # or formats containing '-ar', '->ar', or '/ar' as a target marker (e.g. 'en-ar', 'en->ar')
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = False
+ if lang_out in ("en-ar, ar", "ara", "arabic"):
+ is_arabic = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic = True
+
+ if is_arabic:
+ logger.debug("Shaping Arabic text")
+ # Flip parentheses and brackets for RTL display
+ # text = text.replace("(", "\x00")
+ # text = text.replace(")", "(")
+ # text = text.replace("\x00", ")")
+ # text = text.replace("[", "\x01")
+ # text = text.replace("]", "[")
+ # text = text.replace("\x01", "]")
+ # text = text.replace("{", "\x02")
+ # text = text.replace("}", "{")
+ # text = text.replace("\x02", "}")
+ try:
+ if not re.search(r'[\uFB50-\uFDFF\uFE70-\uFEFF]', text):
+ # Reshape Arabic text for proper character joining
+ from arabic_reshaper import ArabicReshaper
+ configuration = {
+ 'delete_harakat': False, # Keep diacritical marks
+ 'support_ligatures': True, # Support Arabic ligatures
+ 'RIAL SIGN': True,
+ 'ARABIC COMMA': True,
+ 'ARABIC SEMICOLON': True,
+ 'ARABIC QUESTION MARK': True,
+ 'ZWNJ': True, # Zero Width Non-Joiner
+ }
+
+ reshaper = ArabicReshaper(configuration=configuration)
+ reshaped_text = reshaper.reshape(text)
+ display_text = get_display(reshaped_text, base_dir='R')
+ else:
+ display_text = text
+ return display_text
+ except Exception as e:
+ logger.warning(f"Failed to shape Arabic text: {e}")
+ return text
+
+ return text
+
+ def _find_optimal_scale_and_layout(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ initial_scale: float = 1.0,
+ use_english_line_break: bool = True,
+ apply_layout: bool = False,
+ ) -> tuple[float, list[TypesettingUnit] | None]:
+ """查找最优缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ并å¯选择性地执行布局
+
+ Args:
+ paragraph: 段è½对象
+ page: 页é¢对象
+ typesetting_units: 排版å•元列表
+ initial_scale: åˆÂ始缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ
+ use_english_line_break: 是å¦使çâ€Â¨Ã¨â€¹Â±Ã¦â€“‡æ¢行规则
+ apply_layout: 是å¦åºâ€Ã§â€Â¨Ã¥Â¸Æ’局到 paragraph（True 时执行实际排版）
+
+ Returns:
+ tuple[float, list[TypesettingUnit] | None]: (最终缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ，排版åÂŽçš„å•元列表或 None)
+ """
+ if not paragraph.box:
+ return initial_scale, None
+
+ box = paragraph.box
+ scale = initial_scale
+ line_skip = 1.50 if self.is_cjk else 1.3
+ min_scale = 0.1
+ expand_space_flag = 0
+ final_typeset_units = None
+
+ while scale >= min_scale:
+ try:
+ # å°Â试布局排版å•元
+ typeset_units, all_units_fit = self._layout_typesetting_units(
+ typesetting_units,
+ box,
+ scale,
+ line_skip,
+ paragraph,
+ use_english_line_break,
+ )
+
+ # 如果所有å•元都æâ€Â¾Ã¥Â¾â€”下
+ if all_units_fit:
+ if apply_layout:
+ # 实际åºâ€Ã§â€Â¨Ã¦Å½â€™Ã§â€°Ë†Ã§Â»â€œÃ¦Å¾Å“
+ paragraph.scale = scale
+ paragraph.pdf_paragraph_composition = []
+ for unit in typeset_units:
+ chars, curves, forms = unit.render()
+ for char in chars:
+ paragraph.pdf_paragraph_composition.append(
+ PdfParagraphComposition(pdf_character=char),
+ )
+ for curve in curves:
+ page.pdf_curve.append(curve)
+ for form in forms:
+ page.pdf_form.append(form)
+ final_typeset_units = typeset_units
+ return scale, final_typeset_units
+ except Exception:
+ # 如果布局检查出éâ€â„¢Ã¯Â¼Å’继ç»Âå°Â试下一个缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ
+ pass
+
+ # 添加与原 retypeset 一致的逻辑检查
+ if not hasattr(paragraph, "debug_id") or not paragraph.debug_id:
+ return scale, final_typeset_units
+
+ # å‡Âå°Â缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ
+ if scale > 0.6:
+ scale -= 0.05
+ else:
+ scale -= 0.1
+
+ if scale < 0.7:
+ space_expanded = False # 标记是å¦æˆÂ功扩展了空间
+
+ if expand_space_flag == 0:
+ # å°Â试å‘下扩展
+ try:
+ min_y = self.get_max_bottom_space(box, page) + 2
+ if min_y < box.y:
+ expanded_box = Box(x=box.x, y=min_y, x2=box.x2, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # 更新段è½的边界æ¡â€
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 1
+
+ # åª有æˆÂ功扩展空间时扠continue，å¦则继ç»Âå‡Âå° scale
+ if space_expanded:
+ continue
+
+ elif expand_space_flag == 1:
+ # å°Â试å‘å³扩展
+ try:
+ max_x = self.get_max_right_space(box, page) - 5
+ if max_x > box.x2:
+ expanded_box = Box(x=box.x, y=box.y, x2=max_x, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # 更新段è½的边界æ¡â€
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 2
+
+ # åª有æˆÂ功扩展空间时扠continue，å¦则继ç»Âå‡Âå° scale
+ if space_expanded:
+ continue
+
+ # åª有在扩展å°Â试阶段 (expand_space_flag < 2) ä¸â€Ã¦â€°Â©Ã¥Â±â€¢Ã¥Â¤Â±Ã¨Â´Â¥Ã¦â€”¶æ‰Âé‡Âç½® scale
+ # 当 expand_space_flag >= 2 时，说明已ç»Âå°Â试过所有扩展，åºâ€Ã¨Â¯Â¥Ã§Â»Â§Ã§Â»Âæ£常的 scale å‡Âå°Â
+ if expand_space_flag < 2:
+ # 如果无法扩展空间，é‡Âç½® scale 并继ç»Â循环
+ scale = 1.0
+
+ # 如果ä»Â然æâ€Â¾Ã¤Â¸Â下，å°Â试去除英文æ¢行é™Â制
+ if use_english_line_break:
+ return self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ initial_scale,
+ use_english_line_break=False,
+ apply_layout=apply_layout,
+ )
+
+ # 最åÂŽè¿â€Ã¥â€ºÅ¾Ã¦Å“€å°Â缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ
+ return min_scale, final_typeset_units
+
+ def _get_optimal_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ use_english_line_break: bool = True,
+ ) -> float:
+ """获å–段è½的最优缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ，ä¸Â执行实际排版"""
+ scale, _ = self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ 1.0,
+ use_english_line_break,
+ apply_layout=False,
+ )
+ return scale
+
+ def retypeset_with_precomputed_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ precomputed_scale: float,
+ use_english_line_break: bool = True,
+ ):
+ """使çâ€Â¨Ã©Â¢â€žÃ¨Â®Â¡Ã§Â®â€”的缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ进行排版"""
+ if not paragraph.box:
+ return
+
+ # 使çâ€Â¨Ã©â‚¬Å¡Ã§â€Â¨Ã¦â€“¹æ³•è¿›è¡ŒæŽ’版，传入预计算的缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ作为åˆÂ始值
+ self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ precomputed_scale,
+ use_english_line_break,
+ apply_layout=True,
+ )
+
+ def typesetting_document(self, document: il_version_1.Document):
+ # Add detailed logging at the start
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Started")
+
+ # 原有的æŽ'版逻è¾'
+ if self.translation_config.progress_monitor:
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(document.page) * 2,
+ ) as pbar:
+ # 预处ç†ï¼šèŽ·å–所有段è½çš„最优缩æ"¾å› Ã¥ÂÂ
+ self.preprocess_document(document, pbar)
+
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+ pbar.advance()
+ else:
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+
+ # Add detailed logging at the end
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Complete")
+
+ def render_page(self, page: il_version_1.Page):
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if font.font_id:
+ fonts[xobj.xobj_id][font.font_id] = font
+ if (
+ page.page_number == 0
+ and self.translation_config.watermark_output_mode
+ == WatermarkOutputMode.Watermarked
+ ):
+ self.add_watermark(page)
+ try:
+ para_index = index.Index()
+ para_map = {}
+ #
+ valid_paras = [
+ p
+ for p in page.pdf_paragraph
+ if p.box
+ and all(c is not None for c in [p.box.x, p.box.y, p.box.x2, p.box.y2])
+ ]
+
+ for i, para in enumerate(valid_paras):
+ para_map[i] = para
+ para_index.insert(i, box_to_tuple(para.box))
+
+ for i, p_upper in para_map.items():
+ if not (p_upper.box and p_upper.box.y is not None):
+ continue
+
+ # Calculate paragraph height and set required gap accordingly
+ para_height = p_upper.box.y2 - p_upper.box.y
+ required_gap = 0.5 if para_height < 36 else 3
+
+ check_area = il_version_1.Box(
+ x=p_upper.box.x,
+ y=p_upper.box.y - required_gap,
+ x2=p_upper.box.x2,
+ y2=p_upper.box.y,
+ )
+
+ candidate_ids = list(para_index.intersection(box_to_tuple(check_area)))
+
+ conflicting_paras = []
+ for para_id in candidate_ids:
+ if para_id == i:
+ continue
+ p_lower = para_map[para_id]
+ if not (
+ p_lower.box
+ and p_upper.box
+ and p_lower.box.x2 < p_upper.box.x
+ or p_lower.box.x > p_upper.box.x2
+ ):
+ conflicting_paras.append(p_lower)
+
+ if conflicting_paras:
+ max_y2 = max(
+ p.box.y2
+ for p in conflicting_paras
+ if p.box and p.box.y2 is not None
+ )
+
+ new_y = max_y2 + required_gap
+ if p_upper.box and new_y < p_upper.box.y2:
+ p_upper.box.y = new_y
+ except Exception as e:
+ logger.warning(
+ f"Failed to adjust paragraph positions on page {page.page_number}: {e}"
+ )
+ # 开始实际的渲染过程
+ for paragraph in page.pdf_paragraph:
+ self.render_paragraph(paragraph, page, fonts)
+
+ def add_watermark(self, page: il_version_1.Page):
+ page_width = page.cropbox.box.x2 - page.cropbox.box.x
+ page_height = page.cropbox.box.y2 - page.cropbox.box.y
+ style = il_version_1.PdfStyle(
+ font_id="base",
+ font_size=6,
+ graphic_state=il_version_1.GraphicState(),
+ )
+ text = f"BabelDOC {WATERMARK_VERSION} (http://yadt.io)"
+ if self.translation_config.debug:
+ text += "\n "
+ page.pdf_paragraph.append(
+ il_version_1.PdfParagraph(
+ first_line_indent=False,
+ box=il_version_1.Box(
+ x=page.cropbox.box.x + page_width * 0.05,
+ y=page.cropbox.box.y,
+ x2=page.cropbox.box.x2,
+ y2=page.cropbox.box.y2 - page_height * 0.05,
+ ),
+ vertical=False,
+ pdf_style=style,
+ pdf_paragraph_composition=[
+ il_version_1.PdfParagraphComposition(
+ pdf_same_style_unicode_characters=il_version_1.PdfSameStyleUnicodeCharacters(
+ unicode=text,
+ pdf_style=style,
+ ),
+ ),
+ ],
+ xobj_id=-1,
+ ),
+ )
+
+ def render_paragraph(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ],
+ ):
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ # 如果所有å•元都å¯以直接传递，则直接传递
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.scale = 1.0
+ paragraph.pdf_paragraph_composition = self.create_passthrough_composition(
+ typesetting_units,
+ )
+ else:
+ # 使çâ€Â¨Ã©Â¢â€žÃ¨Â®Â¡Ã§Â®â€”的缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ进行é‡Â排版
+ precomputed_scale = (
+ paragraph.optimal_scale if paragraph.optimal_scale is not None else 1.0
+ )
+
+ # 如果有å•元无法直接传递，则进行é‡Â排版
+ paragraph.pdf_paragraph_composition = []
+ self.retypeset_with_precomputed_scale(
+ paragraph, page, typesetting_units, precomputed_scale
+ )
+
+ # é‡Â排版åŽ，é‡Â新设置段è½å„å—符的 render order
+ self._update_paragraph_render_order(paragraph)
+
+ def _is_arabic_char(self, char: str) -> bool:
+ """Check if character is Arabic - OPTIMIZED"""
+ if not char:
+ return False
+ try:
+ code_point = ord(char[0])
+ return (0x0600 <= code_point <= 0x06FF) or (0xFB50 <= code_point <= 0xFDFF) or (0xFE70 <= code_point <= 0xFEFF)
+ except:
+ return False
+
+ def _layout_typesetting_units(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ box: Box,
+ scale: float,
+ line_skip: float,
+ paragraph: il_version_1.PdfParagraph,
+ use_english_line_break: bool = True,
+ ) -> tuple[list[TypesettingUnit], bool]:
+ """布局排版单元 - OPTIMIZED FOR ARABIC RTL"""
+
+ # Detect Arabic FIRST
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = any(marker in lang_out for marker in ["ar", "arabic", "ara"])
+
+ # 计算字体大小
+ font_sizes = []
+ for unit in typesetting_units:
+ if unit.font_size:
+ font_sizes.append(unit.font_size)
+ if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ font_sizes.append(unit.char.pdf_style.font_size)
+ if not font_sizes:
+ font_sizes = [12]
+ font_sizes.sort()
+ font_size = statistics.mode(font_sizes)
+
+ space_width = (
+ self.font_mapper.base_font.char_lengths("你 ", font_size * scale)[0] * 0.5
+ )
+
+ # 计算行高
+ unit_heights = [unit.height for unit in typesetting_units] if typesetting_units else []
+ if not unit_heights:
+ avg_height = 0
+ elif len(unit_heights) == 1:
+ avg_height = unit_heights[0] * scale
+ else:
+ try:
+ avg_height = statistics.mode(unit_heights) * scale
+ except statistics.StatisticsError:
+ avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # 初始化
+ current_x = box.x
+ current_y = box.y2 - avg_height
+ box = copy.deepcopy(box)
+ line_height = 0
+ current_line_heights = []
+ typeset_units = []
+ all_units_fit = True
+ last_unit: TypesettingUnit | None = None
+ line_ys = [current_y]
+
+ if paragraph.first_line_indent:
+ current_x += space_width * 4
+
+ # OPTIMIZED ARABIC WORD-LEVEL PROCESSING
+ if is_arabic:
+ i = 0
+ safety_counter = 0
+ max_iterations = len(typesetting_units) * 2 # Safety limit
+
+ while i < len(typesetting_units) and safety_counter < max_iterations:
+ safety_counter += 1
+
+ # Collect word (simple: until space or end)
+ word_units = []
+ while i < len(typesetting_units):
+ unit = typesetting_units[i]
+ if unit.is_space:
+ if word_units:
+ i += 1
+ break
+ word_units.append(unit)
+ i += 1
+ if len(word_units) > 100: # Safety: max word length
+ break
+
+ if not word_units:
+ continue
+
+ # Calculate word width
+ word_width = sum(u.width * scale for u in word_units)
+
+ # Skip leading spaces
+ if current_x == box.x and word_units and word_units[0].is_space:
+ continue
+
+ # Check if needs new line
+ if current_x + word_width > box.x2 and current_x > box.x:
+ current_x = box.x
+ if current_line_heights:
+ max_height = max(current_line_heights)
+ mode_height = statistics.mode(current_line_heights) if len(current_line_heights) > 1 else max_height
+ current_y -= max(mode_height * line_skip, max_height * 1.05)
+ line_ys.append(current_y)
+ current_line_heights = []
+
+ if current_y < box.y:
+ all_units_fit = False
+
+ # Place word units
+ for unit in word_units:
+ if unit.is_space and current_x == box.x:
+ continue
+
+ unit_width = unit.width * scale
+ unit_height = unit.height * scale
+
+ # CJK spacing
+ if (last_unit and last_unit.is_cjk_char ^ unit.is_cjk_char
+ and not unit.is_space and current_x > box.x):
+ current_x += space_width * 0.5
+
+ relocated_unit = unit.relocate(current_x, current_y, scale)
+ typeset_units.append(relocated_unit)
+
+ if not unit.is_space:
+ current_line_heights.append(unit_height)
+
+ current_x = relocated_unit.box.x2
+ last_unit = relocated_unit
+
+ # Right-align Arabic lines
+ if typeset_units:
+ lines_dict = {}
+ for unit in typeset_units:
+ if unit.box and unit.box.y is not None:
+ line_y = round(unit.box.y, 1)
+ if line_y not in lines_dict:
+ lines_dict[line_y] = []
+ lines_dict[line_y].append(unit)
+
+ for line_y, line_units in lines_dict.items():
+ if line_units:
+ line_max_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+ shift_x = box.x2 - line_max_x
+
+ for unit in line_units:
+ if unit.box:
+ unit.box.x += shift_x
+ unit.box.x2 += shift_x
+ if unit.x is not None:
+ unit.x += shift_x
+ if unit.char and unit.char.box:
+ unit.char.box.x += shift_x
+ unit.char.box.x2 += shift_x
+ if unit.char and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ unit.char.visual_bbox.box.x += shift_x
+ unit.char.visual_bbox.box.x2 += shift_x
+ else:
+ # ORIGINAL NON-ARABIC LOGIC (UNCHANGED)
+ for i, unit in enumerate(typesetting_units):
+ unit_width = unit.width * scale
+ unit_height = unit.height * scale
+
+ if current_x == box.x and unit.is_space:
+ continue
+
+ if (last_unit and last_unit.is_cjk_char ^ unit.is_cjk_char
+ and last_unit.box and last_unit.box.y
+ and current_y - 0.1 <= last_unit.box.y2 <= current_y + line_height + 0.1
+ and not last_unit.mixed_character_blacklist and not unit.mixed_character_blacklist
+ and current_x > box.x and unit.try_get_unicode() != " "
+ and last_unit.try_get_unicode() != " "
+ and last_unit.try_get_unicode() not in ["。", "，", "、", "；", "！", "？"]):
+ current_x += space_width * 0.5
+
+ if use_english_line_break:
+ width_before_next_break_point = self._get_width_before_next_break_point(typesetting_units[i:], scale)
+ else:
+ width_before_next_break_point = 0
+
+ if not unit.is_hung_punctuation and (
+ (current_x + unit_width > box.x2) or
+ (use_english_line_break and current_x + unit_width + width_before_next_break_point > box.x2) or
+ (unit.is_cannot_appear_in_line_end_punctuation and current_x + unit_width * 2 > box.x2)):
+
+ current_x = box.x
+ if not current_line_heights:
+ return [], False
+ max_height = max(current_line_heights)
+ mode_height = statistics.mode(current_line_heights)
+ current_y -= max(mode_height * line_skip, max_height * 1.05)
+ line_ys.append(current_y)
+ line_height = 0.0
+ current_line_heights = []
+
+ if current_y < box.y:
+ all_units_fit = False
+
+ if unit.is_space:
+ line_height = max(line_height, unit_height)
+ continue
+
+ relocated_unit = unit.relocate(current_x, current_y, scale)
+ typeset_units.append(relocated_unit)
+
+ if not unit.is_space:
+ current_line_heights.append(unit_height)
+
+ prev_x = current_x
+ current_x = relocated_unit.box.x2
+ if prev_x > current_x:
+ logger.warning(f"坐标回退！！！TypesettingUnit: {unit.box}, ")
+
+ last_unit = relocated_unit
+ # If Arabic, reverse the line order
+ if is_arabic and typeset_units:
+ # Group units by line (using Y coordinates)
+ lines_dict = {}
+ for unit in typeset_units:
+ if unit.box and unit.box.y is not None:
+ # Round Y coordinate to group units on the same line
+ line_y = round(unit.box.y, 1)
+ if line_y not in lines_dict:
+ lines_dict[line_y] = []
+ lines_dict[line_y].append(unit)
+
+ # Sort lines by Y coordinate (top to bottom) and reverse
+ sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # Rebuild typeset_units with reversed line order
+ reversed_typeset_units = []
+ for line_y in reversed(sorted_line_ys):
+ reversed_typeset_units.extend(lines_dict[line_y])
+
+ # Now reposition all units to swap their Y coordinates
+ # Map old Y positions to new Y positions
+ y_mapping = {}
+ for i, old_y in enumerate(sorted_line_ys):
+ new_y = sorted_line_ys[len(sorted_line_ys) - 1 - i]
+ y_mapping[old_y] = new_y
+
+ # Update Y coordinates for all units
+ for unit in reversed_typeset_units:
+ if unit.box and unit.box.y is not None:
+ old_y = round(unit.box.y, 1)
+ if old_y in y_mapping:
+ new_y = y_mapping[old_y]
+ y_diff = new_y - old_y
+ # Update the unit's Y position
+ if unit.y is not None:
+ unit.y += y_diff
+ if unit.box:
+ unit.box.y += y_diff
+ unit.box.y2 += y_diff
+
+ typeset_units = reversed_typeset_units
+
+ return typeset_units, all_units_fit
+
+# CORRECT FIX FOR ARABIC TEXT LAYOUT
+# Replace the _layout_typesetting_units function in typesetting.py (lines 1346-1502)
+
+ # def _layout_typesetting_units(
+ # self,
+ # typesetting_units: list[TypesettingUnit],
+ # box: Box,
+ # scale: float,
+ # line_skip: float,
+ # paragraph: il_version_1.PdfParagraph,
+ # use_english_line_break: bool = True,
+ # ) -> tuple[list[TypesettingUnit], bool]:
+ # """布局排版å•元。
+
+ # Args:
+ # typesetting_units: è¦Â布局的排版å•元列表
+ # box: 布局边界æ¡â€
+ # scale: 缩æâ€Â¾Ã¥â€ºÂ Ã¥ÂÂ
+
+ # Returns:
+ # tuple[list[TypesettingUnit], bool]: (已布局的排版å•元列表，是å¦所有å•元都æâ€Â¾Ã¥Â¾â€”下)
+ # """
+ # # 计算å—å·众数
+ # font_sizes = []
+ # for unit in typesetting_units:
+ # if unit.font_size:
+ # font_sizes.append(unit.font_size)
+ # if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ # font_sizes.append(unit.char.pdf_style.font_size)
+ # font_sizes.sort()
+ # font_size = statistics.mode(font_sizes)
+
+ # space_width = (
+ # self.font_mapper.base_font.char_lengths("ä½ ", font_size * scale)[0] * 0.5
+ # )
+
+ # # 计算行高（使çâ€Â¨Ã¤Â¼â€”数）
+ # unit_heights = (
+ # [unit.height for unit in typesetting_units] if typesetting_units else []
+ # )
+ # if not unit_heights:
+ # avg_height = 0
+ # elif len(unit_heights) == 1:
+ # avg_height = unit_heights[0] * scale
+ # else:
+ # try:
+ # avg_height = statistics.mode(unit_heights) * scale
+ # except statistics.StatisticsError:
+ # # 如果没有众数（所有值都出现相åŒ次数），则使çâ€Â¨Ã¥Â¹Â³Ã¥Â‡å€¼
+ # avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # # *** NEW: Detect Arabic language ***
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # # åˆÂ始化ä½Â置为å³上角，并å‡Â去一个平å‡行高
+ # # *** CHANGED: For Arabic, calculate total line width first and start from right ***
+ # current_x = box.x
+ # current_y = box.y2 - avg_height
+ # box = copy.deepcopy(box)
+ # line_height = 0
+ # current_line_heights = [] # å˜储当å‰Â行所有元素的高度
+
+ # # å˜储已排版的å•元
+ # typeset_units = []
+ # all_units_fit = True
+ # last_unit: TypesettingUnit | None = None
+ # line_ys = [current_y]
+ # if paragraph.first_line_indent:
+ # current_x += space_width * 4
+ # # éÂÂ历所有排版å•元
+ # for i, unit in enumerate(typesetting_units):
+ # # 计算当å‰Âå•元在当å‰Â缩æâ€Â¾Ã¤Â¸â€¹Ã§Å¡â€žÃ¥Â°ÂºÃ¥Â¯Â¸
+ # unit_width = unit.width * scale
+ # unit_height = unit.height * scale
+
+ # # 跳过行首的空格
+ # if current_x == box.x and unit.is_space:
+ # continue
+
+ # if (
+ # last_unit # 有上一个å•元
+ # and last_unit.is_cjk_char ^ unit.is_cjk_char # ä¸Â英文交界处
+ # and (
+ # last_unit.box
+ # and last_unit.box.y
+ # and current_y - 0.1
+ # <= last_unit.box.y2
+ # <= current_y + line_height + 0.1
+ # ) # 在åŒ一行，ä¸â€Ã¦Å“‰åž‚ç›´é‡ÂÃ¥ÂÂ
+ # and not last_unit.mixed_character_blacklist # ä¸Â是混排空格黑åÂÂå•å—符
+ # and not unit.mixed_character_blacklist # åŒä¸Å
+ # and current_x > box.x # ä¸Â是行首
+ # and unit.try_get_unicode() != " " # ä¸Â是空格
+ # and last_unit.try_get_unicode() != " " # ä¸Â是空格
+ # and last_unit.try_get_unicode()
+ # not in [
+ # "。",
+ # "ï¼Â",
+ # "？",
+ # "ï¼›",
+ # "：",
+ # "，",
+ # ]
+ # ):
+ # current_x += space_width * 0.5
+ # if use_english_line_break:
+ # width_before_next_break_point = self._get_width_before_next_break_point(
+ # typesetting_units[i:], scale
+ # )
+ # else:
+ # width_before_next_break_point = 0
+
+ # # 如果当å‰Âè¡Œæâ€Â¾Ã¤Â¸Â下这个元素，æ¢行
+ # if not unit.is_hung_punctuation and (
+ # (current_x + unit_width > box.x2)
+ # or (
+ # use_english_line_break
+ # and current_x + unit_width + width_before_next_break_point > box.x2
+ # )
+ # or (
+ # unit.is_cannot_appear_in_line_end_punctuation
+ # and current_x + unit_width * 2 > box.x2
+ # )
+ # ):
+ # # æ¢行
+ # current_x = box.x
+ # if not current_line_heights:
+ # return [], False
+ # max_height = max(current_line_heights)
+ # mode_height = statistics.mode(current_line_heights)
+
+ # current_y -= max(mode_height * line_skip, max_height * 1.05)
+ # line_ys.append(current_y)
+ # line_height = 0.0
+ # current_line_heights = [] # 清空当å‰Â行高度列表
+
+ # # 检查是å¦超出底部边界
+ # # if current_y - unit_height < box.y:
+ # if current_y < box.y:
+ # all_units_fit = False
+ # # 这里ä¸Â覠break，继ç»Â排版剩余内容
+
+ # if unit.is_space:
+ # line_height = max(line_height, unit_height)
+ # continue
+
+ # # æâ€Â¾Ã§Â½Â®Ã¥Â½â€œÃ¥â€°Âå•元
+ # relocated_unit = unit.relocate(current_x, current_y, scale)
+ # typeset_units.append(relocated_unit)
+
+ # # 添加当å‰Âå•元的高度到当å‰Â行高度列表
+ # if not unit.is_space:
+ # current_line_heights.append(unit_height)
+
+ # prev_x = current_x
+ # # æ›´æ–° x Ã¥ÂÂæ ‡
+ # current_x = relocated_unit.box.x2
+ # if prev_x > current_x:
+ # logger.warning(f"Ã¥ÂÂ标回绕ï¼Âï¼Âï¼ÂTypesettingUnit: {unit.box}, ")
+
+ # last_unit = relocated_unit
+
+ # # *** NEW: For Arabic, right-align each line ***
+ # if is_arabic and typeset_units:
+ # # Group units by line (Y coordinate)
+ # lines = {}
+ # for unit in typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # line_y = round(unit.box.y, 1)
+ # if line_y not in lines:
+ # lines[line_y] = []
+ # lines[line_y].append(unit)
+
+ # # Right-align each line
+ # for line_y, line_units in lines.items():
+ # if not line_units:
+ # continue
+
+ # # Find the rightmost position of this line
+ # line_max_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+
+ # # Calculate how much to shift right
+ # shift_x = box.x2 - line_max_x
+
+ # # Shift all units in this line to the right
+ # for unit in line_units:
+ # if unit.box:
+ # unit.box.x += shift_x
+ # unit.box.x2 += shift_x
+ # if unit.x is not None:
+ # unit.x += shift_x
+ # # Update character box if present
+ # if unit.char and unit.char.box:
+ # unit.char.box.x += shift_x
+ # unit.char.box.x2 += shift_x
+ # if unit.char and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ # unit.char.visual_bbox.box.x += shift_x
+ # unit.char.visual_bbox.box.x2 += shift_x
+ # # Check if output language is Arabic
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # # If Arabic, reverse the line order
+ # if is_arabic and typeset_units:
+ # # Group units by line (using Y coordinates)
+ # lines_dict = {}
+ # for unit in typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # # Round Y coordinate to group units on the same line
+ # line_y = round(unit.box.y, 1)
+ # if line_y not in lines_dict:
+ # lines_dict[line_y] = []
+ # lines_dict[line_y].append(unit)
+
+ # # Sort lines by Y coordinate (top to bottom) and reverse
+ # sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # # Rebuild typeset_units with reversed line order
+ # reversed_typeset_units = []
+ # for line_y in reversed(sorted_line_ys):
+ # reversed_typeset_units.extend(lines_dict[line_y])
+
+ # # Now reposition all units to swap their Y coordinates
+ # # Map old Y positions to new Y positions
+ # y_mapping = {}
+ # for i, old_y in enumerate(sorted_line_ys):
+ # new_y = sorted_line_ys[len(sorted_line_ys) - 1 - i]
+ # y_mapping[old_y] = new_y
+
+ # # Update Y coordinates for all units
+ # for unit in reversed_typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # old_y = round(unit.box.y, 1)
+ # if old_y in y_mapping:
+ # new_y = y_mapping[old_y]
+ # y_diff = new_y - old_y
+ # # Update the unit's Y position
+ # if unit.y is not None:
+ # unit.y += y_diff
+ # if unit.box:
+ # unit.box.y += y_diff
+ # unit.box.y2 += y_diff
+
+ # typeset_units = reversed_typeset_units
+
+ # return typeset_units, all_units_fit
+
+ def create_typesetting_units(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ fonts: dict[str, il_version_1.PdfFont],
+ ) -> list[TypesettingUnit]:
+ if not paragraph.pdf_paragraph_composition:
+ return []
+ result = []
+
+ @cache
+ def get_font(font_id: str, xobj_id: int | None):
+ if xobj_id in fonts:
+ font = fonts[xobj_id][font_id]
+ else:
+ font = fonts[font_id]
+ return font
+
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition is None:
+ continue
+ if composition.pdf_line:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_line.pdf_character
+ ],
+ )
+ elif composition.pdf_character:
+ result.append(
+ TypesettingUnit(
+ char=composition.pdf_character,
+ debug_info=paragraph.debug_info,
+ ),
+ )
+ elif composition.pdf_same_style_characters:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_same_style_characters.pdf_character
+ ],
+ )
+ elif composition.pdf_same_style_unicode_characters:
+ style = composition.pdf_same_style_unicode_characters.pdf_style
+ if style is None:
+ logger.warning(
+ f"Style is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font_id = style.font_id
+ if font_id is None:
+ logger.warning(
+ f"Font ID is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font = get_font(font_id, paragraph.xobj_id)
+ if composition.pdf_same_style_unicode_characters.unicode:
+ unicode_text = composition.pdf_same_style_unicode_characters.unicode
+ shaped_text = self.shape_arabic_text(unicode_text)
+ result.extend(
+ [
+ TypesettingUnit(
+ unicode=char_unicode,
+ font=self.font_mapper.map(
+ font,
+ char_unicode,
+ ),
+ original_font=font,
+ font_size=style.font_size,
+ style=style,
+ xobj_id=paragraph.xobj_id,
+ debug_info=composition.pdf_same_style_unicode_characters.debug_info
+ or False,
+ )
+ for char_unicode in shaped_text # Use shaped_text instead of original
+ if char_unicode not in ("\n",)
+ ],
+ )
+ elif composition.pdf_formula:
+ result.extend([TypesettingUnit(formular=composition.pdf_formula)])
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ result = list(
+ filter(
+ lambda x: x.unicode is None or x.font is not None,
+ result,
+ ),
+ )
+
+ if any(x.width < 0 for x in result):
+ logger.warning("有排版å•元宽度å°Â于 0，请检查å—体映射是å¦æ£确。")
+ return result
+
+ def create_passthrough_composition(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ ) -> list[PdfParagraphComposition]:
+ """从排版å•元创建直接传递的段è½组åˆ。
+
+ Args:
+ typesetting_units: 排版å•元列表
+
+ Returns:
+ 段è½组åˆ列表
+ """
+ composition = []
+ for unit in typesetting_units:
+ if unit.formular:
+ # 对于公å¼Âå•元，直接创建包å«完整公å¼Â的组åˆ
+ composition.append(PdfParagraphComposition(pdf_formula=unit.formular))
+ else:
+ # 对于å—符å•元，使çâ€Â¨Ã¥Å½Å¸Ã¦Å“‰é€»è¾‘
+ chars, curves, forms = unit.passthrough()
+ composition.extend(
+ [PdfParagraphComposition(pdf_character=char) for char in chars],
+ )
+ return composition
+
+ def get_max_right_space(self, current_box: Box, page) -> float:
+ """获å–段è½å³侧最大å¯çâ€Â¨Ã§Â©ÂºÃ©â€”´
+
+ Args:
+ current_box: 当å‰Â段è½的边界æ¡â€
+ page: 当å‰Â页é¢
+
+ Returns:
+ å¯以扩展到的最大 x Ã¥ÂÂæ ‡
+ """
+ # 获å–页é¢的è£Â剪框作为åˆÂ始最大é™Â制
+ max_x = page.cropbox.box.x2 * 0.9
+
+ # 检查所有å¯能的阻挡元ç´Â
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # 跳过当å‰Â段è½
+ continue
+ # åª考虑在当å‰Â段è½å³侧ä¸â€Ã¦Å“‰åž‚ç›´é‡Âå 的元ç´Â
+ if para.box.x > current_box.x and not (
+ para.box.y >= current_box.y2 or para.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, para.box.x)
+ for char in page.pdf_character:
+ if char.box.x > current_box.x and not (
+ char.box.y >= current_box.y2 or char.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, char.box.x)
+ # 检查图形
+ for figure in page.pdf_figure:
+ if figure.box.x > current_box.x and not (
+ figure.box.y >= current_box.y2 or figure.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, figure.box.x)
+
+ return max_x
+
+ def get_max_bottom_space(self, current_box: Box, page: il_version_1.Page) -> float:
+ """获å–段è½下方最大å¯çâ€Â¨Ã§Â©ÂºÃ©â€”´
+
+ Args:
+ current_box: 当å‰Â段è½的边界æ¡â€
+ page: 当å‰Â页é¢
+
+ Returns:
+ å¯以扩展到的最尠y Ã¥ÂÂæ ‡
+ """
+ # 获å–页é¢的è£Â剪框作为åˆÂ始最å°Âé™Â制
+ min_y = page.cropbox.box.y * 1.1
+
+ # 检查所有å¯能的阻挡元ç´Â
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # 跳过当å‰Â段è½
+ continue
+ # åª考虑在当å‰Â段è½下方ä¸â€Ã¦Å“‰æ°´å¹³é‡Âå 的元ç´Â
+ if para.box.y2 < current_box.y and not (
+ para.box.x >= current_box.x2 or para.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, para.box.y2)
+ for char in page.pdf_character:
+ if char.box.y2 < current_box.y and not (
+ char.box.x >= current_box.x2 or char.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, char.box.y2)
+ # 检查图形
+ for figure in page.pdf_figure:
+ if figure.box.y2 < current_box.y and not (
+ figure.box.x >= current_box.x2 or figure.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, figure.box.y2)
+
+ return min_y
+
+ def _update_paragraph_render_order(self, paragraph: il_version_1.PdfParagraph):
+ """
+ é‡Â新设置段è½å„å—符的 render order
+ 主 render order ç‰于 paragraph çš„ renderorder，sub render order 从 1 开始自增
+ """
+ if not hasattr(paragraph, "render_order") or paragraph.render_order is None:
+ return
+
+ main_render_order = paragraph.render_order
+ sub_render_order = 1
+
+ # éÂÂ历段è½的所有组æˆÂ部åˆâ€
+ for composition in paragraph.pdf_paragraph_composition:
+ # 检查å•个å—符
+ if composition.pdf_character:
+ char = composition.pdf_character
+ char.render_order = main_render_order
+ char.sub_render_order = sub_render_order
+ sub_render_order += 1
\ No newline at end of file
diff --git a/babeldoc/format/pdf/document_il/midend/typesetting_v4.py b/babeldoc/format/pdf/document_il/midend/typesetting_v4.py
new file mode 100644
index 0000000000000000000000000000000000000000..d80650398693527568a22a14724fcfbc40ca404d
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/midend/typesetting_v4.py
@@ -0,0 +1,2346 @@
+from __future__ import annotations
+
+import copy
+import logging
+import re
+import statistics
+import unicodedata
+from functools import cache
+
+import pymupdf
+import regex
+from rtree import index
+
+from babeldoc.const import WATERMARK_VERSION
+from babeldoc.format.pdf.document_il import Box
+from babeldoc.format.pdf.document_il import PdfCharacter
+from babeldoc.format.pdf.document_il import PdfCurve
+from babeldoc.format.pdf.document_il import PdfForm
+from babeldoc.format.pdf.document_il import PdfFormula
+from babeldoc.format.pdf.document_il import PdfParagraphComposition
+from babeldoc.format.pdf.document_il import PdfStyle
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.formular_helper import update_formula_data
+from babeldoc.format.pdf.document_il.utils.layout_helper import box_to_tuple
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from babeldoc.format.pdf.translation_config import WatermarkOutputMode
+from arabic_reshaper import reshape
+from bidi.algorithm import get_display
+
+
+logger = logging.getLogger(__name__)
+
+LINE_BREAK_REGEX = regex.compile(
+ r"^["
+ r"a-z"
+ r"A-Z"
+ r"0-9"
+ r"\u00C0-\u00FF" # Latin-1 Supplement
+ r"\u0100-\u017F" # Latin Extended A
+ r"\u0180-\u024F" # Latin Extended B
+ r"\u1E00-\u1EFF" # Latin Extended Additional
+ r"\u2C60-\u2C7F" # Latin Extended C
+ r"\uA720-\uA7FF" # Latin Extended D
+ r"\uAB30-\uAB6F" # Latin Extended E
+ r"\u0250-\u02A0" # IPA Extensions
+ r"\u0400-\u04FF" # Cyrillic
+ r"\u0300-\u036F" # Combining Diacritical Marks
+ r"\u0500-\u052F" # Cyrillic Supplement
+ r"\u0370-\u03FF" # Greek and Coptic
+ r"\u2DE0-\u2DFF" # Cyrillic Extended-A
+ r"\uA650-\uA69F" # Cyrillic Extended-B
+ r"\u1200-\u137F" # Ethiopic
+ r"\u1380-\u139F" # Ethiopic Supplement
+ r"\u2D80-\u2DDF" # Ethiopic Extended
+ r"\uAB00-\uAB2F" # Ethiopic Extended-A
+ r"\U0001E7E0-\U0001E7FF" # Ethiopic Extended-B
+ r"\u0E80-\u0EFF" # Lao
+ r"\u0D00-\u0D7F" # Malayalam
+ r"\u0A80-\u0AFF" # Gujarati
+ r"\u0E00-\u0E7F" # Thai
+ r"\u1000-\u109F" # Myanmar
+ r"\uAA60-\uAA7F" # Myanmar Extended-A
+ r"\uA9E0-\uA9FF" # Myanmar Extended-B
+ r"\U000116D0-\U000116FF" # Myanmar Extended-C
+ r"\u0B80-\u0BFF" # Tamil
+ r"\u0C00-\u0C7F" # Telugu
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0530-\u058F" # Armenian
+ r"\u10A0-\u10FF" # Georgian
+ r"\u1C90-\u1CBF" # Georgian Extended
+ r"\u2D00-\u2D2F" # Georgian Supplement
+ r"\u1780-\u17FF" # Khmer
+ r"\u19E0-\u19FF" # Khmer Symbols
+ r"\U00010B00-\U00010B3F" # Avestan
+ r"\u1D00-\u1D7F" # Phonetic Extensions
+ r"\u1400-\u167F" # Unified Canadian Aboriginal Syllabics
+ r"\u0B00-\u0B7F" # Oriya
+ r"\u0780-\u07BF" # Thaana
+ r"\U0001E900-\U0001E95F" # Adlam
+ r"\u1C80-\u1C8F" # Cyrillic Extended-C
+ r"\U0001E030-\U0001E08F" # Cyrillic Extended-D
+ r"\uA000-\uA48F" # Yi Syllables
+ r"\uA490-\uA4CF" # Yi Radicals
+ r"'"
+ r"-" # Hyphen
+ r"·" # Middle Dot (U+00B7) For CatalÃÂÂ
+ r"Ê»" # Spacing Modifier Letters U+02BB
+ r"]+$"
+)
+
+
+class TypesettingUnit:
+ def __str__(self):
+ return self.try_get_unicode() or ""
+
+ def __init__(
+ self,
+ char: PdfCharacter | None = None,
+ formular: PdfFormula | None = None,
+ unicode: str | None = None,
+ font: pymupdf.Font | None = None,
+ original_font: il_version_1.PdfFont | None = None,
+ font_size: float | None = None,
+ style: PdfStyle | None = None,
+ xobj_id: int | None = None,
+ debug_info: bool = False,
+ ):
+ assert (char is not None) + (formular is not None) + (
+ unicode is not None
+ ) == 1, "Only one of chars and formular can be not None"
+ self.char = char
+ self.formular = formular
+ self.unicode = unicode
+ self.x = None
+ self.y = None
+ self.scale = None
+ self.debug_info = debug_info
+
+ # Cache variables
+ self.box_cache: Box | None = None
+ self.can_break_line_cache: bool | None = None
+ self.is_cjk_char_cache: bool | None = None
+ self.mixed_character_blacklist_cache: bool | None = None
+ self.is_space_cache: bool | None = None
+ self.is_hung_punctuation_cache: bool | None = None
+ self.is_cannot_appear_in_line_end_punctuation_cache: bool | None = None
+ self.can_passthrough_cache: bool | None = None
+ self.width_cache: float | None = None
+ self.height_cache: float | None = None
+
+ self.font_size: float | None = None
+
+ if unicode:
+ assert font_size, "Font size must be provided when unicode is provided"
+ assert style, "Style must be provided when unicode is provided"
+ assert len(unicode) == 1, "Unicode must be a single character"
+ assert xobj_id is not None, (
+ "Xobj id must be provided when unicode is provided"
+ )
+
+ self.font = font
+ if font is not None and hasattr(font, "font_id"):
+ self.font_id = font.font_id
+ else:
+ self.font_id = "base"
+ if original_font:
+ self.original_font = original_font
+ else:
+ self.original_font = None
+
+ self.font_size = font_size
+ self.style = style
+ self.xobj_id = xobj_id
+
+ def try_resue_cache(self, old_tu: TypesettingUnit):
+ if old_tu.is_cjk_char_cache is not None:
+ self.is_cjk_char_cache = old_tu.is_cjk_char_cache
+
+ if old_tu.can_break_line_cache is not None:
+ self.can_break_line_cache = old_tu.can_break_line_cache
+
+ if old_tu.is_space_cache is not None:
+ self.is_space_cache = old_tu.is_space_cache
+
+ if old_tu.is_hung_punctuation_cache is not None:
+ self.is_hung_punctuation_cache = old_tu.is_hung_punctuation_cache
+
+ if old_tu.is_cannot_appear_in_line_end_punctuation_cache is not None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ old_tu.is_cannot_appear_in_line_end_punctuation_cache
+ )
+
+ if old_tu.can_passthrough_cache is not None:
+ self.can_passthrough_cache = old_tu.can_passthrough_cache
+
+ if old_tu.mixed_character_blacklist_cache is not None:
+ self.mixed_character_blacklist_cache = (
+ old_tu.mixed_character_blacklist_cache
+ )
+
+
+ def try_get_unicode(self) -> str | None:
+ if self.char:
+ return self.char.char_unicode
+ elif self.formular:
+ return None
+ elif self.unicode:
+ return self.unicode
+
+ @property
+ def mixed_character_blacklist(self):
+ if self.mixed_character_blacklist_cache is None:
+ self.mixed_character_blacklist_cache = self.calc_mixed_character_blacklist()
+
+ return self.mixed_character_blacklist_cache
+
+ def calc_mixed_character_blacklist(self):
+ unicode = self.try_get_unicode()
+ if unicode:
+ return unicode in [
+ "。",
+ "，",
+ "：",
+ "？",
+ "ï¼ÂÂ",
+ ]
+ return False
+
+ @property
+ def can_break_line(self):
+ if self.can_break_line_cache is None:
+ self.can_break_line_cache = self.calc_can_break_line()
+
+ return self.can_break_line_cache
+
+ def calc_can_break_line(self):
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return True
+ if LINE_BREAK_REGEX.match(unicode):
+ return False
+ return True
+
+ @property
+ def is_cjk_char(self):
+ if self.is_cjk_char_cache is None:
+ self.is_cjk_char_cache = self.calc_is_cjk_char()
+
+ return self.is_cjk_char_cache
+
+ def calc_is_cjk_char(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ if "(cid" in unicode:
+ return False
+ if len(unicode) > 1:
+ return False
+ assert len(unicode) == 1, "Unicode must be a single character"
+ if unicode in [
+ "（",
+ "）",
+ "ã€ÂÂ",
+ "】",
+ "《",
+ "》",
+ "ã€â€Â",
+ "〕",
+ "〈",
+ "〉",
+ "〖",
+ "ã€â€â€",
+ "「",
+ "ã€ÂÂ",
+ "『",
+ "ã€ÂÂ",
+ "ã€ÂÂ",
+ "。",
+ "：",
+ "？",
+ "ï¼ÂÂ",
+ "，",
+ ]:
+ return True
+ if unicode:
+ if re.match(
+ r"^["
+ r"\u3000-\u303f" # CJK Symbols and Punctuation
+ r"\u3040-\u309f" # Hiragana
+ r"\u30a0-\u30ff" # Katakana
+ r"\u3100-\u312f" # Bopomofo
+ r"\uac00-\ud7af" # Hangul Syllables
+ r"\u1100-\u11ff" # Hangul Jamo
+ r"\u3130-\u318f" # Hangul Compatibility Jamo
+ r"\ua960-\ua97f" # Hangul Jamo Extended-A
+ r"\ud7b0-\ud7ff" # Hangul Jamo Extended-B
+ r"\u3190-\u319f" # Kanbun
+ r"\u3200-\u32ff" # Enclosed CJK Letters and Months
+ r"\u3300-\u33ff" # CJK Compatibility
+ r"\ufe30-\ufe4f" # CJK Compatibility Forms
+ r"\u4e00-\u9fff" # CJK Unified Ideographs
+ r"\u2e80-\u2eff" # CJK Radicals Supplement
+ r"\u31c0-\u31ef" # CJK Strokes
+ r"\u2f00-\u2fdf" # Kangxi Radicals
+ r"\ufe10-\ufe1f" # Vertical Forms
+ r"]+$",
+ unicode,
+ ):
+ return True
+ try:
+ unicodedata_name = unicodedata.name(unicode)
+ return (
+ "CJK UNIFIED IDEOGRAPH" in unicodedata_name
+ or "FULLWIDTH" in unicodedata_name
+ )
+ except ValueError:
+ return False
+ return False
+
+ @property
+ def is_space(self):
+ if self.is_space_cache is None:
+ self.is_space_cache = self.calc_is_space()
+
+ return self.is_space_cache
+
+ def calc_is_space(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ return unicode == " "
+
+ @property
+ def is_hung_punctuation(self):
+ if self.is_hung_punctuation_cache is None:
+ self.is_hung_punctuation_cache = self.calc_is_hung_punctuation()
+
+ return self.is_hung_punctuation_cache
+
+ def calc_is_hung_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+
+ if unicode:
+ return unicode in [
+ # 英文标点
+ ",",
+ ".",
+ ":",
+ ";",
+ "?",
+ "!",
+ # ä¸ÂÂ文点å·
+ "，", # é€â€â€ÃƒÂ¥Ã‚·
+ "。", # Ã¥ÂÂ¥å·
+ "．", # 全角åÂÂ¥å·
+ "ã€ÂÂ", # é¡¿å·
+ "：", # 冒å·
+ "ï¼›", # 分å·
+ "ï¼ÂÂ", # å¹å·
+ "‼", # Ã¥ÂÂŒå¹å·
+ "？", # éâ€â€Ã‚®Ã¥Â·
+ "â‡", # Ã¥ÂÂΎâ€â€Ã‚®Ã¥Â·
+ # 结æŸ引å·
+ "â€ÂÂ", # å³åÂŒ引å·
+ "’", # å³å•引å·
+ "ã€ÂÂ", # å³直角å•引å·
+ "ã€ÂÂ", # å³直角åÂŒ引å·
+ # 结æŸ括å·
+ ")", # å³圆括å·
+ "]", # å³方括å·
+ "}", # å³花括å·
+ "）", # å³圆括å·
+ "〕", # å³龟çâ€Â²æ‹¬å·
+ "〉", # å³å•书åÂÂÂÂå·
+ "】", # å³黑色方头括å·
+ "ã€â€â€", # å³空白方头括å·
+ "ï¼½", # 全角å³方括å·
+ "ï½ÂÂ", # 全角å³花括å·
+ # 结æŸåÂŒ书åÂÂÂÂå·
+ "》", # å³åÂŒ书åÂÂÂÂå·
+ # 连接å·
+ "～", # 全角波浪å·
+ "-", # è¿žåÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥â€¡ÂÂå·
+ "–", # çŸÂÂ破折å· (EN DASH)
+ "â€â€Â", # 长破折å· (EM DASH)
+ # éâ€â€Ã‚´Ã©Å¡â€Âå·
+ "·", # ä¸ÂÂéâ€â€Ã‚´Ã§â€šÂ¹
+ "・", # 片å‡åÂÂÂÂä¸ÂÂéâ€â€Ã‚´Ã§â€šÂ¹
+ "‧", # è¿žåÂÂâ€â€ÃƒÂ§Ã¢â‚¬Å¡Ã‚¹
+ # 分éšâ€Âå·
+ "/", # æ–œæÂÂÂÂ
+ "ï¼ÂÂ", # 全角斜æÂÂÂÂ
+ "â„", # 分数斜æÂÂÂÂ
+ ]
+ return False
+
+ @property
+ def is_cannot_appear_in_line_end_punctuation(self):
+ if self.is_cannot_appear_in_line_end_punctuation_cache is None:
+ self.is_cannot_appear_in_line_end_punctuation_cache = (
+ self.calc_is_cannot_appear_in_line_end_punctuation()
+ )
+
+ return self.is_cannot_appear_in_line_end_punctuation_cache
+
+ def calc_is_cannot_appear_in_line_end_punctuation(self):
+ if self.formular:
+ return False
+ unicode = self.try_get_unicode()
+ if not unicode:
+ return False
+ return unicode in [
+ # 开始引å·
+ "“", # å·¦åÂŒ引å·
+ "‘", # å·¦å•引å·
+ "「", # 左直角å•引å·
+ "『", # 左直角åÂŒ引å·
+ # 开始括å·
+ "(", # 左圆括å·
+ "[", # 左方括å·
+ "{", # 左花括å·
+ "（", # 左圆括å·
+ "ã€â€Â", # 左龟çâ€Â²æ‹¬å·
+ "〈", # å·¦å•书åÂÂÂÂå·
+ "《", # å·¦åÂŒ书åÂÂÂÂå·
+ # 开始å•åÂŒ书åÂÂÂÂå·
+ "〖", # 左空白方头括å·
+ "〘", # 左黑色方头括å·
+ "〚", # å·¦å•书åÂÂÂÂå·
+ ]
+
+ def passthrough(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ if self.char:
+ return [self.char], [], []
+ elif self.formular:
+ return (
+ self.formular.pdf_character,
+ self.formular.pdf_curve,
+ self.formular.pdf_form,
+ )
+ elif self.unicode:
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ logger.error(f"Cannot passthrough unicode. TypesettingUnit: {self}. ")
+ return [], [], []
+
+ @property
+ def can_passthrough(self):
+ if self.can_passthrough_cache is None:
+ self.can_passthrough_cache = self.calc_can_passthrough()
+
+ return self.can_passthrough_cache
+
+ def calc_can_passthrough(self):
+ return self.unicode is None
+
+ def calculate_box(self):
+ if self.char:
+ box = copy.deepcopy(self.char.box)
+ if self.char.visual_bbox and self.char.visual_bbox.box:
+ box.y = self.char.visual_bbox.box.y
+ box.y2 = self.char.visual_bbox.box.y2
+ # return self.char.visual_bbox.box
+
+ return box
+ elif self.formular:
+ return self.formular.box
+ # if self.formular.x_offset <= 0.5:
+ # return self.formular.box
+ # formular_box = copy.copy(self.formular.box)
+ # formular_box.x2 += self.formular.x_advance
+ # return formular_box
+ elif self.unicode:
+ char_width = self.font.char_lengths(self.unicode, self.font_size)[0]
+ if self.x is None or self.y is None or self.scale is None:
+ return Box(0, 0, char_width, self.font_size)
+ return Box(self.x, self.y, self.x + char_width, self.y + self.font_size)
+
+ @property
+ def box(self):
+ if not self.box_cache:
+ self.box_cache = self.calculate_box()
+
+ return self.box_cache
+
+ @property
+ def width(self):
+ if self.width_cache is None:
+ self.width_cache = self.calc_width()
+
+ return self.width_cache
+
+ def calc_width(self):
+ box = self.box
+ return box.x2 - box.x
+
+ @property
+ def height(self):
+ if self.height_cache is None:
+ self.height_cache = self.calc_height()
+
+ return self.height_cache
+
+ def calc_height(self):
+ box = self.box
+ return box.y2 - box.y
+
+ def relocate(
+ self,
+ x: float,
+ y: float,
+ scale: float,
+ ) -> TypesettingUnit:
+ """é‡ÂÂ定ä½ÂÂ并缩æâ€Â¾æŽ’版å•元
+
+ Args:
+ x: æ–°çš„ x Ã¥ÂÂÂÂæ ‡
+ y: æ–°çš„ y Ã¥ÂÂÂÂæ ‡
+ scale: 缩æâ€Â¾å› åÂÂÂÂ
+
+ Returns:
+ 新的排版å•元
+ """
+ if self.char:
+ # 创建新的åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â¯Â¹Ã¨Â±Â¡
+ new_char = PdfCharacter(
+ pdf_character_id=self.char.pdf_character_id,
+ char_unicode=self.char.char_unicode,
+ box=Box(
+ x=x,
+ y=y,
+ x2=x + self.width * scale,
+ y2=y + self.height * scale,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.char.pdf_style.font_id,
+ font_size=self.char.pdf_style.font_size * scale,
+ graphic_state=self.char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=self.char.vertical,
+ advance=self.char.advance * scale if self.char.advance else None,
+ debug_info=self.debug_info,
+ xobj_id=self.char.xobj_id,
+ )
+ new_tu = TypesettingUnit(char=new_char)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.formular:
+ # 创建新的公å¼ÂÂ对象，ä¿ÂÂæŒÂÂ内部åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã§Å¡â€žÃ§â€ºÂ¸Ã¥Â¯Â¹Ã¤Â½ÂÂç½®
+ new_chars = []
+ min_x = self.formular.box.x
+ min_y = self.formular.box.y
+
+ for char in self.formular.pdf_character:
+ # 计ç®â€â€ÃƒÂ§Ã¢â‚¬ÂºÃ‚¸Ã¥Â¯Â¹Ã¤Â½ÂÂç½®
+ rel_x = char.box.x - min_x
+ rel_y = char.box.y - min_y
+
+ visual_rel_x = char.visual_bbox.box.x - min_x
+ visual_rel_y = char.visual_bbox.box.y - min_y
+
+ # 创建新的åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â¯Â¹Ã¨Â±Â¡
+ new_char = PdfCharacter(
+ pdf_character_id=char.pdf_character_id,
+ char_unicode=char.char_unicode,
+ box=Box(
+ x=x + (rel_x + self.formular.x_offset) * scale,
+ y=y + (rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (rel_x + (char.box.x2 - char.box.x) + self.formular.x_offset)
+ * scale,
+ y2=y
+ + (rel_y + (char.box.y2 - char.box.y) + self.formular.y_offset)
+ * scale,
+ ),
+ visual_bbox=il_version_1.VisualBbox(
+ box=Box(
+ x=x + (visual_rel_x + self.formular.x_offset) * scale,
+ y=y + (visual_rel_y + self.formular.y_offset) * scale,
+ x2=x
+ + (
+ visual_rel_x
+ + (char.visual_bbox.box.x2 - char.visual_bbox.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=y
+ + (
+ visual_rel_y
+ + (char.visual_bbox.box.y2 - char.visual_bbox.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ ),
+ ),
+ pdf_style=PdfStyle(
+ font_id=char.pdf_style.font_id,
+ font_size=char.pdf_style.font_size * scale,
+ graphic_state=char.pdf_style.graphic_state,
+ ),
+ scale=scale,
+ vertical=char.vertical,
+ advance=char.advance * scale if char.advance else None,
+ xobj_id=char.xobj_id,
+ )
+ new_chars.append(new_char)
+
+ # Calculate bounding box from new_chars
+ min_x = min(char.visual_bbox.box.x for char in new_chars)
+ min_y = min(char.visual_bbox.box.y for char in new_chars)
+ max_x = max(char.visual_bbox.box.x2 for char in new_chars)
+ max_y = max(char.visual_bbox.box.y2 for char in new_chars)
+
+ new_formula = PdfFormula(
+ box=Box(
+ x=min_x,
+ y=min_y,
+ x2=max_x,
+ y2=max_y,
+ ),
+ pdf_character=new_chars,
+ x_offset=self.formular.x_offset * scale,
+ y_offset=self.formular.y_offset * scale,
+ x_advance=self.formular.x_advance * scale,
+ )
+
+ # Handle contained curves
+ new_curves = []
+ for curve in self.formular.pdf_curve:
+ new_curve = self._transform_curve_for_relocation(
+ curve,
+ self.formular.box.x,
+ self.formular.box.y,
+ x,
+ y,
+ scale,
+ )
+ new_curves.append(new_curve)
+ new_formula.pdf_curve = new_curves
+
+ # Handle contained forms
+ new_forms = []
+ for form in self.formular.pdf_form:
+ new_form = self._transform_form_for_relocation(
+ form, self.formular.box.x, self.formular.box.y, x, y, scale
+ )
+ new_forms.append(new_form)
+ new_formula.pdf_form = new_forms
+
+ update_formula_data(new_formula)
+
+ new_tu = TypesettingUnit(formular=new_formula)
+ new_tu.try_resue_cache(self)
+ return new_tu
+
+ elif self.unicode:
+ # 对于 Unicode Ã¥ÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¯Â¼Å’我们å˜储新的ä½ÂÂ置信æ¯
+ new_unit = TypesettingUnit(
+ unicode=self.unicode,
+ font=self.font,
+ original_font=self.original_font,
+ font_size=self.font_size * scale,
+ style=self.style,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ new_unit.x = x
+ new_unit.y = y
+ new_unit.scale = scale
+ new_unit.try_resue_cache(self)
+ return new_unit
+
+ def _transform_curve_for_relocation(
+ self,
+ curve,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a curve for formula relocation."""
+ import copy
+
+ new_curve = copy.deepcopy(curve)
+
+ if new_curve.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_curve.box.x - original_formula_x
+ rel_y = new_curve.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_curve.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (
+ rel_x
+ + (new_curve.box.x2 - new_curve.box.x)
+ + self.formular.x_offset
+ )
+ * scale,
+ y2=new_y
+ + (
+ rel_y
+ + (new_curve.box.y2 - new_curve.box.y)
+ + self.formular.y_offset
+ )
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original CTM
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_curve.relocation_transform = list(relocation_matrix)
+
+ return new_curve
+
+ def _transform_form_for_relocation(
+ self,
+ form,
+ original_formula_x: float,
+ original_formula_y: float,
+ new_x: float,
+ new_y: float,
+ scale: float,
+ ):
+ """Transform a form for formula relocation."""
+ import copy
+
+ new_form = copy.deepcopy(form)
+
+ if new_form.box:
+ # Calculate relative position to formula's original position (same as chars)
+ rel_x = new_form.box.x - original_formula_x
+ rel_y = new_form.box.y - original_formula_y
+
+ # Apply same transformation as characters
+ new_form.box = Box(
+ x=new_x + (rel_x + self.formular.x_offset) * scale,
+ y=new_y + (rel_y + self.formular.y_offset) * scale,
+ x2=new_x
+ + (rel_x + (new_form.box.x2 - new_form.box.x) + self.formular.x_offset)
+ * scale,
+ y2=new_y
+ + (rel_y + (new_form.box.y2 - new_form.box.y) + self.formular.y_offset)
+ * scale,
+ )
+
+ # Set relocation transform instead of modifying original matrices
+ translation_x = (
+ new_x + self.formular.x_offset * scale - original_formula_x * scale
+ )
+ translation_y = (
+ new_y + self.formular.y_offset * scale - original_formula_y * scale
+ )
+
+ # Create relocation transformation matrix
+ from babeldoc.format.pdf.document_il.utils.matrix_helper import (
+ create_translation_and_scale_matrix,
+ )
+
+ relocation_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale
+ )
+ new_form.relocation_transform = list(relocation_matrix)
+
+ return new_form
+
+ def render(
+ self,
+ ) -> tuple[list[PdfCharacter], list[PdfCurve], list[PdfForm]]:
+ """渲染排版å•元为 PdfCharacter åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+
+ Returns:
+ PdfCharacter åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ """
+ if self.can_passthrough:
+ return self.passthrough()
+ elif self.unicode:
+ assert self.x is not None, (
+ "x position must be set, should be set by `relocate`"
+ )
+ assert self.y is not None, (
+ "y position must be set, should be set by `relocate`"
+ )
+ assert self.scale is not None, (
+ "scale must be set, should be set by `relocate`"
+ )
+ x = self.x
+ y = self.y
+ # if self.original_font and self.font and hasattr(self.original_font, "descent") and hasattr(self.font, "descent_fontmap"):
+ # original_descent = self.original_font.descent
+ # new_descent = self.font.descent_fontmap
+ # y -= (original_descent - new_descent) * self.font_size / 1000
+
+ # 计ç®â€â€ÃƒÂ¥Ã‚Ââ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â®Â½Ã¥ÂºÂ¦
+ char_width = self.width
+
+ # Handle case when font is None (no suitable font found for this character)
+ if self.font is None:
+ logger.warning(
+ f"No font available for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using font_id='{self.font_id}' with glyph_id=0"
+ )
+ glyph_id = 0 # Use glyph 0 as fallback (usually .notdef)
+ else:
+ glyph_id = self.font.has_glyph(ord(self.unicode))
+ if glyph_id == 0 or glyph_id is None:
+ logger.warning(
+ f"Font '{self.font_id}' doesn't have glyph for character '{self.unicode}' (U+{ord(self.unicode):04X}), "
+ f"using glyph_id=0"
+ )
+ glyph_id = 0
+
+ new_char = PdfCharacter(
+ pdf_character_id=glyph_id,
+ char_unicode=self.unicode,
+ box=Box(
+ x=x, # 使çâ€Â¨å˜储的ä½ÂÂç½®
+ y=y,
+ x2=x + char_width,
+ y2=y + self.font_size,
+ ),
+ pdf_style=PdfStyle(
+ font_id=self.font_id,
+ font_size=self.font_size,
+ graphic_state=self.style.graphic_state,
+ ),
+ scale=self.scale,
+ vertical=False,
+ advance=char_width,
+ xobj_id=self.xobj_id,
+ debug_info=self.debug_info,
+ )
+ return [new_char], [], []
+ else:
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ logger.error(f"Unknown typesetting unit. TypesettingUnit: {self}. ")
+ return [], [], []
+
+
+class Typesetting:
+ stage_name = "Typesetting"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.font_mapper = FontMapper(translation_config)
+ self.translation_config = translation_config
+ self.lang_code = self.translation_config.lang_out.upper()
+ # Ensure detailed_logger attribute exists to avoid attribute access errors
+ self.detailed_logger = None
+ self.is_cjk = (
+ # Why zh-CN/zh-HK/zh-TW here but not zh-Hans and so on?
+ # See https://funstory-ai.github.io/BabelDOC/supported_languages/
+ ("ZH" in self.lang_code) # C
+ or ("JA" in self.lang_code)
+ or ("JP" in self.lang_code) # J
+ or ("KR" in self.lang_code) # K
+ or ("CN" in self.lang_code)
+ or ("HK" in self.lang_code)
+ or ("TW" in self.lang_code)
+ )
+
+ def preprocess_document(self, document: il_version_1.Document, pbar):
+ """预处ç†文档，获å–æ¯ÂÂ个段è½的最优缩æâ€Â¾å› åÂÂÂÂ，ä¸ÂÂ执行实际排版"""
+ all_scales: list[float] = []
+ all_paragraphs: list[il_version_1.PdfParagraph] = []
+
+ for page in document.page:
+ pbar.advance()
+ # 准备åÂÂâ€â€ÃƒÂ¤Ã‚½â€œÃ¤Â¿Â¡Ã¦Â¯（å¤ÂÂ制自 render_page 的逻辑）
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if (
+ xobj.xobj_id in fonts
+ and isinstance(fonts[xobj.xobj_id], dict)
+ and font.font_id
+ ):
+ fonts[xobj.xobj_id][font.font_id] = font
+
+ # 处ç†æ¯ÂÂ个段è½
+ for paragraph in page.pdf_paragraph:
+ all_paragraphs.append(paragraph)
+ unit_count = 0
+ try:
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ unit_count = len(typesetting_units)
+ for unit in typesetting_units:
+ if unit.formular:
+ unit_count += len(unit.formular.pdf_character) - 1
+
+ # 如果所有å•元都å¯以直接传递，则 scale = 1.0
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.optimal_scale = 1.0
+ else:
+ # 获å–最优缩æâ€Â¾å› åÂÂÂÂ
+ optimal_scale = self._get_optimal_scale(
+ paragraph, page, typesetting_units
+ )
+ paragraph.optimal_scale = optimal_scale
+ except Exception as e:
+ # 如果预处ç†出éâ€Â™ï¼Œé»˜è®¤ä½¿çâ€Â¨ 1.0 缩æâ€Â¾å› åÂÂÂÂ
+ logger.warning(f"预处ç†段è½æâ€â€Ã‚¶Ã¥â€¡ÂºÃ©â€Â™ï¼š{e}")
+ paragraph.optimal_scale = 1.0
+
+ if paragraph.optimal_scale is not None:
+ all_scales.extend([paragraph.optimal_scale] * unit_count)
+
+ # 获å–缩æâ€Â¾å› åÂÂÂÂçš„ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ if all_scales:
+ try:
+ modes = statistics.multimode(all_scales)
+ mode_scale = min(modes)
+ except statistics.StatisticsError:
+ logger.warning(
+ "Could not find a mode for paragraph scales. Falling back to median."
+ )
+ mode_scale = statistics.median(all_scales)
+ # 将所有大于ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã§Å¡â€žÃ¥â‚¬Â¼Ã¤Â¿Â®Ã¦â€Â¹ä¸ºä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ for paragraph in all_paragraphs:
+ if (
+ paragraph.optimal_scale is not None
+ and paragraph.optimal_scale > mode_scale
+ ):
+ paragraph.optimal_scale = mode_scale
+ else:
+ logger.error(
+ "document_scales is empty, there seems no paragraph in this PDF"
+ )
+
+ def shape_arabic_text(self, text: str) -> str:
+ """Shape and reorder Arabic text if output language is Arabic.
+
+ Args:
+ text: Input text to shape
+
+ Returns:
+ Shaped and reordered text if language is Arabic, original text otherwise
+ """
+ if not text:
+ return text
+
+ # Robust Arabic output detection: accept explicit 'ar', 'ara', 'arabic'
+ # or formats containing '-ar', '->ar', or '/ar' as a target marker (e.g. 'en-ar', 'en->ar')
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = False
+ if lang_out in ("en-ar, ar", "ara", "arabic"):
+ is_arabic = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic = True
+
+ if is_arabic:
+ logger.debug("Shaping Arabic text")
+ # Flip parentheses and brackets for RTL display
+ # text = text.replace("(", "\x00")
+ # text = text.replace(")", "(")
+ # text = text.replace("\x00", ")")
+ # text = text.replace("[", "\x01")
+ # text = text.replace("]", "[")
+ # text = text.replace("\x01", "]")
+ # text = text.replace("{", "\x02")
+ # text = text.replace("}", "{")
+ # text = text.replace("\x02", "}")
+ try:
+ if not re.search(r'[\uFB50-\uFDFF\uFE70-\uFEFF]', text):
+ # Extract inline tags before shaping to prevent corruption
+ tag_pattern = r'<[^>]+>'
+ tags = []
+ tag_positions = []
+ for match in re.finditer(tag_pattern, text):
+ tags.append(match.group(0))
+ tag_positions.append((match.start(), match.end()))
+
+ if tags:
+ text_without_tags = text
+ placeholder_map = {}
+ for i in range(len(tags) - 1, -1, -1):
+ start, end = tag_positions[i]
+ placeholder = f"\u200D{i}\u200D"
+ placeholder_map[placeholder] = tags[i]
+ text_without_tags = text_without_tags[:start] + placeholder + text_without_tags[end:]
+
+ # Reshape Arabic text for proper character joining
+ from arabic_reshaper import ArabicReshaper
+ configuration = {
+ 'delete_harakat': False, # Keep diacritical marks
+ 'support_ligatures': True, # Support Arabic ligatures
+ 'RIAL SIGN': True,
+ 'ARABIC COMMA': True,
+ 'ARABIC SEMICOLON': True,
+ 'ARABIC QUESTION MARK': True,
+ 'ZWNJ': True, # Zero Width Non-Joiner
+ }
+
+ reshaper = ArabicReshaper(configuration=configuration)
+ reshaped_text = reshaper.reshape(text_without_tags)
+ display_text = get_display(reshaped_text, base_dir='R')
+
+ # Restore tags
+ # for placeholder, tag in placeholder_map.items():
+ # display_text = display_text.replace(placeholder, tag)
+ return display_text
+ else:
+ # No tags, process normally
+ # Reshape Arabic text for proper character joining
+ from arabic_reshaper import ArabicReshaper
+ configuration = {
+ 'delete_harakat': False, # Keep diacritical marks
+ 'support_ligatures': True, # Support Arabic ligatures
+ 'RIAL SIGN': True,
+ 'ARABIC COMMA': True,
+ 'ARABIC SEMICOLON': True,
+ 'ARABIC QUESTION MARK': True,
+ 'ZWNJ': True, # Zero Width Non-Joiner
+ }
+
+ reshaper = ArabicReshaper(configuration=configuration)
+ reshaped_text = reshaper.reshape(text)
+ display_text = get_display(reshaped_text, base_dir='R')
+ return display_text
+ else:
+ display_text = text
+ return display_text
+ except Exception as e:
+ logger.warning(f"Failed to shape Arabic text: {e}")
+ return text
+
+ return text
+
+ # # Robust Arabic output detection: accept explicit 'ar', 'ara', 'arabic'
+ # # or formats containing '-ar', '->ar', or '/ar' as a target marker (e.g. 'en-ar', 'en->ar')
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar, ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # if is_arabic:
+ # logger.debug("Shaping Arabic text")
+ # # Flip parentheses and brackets for RTL display
+ # # text = text.replace("(", "\x00")
+ # # text = text.replace(")", "(")
+ # # text = text.replace("\x00", ")")
+ # # text = text.replace("[", "\x01")
+ # # text = text.replace("]", "[")
+ # # text = text.replace("\x01", "]")
+ # # text = text.replace("{", "\x02")
+ # # text = text.replace("}", "{")
+ # # text = text.replace("\x02", "}")
+ # try:
+ # if not re.search(r'[\uFB50-\uFDFF\uFE70-\uFEFF]', text):
+ # # Reshape Arabic text for proper character joining
+ # from arabic_reshaper import ArabicReshaper
+ # configuration = {
+ # 'delete_harakat': False, # Keep diacritical marks
+ # 'support_ligatures': True, # Support Arabic ligatures
+ # 'RIAL SIGN': True,
+ # 'ARABIC COMMA': True,
+ # 'ARABIC SEMICOLON': True,
+ # 'ARABIC QUESTION MARK': True,
+ # 'ZWNJ': True, # Zero Width Non-Joiner
+ # }
+
+ # reshaper = ArabicReshaper(configuration=configuration)
+ # reshaped_text = reshaper.reshape(text)
+ # display_text = get_display(reshaped_text, base_dir='R')
+ # else:
+ # display_text = text
+ # return display_text
+ # except Exception as e:
+ # logger.warning(f"Failed to shape Arabic text: {e}")
+ # return text
+
+ # return text
+
+ def _find_optimal_scale_and_layout(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ initial_scale: float = 1.0,
+ use_english_line_break: bool = True,
+ apply_layout: bool = False,
+ ) -> tuple[float, list[TypesettingUnit] | None]:
+ """查找最优缩æâ€Â¾å› åÂÂÂÂ并å¯选择性地执行布局
+
+ Args:
+ paragraph: 段è½对象
+ page: 页é¢对象
+ typesetting_units: 排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ initial_scale: åˆÂÂ始缩æâ€Â¾å› åÂÂÂÂ
+ use_english_line_break: 是å¦使çâ€Â¨è‹±æ–‡æ¢行规则
+ apply_layout: 是å¦åºâ€Âçâ€Â¨å¸ƒå±€åˆ° paragraph（True æâ€â€Ã‚¶Ã¦â€°Â§Ã¨Â¡Å’实际排版）
+
+ Returns:
+ tuple[float, list[TypesettingUnit] | None]: (最终缩æâ€Â¾å› åÂÂÂÂ，排版åÂÂŽçš„å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨Ã¦Ë†â€“ None)
+ """
+ if not paragraph.box:
+ return initial_scale, None
+
+ box = paragraph.box
+ scale = initial_scale
+ line_skip = 1.50 if self.is_cjk else 1.3
+ min_scale = 0.1
+ expand_space_flag = 0
+ final_typeset_units = None
+
+ while scale >= min_scale:
+ try:
+ # Check if Arabic to disable English line breaking
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic_layout = False
+ if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ is_arabic_layout = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic_layout = True
+
+ # For Arabic, disable English line breaking to prevent premature breaks
+ effective_line_break = use_english_line_break and not is_arabic_layout
+
+ # å°ÂÂ试布局排版å•元
+ typeset_units, all_units_fit = self._layout_typesetting_units(
+ typesetting_units,
+ box,
+ scale,
+ line_skip,
+ paragraph,
+ effective_line_break,
+ )
+
+ # 如果所有å•元都æâ€Â¾å¾â€â€ÃƒÂ¤Ã‚¸â€¹
+ if all_units_fit:
+ # Apply RTL margin mirroring for Arabic documents
+ if is_arabic_layout:
+ typeset_units = self._mirror_margins_for_rtl(
+ typeset_units,
+ box,
+ paragraph
+ )
+
+ if apply_layout:
+ # 实际åºâ€Âçâ€Â¨æŽ’版结果
+ paragraph.scale = scale
+ paragraph.pdf_paragraph_composition = []
+ for unit in typeset_units:
+ chars, curves, forms = unit.render()
+ for char in chars:
+ paragraph.pdf_paragraph_composition.append(
+ PdfParagraphComposition(pdf_character=char),
+ )
+ for curve in curves:
+ page.pdf_curve.append(curve)
+ for form in forms:
+ page.pdf_form.append(form)
+ final_typeset_units = typeset_units
+ return scale, final_typeset_units
+ except Exception:
+ # 如果布局检查出éâ€Â™ï¼Œç»§ç»ÂÂå°ÂÂ试下一个缩æâ€Â¾å› åÂÂÂÂ
+ pass
+
+ # 添加与原 retypeset 一致的逻辑检查
+ if not hasattr(paragraph, "debug_id") or not paragraph.debug_id:
+ return scale, final_typeset_units
+
+ # å‡ÂÂå°ÂÂ缩æâ€Â¾å› åÂÂÂÂ
+ if scale > 0.6:
+ scale -= 0.05
+ else:
+ scale -= 0.1
+
+ if scale < 0.7:
+ space_expanded = False # 标记是å¦æˆÂÂ功扩展了空éâ€â€Ã‚´
+
+ if expand_space_flag == 0:
+ # å°ÂÂ试å‘下扩展
+ try:
+ min_y = self.get_max_bottom_space(box, page) + 2
+ if min_y < box.y:
+ expanded_box = Box(x=box.x, y=min_y, x2=box.x2, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # 更新段è½的边界æ¡â€Â
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 1
+
+ # åª有æˆÂÂ功扩展空éâ€â€Ã‚´Ã¦â€â€Ã‚¶Ã¦â€°Â continue，å¦则继ç»ÂÂå‡ÂÂå° scale
+ if space_expanded:
+ continue
+
+ elif expand_space_flag == 1:
+ # å°ÂÂ试å‘å³扩展
+ try:
+ max_x = self.get_max_right_space(box, page) - 5
+ if max_x > box.x2:
+ expanded_box = Box(x=box.x, y=box.y, x2=max_x, y2=box.y2)
+ box = expanded_box
+ if apply_layout:
+ # 更新段è½的边界æ¡â€Â
+ paragraph.box = expanded_box
+ space_expanded = True
+ except Exception:
+ pass
+ expand_space_flag = 2
+
+ # åª有æˆÂÂ功扩展空éâ€â€Ã‚´Ã¦â€â€Ã‚¶Ã¦â€°Â continue，å¦则继ç»ÂÂå‡ÂÂå° scale
+ if space_expanded:
+ continue
+
+ # åª有在扩展å°ÂÂ试阶段 (expand_space_flag < 2) ä¸â€Â扩展失败æâ€â€Ã‚¶Ã¦â€°ÂÂé‡ÂÂç½® scale
+ # 当 expand_space_flag >= 2 æâ€â€Ã‚¶Ã¯Â¼Å’说明已ç»ÂÂå°ÂÂ试过所有扩展，åºâ€Â该继ç»ÂÂæ£常的 scale å‡ÂÂå°ÂÂ
+ if expand_space_flag < 2:
+ # 如果æâ€â€Ã‚ Ã¦Â³â€¢Ã¦â€°Â©Ã¥Â±â€¢Ã§Â©ÂºÃ©â€â€Ã‚´Ã¯Â¼Å’é‡ÂÂç½® scale 并继ç»ÂÂ循环
+ scale = 1.0
+
+ # 如果ä»ÂÂ然æâ€Â¾ä¸ÂÂ下，å°ÂÂ试去除英文æ¢行é™ÂÂ制
+ if use_english_line_break:
+ return self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ initial_scale,
+ use_english_line_break=False,
+ apply_layout=apply_layout,
+ )
+
+ # 最åÂÂŽè¿â€Â回最å°ÂÂ缩æâ€Â¾å› åÂÂÂÂ
+ return min_scale, final_typeset_units
+
+ def _get_optimal_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ use_english_line_break: bool = True,
+ ) -> float:
+ """获å–段è½的最优缩æâ€Â¾å› åÂÂÂÂ，ä¸ÂÂ执行实际排版"""
+ scale, _ = self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ 1.0,
+ use_english_line_break,
+ apply_layout=False,
+ )
+ return scale
+
+ def retypeset_with_precomputed_scale(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ typesetting_units: list[TypesettingUnit],
+ precomputed_scale: float,
+ use_english_line_break: bool = True,
+ ):
+ """使çâ€Â¨é¢„计ç®â€â€ÃƒÂ§Ã…¡â€žÃ§Â¼Â©Ã¦â€Â¾å› åÂÂÂÂ进行排版"""
+ if not paragraph.box:
+ return
+
+ # 使çâ€Â¨é€šçâ€Â¨æ–¹æ³•è¿›è¡ŒæŽ’版，传入预计ç®â€â€ÃƒÂ§Ã…¡â€žÃ§Â¼Â©Ã¦â€Â¾å› åÂÂÂÂ作为åˆÂÂ始值
+ self._find_optimal_scale_and_layout(
+ paragraph,
+ page,
+ typesetting_units,
+ precomputed_scale,
+ use_english_line_break,
+ apply_layout=True,
+ )
+
+ def typesetting_document(self, document: il_version_1.Document):
+ # Add detailed logging at the start
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Started")
+
+ # 原有的æŽ'版逻è¾'
+ if self.translation_config.progress_monitor:
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ len(document.page) * 2,
+ ) as pbar:
+ # 预处ç†ï¼šèŽ·å–所有段è½çš„最优缩æ"¾å› Ã¥ÂÂÂ
+ self.preprocess_document(document, pbar)
+
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+ pbar.advance()
+ else:
+ for page_idx, page in enumerate(document.page):
+ self.translation_config.raise_if_cancelled()
+
+ # Add detailed logging for each page
+ if self.detailed_logger:
+ self.detailed_logger.log_step(
+ f"Typesetting Page {page_idx + 1}",
+ f"Paragraphs to typeset: {len(page.pdf_paragraph) if hasattr(page, 'pdf_paragraph') else 0}"
+ )
+
+ self.render_page(page)
+
+ # Add detailed logging at the end
+ if self.detailed_logger:
+ self.detailed_logger.log_step("Typesetting Complete")
+
+ def render_page(self, page: il_version_1.Page):
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ] = {f.font_id: f for f in page.pdf_font if f.font_id}
+ page_fonts = {f.font_id: f for f in page.pdf_font if f.font_id}
+ for k, v in self.font_mapper.fontid2font.items():
+ fonts[k] = v
+ for xobj in page.pdf_xobject:
+ if xobj.xobj_id is not None:
+ fonts[xobj.xobj_id] = page_fonts.copy()
+ for font in xobj.pdf_font:
+ if font.font_id:
+ fonts[xobj.xobj_id][font.font_id] = font
+ if (
+ page.page_number == 0
+ and self.translation_config.watermark_output_mode
+ == WatermarkOutputMode.Watermarked
+ ):
+ self.add_watermark(page)
+ try:
+ para_index = index.Index()
+ para_map = {}
+ #
+ valid_paras = [
+ p
+ for p in page.pdf_paragraph
+ if p.box
+ and all(c is not None for c in [p.box.x, p.box.y, p.box.x2, p.box.y2])
+ ]
+
+ for i, para in enumerate(valid_paras):
+ para_map[i] = para
+ para_index.insert(i, box_to_tuple(para.box))
+
+ for i, p_upper in para_map.items():
+ if not (p_upper.box and p_upper.box.y is not None):
+ continue
+
+ # Calculate paragraph height and set required gap accordingly
+ para_height = p_upper.box.y2 - p_upper.box.y
+ required_gap = 0.5 if para_height < 36 else 3
+
+ check_area = il_version_1.Box(
+ x=p_upper.box.x,
+ y=p_upper.box.y - required_gap,
+ x2=p_upper.box.x2,
+ y2=p_upper.box.y,
+ )
+
+ candidate_ids = list(para_index.intersection(box_to_tuple(check_area)))
+
+ conflicting_paras = []
+ for para_id in candidate_ids:
+ if para_id == i:
+ continue
+ p_lower = para_map[para_id]
+ if not (
+ p_lower.box
+ and p_upper.box
+ and p_lower.box.x2 < p_upper.box.x
+ or p_lower.box.x > p_upper.box.x2
+ ):
+ conflicting_paras.append(p_lower)
+
+ if conflicting_paras:
+ max_y2 = max(
+ p.box.y2
+ for p in conflicting_paras
+ if p.box and p.box.y2 is not None
+ )
+
+ new_y = max_y2 + required_gap
+ if p_upper.box and new_y < p_upper.box.y2:
+ p_upper.box.y = new_y
+ except Exception as e:
+ logger.warning(
+ f"Failed to adjust paragraph positions on page {page.page_number}: {e}"
+ )
+ # 开始实际的渲染过程
+ for paragraph in page.pdf_paragraph:
+ self.render_paragraph(paragraph, page, fonts)
+
+ def add_watermark(self, page: il_version_1.Page):
+ page_width = page.cropbox.box.x2 - page.cropbox.box.x
+ page_height = page.cropbox.box.y2 - page.cropbox.box.y
+ style = il_version_1.PdfStyle(
+ font_id="base",
+ font_size=6,
+ graphic_state=il_version_1.GraphicState(),
+ )
+ text = f"本文档çâ€Â± funstory.ai 的开溠PDF 翻译库 BabelDOC {WATERMARK_VERSION} (http://yadt.io) 翻译，本仓库æ£在积æžÂÂ的建设当ä¸ÂÂ，欢迎 star 和关注。"
+ if self.translation_config.debug:
+ text += "\n 当å‰ÂÂ为 DEBUG 模å¼ÂÂ，将显示更多辅助信æ¯。请注æ„ÂÂ，部分框的ä½ÂÂ置对åºâ€Â原文，但在译文ä¸ÂÂå¯能ä¸ÂÂæ£确。"
+ page.pdf_paragraph.append(
+ il_version_1.PdfParagraph(
+ first_line_indent=False,
+ box=il_version_1.Box(
+ x=page.cropbox.box.x + page_width * 0.05,
+ y=page.cropbox.box.y,
+ x2=page.cropbox.box.x2,
+ y2=page.cropbox.box.y2 - page_height * 0.05,
+ ),
+ vertical=False,
+ pdf_style=style,
+ pdf_paragraph_composition=[
+ il_version_1.PdfParagraphComposition(
+ pdf_same_style_unicode_characters=il_version_1.PdfSameStyleUnicodeCharacters(
+ unicode=text,
+ pdf_style=style,
+ ),
+ ),
+ ],
+ xobj_id=-1,
+ ),
+ )
+
+ def render_paragraph(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ page: il_version_1.Page,
+ fonts: dict[
+ str | int,
+ il_version_1.PdfFont | dict[str, il_version_1.PdfFont],
+ ],
+ ):
+ typesetting_units = self.create_typesetting_units(paragraph, fonts)
+ # 如果所有å•元都å¯以直接传递，则直接传递
+ if all(unit.can_passthrough for unit in typesetting_units):
+ paragraph.scale = 1.0
+ paragraph.pdf_paragraph_composition = self.create_passthrough_composition(
+ typesetting_units,
+ )
+ else:
+ # 使çâ€Â¨é¢„计ç®â€â€ÃƒÂ§Ã…¡â€žÃ§Â¼Â©Ã¦â€Â¾å› åÂÂÂÂ进行é‡ÂÂ排ç‰Ëâ€
+ precomputed_scale = (
+ paragraph.optimal_scale if paragraph.optimal_scale is not None else 1.0
+ )
+
+ # 如果有å•元æâ€â€Ã‚ Ã¦Â³â€¢Ã§â€ºÂ´Ã¦Å½Â¥Ã¤Â¼Â Ã©â‚¬â€™Ã¯Â¼Å’则进行é‡ÂÂ排ç‰Ëâ€
+ paragraph.pdf_paragraph_composition = []
+ self.retypeset_with_precomputed_scale(
+ paragraph, page, typesetting_units, precomputed_scale
+ )
+
+ # é‡ÂÂ排版åÂŽ，é‡ÂÂ新设置段è½å„åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã§Å¡â€ž render order
+ self._update_paragraph_render_order(paragraph)
+ # Log the typeset text block with coordinates
+ if hasattr(self, 'detailed_logger') and self.detailed_logger:
+ try:
+ # Extract the complete text from the paragraph
+ paragraph_text = ""
+ if hasattr(paragraph, 'unicode') and paragraph.unicode:
+ paragraph_text = paragraph.unicode
+ elif hasattr(paragraph, 'pdf_paragraph_composition'):
+ text_parts = []
+ for comp in paragraph.pdf_paragraph_composition:
+ if comp.pdf_character and hasattr(comp.pdf_character, 'char_unicode'):
+ if comp.pdf_character.char_unicode:
+ text_parts.append(comp.pdf_character.char_unicode)
+ elif comp.pdf_line and hasattr(comp.pdf_line, 'pdf_character'):
+ for char in comp.pdf_line.pdf_character:
+ if hasattr(char, 'char_unicode') and char.char_unicode:
+ text_parts.append(char.char_unicode)
+ elif comp.pdf_same_style_unicode_characters:
+ if comp.pdf_same_style_unicode_characters.unicode:
+ text_parts.append(comp.pdf_same_style_unicode_characters.unicode)
+ paragraph_text = "".join(text_parts)
+
+ # Determine paragraph type based on layout
+ paragraph_type = "paragraph" # default
+ if hasattr(paragraph, 'layout') and paragraph.layout:
+ layout_name = paragraph.layout.class_name if hasattr(paragraph.layout, 'class_name') else str(paragraph.layout)
+ if 'title' in layout_name.lower() or 'heading' in layout_name.lower():
+ paragraph_type = "heading"
+ elif 'list' in layout_name.lower():
+ paragraph_type = "list_item"
+ # Check if text starts with bullet point
+ if paragraph_text and len(paragraph_text) > 0:
+ first_char = paragraph_text[0]
+ if first_char in ['•', '◦', '▪', '▫', '●', '○', '■', '□', '▶', '▷', '-', '·']:
+ paragraph_type = "bullet_point"
+
+ # Get box coordinates
+ if hasattr(paragraph, 'box') and paragraph.box:
+ box_coords = {
+ 'x': paragraph.box.x,
+ 'y': paragraph.box.y,
+ 'x2': paragraph.box.x2,
+ 'y2': paragraph.box.y2
+ }
+
+ # Get page number
+ page_num = page.page_number if hasattr(page, 'page_number') else 0
+
+ # Get scale
+ scale = paragraph.scale if hasattr(paragraph, 'scale') else None
+
+ # Log the typeset text block
+ self.detailed_logger.log_typeset_text_block(
+ page_num=page_num,
+ paragraph_type=paragraph_type,
+ text=paragraph_text,
+ box_coords=box_coords,
+ scale=scale
+ )
+ except Exception as e:
+ # Silently fail if logging has issues
+ pass
+
+ def _get_width_before_next_break_point(
+ self, typesetting_units: list[TypesettingUnit], scale: float
+ ) -> float:
+ if not typesetting_units:
+ return 0
+ if typesetting_units[0].can_break_line:
+ return 0
+
+ total_width = 0
+ for unit in typesetting_units:
+ if unit.can_break_line:
+ return total_width * scale
+ total_width += unit.width
+ return total_width * scale
+
+ def _layout_typesetting_units(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ box: Box,
+ scale: float,
+ line_skip: float,
+ paragraph: il_version_1.PdfParagraph,
+ use_english_line_break: bool = True,
+ ) -> tuple[list[TypesettingUnit], bool]:
+ """布局排版å•元。
+
+ Args:
+ typesetting_units: è¦ÂÂ布局的排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ box: 布局边界æ¡â€Â
+ scale: 缩æâ€Â¾å› åÂÂÂÂ
+
+ Returns:
+ tuple[list[TypesettingUnit], bool]: (已布局的排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨Ã¯Â¼Å’是å¦所有å•元都æâ€Â¾å¾â€â€ÃƒÂ¤Ã‚¸â€¹)
+ """
+ # 计ç®â€â€ÃƒÂ¥Ã‚Ââ€â€ÃƒÂ¥Ã‚·ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ font_sizes = []
+ for unit in typesetting_units:
+ if unit.font_size:
+ font_sizes.append(unit.font_size)
+ if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ font_sizes.append(unit.char.pdf_style.font_size)
+ font_sizes.sort()
+ font_size = statistics.mode(font_sizes)
+
+ space_width = (
+ self.font_mapper.base_font.char_lengths("ä½ ", font_size * scale)[0] * 0.5
+ )
+
+ # 计ç®â€â€ÃƒÂ¨Ã‚¡Å’高（使çâ€Â¨ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã¯Â¼â€°
+ unit_heights = (
+ [unit.height for unit in typesetting_units] if typesetting_units else []
+ )
+ if not unit_heights:
+ avg_height = 0
+ elif len(unit_heights) == 1:
+ avg_height = unit_heights[0] * scale
+ else:
+ try:
+ avg_height = statistics.mode(unit_heights) * scale
+ except statistics.StatisticsError:
+ # 如果没有ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã¯Â¼Ë†Ã¦â€°â‚¬Ã¦Å“‰å€¼éƒ½å‡ºçŽ°ç›¸åÂŒ次数），则使çâ€Â¨å¹³å‡值
+ avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # Check if output language is Arabic for RTL layout
+ lang_out = (self.translation_config.lang_out or "").lower()
+ is_arabic = False
+ if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ is_arabic = True
+ elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ is_arabic = True
+
+ # Initialize position - for Arabic (RTL), start from right; for LTR, start from left
+ if is_arabic:
+ # For RTL: start from right edge and work left
+ current_x = box.x2
+ current_y = box.y2 - avg_height
+ else:
+ # For LTR: start from left edge and work right
+ current_x = box.x
+ current_y = box.y2 - avg_height
+
+ box = copy.deepcopy(box)
+ # box.y -= avg_height * (line_spacing - 1.01) # line_spacing 已被替æ¢为 line_skip
+ line_height = 0
+ current_line_heights = [] # Ã¥ÂÂËÅ"储å½â€Å"å‰ÂÂ行所有元素的é«ËÅ"度
+
+ # Ã¥ÂÂËÅ"储已排版的å•元
+ typeset_units = []
+ all_units_fit = True
+ last_unit: TypesettingUnit | None = None
+ line_ys = [current_y]
+ is_first_line = True
+ prev_x = None
+ if paragraph.first_line_indent:
+ if is_arabic:
+ # For RTL: apply indent from right side
+ current_x -= space_width * 4
+ else:
+ # For LTR: apply indent from left side
+ current_x += space_width * 4
+ # For Arabic (RTL), process units in reverse order; for LTR, process normally
+ units_to_process = list(reversed(typesetting_units)) if is_arabic else typesetting_units
+
+ # éÂÂÂÂ历所有排版å•元
+ for i, unit in enumerate(units_to_process):
+ # Get original index for width calculation
+ orig_idx = len(typesetting_units) - 1 - i if is_arabic else i
+
+ # 计ç®â€â€ÃƒÂ¥Ã‚½â€Å"å‰ÂÂå•元在å½â€Å"å‰ÂÂ缩æâ€Â¾ä¸‹çš„尺寸
+ unit_width = unit.width * scale
+ unit_height = unit.height * scale
+
+ # 跳过行首的空格
+ if is_arabic:
+ # For RTL: skip leading spaces at right edge
+ if current_x == box.x2 and unit.is_space:
+ continue
+ else:
+ # For LTR: skip leading spaces at left edge
+ if current_x == box.x and unit.is_space:
+ continue
+
+ # Apply spacing between CJK and non-CJK characters (only for LTR)
+ if not is_arabic and (
+ last_unit # 有上一个å•元
+ and last_unit.is_cjk_char ^ unit.is_cjk_char # ä¸ÂÂ英文交界处
+ and (
+ last_unit.box
+ and last_unit.box.y
+ and current_y - 0.1
+ <= last_unit.box.y2
+ <= current_y + line_height + 0.1
+ ) # 在åÂŒ一行，ä¸â€Â有垂直é‡ÂÂÃ¥ÂÂÂÂ
+ and not last_unit.mixed_character_blacklist # ä¸ÂÂæËÅ"¯æ··æŽ’空格黑åÂÂÂÂå•åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦
+ and not unit.mixed_character_blacklist # Ã¥ÂÂŒä¸ÅÂ
+ and current_x > box.x # ä¸ÂÂæËÅ"¯è¡Œé¦–
+ and unit.try_get_unicode() != " " # ä¸ÂÂæËÅ"¯ç©ºæ ¼
+ and last_unit.try_get_unicode() != " " # ä¸ÂÂæËÅ"¯ç©ºæ ¼
+ and last_unit.try_get_unicode()
+ not in [
+ "。",
+ "ï¼ÂÂ",
+ "？",
+ "ï¼›",
+ "：",
+ "，",
+ ]
+ ):
+ current_x += space_width * 0.5
+ # Calculate width before next break point (for LTR only)
+ if use_english_line_break and not is_arabic:
+ width_before_next_break_point = self._get_width_before_next_break_point(
+ typesetting_units[orig_idx:], scale
+ )
+ else:
+ width_before_next_break_point = 0
+
+ # Check if we need to break line - different logic for RTL vs LTR
+ need_line_break = False
+ if not unit.is_hung_punctuation:
+ if is_arabic:
+ # For RTL: check if we've gone past the left boundary
+ # Position unit so its left edge is at current_x - unit_width
+ if (current_x - unit_width < box.x):
+ need_line_break = True
+ elif (
+ unit.is_cannot_appear_in_line_end_punctuation
+ and current_x - unit_width * 2 < box.x
+ ):
+ need_line_break = True
+ else:
+ # For LTR: check if we've gone past the right boundary
+ if (current_x + unit_width > box.x2):
+ need_line_break = True
+ elif (
+ use_english_line_break
+ and current_x + unit_width + width_before_next_break_point > box.x2
+ ):
+ need_line_break = True
+ elif (
+ unit.is_cannot_appear_in_line_end_punctuation
+ and current_x + unit_width * 2 > box.x2
+ ):
+ need_line_break = True
+
+ if need_line_break:
+ # æ¢行
+ if is_arabic:
+ current_x = box.x2
+ else:
+ current_x = box.x
+
+ if not current_line_heights:
+ return [], False
+ max_height = max(current_line_heights)
+ mode_height = statistics.mode(current_line_heights)
+
+ current_y -= max(mode_height * line_skip, max_height * 1.05)
+ line_ys.append(current_y)
+ line_height = 0.0
+ current_line_heights = [] # 清空å½â€Å"å‰ÂÂè¡Œé«ËÅ"度åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ is_first_line = False
+
+ # 检查æËÅ"¯å¦超出底部边界
+ # if current_y - unit_height < box.y:
+ if current_y < box.y:
+ all_units_fit = False
+ # 这里ä¸ÂÂ覠break，继ç»ÂÂ排版剩余内容
+
+ if unit.is_space:
+ line_height = max(line_height, unit_height)
+ continue
+
+ # Position unit - for RTL, place from right to left; for LTR, place from left to right
+ if is_arabic:
+ # For RTL: position unit so its right edge is at current_x
+ # The unit's x position will be current_x - unit_width
+ unit_x = current_x - unit_width
+ relocated_unit = unit.relocate(unit_x, current_y, scale)
+ # Update current_x to the left edge of the unit (for next unit)
+ current_x = unit_x
+ else:
+ # For LTR: position unit at current_x
+ relocated_unit = unit.relocate(current_x, current_y, scale)
+ # Update current_x to the right edge of the unit (for next unit)
+ current_x = relocated_unit.box.x2
+
+ typeset_units.append(relocated_unit)
+
+ # 添加å½â€Å"å‰ÂÂå•元的é«ËÅ"度到å½â€Å"å‰ÂÂè¡Œé«ËÅ"度åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ if not unit.is_space:
+ current_line_heights.append(unit_height)
+
+ if is_arabic and prev_x is not None and current_x > prev_x:
+ logger.warning(f"RTL position error: current_x ({current_x}) > prev_x ({prev_x})")
+
+ last_unit = relocated_unit
+ prev_x = current_x
+
+ # For Arabic, reverse the units order since we processed them in reverse
+ # This ensures the final order matches the logical text order
+ if is_arabic and typeset_units:
+ typeset_units = list(reversed(typeset_units))
+
+ return typeset_units, all_units_fit
+
+ def _mirror_margins_for_rtl(
+ self,
+ typeset_units: list[TypesettingUnit],
+ box: Box,
+ paragraph: il_version_1.PdfParagraph,
+ ) -> list[TypesettingUnit]:
+ """
+ Mirror left margins to right margins for RTL languages (Arabic).
+ This function ensures that any left margin/indentation in the original
+ is mirrored to the right side in the Arabic output.
+
+ Args:
+ typeset_units: Already laid out typesetting units (RTL layout)
+ box: The paragraph's bounding box
+ paragraph: The paragraph object containing metadata
+
+ Returns:
+ list[TypesettingUnit]: Units with properly mirrored margins
+ """
+ if not typeset_units or not box:
+ return typeset_units
+
+ # Check if this is a table paragraph (tables have their own layout)
+ is_table_paragraph = False
+ if hasattr(paragraph, 'pdf_paragraph_composition'):
+ for comp in paragraph.pdf_paragraph_composition:
+ if hasattr(comp, 'pdf_table') and comp.pdf_table:
+ is_table_paragraph = True
+ break
+
+ # Don't adjust table content
+ if is_table_paragraph:
+ return typeset_units
+
+ # Group units by line (Y coordinate) and sort by Y (top to bottom)
+ lines_dict = {}
+ for unit in typeset_units:
+ if unit.box and unit.box.y is not None:
+ line_y = round(unit.box.y, 1)
+ if line_y not in lines_dict:
+ lines_dict[line_y] = []
+ lines_dict[line_y].append(unit)
+
+ # Sort lines by Y coordinate (top to bottom)
+ sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # Process each line to mirror margins
+ for line_idx, line_y in enumerate(sorted_line_ys):
+ line_units = lines_dict[line_y]
+ if not line_units:
+ continue
+
+ # Find the rightmost position in this line (current right edge of text)
+ rightmost_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+
+ # Find the leftmost position in this line (current left edge of text)
+ leftmost_x = min(u.box.x for u in line_units if u.box and u.box.x is not None)
+
+ # Calculate the current right margin (distance from text to box.x2)
+ current_right_margin = box.x2 - rightmost_x
+
+ # Calculate the current left margin (distance from box.x to text)
+ # This is what we want to mirror to the right
+ current_left_margin = leftmost_x - box.x
+
+ # For RTL, we want the right margin to equal the original left margin
+ # So we shift the entire line so that the right margin matches the left margin
+ target_right_margin = current_left_margin
+ target_rightmost_x = box.x2 - target_right_margin
+
+ # Calculate the shift needed
+ shift_x = target_rightmost_x - rightmost_x
+
+ # Apply the shift to all units in this line
+ for unit in line_units:
+ if unit.box:
+ unit.box.x += shift_x
+ unit.box.x2 += shift_x
+ if unit.x is not None:
+ unit.x += shift_x
+
+ # Update character box if present
+ if unit.char:
+ if unit.char.box:
+ unit.char.box.x += shift_x
+ unit.char.box.x2 += shift_x
+ if hasattr(unit.char, 'visual_bbox') and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ unit.char.visual_bbox.box.x += shift_x
+ unit.char.visual_bbox.box.x2 += shift_x
+
+ return typeset_units
+
+# CORRECT FIX FOR ARABIC TEXT LAYOUT
+# Replace the _layout_typesetting_units function in typesetting.py (lines 1346-1502)
+
+ # def _layout_typesetting_units(
+ # self,
+ # typesetting_units: list[TypesettingUnit],
+ # box: Box,
+ # scale: float,
+ # line_skip: float,
+ # paragraph: il_version_1.PdfParagraph,
+ # use_english_line_break: bool = True,
+ # ) -> tuple[list[TypesettingUnit], bool]:
+ # """布局排版å•元。
+
+ # Args:
+ # typesetting_units: è¦ÂÂ布局的排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ # box: 布局边界æ¡â€Â
+ # scale: 缩æâ€Â¾å› åÂÂÂÂ
+
+ # Returns:
+ # tuple[list[TypesettingUnit], bool]: (已布局的排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨Ã¯Â¼Å’是å¦所有å•元都æâ€Â¾å¾â€â€ÃƒÂ¤Ã‚¸â€¹)
+ # """
+ # # 计ç®â€â€ÃƒÂ¥Ã‚Ââ€â€ÃƒÂ¥Ã‚·ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°
+ # font_sizes = []
+ # for unit in typesetting_units:
+ # if unit.font_size:
+ # font_sizes.append(unit.font_size)
+ # if unit.char and unit.char.pdf_style and unit.char.pdf_style.font_size:
+ # font_sizes.append(unit.char.pdf_style.font_size)
+ # font_sizes.sort()
+ # font_size = statistics.mode(font_sizes)
+
+ # space_width = (
+ # self.font_mapper.base_font.char_lengths("ä½ ", font_size * scale)[0] * 0.5
+ # )
+
+ # # 计ç®â€â€ÃƒÂ¨Ã‚¡Å’高（使çâ€Â¨ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã¯Â¼â€°
+ # unit_heights = (
+ # [unit.height for unit in typesetting_units] if typesetting_units else []
+ # )
+ # if not unit_heights:
+ # avg_height = 0
+ # elif len(unit_heights) == 1:
+ # avg_height = unit_heights[0] * scale
+ # else:
+ # try:
+ # avg_height = statistics.mode(unit_heights) * scale
+ # except statistics.StatisticsError:
+ # # 如果没有ä¼â€â€ÃƒÂ¦Ã¢â‚¬Â¢Ã‚°Ã¯Â¼Ë†Ã¦â€°â‚¬Ã¦Å“‰å€¼éƒ½å‡ºçŽ°ç›¸åÂŒ次数），则使çâ€Â¨å¹³å‡值
+ # avg_height = sum(unit_heights) / len(unit_heights) * scale
+
+ # # *** NEW: Detect Arabic language ***
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # # åˆÂÂ始化ä½ÂÂ置为å³上角，并å‡ÂÂ去一个平å‡行高
+ # # *** CHANGED: For Arabic, calculate total line width first and start from right ***
+ # current_x = box.x
+ # current_y = box.y2 - avg_height
+ # box = copy.deepcopy(box)
+ # line_height = 0
+ # current_line_heights = [] # å˜储当å‰ÂÂ行所有元素的高度
+
+ # # å˜储已排版的å•元
+ # typeset_units = []
+ # all_units_fit = True
+ # last_unit: TypesettingUnit | None = None
+ # line_ys = [current_y]
+ # if paragraph.first_line_indent:
+ # current_x += space_width * 4
+ # # éÂÂÂÂ历所有排版å•元
+ # for i, unit in enumerate(typesetting_units):
+ # # 计ç®â€â€ÃƒÂ¥Ã‚½â€œÃ¥â€°ÂÂå•元在当å‰ÂÂ缩æâ€Â¾ä¸‹çš„尺寸
+ # unit_width = unit.width * scale
+ # unit_height = unit.height * scale
+
+ # # 跳过行首的空格
+ # if current_x == box.x and unit.is_space:
+ # continue
+
+ # if (
+ # last_unit # 有上一个å•元
+ # and last_unit.is_cjk_char ^ unit.is_cjk_char # ä¸ÂÂ英文交界处
+ # and (
+ # last_unit.box
+ # and last_unit.box.y
+ # and current_y - 0.1
+ # <= last_unit.box.y2
+ # <= current_y + line_height + 0.1
+ # ) # 在åÂŒ一行，ä¸â€Â有垂直é‡ÂÂÃ¥ÂÂÂÂ
+ # and not last_unit.mixed_character_blacklist # ä¸ÂÂ是混排空格黑åÂÂÂÂå•åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦
+ # and not unit.mixed_character_blacklist # Ã¥ÂÂŒä¸ÅÂ
+ # and current_x > box.x # ä¸ÂÂ是行首
+ # and unit.try_get_unicode() != " " # ä¸ÂÂ是空格
+ # and last_unit.try_get_unicode() != " " # ä¸ÂÂ是空格
+ # and last_unit.try_get_unicode()
+ # not in [
+ # "。",
+ # "ï¼ÂÂ",
+ # "？",
+ # "ï¼›",
+ # "：",
+ # "，",
+ # ]
+ # ):
+ # current_x += space_width * 0.5
+ # if use_english_line_break:
+ # width_before_next_break_point = self._get_width_before_next_break_point(
+ # typesetting_units[i:], scale
+ # )
+ # else:
+ # width_before_next_break_point = 0
+
+ # # 如果当å‰ÂÂè¡Œæâ€Â¾ä¸ÂÂ下这个元素，æ¢行
+ # if not unit.is_hung_punctuation and (
+ # (current_x + unit_width > box.x2)
+ # or (
+ # use_english_line_break
+ # and current_x + unit_width + width_before_next_break_point > box.x2
+ # )
+ # or (
+ # unit.is_cannot_appear_in_line_end_punctuation
+ # and current_x + unit_width * 2 > box.x2
+ # )
+ # ):
+ # # æ¢行
+ # current_x = box.x
+ # if not current_line_heights:
+ # return [], False
+ # max_height = max(current_line_heights)
+ # mode_height = statistics.mode(current_line_heights)
+
+ # current_y -= max(mode_height * line_skip, max_height * 1.05)
+ # line_ys.append(current_y)
+ # line_height = 0.0
+ # current_line_heights = [] # 清空当å‰ÂÂ行高度åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+
+ # # 检查是å¦超出底部边界
+ # # if current_y - unit_height < box.y:
+ # if current_y < box.y:
+ # all_units_fit = False
+ # # 这里ä¸ÂÂ覠break，继ç»ÂÂ排版剩余内容
+
+ # if unit.is_space:
+ # line_height = max(line_height, unit_height)
+ # continue
+
+ # # æâ€Â¾ç½®å½“å‰ÂÂå•元
+ # relocated_unit = unit.relocate(current_x, current_y, scale)
+ # typeset_units.append(relocated_unit)
+
+ # # 添加当å‰ÂÂå•元的高度到当å‰ÂÂ行高度åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ # if not unit.is_space:
+ # current_line_heights.append(unit_height)
+
+ # prev_x = current_x
+ # # æ›´æ–° x Ã¥ÂÂÂÂæ ‡
+ # current_x = relocated_unit.box.x2
+ # if prev_x > current_x:
+ # logger.warning(f"Ã¥ÂÂÂÂ标回绕ï¼ÂÂï¼ÂÂï¼ÂÂTypesettingUnit: {unit.box}, ")
+
+ # last_unit = relocated_unit
+
+ # # *** NEW: For Arabic, right-align each line ***
+ # if is_arabic and typeset_units:
+ # # Group units by line (Y coordinate)
+ # lines = {}
+ # for unit in typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # line_y = round(unit.box.y, 1)
+ # if line_y not in lines:
+ # lines[line_y] = []
+ # lines[line_y].append(unit)
+
+ # # Right-align each line
+ # for line_y, line_units in lines.items():
+ # if not line_units:
+ # continue
+
+ # # Find the rightmost position of this line
+ # line_max_x = max(u.box.x2 for u in line_units if u.box and u.box.x2 is not None)
+
+ # # Calculate how much to shift right
+ # shift_x = box.x2 - line_max_x
+
+ # # Shift all units in this line to the right
+ # for unit in line_units:
+ # if unit.box:
+ # unit.box.x += shift_x
+ # unit.box.x2 += shift_x
+ # if unit.x is not None:
+ # unit.x += shift_x
+ # # Update character box if present
+ # if unit.char and unit.char.box:
+ # unit.char.box.x += shift_x
+ # unit.char.box.x2 += shift_x
+ # if unit.char and unit.char.visual_bbox and unit.char.visual_bbox.box:
+ # unit.char.visual_bbox.box.x += shift_x
+ # unit.char.visual_bbox.box.x2 += shift_x
+ # # Check if output language is Arabic
+ # lang_out = (self.translation_config.lang_out or "").lower()
+ # is_arabic = False
+ # if lang_out in ("en-ar", "ar", "ara", "arabic"):
+ # is_arabic = True
+ # elif "-ar" in lang_out or "->ar" in lang_out or "/ar" in lang_out:
+ # is_arabic = True
+
+ # # If Arabic, reverse the line order
+ # if is_arabic and typeset_units:
+ # # Group units by line (using Y coordinates)
+ # lines_dict = {}
+ # for unit in typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # # Round Y coordinate to group units on the same line
+ # line_y = round(unit.box.y, 1)
+ # if line_y not in lines_dict:
+ # lines_dict[line_y] = []
+ # lines_dict[line_y].append(unit)
+
+ # # Sort lines by Y coordinate (top to bottom) and reverse
+ # sorted_line_ys = sorted(lines_dict.keys(), reverse=True)
+
+ # # Rebuild typeset_units with reversed line order
+ # reversed_typeset_units = []
+ # for line_y in reversed(sorted_line_ys):
+ # reversed_typeset_units.extend(lines_dict[line_y])
+
+ # # Now reposition all units to swap their Y coordinates
+ # # Map old Y positions to new Y positions
+ # y_mapping = {}
+ # for i, old_y in enumerate(sorted_line_ys):
+ # new_y = sorted_line_ys[len(sorted_line_ys) - 1 - i]
+ # y_mapping[old_y] = new_y
+
+ # # Update Y coordinates for all units
+ # for unit in reversed_typeset_units:
+ # if unit.box and unit.box.y is not None:
+ # old_y = round(unit.box.y, 1)
+ # if old_y in y_mapping:
+ # new_y = y_mapping[old_y]
+ # y_diff = new_y - old_y
+ # # Update the unit's Y position
+ # if unit.y is not None:
+ # unit.y += y_diff
+ # if unit.box:
+ # unit.box.y += y_diff
+ # unit.box.y2 += y_diff
+
+ # typeset_units = reversed_typeset_units
+
+ # return typeset_units, all_units_fit
+
+ def create_typesetting_units(
+ self,
+ paragraph: il_version_1.PdfParagraph,
+ fonts: dict[str, il_version_1.PdfFont],
+ ) -> list[TypesettingUnit]:
+ if not paragraph.pdf_paragraph_composition:
+ return []
+ result = []
+
+ @cache
+ def get_font(font_id: str, xobj_id: int | None):
+ if xobj_id in fonts:
+ font = fonts[xobj_id][font_id]
+ else:
+ font = fonts[font_id]
+ return font
+
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition is None:
+ continue
+ if composition.pdf_line:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_line.pdf_character
+ ],
+ )
+ elif composition.pdf_character:
+ result.append(
+ TypesettingUnit(
+ char=composition.pdf_character,
+ debug_info=paragraph.debug_info,
+ ),
+ )
+ elif composition.pdf_same_style_characters:
+ result.extend(
+ [
+ TypesettingUnit(char=char)
+ for char in composition.pdf_same_style_characters.pdf_character
+ ],
+ )
+ elif composition.pdf_same_style_unicode_characters:
+ style = composition.pdf_same_style_unicode_characters.pdf_style
+ if style is None:
+ logger.warning(
+ f"Style is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font_id = style.font_id
+ if font_id is None:
+ logger.warning(
+ f"Font ID is None. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ font = get_font(font_id, paragraph.xobj_id)
+ if composition.pdf_same_style_unicode_characters.unicode:
+ unicode_text = composition.pdf_same_style_unicode_characters.unicode
+ shaped_text = self.shape_arabic_text(unicode_text)
+ result.extend(
+ [
+ TypesettingUnit(
+ unicode=char_unicode,
+ font=self.font_mapper.map(
+ font,
+ char_unicode,
+ ),
+ original_font=font,
+ font_size=style.font_size,
+ style=style,
+ xobj_id=paragraph.xobj_id,
+ debug_info=composition.pdf_same_style_unicode_characters.debug_info
+ or False,
+ )
+ for char_unicode in shaped_text # Use shaped_text instead of original
+ if char_unicode not in ("\n",)
+ ],
+ )
+ elif composition.pdf_formula:
+ result.extend([TypesettingUnit(formular=composition.pdf_formula)])
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ result = list(
+ filter(
+ lambda x: x.unicode is None or x.font is not None,
+ result,
+ ),
+ )
+
+ if any(x.width < 0 for x in result):
+ logger.warning("有排版å•元宽度å°ÂÂ于 0，请检查åÂÂâ€â€ÃƒÂ¤Ã‚½â€œÃ¦ËœÂ Ã¥Â°â€žÃ¦ËœÂ¯Ã¥Â¦æ£确。")
+ return result
+
+ def create_passthrough_composition(
+ self,
+ typesetting_units: list[TypesettingUnit],
+ ) -> list[PdfParagraphComposition]:
+ """从排版å•元创建直接传递的段è½组åˆ。
+
+ Args:
+ typesetting_units: 排版å•元åˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+
+ Returns:
+ 段è½组åˆåˆâ€â€ÃƒÂ¨Ã‚¡Â¨
+ """
+ composition = []
+ for unit in typesetting_units:
+ if unit.formular:
+ # 对于公å¼ÂÂå•元，直接创建包å«完整公å¼ÂÂ的组åÂÂËâ€
+ composition.append(PdfParagraphComposition(pdf_formula=unit.formular))
+ else:
+ # 对于åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã¥Â•元，使çâ€Â¨åŽŸæœ‰é€»è¾‘
+ chars, curves, forms = unit.passthrough()
+ composition.extend(
+ [PdfParagraphComposition(pdf_character=char) for char in chars],
+ )
+ return composition
+
+ def get_max_right_space(self, current_box: Box, page) -> float:
+ """获å–段è½å³侧最大å¯çâ€Â¨ç©ºéâ€â€Ã‚´
+
+ Args:
+ current_box: 当å‰ÂÂ段è½的边界æ¡â€Â
+ page: 当å‰ÂÂ页é¢
+
+ Returns:
+ å¯以扩展到的最大 x Ã¥ÂÂÂÂæ ‡
+ """
+ # 获å–页é¢的è£ÂÂ剪框作为åˆÂÂ始最大é™ÂÂ制
+ max_x = page.cropbox.box.x2 * 0.9
+
+ # 检查所有å¯能的阻挡元ç´ÂÂ
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # 跳过当å‰ÂÂ段è½
+ continue
+ # åª考虑在当å‰ÂÂ段è½å³侧ä¸â€Â有垂直é‡ÂÂå 的元ç´ÂÂ
+ if para.box.x > current_box.x and not (
+ para.box.y >= current_box.y2 or para.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, para.box.x)
+ for char in page.pdf_character:
+ if char.box.x > current_box.x and not (
+ char.box.y >= current_box.y2 or char.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, char.box.x)
+ # 检查图形
+ for figure in page.pdf_figure:
+ if figure.box.x > current_box.x and not (
+ figure.box.y >= current_box.y2 or figure.box.y2 <= current_box.y
+ ):
+ max_x = min(max_x, figure.box.x)
+
+ return max_x
+
+ def get_max_bottom_space(self, current_box: Box, page: il_version_1.Page) -> float:
+ """获å–段è½下方最大å¯çâ€Â¨ç©ºéâ€â€Ã‚´
+
+ Args:
+ current_box: 当å‰ÂÂ段è½的边界æ¡â€Â
+ page: 当å‰ÂÂ页é¢
+
+ Returns:
+ å¯以扩展到的最尠y Ã¥ÂÂÂÂæ ‡
+ """
+ # 获å–页é¢的è£ÂÂ剪框作为åˆÂÂ始最å°ÂÂé™ÂÂ制
+ min_y = page.cropbox.box.y * 1.1
+
+ # 检查所有å¯能的阻挡元ç´ÂÂ
+ for para in page.pdf_paragraph:
+ if para.box == current_box or para.box is None: # 跳过当å‰ÂÂ段è½
+ continue
+ # åª考虑在当å‰ÂÂ段è½下方ä¸â€Â有水平é‡ÂÂå 的元ç´ÂÂ
+ if para.box.y2 < current_box.y and not (
+ para.box.x >= current_box.x2 or para.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, para.box.y2)
+ for char in page.pdf_character:
+ if char.box.y2 < current_box.y and not (
+ char.box.x >= current_box.x2 or char.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, char.box.y2)
+ # 检查图形
+ for figure in page.pdf_figure:
+ if figure.box.y2 < current_box.y and not (
+ figure.box.x >= current_box.x2 or figure.box.x2 <= current_box.x
+ ):
+ min_y = max(min_y, figure.box.y2)
+
+ return min_y
+
+ def _update_paragraph_render_order(self, paragraph: il_version_1.PdfParagraph):
+ """
+ é‡ÂÂ新设置段è½å„åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦Ã§Å¡â€ž render order
+ 主 render order ç‰于 paragraph çš„ renderorder，sub render order 从 1 开始自增
+ """
+ if not hasattr(paragraph, "render_order") or paragraph.render_order is None:
+ return
+
+ main_render_order = paragraph.render_order
+ sub_render_order = 1
+
+ # éÂÂÂÂ历段è½的所有组æˆÂÂ部åˆâ€Â
+ for composition in paragraph.pdf_paragraph_composition:
+ # 检查å•个åÂÂâ€â€ÃƒÂ§Ã‚¬Â¦
+ if composition.pdf_character:
+ char = composition.pdf_character
+ char.render_order = main_render_order
+ char.sub_render_order = sub_render_order
+ sub_render_order += 1
\ No newline at end of file
diff --git a/babeldoc/format/pdf/document_il/utils/__init__.py b/babeldoc/format/pdf/document_il/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/babeldoc/format/pdf/document_il/utils/extract_char.py b/babeldoc/format/pdf/document_il/utils/extract_char.py
new file mode 100644
index 0000000000000000000000000000000000000000..3432d16727404cfd15bd977d7a37ded854d9d077
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/utils/extract_char.py
@@ -0,0 +1,763 @@
+import logging
+import shutil
+from collections import defaultdict
+from pathlib import Path
+
+import cv2
+import numpy as np
+import pymupdf
+from rich.logging import RichHandler
+from sklearn.cluster import DBSCAN
+
+import babeldoc.format.pdf.high_level
+import babeldoc.format.pdf.translation_config
+from babeldoc.const import get_process_pool
+from babeldoc.format.pdf.document_il import il_version_1
+
+logger = logging.getLogger(__name__)
+
+# --- Algorithm Tuning Parameters ---
+
+# --- Band Creation ---
+# Minimum vertical overlap ratio for a character to be added to an existing band.
+BAND_CREATION_OVERLAP_THRESHOLD = 0.5
+
+# --- Line Clustering (within a band) ---
+# Epsilon for DBSCAN, as a multiplier of the average character width/height.
+LINE_CLUSTERING_EPS_MULTIPLIER = 3.5
+
+# --- Line Splitting (for tall/wide lines) ---
+# A line is considered for splitting if its height/width is > X times the max char size.
+LINE_SPLIT_SIZE_RATIO_THRESHOLD = 1.5
+# Epsilon for DBSCAN when splitting lines, as a multiplier of the max char size.
+LINE_SPLIT_DBSCAN_EPS_MULTIPLIER = 0.5
+
+# --- Space Insertion (in a finalized line) ---
+# A space is inserted if the gap between chars is > X times the average char width.
+SPACE_INSERTION_GAP_MULTIPLIER = 0.45
+
+# --- Line Merging (across the page) ---
+# --- Optimization ---
+# Maximum vertical gap to search for potential merges, as a multiplier of avg char height.
+MERGE_VERTICAL_GAP_MULTIPLIER = 1.5
+# --- Containment Merge ---
+# Intersection-over-area threshold to consider one line as contained within another.
+MERGE_CONTAINMENT_IOU_THRESHOLD = 0.6
+# --- Adjacency Merge ---
+# Minimum vertical/horizontal overlap for adjacent lines to be considered for merging.
+MERGE_ADJACENCY_OVERLAP_THRESHOLD = 0.7
+# Maximum gap between adjacent lines to merge, as a multiplier of avg char size.
+MERGE_ADJACENCY_GAP_MULTIPLIER = 1.5
+
+
+# --- End of Parameters ---
+
+
+def parse_pdf(pdf_path, page_ranges=None) -> il_version_1.Document:
+ translation_config = babeldoc.format.pdf.translation_config.TranslationConfig(
+ *[None for _ in range(4)], doc_layout_model=None
+ )
+ if page_ranges:
+ translation_config.page_ranges = [page_ranges]
+ translation_config.progress_monitor = (
+ babeldoc.format.pdf.high_level.ProgressMonitor(
+ babeldoc.format.pdf.high_level.TRANSLATE_STAGES
+ )
+ )
+ try:
+ shutil.copy(pdf_path, translation_config.get_working_file_path("input.pdf"))
+ doc = pymupdf.open(pdf_path)
+ il_creater = babeldoc.format.pdf.high_level.ILCreater(translation_config)
+ il_creater.mupdf = doc
+ with Path(translation_config.get_working_file_path("input.pdf")).open(
+ "rb"
+ ) as f:
+ babeldoc.format.pdf.high_level.start_parse_il(
+ f,
+ doc_zh=doc,
+ resfont="test_font",
+ il_creater=il_creater,
+ translation_config=translation_config,
+ )
+ il = il_creater.create_il()
+ doc.close()
+ return il
+ finally:
+ translation_config.cleanup_temp_files()
+ return None
+
+
+class Line:
+ def __init__(self, chars: list[tuple[il_version_1.Box, str, bool]]):
+ self.chars = chars
+ self.text = "".join([c[1] for c in chars])
+
+
+def _recalculate_line_text_with_spacing(line, orientation):
+ if not line.chars:
+ line.text = ""
+ return
+
+ if orientation == "horizontal":
+
+ def get_main_start(c):
+ return c[0].x
+
+ def get_main_end(c):
+ return c[0].x2
+
+ def get_main_size(c):
+ return c[0].x2 - c[0].x
+
+ else: # vertical
+
+ def get_main_start(c):
+ return c[0].y
+
+ def get_main_end(c):
+ return c[0].y2
+
+ def get_main_size(c):
+ return c[0].y2 - c[0].y
+
+ line_text = ""
+ avg_width = np.mean(
+ [get_main_size(c) for c in line.chars if get_main_size(c) > 0] or [0]
+ )
+
+ if len(line.chars) > 1 and avg_width > 0:
+ for i in range(len(line.chars) - 1):
+ c1, c2 = line.chars[i], line.chars[i + 1]
+ gap = get_main_start(c2) - get_main_end(c1)
+
+ if gap > avg_width * SPACE_INSERTION_GAP_MULTIPLIER:
+ line_text += c1[1] + " "
+ else:
+ line_text += c1[1]
+
+ if line.chars:
+ line_text += line.chars[-1][1]
+
+ line.text = line_text
+
+
+# [box, char_unicode, vertical]
+# vertical: True if the char is vertical, False if the char is horizontal
+def extract_paragraph_line(
+ pdf_path,
+) -> dict[int, list[tuple[il_version_1.Box, str, bool]]]:
+ il = parse_pdf(pdf_path)
+ if il is None:
+ return None
+ line_boxes = {}
+ for page in il.page:
+ line_boxes[page.page_number] = convert_page_to_char_boxes(page)
+ return line_boxes
+
+
+def convert_page_to_char_boxes(
+ page: il_version_1.Page,
+) -> list[tuple[il_version_1.Box, str, bool]]:
+ return [
+ (char.visual_bbox.box, char.char_unicode, char.vertical)
+ for char in page.pdf_character
+ ]
+
+
+def _cluster_by_axis(chars: list[tuple[il_version_1.Box, str, bool]], orientation: str):
+ """
+ A generalized function to cluster characters into lines based on main and secondary axes.
+ """
+ if not chars:
+ return []
+
+ # Define main and secondary axes based on orientation
+ if orientation == "horizontal":
+
+ def get_secondary_start(c):
+ return c[0].y
+
+ def get_secondary_end(c):
+ return c[0].y2
+
+ def get_main_start(c):
+ return c[0].x
+
+ def get_main_end(c):
+ return c[0].x2
+
+ def get_main_size(c):
+ return c[0].x2 - c[0].x
+
+ else: # vertical
+
+ def get_secondary_start(c):
+ return c[0].x
+
+ def get_secondary_end(c):
+ return c[0].x2
+
+ def get_main_start(c):
+ return c[0].y
+
+ def get_main_end(c):
+ return c[0].y2
+
+ def get_main_size(c):
+ return c[0].y2 - c[0].y
+
+ # Step 1: Group chars into bands along the secondary axis based on overlap.
+ # This is an optimized version of the band clustering algorithm.
+ # It avoids the O(N^2) complexity of the naive approach by making
+ # assumptions based on the sorted order of characters.
+ chars.sort(key=get_secondary_start)
+
+ # Each band is a tuple: (list_of_chars, min_secondary_coord, max_secondary_coord)
+ bands_data: list[tuple[list, float, float]] = []
+
+ for char in chars:
+ char_secondary_start = get_secondary_start(char)
+ char_secondary_end = get_secondary_end(char)
+ char_secondary_size = char_secondary_end - char_secondary_start
+
+ best_band_index = -1
+ max_overlap_ratio = (
+ BAND_CREATION_OVERLAP_THRESHOLD # Minimum overlap ratio to be considered
+ )
+
+ # Iterate backwards over bands, as recent bands are more likely to overlap.
+ for i in range(len(bands_data) - 1, -1, -1):
+ band_chars, band_secondary_start, band_secondary_end = bands_data[i]
+
+ # Optimization: If the band is already far above the current char,
+ # and since chars are sorted by start, no further bands will match.
+ if band_secondary_end < char_secondary_start:
+ break
+
+ overlap = max(
+ 0,
+ min(char_secondary_end, band_secondary_end)
+ - max(char_secondary_start, band_secondary_start),
+ )
+
+ if char_secondary_size > 0:
+ overlap_ratio = overlap / char_secondary_size
+ if overlap_ratio > max_overlap_ratio:
+ max_overlap_ratio = overlap_ratio
+ best_band_index = i
+
+ if best_band_index != -1:
+ # Add char to the best matching band and update its boundaries
+ band_chars, band_start, band_end = bands_data[best_band_index]
+ band_chars.append(char)
+ updated_band = (
+ band_chars,
+ min(band_start, char_secondary_start),
+ max(band_end, char_secondary_end),
+ )
+ bands_data[best_band_index] = updated_band
+ # Move the updated band to the end to maintain rough locality
+ bands_data.append(bands_data.pop(best_band_index))
+ else:
+ # No suitable band found, create a new one
+ bands_data.append(([char], char_secondary_start, char_secondary_end))
+
+ # Extract final bands from the data structure
+ bands = [b[0] for b in bands_data]
+
+ # Step 2: For each band, cluster along the main axis using DBSCAN
+ final_lines = []
+ for band in bands:
+ if len(band) < 1:
+ continue
+
+ main_axis_sizes = [get_main_size(c) for c in band if get_main_size(c) > 0]
+ avg_main_size = np.mean(main_axis_sizes) if main_axis_sizes else 10
+
+ # Epsilon for main-axis clustering is twice the average character size in that dimension
+ eps = avg_main_size * LINE_CLUSTERING_EPS_MULTIPLIER
+
+ centroids = np.array(
+ [((c[0].x + c[0].x2) / 2, (c[0].y + c[0].y2) / 2) for c in band]
+ )
+
+ if centroids.size > 0:
+ db = DBSCAN(eps=eps, min_samples=1, metric="manhattan").fit(centroids)
+
+ line_groups = defaultdict(list)
+ for i, label in enumerate(db.labels_):
+ if label != -1:
+ line_groups[label].append(band[i])
+
+ for _, line in line_groups.items():
+ line.sort(key=get_main_start)
+ final_lines.append(Line(line))
+
+ # Step 3: Split lines that are too tall/wide, which likely contain multiple distinct lines from different columns
+ processed_lines = []
+ for line in final_lines:
+ if not line.chars:
+ continue
+
+ line_secondary_start = min(get_secondary_start(c) for c in line.chars)
+ line_secondary_end = max(get_secondary_end(c) for c in line.chars)
+ line_secondary_size = line_secondary_end - line_secondary_start
+
+ char_secondary_sizes = [
+ get_secondary_end(c) - get_secondary_start(c)
+ for c in line.chars
+ if get_secondary_end(c) - get_secondary_start(c) > 0
+ ]
+ if not char_secondary_sizes:
+ processed_lines.append(line)
+ continue
+
+ max_char_secondary_size = np.max(char_secondary_sizes)
+
+ if (
+ line_secondary_size
+ > max_char_secondary_size * LINE_SPLIT_SIZE_RATIO_THRESHOLD
+ and len(line.chars) > 1
+ ):
+ # logger.debug(
+ # f"Splitting line '{line.text}' which seems to contain multiple lines."
+ # )
+
+ # Use DBSCAN on the secondary axis centers to split the line
+ centers = np.array(
+ [
+ [(get_secondary_start(c) + get_secondary_end(c)) / 2]
+ for c in line.chars
+ ]
+ )
+ db = DBSCAN(
+ eps=max_char_secondary_size * LINE_SPLIT_DBSCAN_EPS_MULTIPLIER,
+ min_samples=1,
+ ).fit(centers)
+
+ sub_lines = defaultdict(list)
+ for i, label in enumerate(db.labels_):
+ sub_lines[label].append(line.chars[i])
+
+ for _, sub_line_chars in sub_lines.items():
+ sub_line_chars.sort(key=get_main_start)
+ processed_lines.append(Line(sub_line_chars))
+ else:
+ processed_lines.append(line)
+ final_lines = processed_lines
+
+ for line in final_lines:
+ _recalculate_line_text_with_spacing(line, orientation)
+
+ return final_lines
+
+
+def _merge_lines_on_page(page_lines: list[Line]) -> list[Line]:
+ """
+ Merge lines on a page that are either contained within or adjacent to each other.
+ This function contains both containment and adjacency merge logic.
+ """
+ if not page_lines:
+ return []
+
+ merged_lines = []
+ lines_to_skip = set()
+
+ for i in range(len(page_lines)):
+ if i in lines_to_skip:
+ continue
+
+ line1 = page_lines[i]
+ if not line1.chars:
+ merged_lines.append(line1)
+ continue
+
+ bbox1 = (
+ min(c[0].x for c in line1.chars),
+ min(c[0].y for c in line1.chars),
+ max(c[0].x2 for c in line1.chars),
+ max(c[0].y2 for c in line1.chars),
+ )
+
+ # Optimization: Calculate a vertical gap threshold to prune the search space.
+ # Based on the vertical adjacency merge condition.
+ line1_avg_char_height = np.mean(
+ [c[0].y2 - c[0].y for c in line1.chars if c[0].y2 > c[0].y] or [0]
+ )
+ max_v_gap = line1_avg_char_height * MERGE_VERTICAL_GAP_MULTIPLIER
+
+ merged = False
+ for j in range(i + 1, len(page_lines)):
+ if j in lines_to_skip:
+ continue
+
+ line2 = page_lines[j]
+ if not line2.chars:
+ continue
+
+ bbox2 = (
+ min(c[0].x for c in line2.chars),
+ min(c[0].y for c in line2.chars),
+ max(c[0].x2 for c in line2.chars),
+ max(c[0].y2 for c in line2.chars),
+ )
+
+ # Optimization: if line2 is too far below line1, no more merges with line1 are possible.
+ # The list is sorted top-to-bottom, so we can break early.
+ v_gap = bbox1[1] - bbox2[3] # y_min_1 - y_max_2
+ if v_gap > max_v_gap:
+ break
+
+ # Check for "mostly contained" by checking intersection over area
+ inter_x0 = max(bbox1[0], bbox2[0])
+ inter_y0 = max(bbox1[1], bbox2[1])
+ inter_x1 = min(bbox1[2], bbox2[2])
+ inter_y1 = min(bbox1[3], bbox2[3])
+
+ inter_area = max(0, inter_x1 - inter_x0) * max(0, inter_y1 - inter_y0)
+
+ area1 = (
+ (bbox1[2] - bbox1[0]) * (bbox1[3] - bbox1[1])
+ if (bbox1[2] > bbox1[0] and bbox1[3] > bbox1[1])
+ else 0
+ )
+ area2 = (
+ (bbox2[2] - bbox2[0]) * (bbox2[3] - bbox2[1])
+ if (bbox2[2] > bbox2[0] and bbox2[3] > bbox2[1])
+ else 0
+ )
+
+ # Heuristic for merging:
+ # 1. By containment: if one line is mostly inside another.
+ # 2. By adjacency: if two lines are close and aligned.
+ if (
+ area2 > 0
+ and area1 >= area2
+ and (inter_area / area2) > MERGE_CONTAINMENT_IOU_THRESHOLD
+ ):
+ # Case 1: Merge line2 (smaller) into line1 (larger) by containment
+ # logger.debug(
+ # f"Merging line '{line2.text}' into '{line1.text}' (mostly contained)"
+ # )
+ line1.chars.extend(line2.chars)
+ lines_to_skip.add(j)
+ merged = True
+ bbox1 = (
+ min(bbox1[0], bbox2[0]),
+ min(bbox1[1], bbox2[1]),
+ max(bbox1[2], bbox2[2]),
+ max(bbox1[3], bbox2[3]),
+ )
+
+ elif (
+ area1 > 0
+ and area2 > area1
+ and (inter_area / area1) > MERGE_CONTAINMENT_IOU_THRESHOLD
+ ):
+ # Case 2: Merge line1 (smaller) into line2 (larger) by containment
+ # logger.debug(
+ # f"Merging line '{line1.text}' into '{line2.text}' (mostly contained)"
+ # )
+ line2.chars.extend(line1.chars)
+ page_lines[i], page_lines[j] = page_lines[j], page_lines[i]
+ line1 = page_lines[i]
+ lines_to_skip.add(j)
+ merged = True
+ bbox1 = (
+ min(bbox1[0], bbox2[0]),
+ min(bbox1[1], bbox2[1]),
+ max(bbox1[2], bbox2[2]),
+ max(bbox1[3], bbox2[3]),
+ )
+
+ else:
+ # Case 3: Merge by adjacency for lines that are close to each other
+ orientation = "horizontal" if not line1.chars[0][2] else "vertical"
+ if orientation == "horizontal":
+ height1 = bbox1[3] - bbox1[1]
+ height2 = bbox2[3] - bbox2[1]
+ if height1 > 0 and height2 > 0:
+ v_overlap = max(
+ 0,
+ min(bbox1[3], bbox2[3]) - max(bbox1[1], bbox2[1]),
+ )
+ if (
+ v_overlap / height1
+ ) > MERGE_ADJACENCY_OVERLAP_THRESHOLD and (
+ v_overlap / height2
+ ) > MERGE_ADJACENCY_OVERLAP_THRESHOLD:
+ h_gap = max(bbox1[0], bbox2[0]) - min(bbox1[2], bbox2[2])
+ if h_gap >= 0:
+ avg_char_width = np.mean(
+ [
+ c[0].x2 - c[0].x
+ for c in (line1.chars + line2.chars)
+ if c[0].x2 > c[0].x
+ ]
+ or [0]
+ )
+ if (
+ avg_char_width > 0
+ and h_gap
+ < avg_char_width * MERGE_ADJACENCY_GAP_MULTIPLIER
+ ):
+ # logger.debug(
+ # f"Merging adjacent lines '{line1.text}' and '{line2.text}'"
+ # )
+ line1.chars.extend(line2.chars)
+ lines_to_skip.add(j)
+ merged = True
+ bbox1 = (
+ min(bbox1[0], bbox2[0]),
+ min(bbox1[1], bbox2[1]),
+ max(bbox1[2], bbox2[2]),
+ max(bbox1[3], bbox2[3]),
+ )
+ else: # Vertical
+ width1 = bbox1[2] - bbox1[0]
+ width2 = bbox2[2] - bbox2[0]
+ if width1 > 0 and width2 > 0:
+ h_overlap = max(
+ 0,
+ min(bbox1[2], bbox2[2]) - max(bbox1[0], bbox2[0]),
+ )
+ if (
+ h_overlap / width1
+ ) > MERGE_ADJACENCY_OVERLAP_THRESHOLD and (
+ h_overlap / width2
+ ) > MERGE_ADJACENCY_OVERLAP_THRESHOLD:
+ v_gap = max(bbox1[1], bbox2[1]) - min(bbox1[3], bbox2[3])
+ if v_gap >= 0:
+ avg_char_height = np.mean(
+ [
+ c[0].y2 - c[0].y
+ for c in (line1.chars + line2.chars)
+ if c[0].y2 > c[0].y
+ ]
+ or [0]
+ )
+ if (
+ avg_char_height > 0
+ and v_gap
+ < avg_char_height * MERGE_ADJACENCY_GAP_MULTIPLIER
+ ):
+ # logger.debug(
+ # f"Merging adjacent vertical lines '{line1.text}' and '{line2.text}'"
+ # )
+ line1.chars.extend(line2.chars)
+ lines_to_skip.add(j)
+ merged = True
+ bbox1 = (
+ min(bbox1[0], bbox2[0]),
+ min(bbox1[1], bbox2[1]),
+ max(bbox1[2], bbox2[2]),
+ max(bbox1[3], bbox2[3]),
+ )
+
+ if merged:
+ # Re-sort and recalculate text for the merged line
+ orientation = (
+ "horizontal" if not line1.chars[0][2] else "vertical"
+ ) # Guess orientation from first char
+ if orientation == "horizontal":
+ line1.chars.sort(key=lambda c: c[0].x)
+ else: # vertical
+ line1.chars.sort(key=lambda c: c[0].y)
+ _recalculate_line_text_with_spacing(line1, orientation)
+
+ merged_lines.append(line1)
+
+ return merged_lines
+
+
+def process_page_chars_to_lines(
+ chars: list[tuple[il_version_1.Box, str, bool]],
+) -> list[Line]:
+ pool = get_process_pool()
+ if pool is None:
+ return process_page_chars_to_lines_internal(chars)
+ return pool.apply(process_page_chars_to_lines_internal, (chars,))
+
+
+def process_page_chars_to_lines_internal(
+ chars: list[tuple[il_version_1.Box, str, bool]],
+) -> list[Line]:
+ """
+ Process characters on a single page to cluster them into lines.
+
+ Args:
+ chars: List of character tuples (box, char_unicode, is_vertical)
+
+ Returns:
+ List of Line objects representing clustered and merged lines
+ """
+ if not chars:
+ return []
+
+ horizontal_chars = [c for c in chars if not c[2]]
+ vertical_chars = [c for c in chars if c[2]]
+
+ horizontal_lines = _cluster_by_axis(horizontal_chars, "horizontal")
+ vertical_lines = _cluster_by_axis(vertical_chars, "vertical")
+
+ page_lines = horizontal_lines + vertical_lines
+
+ # Sort all found lines by their position on the page (top-to-bottom, left-to-right)
+ def get_line_position(line):
+ if not line:
+ return (0, 0)
+ # PDF coordinate system: Y increases upwards. We negate it for top-to-bottom sort.
+ avg_y = np.mean([(c[0].y + c[0].y2) / 2 for c in line])
+ avg_x = np.mean([(c[0].x + c[0].x2) / 2 for c in line])
+ return (-avg_y, avg_x)
+
+ page_lines.sort(key=lambda line: get_line_position(line.chars))
+
+ # Merge lines on the page
+ merged_page_lines = _merge_lines_on_page(page_lines)
+ return merged_page_lines
+
+
+def cluster_chars_to_lines(
+ char_boxes: dict[int, list[tuple[il_version_1.Box, str, bool]]],
+) -> dict[int, list[Line]]:
+ clustered_lines = {}
+ if not char_boxes:
+ return clustered_lines
+
+ for page_num, chars in char_boxes.items():
+ merged_page_lines = process_page_chars_to_lines(chars)
+ clustered_lines[page_num] = merged_page_lines
+
+ return clustered_lines
+
+
+def draw_clustered_lines_to_image(pdf_path, clustered_lines: dict[int, list[Line]]):
+ doc = pymupdf.open(pdf_path)
+ debug_dir = Path("ocr-box-image-clustered") / Path(pdf_path).stem
+ debug_dir.mkdir(parents=True, exist_ok=True)
+
+ for page_number, lines in clustered_lines.items():
+ if not lines:
+ continue
+
+ page = doc[page_number]
+ pixmap = page.get_pixmap(dpi=300)
+ image_height = pixmap.height
+ image_width = pixmap.width
+
+ samples = bytearray(pixmap.samples)
+ image_array = np.frombuffer(samples, dtype=np.uint8).reshape(
+ image_height, image_width, pixmap.n
+ )
+
+ if pixmap.n in [3, 4]:
+ image_array = cv2.cvtColor(image_array, cv2.COLOR_RGB2BGR)
+
+ # cv2.imwrite(str(debug_dir / f"{page_number}.png"), image_array)
+
+ annotated_image = image_array.copy()
+
+ page_rect = page.rect
+ x_scale = image_width / page_rect.width
+ y_scale = image_height / page_rect.height
+
+ for i, line in enumerate(lines):
+ if not line:
+ continue
+
+ # Draw the encompassing line box first (red)
+ char_boxes_in_line = [item[0] for item in line.chars]
+ min_x = min(b.x for b in char_boxes_in_line)
+ min_y = min(b.y for b in char_boxes_in_line)
+ max_x2 = max(b.x2 for b in char_boxes_in_line)
+ max_y2 = max(b.y2 for b in char_boxes_in_line)
+
+ img_x0_line = int(min_x * x_scale)
+ img_y1_line = int(image_height - (max_y2 * y_scale))
+ img_x1_line = int(max_x2 * x_scale)
+ img_y0_line = int(image_height - (min_y * y_scale))
+
+ cv2.rectangle(
+ annotated_image,
+ (img_x0_line, img_y1_line),
+ (img_x1_line, img_y0_line),
+ (0, 0, 255), # Red for lines
+ 2,
+ )
+
+ cv2.putText(
+ annotated_image,
+ f"line {i}: {line.text}",
+ (img_x0_line, img_y1_line - 10),
+ cv2.FONT_HERSHEY_SIMPLEX,
+ 0.7,
+ (0, 0, 255),
+ 2,
+ )
+
+ # Then, draw the individual character boxes on top (green)
+ for char_box, _, _ in line.chars:
+ pdf_x0, pdf_y0, pdf_x1, pdf_y1 = (
+ char_box.x,
+ char_box.y,
+ char_box.x2,
+ char_box.y2,
+ )
+
+ img_x0_char = int(pdf_x0 * x_scale)
+ img_y0_char_pdf = int(pdf_y0 * y_scale)
+ img_x1_char = int(pdf_x1 * x_scale)
+ img_y1_char_pdf = int(pdf_y1 * y_scale)
+
+ img_y0_char = image_height - img_y0_char_pdf
+ img_y1_char = image_height - img_y1_char_pdf
+
+ cv2.rectangle(
+ annotated_image,
+ (img_x0_char, img_y1_char),
+ (img_x1_char, img_y0_char),
+ (0, 255, 0), # Green for characters
+ 1, # Thinner line
+ )
+
+ cv2.imwrite(str(debug_dir / f"{page_number}_annotated.png"), annotated_image)
+
+ doc.close()
+
+
+def main():
+ logging.basicConfig(level=logging.INFO, handlers=[RichHandler()])
+ for pdf_path in (
+ "2404.16109v1.pdf",
+ "2022 - Bortoli_Valentin De, Mathieu_Emile - Riemannian Score-Based Generative Modelling.pdf",
+ "2024 - Regev_Oded - On Lattices, Learning with Errors, Random Linear Codes, and Cryptography.pdf",
+ "2024 - Yang_Tian-Le, Lee_Kuang-Yao - Functional Linear Non-Gaussian Acyclic Model for Causal Discovery.pdf",
+ ):
+ logger.info(f"Processing {pdf_path}")
+ char_boxes = extract_paragraph_line(pdf_path)
+ if not char_boxes:
+ logger.warning(f"No character boxes extracted from {pdf_path}")
+ continue
+
+ logger.info(
+ f"Extracted {sum(len(c) for c in char_boxes.values())} characters. Clustering them into lines..."
+ )
+ lines = cluster_chars_to_lines(char_boxes)
+
+ total_lines = sum(len(l) for l in lines.values())
+ logger.info(f"Clustered into {total_lines} lines. Drawing boxes...")
+
+ # logger.info("--- Clustered Lines Text ---")
+ # for page_num, page_lines in lines.items():
+ # logger.info(f"Page {page_num}:")
+ # for i, line in enumerate(page_lines):
+ # logger.info(f" Line {i}: {line.text}")
+ # logger.info("----------------------------")
+
+ draw_clustered_lines_to_image(pdf_path, lines)
+ logger.info("Annotated images saved in 'ocr-box-image-clustered' directory.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/babeldoc/format/pdf/document_il/utils/fontmap.py b/babeldoc/format/pdf/document_il/utils/fontmap.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e3dd2a557703baa236d3c425b08816db98c2b5a
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/utils/fontmap.py
@@ -0,0 +1,315 @@
+import enum
+import functools
+import logging
+import re
+from pathlib import Path
+
+import pymupdf
+
+from babeldoc.assets import assets
+from babeldoc.format.pdf.document_il import PdfFont
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.translation_config import TranslationConfig
+
+logger = logging.getLogger(__name__)
+
+
+class PrimaryFontFamily(enum.IntEnum):
+ SERIF = 1
+ SANS_SERIF = 2
+ SCRIPT = 3
+ NONE = 4
+
+ @classmethod
+ def from_str(cls, value: str):
+ if value == "serif":
+ return cls.SERIF
+ elif value == "sans-serif":
+ return cls.SANS_SERIF
+ elif value == "script":
+ return cls.SCRIPT
+ else:
+ return cls.NONE
+
+
+class FontMapper:
+ stage_name = "Add Fonts"
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.translation_config = translation_config
+ assert translation_config.primary_font_family in [
+ None,
+ "serif",
+ "sans-serif",
+ "script",
+ ]
+ self.primary_font_family = PrimaryFontFamily.from_str(
+ translation_config.primary_font_family,
+ )
+
+ font_family = assets.get_font_family(translation_config.lang_out)
+ self.font_file_names = []
+ for k in (
+ "normal",
+ "script",
+ "fallback",
+ "base",
+ ):
+ self.font_file_names.extend(font_family[k])
+
+ self.fonts: dict[str, pymupdf.Font] = {}
+ self.fontid2fontpath: dict[str, Path] = {}
+ for font_file_name in self.font_file_names:
+ if font_file_name in self.fontid2fontpath:
+ continue
+ font_path, font_metadata = assets.get_font_and_metadata(font_file_name)
+ pymupdf_font = pymupdf.Font(fontfile=str(font_path))
+ pymupdf_font.has_glyph = functools.lru_cache(maxsize=10240, typed=True)(
+ pymupdf_font.has_glyph,
+ )
+ pymupdf_font.char_lengths = functools.lru_cache(maxsize=10240, typed=True)(
+ pymupdf_font.char_lengths,
+ )
+ self.fonts[font_file_name] = pymupdf_font
+ self.fontid2fontpath[font_file_name] = font_path
+ self.fonts[font_file_name].font_id = font_file_name
+ self.fonts[font_file_name].font_path = font_path
+ self.fonts[font_file_name].ascent_fontmap = font_metadata["ascent"]
+ self.fonts[font_file_name].descent_fontmap = font_metadata["descent"]
+ self.fonts[font_file_name].encoding_length = font_metadata[
+ "encoding_length"
+ ]
+
+ self.normal_font_ids: list[str] = font_family["normal"]
+ self.script_font_ids: list[str] = font_family["script"]
+ self.fallback_font_ids: list[str] = font_family["fallback"]
+ self.base_font_ids: list[str] = font_family["base"]
+ self.fontid2fontpath["base"] = self.fontid2fontpath[font_family["base"][0]]
+
+ self.fontid2font: dict[str, pymupdf.Font] = {
+ f.font_id: f for f in self.fonts.values()
+ }
+
+ self.fontid2font["base"] = self.fontid2font[self.base_font_ids[0]]
+
+ self.normal_fonts: list[pymupdf.Font] = [
+ self.fontid2font[font_id] for font_id in self.normal_font_ids
+ ]
+ self.script_fonts: list[pymupdf.Font] = [
+ self.fontid2font[font_id] for font_id in self.script_font_ids
+ ]
+ self.fallback_fonts: list[pymupdf.Font] = [
+ self.fontid2font[font_id] for font_id in self.fallback_font_ids
+ ]
+
+ self.base_font = self.fontid2font["base"]
+
+ self.type2font: dict[str, list[pymupdf.Font]] = {
+ "normal": self.normal_fonts,
+ "script": self.script_fonts,
+ "fallback": self.fallback_fonts,
+ "base": [self.base_font],
+ }
+
+ self.has_char = functools.lru_cache(maxsize=10240, typed=True)(self.has_char)
+ self.map_in_type = functools.lru_cache(maxsize=10240, typed=True)(
+ self.map_in_type
+ )
+
+ def has_char(self, char_unicode: str):
+ if len(char_unicode) != 1:
+ return False
+ current_char = ord(char_unicode)
+ for font in self.fonts.values():
+ if font.has_glyph(current_char):
+ return True
+ return False
+
+ def map_in_type(
+ self,
+ bold: bool,
+ italic: bool,
+ monospaced: bool,
+ serif: bool,
+ char_unicode: str,
+ font_type: str,
+ ):
+ if font_type == "script" and not italic:
+ return None
+ current_char = ord(char_unicode)
+ for font in self.type2font[font_type]:
+ if not font.has_glyph(current_char):
+ continue
+ if bool(bold) != bool(font.is_bold):
+ continue
+ # 不知道什么原因，思源黑体的 serif 属性为 1，先 workaround
+ if bool(serif) and "serif" not in font.font_id.lower():
+ continue
+ if not bool(serif) and "serif" in font.font_id.lower():
+ continue
+ return font
+
+ return None
+
+ def map(self, original_font: PdfFont, char_unicode: str):
+ current_char = ord(char_unicode)
+ if isinstance(original_font, pymupdf.Font):
+ bold = original_font.is_bold
+ italic = original_font.is_italic
+ monospaced = original_font.is_monospaced
+ serif = original_font.is_serif
+ elif isinstance(original_font, PdfFont):
+ bold = original_font.bold
+ italic = original_font.italic
+ monospaced = original_font.monospace
+ serif = original_font.serif
+ else:
+ logger.error(
+ f"Unknown font type: {type(original_font)}. "
+ f"Original font: {original_font}. "
+ f"Char unicode: {char_unicode}. ",
+ )
+ return None
+
+ if self.primary_font_family == PrimaryFontFamily.SERIF:
+ serif = True
+ elif self.primary_font_family == PrimaryFontFamily.SANS_SERIF:
+ serif = False
+ elif self.primary_font_family == PrimaryFontFamily.SCRIPT:
+ serif = False
+ italic = True
+
+ script_font_map_result = self.map_in_type(
+ bold, italic, monospaced, serif, char_unicode, "script"
+ )
+ if script_font_map_result:
+ return script_font_map_result
+
+ for script_font in self.script_fonts:
+ if italic and script_font.has_glyph(current_char):
+ return script_font
+
+ normal_font_map_result = self.map_in_type(
+ bold, italic, monospaced, serif, char_unicode, "normal"
+ )
+ if normal_font_map_result is not None:
+ return normal_font_map_result
+
+ fallback_font_map_result = self.map_in_type(
+ bold, italic, monospaced, serif, char_unicode, "fallback"
+ )
+ if fallback_font_map_result is not None:
+ return fallback_font_map_result
+
+ for font in self.fallback_fonts:
+ if font.has_glyph(current_char):
+ return font
+
+ logger.warning(
+ f"Can't find font for {char_unicode}({current_char}). "
+ f"Original font: {original_font.name}[{original_font.font_id}]. "
+ f"Char unicode: {char_unicode}. ",
+ )
+ return None
+
+ def get_used_font_ids(self, il: il_version_1.Document) -> set[str]:
+ result = set()
+ for page in il.page:
+ for char in page.pdf_character:
+ if char.pdf_style and char.pdf_style.font_id:
+ result.add(char.pdf_style.font_id)
+ for para in page.pdf_paragraph:
+ for comp in para.pdf_paragraph_composition:
+ if char := comp.pdf_character:
+ if char.pdf_style and char.pdf_style.font_id:
+ result.add(char.pdf_style.font_id)
+ return result
+
+ def add_font(self, doc_zh: pymupdf.Document, il: il_version_1.Document):
+ used_font_ids = self.get_used_font_ids(il)
+ font_list = [
+ (k, v) for k, v in self.fontid2fontpath.items() if k in used_font_ids
+ ]
+
+ font_id = {}
+ xreflen = doc_zh.xref_length()
+ total = xreflen - 1 + len(font_list) + len(il.page) + len(font_list)
+ with self.translation_config.progress_monitor.stage_start(
+ self.stage_name,
+ total,
+ ) as pbar:
+ if not il.page:
+ pbar.advance(total)
+ return
+ for font in font_list:
+ if font[0] in font_id:
+ continue
+ font_id[font[0]] = doc_zh[0].insert_font(font[0], font[1])
+ pbar.advance(1)
+ for xref in range(1, xreflen):
+ pbar.advance(1)
+ # xref_type = doc_zh.xref_get_key(xref, "Type")
+ # if xref_type[1] == "/Page":
+ # resources_xref = doc_zh.xref_get_key(xref, "Resources")
+ # if resources_xref[0] == 'null':
+ # doc_zh.xref_set_key(xref, "Resources", f"<>>>")
+ for label in ["Resources/", ""]: # 可能是基于 xobj 的 res
+ try: # xref 读写可能出错
+ font_res = doc_zh.xref_get_key(xref, f"{label}Font")
+ if font_res is None:
+ continue
+ target_key_prefix = f"{label}Font/"
+ if font_res[0] == "xref":
+ resource_xref_id = re.search(
+ "(\\d+) 0 R",
+ font_res[1],
+ ).group(1)
+ xref = int(resource_xref_id)
+ font_res = ("dict", doc_zh.xref_object(xref))
+ target_key_prefix = ""
+ if font_res[0] == "dict":
+ for font in font_list:
+ target_key = f"{target_key_prefix}{font[0]}"
+ font_exist = doc_zh.xref_get_key(xref, target_key)
+ if font_exist[0] == "null":
+ doc_zh.xref_set_key(
+ xref,
+ target_key,
+ f"{font_id[font[0]]} 0 R",
+ )
+ except Exception:
+ pass
+
+ # Create PdfFont for each font
+ # 预先创建所有字体对象
+ pdf_fonts = []
+ for font_name, _ in font_list:
+ # Get descent_fontmap from fontid2font
+ assert font_name in self.fontid2font, f"Font {font_name} not found"
+ mupdf_font = self.fontid2font[font_name]
+ descent_fontmap = mupdf_font.descent_fontmap
+ ascent_fontmap = mupdf_font.ascent_fontmap
+ encoding_length = mupdf_font.encoding_length
+
+ pdf_fonts.append(
+ il_version_1.PdfFont(
+ name=font_name,
+ xref_id=font_id[font_name],
+ font_id=font_name,
+ encoding_length=encoding_length,
+ bold=mupdf_font.is_bold,
+ italic=mupdf_font.is_italic,
+ monospace=mupdf_font.is_monospaced,
+ serif=mupdf_font.is_serif,
+ descent=descent_fontmap,
+ ascent=ascent_fontmap,
+ ),
+ )
+ pbar.advance(1)
+
+ # 批量添加字体到页面和 XObject
+ for page in il.page:
+ page.pdf_font.extend(pdf_fonts)
+ for xobj in page.pdf_xobject:
+ xobj.pdf_font.extend(pdf_fonts)
+ pbar.advance(1)
diff --git a/babeldoc/format/pdf/document_il/utils/formular_helper.py b/babeldoc/format/pdf/document_il/utils/formular_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a444a74c08f4bb6a103d21ee6a0bfff20ceb5b6
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/utils/formular_helper.py
@@ -0,0 +1,335 @@
+import base64
+import functools
+import re
+import unicodedata
+
+from babeldoc.format.pdf.document_il.il_version_1 import Box
+from babeldoc.format.pdf.document_il.il_version_1 import Page
+from babeldoc.format.pdf.document_il.il_version_1 import PdfFormula
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.layout_helper import (
+ formular_height_ignore_char,
+)
+from babeldoc.format.pdf.translation_config import TranslationConfig
+
+
+def is_formulas_start_char(
+ char: str,
+ font_mapper: FontMapper,
+ translation_config: TranslationConfig,
+) -> bool:
+ if not char:
+ return False
+ if "(cid:" in char:
+ return True
+ if not font_mapper.has_char(char):
+ if len(char) > 1 and all(font_mapper.has_char(x) for x in char):
+ return False
+ return True
+ if translation_config.formular_char_pattern:
+ pattern = translation_config.formular_char_pattern
+ if re.match(pattern, char):
+ return True
+ if char != " " and (
+ unicodedata.category(char[0])
+ in [
+ # "Lm",
+ "Mn",
+ "Sk",
+ "Sm",
+ "Zl",
+ "Zp",
+ "Zs",
+ "Co", # private use character
+ # "So", # symbol
+ ] # 文字修饰符、数学符号、分隔符号
+ or ord(char[0]) in range(0x370, 0x400) # 希腊字母
+ ):
+ return True
+ if re.match("[0-9\\[\\]•]", char):
+ return True
+ return False
+
+
+def is_formulas_middle_char(
+ char: str,
+ font_mapper: FontMapper,
+ translation_config: TranslationConfig,
+) -> bool:
+ if is_formulas_start_char(char, font_mapper, translation_config):
+ return True
+
+ if re.match(",", char):
+ return True
+
+ return False
+
+
+def collect_page_formula_font_ids(
+ page: Page, formular_font_pattern: str | None
+) -> tuple[set[int], dict[str, set[int]]]:
+ """
+ Collects formula font IDs from page fonts and XObject fonts.
+
+ Args:
+ page: The Page object to process.
+ formular_font_pattern: The regex pattern to identify formula fonts by name.
+
+ Returns:
+ A tuple containing:
+ - A set of font_ids considered formula fonts at the page level.
+ - A dictionary mapping xobj_id to a set of font_ids considered
+ formula fonts for that specific XObject.
+ """
+ # Page-level formula font IDs
+ page_formula_font_ids = set()
+ if page.pdf_font:
+ for font in page.pdf_font:
+ if is_formulas_font(font.name, formular_font_pattern):
+ page_formula_font_ids.add(font.font_id)
+
+ # XObject-level formula font IDs
+ xobj_formula_font_ids_map = {}
+ if page.pdf_xobject:
+ for xobj in page.pdf_xobject:
+ # Start with a copy of page-level formula fonts for this XObject
+ current_xobj_fonts = page_formula_font_ids.copy()
+ if xobj.pdf_font:
+ for font in xobj.pdf_font:
+ if is_formulas_font(font.name, formular_font_pattern):
+ current_xobj_fonts.add(font.font_id)
+ else:
+ # If a font within an XObject is explicitly not a formula font,
+ # remove it from this XObject's set.
+ current_xobj_fonts.discard(font.font_id)
+ xobj_formula_font_ids_map[xobj.xobj_id] = current_xobj_fonts
+
+ return page_formula_font_ids, xobj_formula_font_ids_map
+
+
+@functools.cache
+def is_formulas_font(font_name: str, formular_font_pattern: str | None) -> bool:
+ pattern_text = (
+ r"^("
+ r"|BLKFort.*"
+ r"|Cambria.*"
+ r"|EUAlbertina.*"
+ r"|NimbusRomNo9L.*"
+ r"|GlosaMath.*"
+ r"|URWPalladioL.*"
+ r"|CMSS.+"
+ r"|Arial.*"
+ r"|TimesNewRoman.*"
+ r"|SegoeUI.*"
+ r"|CMTT9.*"
+ r"|CMSL10.*"
+ r"|CMTI10.*"
+ r"|CMTT10.*"
+ r"|CMTI12.*"
+ r"|CMR12.*"
+ r"|MeridienLTStd.*"
+ r"|Calibri.*"
+ r"|STIXMathJax_Main.*"
+ r"|.*NewBaskerville.*"
+ r"|.*FranklinGothic.*"
+ r"|.*AGaramondPro.*"
+ r"|.*PalatinoItalCOR.*"
+ r"|.*ITCSymbolStd.*"
+ r"|.*PlantinStd.*"
+ r"|.*DJ5EscrowCond.*"
+ r"|.*ExchangeBook.*"
+ r"|.*DJ5Exchange.*"
+ r"|.*Times.*"
+ r"|.*PalatinoLTStd.*"
+ r"|.*Times New Roman,Italic.*"
+ r"|.*EhrhardtMT.*"
+ r"|.*GillSansMTStd.*"
+ r"|.*MedicineSymbols3.*"
+ r"|.*HardingText.*"
+ r"|.*GraphikNaturel.*"
+ r"|.*HelveticaNeue.*"
+ r"|.*GoudyOldStyleT.*"
+ r"|.*Symbol.*"
+ r"|.*ScalaSansLF.*"
+ r"|.*ScalaLF.*"
+ r"|.*ScalaSansPro.*"
+ r"|.*PetersburgC.*"
+ r"|.*ColiseumC.*"
+ r"|.*Gantari.*"
+ r"|.*OptimaLTStd.*"
+ r"|.*CronosPro.*"
+ r"|.*ACaslon.*"
+ r"|.*Frutiger.*"
+ r"|.*BrandonGrotesque.*"
+ r"|.*FairfieldLH.*"
+ r"|.*CaeciliaLTStd.*"
+ r"|.*Whitney.*"
+ r"|.*Mercury.*"
+ r"|.*SabonLTStd.*"
+ r"|.*AnonymousPro.*"
+ r"|.*SabonLTPro.*"
+ r"|.*ArnoPro.*"
+ r"|.*CharisSIL.*"
+ r"|.*MSReference.*"
+ r"|.*CMUSerif-Roman.*"
+ r"|.*CourierNewPS.*"
+ r"|.*XCharter.*"
+ r"|.*GillSans.*"
+ r"|.*Perpetua.*"
+ r"|.*GEInspira.*"
+ r"|.*AGaramond.*"
+ r"|.*BMath.*"
+ r"|.*MSTT.*"
+ r"|.*Bookinsanity.*"
+ r"|.*ScalySans.*"
+ r"|.*Code2000.*"
+ r"|.*Minion.*"
+ r"|.*JansonTextLT.*"
+ r"|.*MathPack.*"
+ r"|.*Macmillan.*"
+ r"|.*NimbusSan.*"
+ r"|.*Mincho.*"
+ r"|.*Amerigo.*"
+ r"|.*MSGloriolaIIStd.*"
+ r"|.*CMU.+"
+ r"|.*LinLibertine.*"
+ r"|.*txsys.*"
+ r")$"
+ )
+ precise_formula_font_pattern = (
+ r"^("
+ # r"|.*CambriaMath.*"
+ # r"|.*Cambria Math.*"
+ r"|.*Asana.*"
+ r"|.*MiriamMonoCLM-BookOblique.*"
+ r"|.*Miriam Mono CLM.*"
+ r"|.*Logix.*"
+ r"|.*AeBonum.*"
+ r"|.*AeMRoman.*"
+ r"|.*AePagella.*"
+ r"|.*AeSchola.*"
+ r"|.*Concrete.*"
+ r"|.*LatinModernMathCompanion.*"
+ r"|.*Latin Modern Math Companion.*"
+ r"|.*RalphSmithsFormalScriptCompanion.*"
+ r"|.*Ralph Smiths Formal Script Companion.*"
+ r"|.*TeXGyreBonumMathCompanion.*"
+ r"|.*TeX Gyre Bonum Companion.*"
+ r"|.*TeXGyrePagellaMathCompanion.*"
+ r"|.*TeX Gyre Pagella Math Companion.*"
+ r"|.*TeXGyreTermesMathCompanion.*"
+ r"|.*TeX Gyre Termes Math Companion.*"
+ r"|.*XITSMathCompanion.*"
+ r"|.*XITS Math Companion.*"
+ r"|.*Erewhon.*"
+ r"|.*Euler-Math.*"
+ r"|.*Euler Math.*"
+ r"|.*FiraMath-Regular.*"
+ r"|.*Fira Math.*"
+ r"|.*Garamond-Math.*"
+ r"|.*GFSNeohellenicMath.*"
+ r"|.*KpMath.*"
+ r"|.*Lete Sans Math.*"
+ r"|.*LeteSansMath.*"
+ # r"|.*LinLibertineO.*"
+ r"|.*Linux Libertine O.*"
+ r"|.*LibertinusMath-Regular.*"
+ r"|.*Libertinus Math.*"
+ r"|.*LatinModernMath-Regular.*"
+ r"|.*Latin Modern Math.*"
+ r"|.*Luciole.*"
+ r"|.*NewCM.*"
+ r"|.*NewComputerModern.*"
+ r"|.*OldStandard-Math.*"
+ r"|.*STIXMath-Regular.*"
+ r"|.*STIX Math.*"
+ r"|.*STIXTwoMath-Regular.*"
+ r"|.*STIX Two Math.*"
+ r"|.*TeXGyreBonumMath.*"
+ r"|.*TeX Gyre Bonum Math.*"
+ r"|.*TeXGyreDejaVuMath.*"
+ r"|.*TeX Gyre DejaVu Math.*"
+ r"|.*TeXGyrePagellaMath.*"
+ r"|.*TeX Gyre Pagella Math.*"
+ r"|.*TeXGyreScholaMath.*"
+ r"|.*TeX Gyre Schola Math.*"
+ r"|.*TeXGyreTermesMath.*"
+ r"|.*TeX Gyre Termes Math.*"
+ r"|.*XCharter-Math.*"
+ r"|.*XCharter Math.*"
+ r"|.*XITSMath-Bold.*"
+ r"|.*XITS Math.*"
+ r"|.*XITSMath.*"
+ r"|.*IBMPlexMath.*"
+ r"|.*IBM Plex Math.*"
+ r")$"
+ )
+ if formular_font_pattern:
+ broad_formula_font_pattern = formular_font_pattern
+ else:
+ broad_formula_font_pattern = (
+ r"(CM[^RB]"
+ r"|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]"
+ r"|LINE"
+ r"|LCIRCLE"
+ r"|TeX-"
+ r"|rsfs"
+ r"|txsy"
+ r"|wasy"
+ r"|stmary"
+ r"|.*Mono"
+ r"|.*Code"
+ # r"|.*Ital"
+ r"|.*Sym"
+ r"|.*Math"
+ r"|AdvP4C4E74"
+ r"|AdvPSSym"
+ r"|AdvP4C4E59"
+ r")"
+ )
+
+ if font_name.startswith("BASE64:"):
+ font_name_bytes = base64.b64decode(font_name[7:])
+ font = font_name_bytes.split(b"+")[-1]
+ pattern_text = pattern_text.encode()
+ broad_formula_font_pattern = broad_formula_font_pattern.encode()
+ else:
+ font = font_name.split("+")[-1]
+
+ if not font:
+ return False
+
+ if re.match(precise_formula_font_pattern, font):
+ return True
+ elif re.match(pattern_text, font):
+ return False
+ elif re.match(broad_formula_font_pattern, font):
+ return True
+
+ return False
+
+
+def update_formula_data(formula: PdfFormula):
+ min_x = min(char.visual_bbox.box.x for char in formula.pdf_character)
+ max_x = max(char.visual_bbox.box.x2 for char in formula.pdf_character)
+ if not all(map(formular_height_ignore_char, formula.pdf_character)):
+ min_y = min(
+ char.visual_bbox.box.y
+ for char in formula.pdf_character
+ if not formular_height_ignore_char(char)
+ )
+ max_y = max(
+ char.visual_bbox.box.y2
+ for char in formula.pdf_character
+ if not formular_height_ignore_char(char)
+ )
+ else:
+ min_y = min(char.visual_bbox.box.y for char in formula.pdf_character)
+ max_y = max(char.visual_bbox.box.y2 for char in formula.pdf_character)
+ formula.box = Box(min_x, min_y, max_x, max_y)
+ if not formula.y_offset:
+ formula.y_offset = 0
+ if not formula.x_offset:
+ formula.x_offset = 0
+ if not formula.x_advance:
+ formula.x_advance = 0
diff --git a/babeldoc/format/pdf/document_il/utils/layout_helper.py b/babeldoc/format/pdf/document_il/utils/layout_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..fce8d38c453bfc30d0149bcef9915aef2be53492
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/utils/layout_helper.py
@@ -0,0 +1,1126 @@
+import logging
+import math
+import re
+import unicodedata
+from typing import Literal
+
+import regex
+from pymupdf import Font
+
+from babeldoc.format.pdf.document_il import GraphicState
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.il_version_1 import Box
+from babeldoc.format.pdf.document_il.il_version_1 import PdfCharacter
+from babeldoc.format.pdf.document_il.il_version_1 import PdfParagraph
+from babeldoc.format.pdf.document_il.il_version_1 import PdfParagraphComposition
+
+logger = logging.getLogger(__name__)
+# HEIGHT_NOT_USFUL_CHAR_IN_CHAR = (
+# "∑ï¸Â",
+# # æš‚æ—¶å‡设 cid:17 å’Œ cid 16 是特殊情况
+# # æÂ¥æºÂ于 arXiv:2310.18608v2 第ä¹Â页公å¼Â大括å·
+# "(cid:17)",
+# "(cid:16)",
+# # arXiv:2411.19509v2 第四页 []
+# "(cid:104)",
+# "(cid:105)",
+# # arXiv:2411.19509v2 第四页 å…¬å¼Âçš„ | 竖线
+# "(cid:13)",
+# "∑ï¸Â",
+# # arXiv:2412.05265 27 页 累加å·
+# "(cid:88)",
+# # arXiv:2412.05265 16 页 累乘å·
+# "(cid:89)",
+# # arXiv:2412.05265 27 页 积åˆâ€
+# "(cid:90)",
+# # arXiv:2412.05265 32 页 å…¬å¼Âå·¦å³的ä¸Â括å·
+# "(cid:2)",
+# "(cid:3)",
+# "·",
+# "√",
+# )
+
+# çâ€Â±Ã¤ÂºÅ½Ã¦Ë†â€˜Ã¤Â»Â¬Ã¦Å“‰ä¸€å¥— bbox 解æžÂ机制了，所以现在ä¸Â需è¦Â这个东西了。
+HEIGHT_NOT_USFUL_CHAR_IN_CHAR = (None,)
+
+
+LEFT_BRACKET = ("(cid:8)", "(", "(cid:16)", "{", "[", "(cid:104)", "(cid:2)")
+RIGHT_BRACKET = ("(cid:9)", ")", "(cid:17)", "}", "]", "(cid:105)", "(cid:3)")
+
+BULLET_POINT_PATTERN = re.compile(
+ r"[•◦▪▫⬤○●◉◎□▷▶◀◁▲▼◆◇★☆✓✔✕✖✗✘✚✛✜✦✧➔→➜➙➛➞–—―‐]"
+)
+
+
+def is_bullet_point(char: PdfCharacter) -> bool:
+ """Check if the character is a bullet point.
+
+ Args:
+ char: The character to check
+
+ Returns:
+ bool: True if the character is a bullet point
+ """
+ is_bullet = bool(BULLET_POINT_PATTERN.match(char.char_unicode))
+ return is_bullet
+
+
+
+def is_list_marker_line(chars: list[PdfCharacter]) -> bool:
+ """Check if a line starts with a list marker (1., a., i., 01, etc.)
+
+ Args:
+ chars: List of characters in the line
+
+ Returns:
+ bool: True if the line starts with a numbered/lettered list marker
+ """
+ if not chars or len(chars) < 1:
+ return False
+
+ # Build the start of the line as a string (first 10 chars to capture longer patterns)
+ line_start = ''
+ for i, char in enumerate(chars[:10]):
+ if hasattr(char, 'char_unicode'):
+ line_start += char.char_unicode
+ if len(line_start) >= 10:
+ break
+
+ if len(line_start) < 1:
+ return False
+
+ # Remove leading spaces
+ line_start = line_start.lstrip()
+
+ if len(line_start) < 1:
+ return False
+
+ # Check various list marker patterns
+ import re
+
+ # Pattern 1: Number(s) followed by . or ) or : or JUST number with space
+ # Examples: "1.", "2)", "10.", "29)", "01 ", "02 ", "001 " (with optional punctuation)
+ # But NOT section numbers like "2.1", "3.2.1" etc.
+ match = re.match(r'^(\d+)([\.\):])?', line_start)
+ if match:
+ number_part = match.group(1)
+ punct_part = match.group(2)
+
+ # Check if it's followed by another digit (section number) - only if there was punctuation
+ remainder = line_start[match.end():]
+ if punct_part:
+ # Has punctuation - check it's not a section number
+ if remainder and not remainder[0].isdigit():
+ return True
+ else:
+ # No punctuation - check if followed by space or is at end (standalone number list marker)
+ # This handles cases like "01 Integrity Is Our Identity" or "02 Excellence"
+ if not remainder or remainder[0].isspace():
+ return True
+
+ # Pattern 2: Single letter followed by . or ) or : AND then a space or end of string
+ # Examples: "a.", "b)", "A.", "B)"
+ # This prevents matching abbreviations like "E.g.", "i.e.", "vs.", "etc."
+ if re.match(r'^[a-zA-Z][\.\):](?:\s|$)', line_start):
+ return True
+
+ # Pattern 3: Roman numerals (basic support for i, ii, iii, iv, v, vi, vii, viii, ix, x)
+ # Examples: "i.", "ii)", "iii.", "iv)"
+ if re.match(r'^(?:i{1,3}|iv|v|vi{0,3}|ix|x)[\.\):]', line_start, re.IGNORECASE):
+ return True
+
+ return False
+
+
+
+def is_bullet_or_list_marker(chars: list[PdfCharacter]) -> bool:
+ """Check if line starts with bullet point or list marker
+
+ Args:
+ chars: List of characters in the line
+
+ Returns:
+ bool: True if line starts with a bullet or list marker
+ """
+ if not chars:
+ return False
+ return is_bullet_point(chars[0]) or is_list_marker_line(chars)
+
+
+def could_be_list_marker_start(char: PdfCharacter) -> bool:
+ """Check if a character could be the start of a list marker (digit or single letter)
+
+ This is a preliminary check used during character-by-character processing.
+ It's more permissive than is_list_marker_line() since we don't have the full context yet.
+
+ Args:
+ char: The character to check
+
+ Returns:
+ bool: True if the character could start a list marker
+ """
+ if not char or not hasattr(char, 'char_unicode'):
+ return False
+
+ c = char.char_unicode
+
+ # Check if it's a digit (0-9)
+ if len(c) == 1 and c.isdigit():
+ return True
+
+ # Check if it's a single ASCII letter (a-z, A-Z) - common for Latin lists
+ if len(c) == 1 and c.isalpha() and ord(c) < 128:
+ return True
+
+ # Also check for Arabic/other script list markers if needed
+ # Add more patterns here for other languages
+
+ return False
+
+
+def calculate_box_iou(box1: Box, box2: Box) -> float:
+ """Calculate the Intersection over Union (IOU) between two boxes.
+
+ Args:
+ box1: First box
+ box2: Second box
+
+ Returns:
+ float: IOU value between 0 and 1
+ """
+ if box1 is None or box2 is None:
+ return 0.0
+
+ # Calculate intersection
+ x_left = max(box1.x, box2.x)
+ y_top = max(box1.y, box2.y)
+ x_right = min(box1.x2, box2.x2)
+ y_bottom = min(box1.y2, box2.y2)
+
+ # Check if there's no intersection
+ if x_left >= x_right or y_top >= y_bottom:
+ return 0.0
+
+ # Calculate intersection area
+ intersection_area = (x_right - x_left) * (y_bottom - y_top)
+
+ # Calculate areas of both boxes
+ box1_area = (box1.x2 - box1.x) * (box1.y2 - box1.y)
+ box2_area = (box2.x2 - box2.x) * (box2.y2 - box2.y)
+
+ # Calculate union area
+ union_area = box1_area + box2_area - intersection_area
+
+ # Avoid division by zero
+ if union_area <= 0:
+ return 0.0
+
+ return intersection_area / union_area
+
+
+def formular_height_ignore_char(char: PdfCharacter):
+ return (
+ char.pdf_character_id is None
+ or char.char_unicode in HEIGHT_NOT_USFUL_CHAR_IN_CHAR
+ )
+
+
+def box_to_tuple(box: Box) -> tuple[float, float, float, float]:
+ """Converts a Box object to a tuple of its coordinates."""
+ if box is None:
+ return (0, 0, 0, 0)
+ return (box.x, box.y, box.x2, box.y2)
+
+
+class Layout:
+ def __init__(self, layout_id, name):
+ self.id = layout_id
+ self.name = name
+
+ @staticmethod
+ def is_newline(prev_char: PdfCharacter, curr_char: PdfCharacter) -> bool:
+ # 如果没有å‰Â一个å—符，ä¸Â是æ¢行
+ if prev_char is None:
+ return False
+
+ # 获å–两个å—符的ä¸Â心 y Ã¥ÂÂæ ‡
+ # prev_y = (prev_char.box.y + prev_char.box.y2) / 2
+ # curr_y = (curr_char.box.y + curr_char.box.y2) / 2
+
+ # 如果当å‰Âå—符的 y Ã¥ÂÂ标明显低于å‰Â一个å—符，说明æ¢行äºâ€
+ # 这里使çâ€Â¨Ã¥Â—符高度的一åŠ作为阈值
+ char_height = max(
+ curr_char.box.y2 - curr_char.box.y,
+ prev_char.box.y2 - prev_char.box.y,
+ )
+ char_width = max(
+ curr_char.box.x2 - curr_char.box.x,
+ prev_char.box.x2 - prev_char.box.x,
+ )
+ should_new_line = (
+ curr_char.box.y2 < prev_char.box.y
+ or curr_char.box.x2 < prev_char.box.x - char_width * 10
+ )
+ if should_new_line and (
+ formular_height_ignore_char(curr_char)
+ or formular_height_ignore_char(prev_char)
+ ):
+ return False
+ return should_new_line
+
+
+def get_paragraph_length_except(
+ paragraph: PdfParagraph,
+ except_chars: str,
+ font: Font,
+) -> int:
+ length = 0
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_character:
+ length += (
+ composition.pdf_character[0].box.x2 - composition.pdf_character[0].box.x
+ )
+ elif composition.pdf_same_style_characters:
+ for pdf_char in composition.pdf_same_style_characters.pdf_character:
+ if pdf_char.char_unicode in except_chars:
+ continue
+ length += pdf_char.box.x2 - pdf_char.box.x
+ elif composition.pdf_same_style_unicode_characters:
+ for char_unicode in composition.pdf_same_style_unicode_characters.unicode:
+ if char_unicode in except_chars:
+ continue
+ length += font.char_lengths(
+ char_unicode,
+ composition.pdf_same_style_unicode_characters.pdf_style.font_size,
+ )[0]
+ elif composition.pdf_line:
+ for pdf_char in composition.pdf_line.pdf_character:
+ if pdf_char.char_unicode in except_chars:
+ continue
+ length += pdf_char.box.x2 - pdf_char.box.x
+ elif composition.pdf_formula:
+ length += composition.pdf_formula.box.x2 - composition.pdf_formula.box.x
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ return length
+
+
+def get_paragraph_unicode(paragraph: PdfParagraph) -> str:
+ chars = []
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_line:
+ chars.extend(composition.pdf_line.pdf_character)
+ elif composition.pdf_same_style_characters:
+ chars.extend(composition.pdf_same_style_characters.pdf_character)
+ elif composition.pdf_same_style_unicode_characters:
+ chars.extend(composition.pdf_same_style_unicode_characters.unicode)
+ elif composition.pdf_formula:
+ chars.extend(composition.pdf_formula.pdf_character)
+ elif composition.pdf_character:
+ chars.append(composition.pdf_character)
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ return get_char_unicode_string(chars)
+
+
+SPACE_REGEX = regex.compile(r"\s+", regex.UNICODE)
+
+
+def get_char_unicode_string(chars: list[PdfCharacter | str]) -> str:
+ """
+ å°†å—符列表转æ¢为 Unicode å—符串，根æ®å—符间è·Â自动æÂ’入空格。
+ 有些 PDF ä¸Â会显å¼Âç¼–ç Â空格，这时需è¦Âæ ¹æ®间è·Â自动æÂ’入空格。
+
+ Args:
+ chars: å—符列表，å¯以是 PdfCharacter 对象或å—符串
+
+ Returns:
+ str: 处ç†åÂŽçš„ Unicode å—符串
+ """
+ # 计算å—符间è·Âçš„ä¸Âä½Âæ•°
+ distances = []
+ for i in range(len(chars) - 1):
+ if not (
+ isinstance(chars[i], PdfCharacter)
+ and isinstance(chars[i + 1], PdfCharacter)
+ ):
+ continue
+ distance = chars[i + 1].box.x - chars[i].box.x2
+ if distance > 1: # åª考虑æ£å‘è·Â离
+ distances.append(distance)
+
+ # 去é‡ÂåŽ的è·Â离
+ distinct_distances = sorted(set(distances))
+
+ if not distinct_distances:
+ median_distance = 1
+ elif len(distinct_distances) == 1:
+ median_distance = distinct_distances[0]
+ else:
+ median_distance = distinct_distances[1]
+
+ # 构建 unicode å—符串，根æ®间è·ÂæÂ’入空格
+ unicode_chars = []
+ for i in range(len(chars)):
+ # 如果ä¸Â是å—符对象，直接添加，一般æ¥说这个时候 chars[i] 是å—符串
+ if not isinstance(chars[i], PdfCharacter):
+ unicode_chars.append(chars[i])
+ continue
+
+ # use unicode regex to replace all space with " "
+ unicode_chars.append(
+ regex.sub(
+ r"\s+",
+ " ",
+ unicodedata.normalize("NFKC", chars[i].char_unicode),
+ )
+ )
+
+ # 如果是空格，跳过
+ if chars[i].char_unicode == " ":
+ continue
+
+ # 如果两个å—符都是 PdfCharacter，检查间è·Â
+ if i < len(chars) - 1 and isinstance(chars[i + 1], PdfCharacter):
+ distance = chars[i + 1].box.x - chars[i].box.x2
+ if distance >= median_distance or Layout.is_newline( # é—´è·Â大于ä¸Âä½Âæ•°
+ chars[i],
+ chars[i + 1],
+ ): # æ¢行
+ unicode_chars.append(" ") # 添加空格
+
+ result = "".join(unicode_chars)
+ # use unicode regex to replace all space with " "
+ normalize = unicodedata.normalize("NFKC", result)
+ result = SPACE_REGEX.sub(" ", normalize).strip()
+ return result
+
+
+def get_paragraph_max_height(paragraph: PdfParagraph) -> float:
+ """
+ 获å–段è½ä¸Â最高的排版å•元高度。
+
+ Args:
+ paragraph: PDF 段è½对象
+
+ Returns:
+ float: 最大高度值
+ """
+ max_height = 0.0
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition is None:
+ continue
+ if composition.pdf_character:
+ char_height = (
+ composition.pdf_character[0].box.y2 - composition.pdf_character[0].box.y
+ )
+ max_height = max(max_height, char_height)
+ elif composition.pdf_same_style_characters:
+ for pdf_char in composition.pdf_same_style_characters.pdf_character:
+ char_height = pdf_char.box.y2 - pdf_char.box.y
+ max_height = max(max_height, char_height)
+ elif composition.pdf_same_style_unicode_characters:
+ # 对于纯 Unicode å—符，我们使çâ€Â¨Ã¥â€¦Â¶Ã¦Â ·å¼Âä¸Âçš„å—体大å°Â作为高度估计
+ font_size = (
+ composition.pdf_same_style_unicode_characters.pdf_style.font_size
+ )
+ max_height = max(max_height, font_size)
+ elif composition.pdf_line:
+ for pdf_char in composition.pdf_line.pdf_character:
+ char_height = pdf_char.box.y2 - pdf_char.box.y
+ max_height = max(max_height, char_height)
+ elif composition.pdf_formula:
+ formula_height = (
+ composition.pdf_formula.box.y2 - composition.pdf_formula.box.y
+ )
+ max_height = max(max_height, formula_height)
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+ return max_height
+
+
+def is_same_style(style1, style2) -> bool:
+ """判æ–Â两个样å¼Â是å¦相åÂÅ’"""
+ if style1 is None or style2 is None:
+ return style1 is style2
+
+ return (
+ style1.font_id == style2.font_id
+ and math.fabs(style1.font_size - style2.font_size) < 0.02
+ and is_same_graphic_state(style1.graphic_state, style2.graphic_state)
+ )
+
+
+def is_same_style_except_size(style1, style2) -> bool:
+ """判æ–Â两个样å¼Â是å¦相åÂÅ’"""
+ if style1 is None or style2 is None:
+ return style1 is style2
+
+ return (
+ style1.font_id == style2.font_id
+ and 0.7 < math.fabs(style1.font_size / style2.font_size) < 1.3
+ and is_same_graphic_state(style1.graphic_state, style2.graphic_state)
+ )
+
+
+def is_same_style_except_font(style1, style2) -> bool:
+ """判æ–Â两个样å¼Â是å¦相åÂÅ’"""
+ if style1 is None or style2 is None:
+ return style1 is style2
+
+ return math.fabs(
+ style1.font_size - style2.font_size,
+ ) < 0.02 and is_same_graphic_state(style1.graphic_state, style2.graphic_state)
+
+
+def is_same_graphic_state(state1: GraphicState, state2: GraphicState) -> bool:
+ """判æ–Â两个 GraphicState 是å¦相åÂÅ’"""
+ if state1 is None or state2 is None:
+ return state1 is state2
+
+ return (
+ state1.passthrough_per_char_instruction
+ == state2.passthrough_per_char_instruction
+ )
+
+
+def add_space_dummy_chars(paragraph: PdfParagraph) -> None:
+ """
+ 在 PDF 段è½ä¸Â添加表示空格的 dummy å—符。
+ 这个函数会直接修æâ€Â¹Ã¤Â¼Â 入的 paragraph 对象，在需è¦Â空格的地方添加dummy å—符。
+ åŒ时也会处ç†ä¸ÂåŒ组æˆÂ部分之间的空格。
+
+ Args:
+ paragraph: 需è¦Â处ç†的 PDF 段è½对象
+ """
+ # 首先处ç†æ¯Â个组æˆÂ部分内部的空格
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_line:
+ chars = composition.pdf_line.pdf_character
+ _add_space_dummy_chars_to_list(chars)
+ elif composition.pdf_same_style_characters:
+ chars = composition.pdf_same_style_characters.pdf_character
+ _add_space_dummy_chars_to_list(chars)
+ elif composition.pdf_same_style_unicode_characters:
+ # 对于 unicode å—符，ä¸Â需è¦Â处ç†。
+ # è¿™ç§Â类型åª会出现在翻译好的结果ä¸Â
+ continue
+ elif composition.pdf_formula:
+ chars = composition.pdf_formula.pdf_character
+ _add_space_dummy_chars_to_list(chars)
+
+ # 然åŽ处ç†组æˆÂ部分之间的空格
+ for i in range(len(paragraph.pdf_paragraph_composition) - 1):
+ curr_comp = paragraph.pdf_paragraph_composition[i]
+ next_comp = paragraph.pdf_paragraph_composition[i + 1]
+
+ # 获å–当å‰Â组æˆÂ部分的最åŽ一个å—符
+ curr_last_char = _get_last_char_from_composition(curr_comp)
+ if not curr_last_char:
+ continue
+
+ # 获å–下一个组æˆÂ部分的第一个å—符
+ next_first_char = _get_first_char_from_composition(next_comp)
+ if not next_first_char:
+ continue
+
+ # 检查两个组æˆÂ部分之间是å¦需è¦Â添加空格
+ distance = next_first_char.box.x - curr_last_char.box.x2
+ if distance > 1: # åª考虑æ£å‘è·Â离
+ # 创建一个 dummy å—符作为空格
+ space_box = Box(
+ x=curr_last_char.box.x2,
+ y=curr_last_char.box.y,
+ x2=curr_last_char.box.x2 + distance,
+ y2=curr_last_char.box.y2,
+ )
+
+ space_char = PdfCharacter(
+ pdf_style=curr_last_char.pdf_style,
+ box=space_box,
+ char_unicode=" ",
+ scale=curr_last_char.scale,
+ advance=space_box.x2 - space_box.x,
+ visual_bbox=il_version_1.VisualBbox(box=space_box),
+ )
+
+ # 将空格添加到当å‰Â组æˆÂ部分的末尾
+ if curr_comp.pdf_line:
+ curr_comp.pdf_line.pdf_character.append(space_char)
+ elif curr_comp.pdf_same_style_characters:
+ curr_comp.pdf_same_style_characters.pdf_character.append(space_char)
+ elif curr_comp.pdf_formula:
+ curr_comp.pdf_formula.pdf_character.append(space_char)
+
+
+def _get_first_char_from_composition(
+ comp: PdfParagraphComposition,
+) -> PdfCharacter | None:
+ """获å–组æˆÂ部分的第一个å—符"""
+ if comp.pdf_line and comp.pdf_line.pdf_character:
+ return comp.pdf_line.pdf_character[0]
+ elif (
+ comp.pdf_same_style_characters and comp.pdf_same_style_characters.pdf_character
+ ):
+ return comp.pdf_same_style_characters.pdf_character[0]
+ elif comp.pdf_formula and comp.pdf_formula.pdf_character:
+ return comp.pdf_formula.pdf_character[0]
+ elif comp.pdf_character:
+ return comp.pdf_character
+ return None
+
+
+def _get_last_char_from_composition(
+ comp: PdfParagraphComposition,
+) -> PdfCharacter | None:
+ """获å–组æˆÂ部分的最åŽ一个å—符"""
+ if comp.pdf_line and comp.pdf_line.pdf_character:
+ return comp.pdf_line.pdf_character[-1]
+ elif (
+ comp.pdf_same_style_characters and comp.pdf_same_style_characters.pdf_character
+ ):
+ return comp.pdf_same_style_characters.pdf_character[-1]
+ elif comp.pdf_formula and comp.pdf_formula.pdf_character:
+ return comp.pdf_formula.pdf_character[-1]
+ elif comp.pdf_character:
+ return comp.pdf_character
+ return None
+
+
+def _add_space_dummy_chars_to_list(chars: list[PdfCharacter]) -> None:
+ """
+ 在å—符列表ä¸Â的适当ä½Â置添加表示空格的 dummy å—符。
+
+ Args:
+ chars: PdfCharacter 对象列表
+ """
+ if not chars:
+ return
+
+ # 计算å—符间è·Âçš„ä¸Âä½Âæ•°
+ distances = []
+ for i in range(len(chars) - 1):
+ distance = chars[i + 1].box.x - chars[i].box.x2
+ if distance > 1: # åª考虑æ£å‘è·Â离
+ distances.append(distance)
+
+ # 去é‡ÂåŽ的è·Â离
+ distinct_distances = sorted(set(distances))
+
+ if not distinct_distances:
+ median_distance = 1
+ elif len(distinct_distances) == 1:
+ median_distance = distinct_distances[0]
+ else:
+ median_distance = distinct_distances[1]
+
+ # 在需è¦Â的地方æÂ’入空格å—符
+ i = 0
+ while i < len(chars) - 1:
+ curr_char = chars[i]
+ next_char = chars[i + 1]
+
+ distance = next_char.box.x - curr_char.box.x2
+ if distance >= median_distance or Layout.is_newline(curr_char, next_char):
+ if distance < 0:
+ distance = -distance
+ # 创建一个 dummy å—符作为空格
+ space_box = Box(
+ x=curr_char.box.x2,
+ y=curr_char.box.y,
+ x2=curr_char.box.x2 + min(distance, median_distance),
+ y2=curr_char.box.y2,
+ )
+
+ space_char = PdfCharacter(
+ pdf_style=curr_char.pdf_style,
+ box=space_box,
+ char_unicode=" ",
+ scale=curr_char.scale,
+ advance=space_box.x2 - space_box.x,
+ visual_bbox=il_version_1.VisualBbox(box=space_box),
+ )
+
+ # 在当å‰Âä½Âç½®åÂŽæÂ’入空格å—符
+ chars.insert(i + 1, space_char)
+ i += 2 # 跳过刚æÂ’入的空格
+ else:
+ i += 1
+
+
+def build_layout_index(page):
+ """Builds an R-tree index for all layouts on the page."""
+ from rtree import index
+
+ layout_index = index.Index()
+ layout_map = {}
+ for i, layout in enumerate(page.page_layout):
+ layout_map[i] = layout
+ if layout.box:
+ layout_index.insert(i, box_to_tuple(layout.box))
+ return layout_index, layout_map
+
+
+def calculate_iou_for_boxes(box1: Box, box2: Box) -> float:
+ """Calculate the intersection area divided by the first box area."""
+ x_left = max(box1.x, box2.x)
+ y_bottom = max(box1.y, box2.y)
+ x_right = min(box1.x2, box2.x2)
+ y_top = min(box1.y2, box2.y2)
+
+ if x_right <= x_left or y_top <= y_bottom:
+ return 0.0
+
+ # Calculate intersection area
+ intersection_area = (x_right - x_left) * (y_top - y_bottom)
+
+ # Calculate area of first box
+ first_box_area = (box1.x2 - box1.x) * (box1.y2 - box1.y)
+
+ # Return intersection divided by first box area, handle division by zero
+ if first_box_area <= 0:
+ return 0.0
+
+ return intersection_area / first_box_area
+
+
+def calculate_y_iou_for_boxes(box1: Box, box2: Box) -> float:
+ """Calculate the intersection ratio in y-axis direction divided by the first box height.
+
+ Args:
+ box1: First box
+ box2: Second box
+
+ Returns:
+ float: Intersection ratio in y-axis direction between 0 and 1
+ """
+ y_bottom = max(box1.y, box2.y)
+ y_top = min(box1.y2, box2.y2)
+
+ if y_top <= y_bottom:
+ return 0.0
+
+ # Calculate intersection height
+ intersection_height = y_top - y_bottom
+
+ # Calculate height of first box
+ first_box_height = box1.y2 - box1.y
+
+ # Return intersection divided by first box height, handle division by zero
+ if first_box_height <= 0:
+ return 0.0
+
+ return intersection_height / first_box_height
+
+
+def calculate_y_true_iou_for_boxes(box1: Box, box2: Box) -> float:
+ """Calculate the intersection ratio in y-axis direction divided by the first box height.
+
+ Args:
+ box1: First box
+ box2: Second box
+
+ Returns:
+ float: Intersection ratio in y-axis direction between 0 and 1
+ """
+ y_bottom = max(box1.y, box2.y)
+ y_top = min(box1.y2, box2.y2)
+
+ if y_top <= y_bottom:
+ return 0.0
+
+ # Calculate intersection height
+ intersection_height = y_top - y_bottom
+
+ # Calculate height of first box
+ first_box_height = box1.y2 - box1.y
+ second_box_height = box2.y2 - box2.y
+
+ min_height = min(first_box_height, second_box_height)
+
+ # Return intersection divided by first box height, handle division by zero
+ if first_box_height <= 0:
+ return 0.0
+
+ return intersection_height / min_height
+
+
+def get_character_layout(
+ char,
+ layout_index,
+ layout_map,
+ layout_priority=None,
+ _bbox_mode: Literal["auto", "visual", "box"] = "auto",
+):
+ """Get the layout for a character based on priority and IoU."""
+ if layout_priority is None:
+ layout_priority = [
+ "number",
+ "reference",
+ "reference_content",
+ "algorithm",
+ "formula_caption",
+ "isolate_formula",
+ "table_footnote",
+ "table_caption",
+ "figure_caption",
+ "figure_title",
+ "chart_title",
+ "table_title",
+ "table_cell_hybrid",
+ "table_text",
+ "wireless_table_cell",
+ "wired_table_cell",
+ "abandon",
+ "title",
+ "abstract",
+ "paragraph_title",
+ "content",
+ "doc_title",
+ "footnote",
+ "header",
+ "footer",
+ "seal",
+ "plain text",
+ "tiny text",
+ "author_info_hybrid",
+ "list_item_hybrid",
+ "text",
+ "paragraph_hybrid",
+ "paragraph",
+ "table_cell",
+ "figure_text",
+ "list_item",
+ "title",
+ "caption",
+ "footnote_hybrid",
+ "footnote",
+ "formula",
+ "formula_hybrid",
+ "page_header",
+ "page_footer",
+ # --- hybrid labels ---
+ "reference_hybrid",
+ "document_hybrid",
+ "academic_paper_hybrid",
+ "form_or_table_hybrid",
+ "presentation_slide_hybrid",
+ "webpage_screenshot_hybrid",
+ "manga_or_comic_hybrid",
+ "advertisement_hybrid",
+ "magazine_or_newspaper_hybrid",
+ "other_hybrid",
+ "table_cell_hybrid",
+ "figure_text_hybrid",
+ "title_hybrid",
+ "caption_hybrid",
+ "code_algo_hybrid",
+ "line_number_hybrid",
+ "page_header_hybrid",
+ "page_footer_hybrid",
+ "page_number_hybrid",
+ "unknown_hybrid",
+ "fallback_line",
+ "table",
+ "figure",
+ "image",
+ ]
+
+ char_box = char.visual_bbox.box
+ # char_box2 = char.box
+ # if bbox_mode == "auto":
+ # # Calculate IOU to decide which box to use
+ # intersection_area = max(
+ # 0, min(char_box.x2, char_box2.x2) - max(char_box.x, char_box2.x)
+ # ) * max(0, min(char_box.y2, char_box2.y2) - max(char_box.y, char_box2.y))
+ # char_box_area = (char_box.x2 - char_box.x) * (char_box.y2 - char_box.y)
+ #
+ # if char_box_area > 0:
+ # iou = intersection_area / char_box_area
+ # if iou < 0.2:
+ # char_box = char_box2
+ # elif bbox_mode == "box":
+ # char_box = char_box2
+
+ # Collect all intersecting layouts and their IoU values
+ matching_layouts = []
+ candidate_ids = list(layout_index.intersection(box_to_tuple(char_box)))
+ candidate_layouts = [layout_map[i] for i in candidate_ids]
+
+ for layout in candidate_layouts:
+ # Calculate IoU
+ intersection_area = max(
+ 0, min(char_box.x2, layout.box.x2) - max(char_box.x, layout.box.x)
+ ) * max(0, min(char_box.y2, layout.box.y2) - max(char_box.y, layout.box.y))
+ char_area = (char_box.x2 - char_box.x) * (char_box.y2 - char_box.y)
+
+ if char_area > 0:
+ iou = intersection_area / char_area
+ if iou > 0:
+ matching_layouts.append(
+ {
+ "layout": Layout(layout.id, layout.class_name),
+ "priority": (
+ layout_priority.index(layout.class_name)
+ if layout.class_name in layout_priority
+ else len(layout_priority)
+ ),
+ "iou": iou,
+ }
+ )
+
+ if not matching_layouts:
+ return None
+
+ # Sort by priority (ascending) and IoU value (descending)
+ matching_layouts.sort(key=lambda x: (x["priority"], -x["iou"]))
+
+ # non_hybrid_table_label = None
+ # for layout in matching_layouts:
+ # layout = layout["layout"]
+ # label = layout.name
+ # if is_text_layout(layout) and label not in (
+ # "table_cell_hybrid",
+ # "table_text",
+ # "wireless_table_cell",
+ # "wired_table_cell",
+ # "fallback_line",
+ # "unknown_hybrid",
+ # ):
+ # non_hybrid_table_label = layout
+ # break
+ #
+ # if non_hybrid_table_label:
+ # return non_hybrid_table_label
+
+ return matching_layouts[0]["layout"]
+
+
+def is_text_layout(layout: Layout):
+ """Check if a layout is a text layout."""
+ return layout is not None and layout.name in [
+ "plain text",
+ "tiny text",
+ "title",
+ "abandon",
+ "figure_caption",
+ "table_caption",
+ "table_text",
+ "table_footnote",
+ # "reference",
+ "title",
+ "paragraph_title",
+ "abstract",
+ "content",
+ "figure_title",
+ "table_title",
+ "doc_title",
+ "footnote",
+ "header",
+ "footer",
+ "seal",
+ "text",
+ "chart_title",
+ "paragraph",
+ "table_cell",
+ "figure_text",
+ "list_item",
+ "title",
+ "caption",
+ "footnote",
+ "page_header",
+ "page_footer",
+ "wired_table_cell",
+ "wireless_table_cell",
+ "paragraph_hybrid",
+ "table_cell_hybrid",
+ "caption_hybrid",
+ "unknown_hybrid",
+ "figure_text_hybrid",
+ "list_item_hybrid",
+ "title_hybrid",
+ "fallback_line",
+ "author_info_hybrid",
+ "page_header_hybrid",
+ "page_footer_hybrid",
+ "footnote_hybrid",
+ ]
+
+
+def is_character_in_formula_layout(
+ char: il_version_1.PdfCharacter,
+ _page: il_version_1.Page,
+ layout_index,
+ layout_map,
+) -> int | None:
+ """Check if character is contained within any formula-related layout."""
+ formula_layout_types = {"formula"}
+
+ char_box = char.visual_bbox.box
+ char_box2 = char.box
+
+ if calculate_iou_for_boxes(char_box, char_box2) < 0.2:
+ char_box = char_box2
+
+ # Get all candidate layouts that intersect with the character
+ candidate_ids = list(layout_index.intersection(box_to_tuple(char_box)))
+ candidate_layouts: list[il_version_1.PageLayout] = [
+ layout_map[i] for i in candidate_ids
+ ]
+
+ # Check if any intersecting layout is a formula type
+ for layout in candidate_layouts:
+ if layout.class_name in formula_layout_types:
+ iou = calculate_iou_for_boxes(char_box, layout.box)
+ if iou > 0.4: # Character has overlap with formula layout
+ return layout.id
+
+ return None
+
+
+def is_curve_in_figure_table_layout(
+ curve, layout_index, layout_map, protection_threshold: float = 0.3
+) -> bool:
+ """Check if curve is within figure/table layout areas.
+
+ Args:
+ curve: The curve object to check
+ layout_index: Spatial index for layouts
+ layout_map: Mapping from layout IDs to layout objects
+ protection_threshold: IoU threshold for figure/table protection
+
+ Returns:
+ True if curve is within figure/table layout areas
+ """
+ if not curve.box:
+ return False
+
+ # Figure/table related layout types
+ figure_table_layouts = {
+ "figure",
+ "table",
+ "figure_text",
+ "table_text",
+ "figure_caption",
+ "table_caption",
+ "figure_title",
+ "table_title",
+ "chart_title",
+ "table_cell",
+ "table_cell_hybrid",
+ "wired_table_cell",
+ "wireless_table_cell",
+ "table_footnote",
+ }
+
+ # Get candidate layouts that intersect with curve
+ candidate_ids = list(layout_index.intersection(box_to_tuple(curve.box)))
+ candidate_layouts = [layout_map[i] for i in candidate_ids]
+
+ for layout in candidate_layouts:
+ if layout.class_name in figure_table_layouts:
+ # Check if curve has significant overlap with figure/table layout
+ iou = calculate_iou_for_boxes(curve.box, layout.box)
+ if iou > protection_threshold:
+ return True
+
+ return False
+
+
+def is_curve_overlapping_with_paragraphs(
+ curve, paragraphs: list, overlap_threshold: float = 0.2
+) -> bool:
+ """Check if curve overlaps with text paragraph areas.
+
+ Args:
+ curve: The curve object to check
+ paragraphs: List of paragraph objects
+ overlap_threshold: IoU threshold for paragraph overlap detection
+
+ Returns:
+ True if curve overlaps with any paragraph area
+ """
+ if not curve.box:
+ return False
+
+ for paragraph in paragraphs:
+ para_box = get_paragraph_bounding_box(paragraph)
+ if para_box:
+ iou = calculate_iou_for_boxes(curve.box, para_box)
+ if iou > overlap_threshold:
+ return True
+
+ return False
+
+
+def get_paragraph_bounding_box(paragraph) -> Box | None:
+ """Calculate the bounding box of a paragraph from its compositions.
+
+ Args:
+ paragraph: The paragraph object
+
+ Returns:
+ Box object representing the paragraph bounds, or None if no valid bounds
+ """
+ if not paragraph.pdf_paragraph_composition:
+ return None
+
+ min_x = float("inf")
+ min_y = float("inf")
+ max_x = float("-inf")
+ max_y = float("-inf")
+
+ has_valid_box = False
+
+ for composition in paragraph.pdf_paragraph_composition:
+ comp_box = None
+
+ if composition.pdf_line and composition.pdf_line.box:
+ comp_box = composition.pdf_line.box
+ elif composition.pdf_formula and composition.pdf_formula.box:
+ comp_box = composition.pdf_formula.box
+ elif (
+ composition.pdf_same_style_characters
+ and composition.pdf_same_style_characters.box
+ ):
+ comp_box = composition.pdf_same_style_characters.box
+ elif composition.pdf_character and len(composition.pdf_character) > 0:
+ # Calculate box from character list
+ char_boxes = [
+ char.visual_bbox.box
+ for char in composition.pdf_character
+ if char.visual_bbox and char.visual_bbox.box
+ ]
+ if char_boxes:
+ comp_min_x = min(box.x for box in char_boxes)
+ comp_min_y = min(box.y for box in char_boxes)
+ comp_max_x = max(box.x2 for box in char_boxes)
+ comp_max_y = max(box.y2 for box in char_boxes)
+ comp_box = Box(comp_min_x, comp_min_y, comp_max_x, comp_max_y)
+
+ if comp_box:
+ min_x = min(min_x, comp_box.x)
+ min_y = min(min_y, comp_box.y)
+ max_x = max(max_x, comp_box.x2)
+ max_y = max(max_y, comp_box.y2)
+ has_valid_box = True
+
+ if not has_valid_box:
+ return None
+
+ return Box(min_x, min_y, max_x, max_y)
\ No newline at end of file
diff --git a/babeldoc/format/pdf/document_il/utils/matrix_helper.py b/babeldoc/format/pdf/document_il/utils/matrix_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..18b6c846530f23cef04108f70152fb8fc7f4af12
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/utils/matrix_helper.py
@@ -0,0 +1,335 @@
+"""Matrix helper utilities for CTM decomposition and composition.
+
+This module provides functions to:
+- Decompose a PDF CTM into translation, rotation, scale, and shear
+- Compose a CTM back from translation, rotation, scale, and shear
+
+All comments and docstrings are in English per project guidelines.
+"""
+
+from __future__ import annotations
+
+import math
+
+from babeldoc.format.pdf.document_il.il_version_1 import PdfAffineTransform
+from babeldoc.format.pdf.document_il.il_version_1 import PdfMatrix
+
+# Local type aliases to avoid importing from pdfminer
+Point = tuple[float, float]
+Matrix = tuple[float, float, float, float, float, float]
+
+
+def decompose_ctm(m: Matrix | PdfMatrix) -> PdfAffineTransform:
+ """Decompose a PDF CTM into a PdfAffineTransform.
+
+ The PDF current transformation matrix (CTM) is represented as
+ ``(a, b, c, d, e, f)`` corresponding to the affine matrix:
+ ``[[a, c, e], [b, d, f], [0, 0, 1]]``.
+
+ This function decomposes it into:
+ - translation: (tx, ty)
+ - rotation: angle in radians (counter-clockwise)
+ - scale: (sx, sy)
+ - shear: x-shear factor (dimensionless, equals tan(shear_angle))
+
+ The decomposition is based on a QR-like approach commonly used for 2D
+ affine matrices. If the linear part is degenerate, sensible fallbacks are
+ applied.
+
+ Args:
+ m: CTM as ``(a, b, c, d, e, f)``.
+
+ Returns:
+ A ``PdfAffineTransform`` instance with fields populated.
+ """
+ if isinstance(m, PdfMatrix):
+ a = m.a
+ b = m.b
+ c = m.c
+ d = m.d
+ e = m.e
+ f = m.f
+ assert a is not None
+ assert b is not None
+ assert c is not None
+ assert d is not None
+ assert e is not None
+ assert f is not None
+ else:
+ (a, b, c, d, e, f) = m
+
+ tx, ty = e, f
+
+ # Linear part
+ m00, m01 = a, c
+ m10, m11 = b, d
+
+ # Scale X is the length of the first column
+ sx = math.hypot(m00, m10)
+
+ eps = 1e-12
+ if sx < eps:
+ # Degenerate first column. Choose rotation = 0, shear = 0, sx = 0.
+ rotation = 0.0
+ shear = 0.0
+ # Then sy is the length of the second column
+ sy = math.hypot(m01, m11)
+ # Handle reflection
+ det = m00 * m11 - m01 * m10
+ if det < 0:
+ sy = -sy if sy != 0 else -0.0
+ return PdfAffineTransform(
+ translation_x=tx,
+ translation_y=ty,
+ rotation=rotation,
+ scale_x=sx,
+ scale_y=sy,
+ shear=shear,
+ )
+
+ # Normalize first column to get rotation axis
+ r0x = m00 / sx
+ r0y = m10 / sx
+
+ # Shear is the projection of the second column onto the first column
+ shear = r0x * m01 + r0y * m11
+
+ # Remove the shear component from the second column
+ m01_ortho = m01 - shear * r0x
+ m11_ortho = m11 - shear * r0y
+
+ # Scale Y is the length of the orthogonalized second column
+ sy = math.hypot(m01_ortho, m11_ortho)
+
+ # Determine reflection by determinant sign
+ det = m00 * m11 - m01 * m10
+ if det < 0:
+ sy = -sy if sy != 0 else -0.0
+ shear = -shear
+ m01_ortho = -m01_ortho
+ m11_ortho = -m11_ortho
+
+ # Rotation is the angle of the first column
+ rotation = math.atan2(m10, m00)
+
+ return PdfAffineTransform(
+ translation_x=tx,
+ translation_y=ty,
+ rotation=rotation,
+ scale_x=sx,
+ scale_y=sy,
+ shear=shear,
+ )
+
+
+def compose_ctm(transform: PdfAffineTransform) -> Matrix:
+ """Compose a PDF CTM from a PdfAffineTransform.
+
+ This composes the 2x2 linear part using the following model:
+ - First column: ``sx * r0`` where ``r0 = (cos(theta), sin(theta))``
+ - Second column: ``shear * r0 + sy * r1`` where ``r1`` is the unit vector
+ orthogonal to ``r0``: ``r1 = (-sin(theta), cos(theta))``
+ - Translation is appended as (e, f) = (tx, ty)
+
+ Args:
+ transform: A ``PdfAffineTransform`` with translation, rotation,
+ scale, and shear populated.
+
+ Returns:
+ The CTM matrix ``(a, b, c, d, e, f)``.
+ """
+ # Extract and validate required values from the dataclass
+ tx = float(transform.translation_x if transform.translation_x is not None else 0.0)
+ ty = float(transform.translation_y if transform.translation_y is not None else 0.0)
+ theta = float(transform.rotation if transform.rotation is not None else 0.0)
+ sx = float(transform.scale_x if transform.scale_x is not None else 1.0)
+ sy = float(transform.scale_y if transform.scale_y is not None else 1.0)
+ shear = float(transform.shear if transform.shear is not None else 0.0)
+
+ cos_t = math.cos(theta)
+ sin_t = math.sin(theta)
+
+ # Unit basis aligned with rotation
+ r0x, r0y = cos_t, sin_t
+ r1x, r1y = -sin_t, cos_t
+
+ # Columns of the linear matrix
+ col0x = sx * r0x
+ col0y = sx * r0y
+ col1x = shear * r0x + sy * r1x
+ col1y = shear * r0y + sy * r1y
+
+ a = col0x
+ b = col0y
+ c = col1x
+ d = col1y
+ e = tx
+ f = ty
+
+ return a, b, c, d, e, f
+
+
+def scale_and_set_translation(
+ m: Matrix | PdfMatrix, scale_factor: float, tx: float, ty: float
+) -> Matrix | PdfMatrix:
+ """Uniformly scale CTM by percentage and set translation to a position.
+
+ This function performs an isotropic scale in X and Y by ``percent`` and
+ then sets the translation components to ``(tx, ty)``. It preserves the
+ input type: if a ``PdfMatrix`` is provided, a ``PdfMatrix`` is returned;
+ if a tuple is provided, a tuple is returned.
+
+ Args:
+ m: Input CTM as ``(a, b, c, d, e, f)`` or ``PdfMatrix``.
+ scale_factor: Scale factor. ``1.0`` keeps size unchanged, ``0.5``
+ halves it, ``2.0`` doubles it.
+ tx: New translation X.
+ ty: New translation Y.
+
+ Returns:
+ A CTM of the same type as the input, scaled and with translation set.
+ """
+
+ if isinstance(m, PdfMatrix):
+ a = m.a
+ b = m.b
+ c = m.c
+ d = m.d
+ # e, f will be overridden by tx, ty
+ assert a is not None
+ assert b is not None
+ assert c is not None
+ assert d is not None
+
+ return PdfMatrix(
+ a=a * scale_factor,
+ b=b * scale_factor,
+ c=c * scale_factor,
+ d=d * scale_factor,
+ e=float(tx),
+ f=float(ty),
+ )
+
+ a, b, c, d, _, _ = m
+ return (
+ a * scale_factor,
+ b * scale_factor,
+ c * scale_factor,
+ d * scale_factor,
+ float(tx),
+ float(ty),
+ )
+
+
+def create_translation_and_scale_matrix(
+ translation_x: float, translation_y: float, scale_factor: float
+) -> Matrix:
+ """Create a transformation matrix for translation and uniform scaling.
+
+ This creates a CTM that first scales uniformly by scale_factor, then translates
+ by (translation_x, translation_y).
+
+ Args:
+ translation_x: Translation in X direction
+ translation_y: Translation in Y direction
+ scale_factor: Uniform scale factor for both X and Y
+
+ Returns:
+ The CTM matrix (a, b, c, d, e, f)
+ """
+ # Matrix for uniform scaling and translation:
+ # [scale 0 tx]
+ # [0 scale ty]
+ # [0 0 1 ]
+ # Which maps to CTM (scale, 0, 0, scale, tx, ty)
+ return (scale_factor, 0.0, 0.0, scale_factor, translation_x, translation_y)
+
+
+def multiply_matrices(m1: Matrix | PdfMatrix, m2: Matrix | PdfMatrix) -> Matrix:
+ """Multiply two transformation matrices (m1 * m2).
+
+ Args:
+ m1: Left matrix in multiplication
+ m2: Right matrix in multiplication
+
+ Returns:
+ Result matrix as tuple (a, b, c, d, e, f)
+ """
+ # Extract components from first matrix
+ if isinstance(m1, PdfMatrix):
+ a1, b1, c1, d1, e1, f1 = m1.a, m1.b, m1.c, m1.d, m1.e, m1.f
+ assert all(x is not None for x in [a1, b1, c1, d1, e1, f1])
+ else:
+ a1, b1, c1, d1, e1, f1 = m1
+
+ # Extract components from second matrix
+ if isinstance(m2, PdfMatrix):
+ a2, b2, c2, d2, e2, f2 = m2.a, m2.b, m2.c, m2.d, m2.e, m2.f
+ assert all(x is not None for x in [a2, b2, c2, d2, e2, f2])
+ else:
+ a2, b2, c2, d2, e2, f2 = m2
+
+ # Matrix multiplication for 2D affine transformations:
+ # [a1 c1 e1] [a2 c2 e2] [a1*a2+c1*b2 a1*c2+c1*d2 a1*e2+c1*f2+e1]
+ # [b1 d1 f1] * [b2 d2 f2] = [b1*a2+d1*b2 b1*c2+d1*d2 b1*e2+d1*f2+f1]
+ # [0 0 1 ] [0 0 1 ] [0 0 1 ]
+
+ a = a1 * a2 + c1 * b2
+ b = b1 * a2 + d1 * b2
+ c = a1 * c2 + c1 * d2
+ d = b1 * c2 + d1 * d2
+ e = a1 * e2 + c1 * f2 + e1
+ f = b1 * e2 + d1 * f2 + f1
+
+ return (a, b, c, d, e, f)
+
+
+def apply_transform_to_ctm(
+ existing_ctm: list[object],
+ translation_x: float,
+ translation_y: float,
+ scale_factor: float,
+) -> list[object]:
+ """Apply translation and scale transformation to an existing CTM.
+
+ Args:
+ existing_ctm: Existing CTM as list of 6 floats
+ translation_x: Translation in X direction
+ translation_y: Translation in Y direction
+ scale_factor: Uniform scale factor
+
+ Returns:
+ New CTM as list of objects
+ """
+ if len(existing_ctm) != 6:
+ # If CTM is invalid, create a new identity matrix with the transform
+ transform_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale_factor
+ )
+ return list(transform_matrix)
+
+ # Convert existing CTM to Matrix format
+ try:
+ existing_matrix = tuple(float(x) for x in existing_ctm)
+ except (ValueError, TypeError):
+ # If conversion fails, use identity matrix
+ existing_matrix = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)
+
+ # Create the transform matrix
+ transform_matrix = create_translation_and_scale_matrix(
+ translation_x, translation_y, scale_factor
+ )
+
+ # Left-multiply: new_ctm = transform_matrix * existing_matrix
+ result_matrix = multiply_matrices(transform_matrix, existing_matrix)
+
+ return list(result_matrix)
+
+
+def matrix_to_bytes(m: Matrix | PdfMatrix) -> bytes:
+ if isinstance(m, PdfMatrix):
+ return (
+ f" {m.a:.6f} {m.b:.6f} {m.c:.6f} {m.d:.6f} {m.e:.6f} {m.f:.6f} cm ".encode()
+ )
+ else:
+ return f" {m[0]:.6f} {m[1]:.6f} {m[2]:.6f} {m[3]:.6f} {m[4]:.6f} {m[5]:.6f} cm ".encode()
diff --git a/babeldoc/format/pdf/document_il/utils/mupdf_helper.py b/babeldoc/format/pdf/document_il/utils/mupdf_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..e4e2aafe2b8bd6f989165b1986c35e114dc18314
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/utils/mupdf_helper.py
@@ -0,0 +1,42 @@
+import numpy as np
+import pymupdf
+
+from babeldoc.const import get_process_pool
+
+
+def get_no_rotation_img(page: pymupdf.Page, dpi: int = 72) -> pymupdf.Pixmap:
+ # return page.get_pixmap(dpi=72)
+ original_rotation = page.rotation
+ page.set_rotation(0)
+ pix = page.get_pixmap(dpi=dpi)
+ page.set_rotation(original_rotation)
+ return pix
+
+
+def get_no_rotation_img_multiprocess_internal(
+ pdf_bytes: str, pagenum: int, dpi: int = 72
+) -> np.ndarray:
+ # return page.get_pixmap(dpi=72)
+ doc = pymupdf.open(pdf_bytes)
+ try:
+ page = doc[pagenum]
+ original_rotation = page.rotation
+ page.set_rotation(0)
+ pix = page.get_pixmap(dpi=dpi)
+ page.set_rotation(original_rotation)
+ return np.frombuffer(pix.samples, np.uint8).reshape(
+ pix.height,
+ pix.width,
+ 3,
+ )[:, :, ::-1]
+ finally:
+ doc.close()
+
+
+def get_no_rotation_img_multiprocess(pdf_bytes: str, pagenum: int, dpi: int = 72):
+ pool = get_process_pool()
+ if pool is None:
+ return get_no_rotation_img_multiprocess_internal(pdf_bytes, pagenum, dpi)
+ return pool.apply(
+ get_no_rotation_img_multiprocess_internal, (pdf_bytes, pagenum, dpi)
+ )
diff --git a/babeldoc/format/pdf/document_il/utils/paragraph_helper.py b/babeldoc/format/pdf/document_il/utils/paragraph_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..0ba671461b2a8c97828a58b52fe9ab88ad93deab
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/utils/paragraph_helper.py
@@ -0,0 +1,94 @@
+import logging
+import re
+
+from babeldoc.format.pdf.document_il import il_version_1
+
+logger = logging.getLogger(__name__)
+
+
+def is_cid_paragraph(paragraph: il_version_1.PdfParagraph):
+ chars: list[il_version_1.PdfCharacter] = []
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_line:
+ chars.extend(composition.pdf_line.pdf_character)
+ elif composition.pdf_same_style_characters:
+ chars.extend(composition.pdf_same_style_characters.pdf_character)
+ elif composition.pdf_same_style_unicode_characters:
+ continue
+ # chars.extend(composition.pdf_same_style_unicode_characters.unicode)
+ elif composition.pdf_formula:
+ chars.extend(composition.pdf_formula.pdf_character)
+ elif composition.pdf_character:
+ chars.append(composition.pdf_character)
+ else:
+ logger.error(
+ f"Unknown composition type. "
+ f"Composition: {composition}. "
+ f"Paragraph: {paragraph}. ",
+ )
+ continue
+
+ cid_count = 0
+ for char in chars:
+ if re.match(r"^\(cid:\d+\)$", char.char_unicode):
+ cid_count += 1
+
+ return cid_count > len(chars) * 0.8
+
+
+NUMERIC_PATTERN = re.compile(r"^-?\d+(\.\d+)?$")
+
+
+def is_pure_numeric_paragraph(paragraph) -> bool:
+ """只检查段落是否为纯数字（支持整数、小数、负数）"""
+
+ if not paragraph or not getattr(paragraph, "unicode", None):
+ return False
+
+ text = paragraph.unicode.strip()
+ if not text:
+ return False
+
+ return bool(NUMERIC_PATTERN.match(text))
+
+
+def is_placeholder_only_paragraph(paragraph: il_version_1.PdfParagraph) -> bool:
+ """Check if a paragraph contains only placeholders and whitespace.
+
+ Args:
+ paragraph: PDF paragraph to check
+
+ Returns:
+ True if the paragraph contains only placeholders (formula or style tags)
+ and whitespace, False otherwise
+ """
+ if not paragraph or not paragraph.unicode:
+ return False
+
+ for composition in paragraph.pdf_paragraph_composition:
+ if composition.pdf_formula:
+ # Formula composition is allowed
+ continue
+ elif composition.pdf_character:
+ # Check if single character is whitespace
+ if not composition.pdf_character.char_unicode.isspace():
+ return False
+ elif composition.pdf_line:
+ # Check if all characters in the line are whitespace
+ for char in composition.pdf_line.pdf_character:
+ if not char.char_unicode.isspace():
+ return False
+ elif composition.pdf_same_style_characters:
+ # Check if all characters in the group are whitespace
+ for char in composition.pdf_same_style_characters.pdf_character:
+ if not char.char_unicode.isspace():
+ return False
+ elif composition.pdf_same_style_unicode_characters:
+ # Check if the unicode content is only whitespace
+ if not composition.pdf_same_style_unicode_characters.unicode.isspace():
+ return False
+ else:
+ # Unknown composition type, conservatively return False
+ return False
+
+ return True
diff --git a/babeldoc/format/pdf/document_il/utils/spatial_analyzer.py b/babeldoc/format/pdf/document_il/utils/spatial_analyzer.py
new file mode 100644
index 0000000000000000000000000000000000000000..6f7fafdec1c2d719040ca2bc5bd2f741079d37af
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/utils/spatial_analyzer.py
@@ -0,0 +1,158 @@
+"""Spatial relationship analyzer for PDF elements.
+
+This module provides functions to analyze spatial relationships between PDF elements,
+particularly for detecting containment relationships between formulas and other elements
+like curves and forms.
+
+All comments and docstrings are in English per project guidelines.
+"""
+
+from __future__ import annotations
+
+from babeldoc.format.pdf.document_il.il_version_1 import Box
+from babeldoc.format.pdf.document_il.il_version_1 import Page
+from babeldoc.format.pdf.document_il.il_version_1 import PdfCurve
+from babeldoc.format.pdf.document_il.il_version_1 import PdfForm
+from babeldoc.format.pdf.document_il.il_version_1 import PdfFormula
+from babeldoc.format.pdf.document_il.utils.layout_helper import calculate_iou_for_boxes
+
+
+def is_element_contained_in_formula(
+ element_box: Box,
+ formula_box: Box,
+ containment_threshold: float = 0.95,
+ tolerance: float = 2.0,
+) -> bool:
+ """Check if an element is completely contained within a formula with tolerance.
+
+ Args:
+ element_box: The bounding box of the element to check
+ formula_box: The bounding box of the formula
+ containment_threshold: Minimum IoU ratio to consider as contained (default: 0.95)
+ tolerance: Tolerance in units to expand formula box for containment check (default: 2.0)
+
+ Returns:
+ True if the element is considered contained within the formula
+ """
+ if element_box is None or formula_box is None:
+ return False
+
+ # Expand formula box by tolerance for more lenient containment check
+ expanded_formula_box = Box(
+ x=formula_box.x - tolerance,
+ y=formula_box.y - tolerance,
+ x2=formula_box.x2 + tolerance,
+ y2=formula_box.y2 + tolerance,
+ )
+
+ # Calculate IoU of element box with respect to expanded formula box
+ iou = calculate_iou_for_boxes(element_box, expanded_formula_box)
+ return iou >= containment_threshold
+
+
+def find_contained_curves(
+ formula: PdfFormula, page: Page, paragraph_xobj_id: int | None = None
+) -> list[PdfCurve]:
+ """Find all curves that are contained within the given formula.
+
+ Args:
+ formula: The formula to check for contained curves
+ page: The page containing the curves
+ paragraph_xobj_id: The xobj_id of the paragraph containing the formula.
+ If provided, only curves with matching xobj_id will be returned.
+
+ Returns:
+ List of curves that are contained within the formula
+ """
+ if not formula.box or not page.pdf_curve:
+ return []
+
+ contained_curves = []
+ for curve in page.pdf_curve:
+ if curve.box and is_element_contained_in_formula(curve.box, formula.box):
+ # If paragraph_xobj_id is specified, only include curves with matching xobj_id
+ if paragraph_xobj_id is not None and curve.xobj_id != paragraph_xobj_id:
+ continue
+ contained_curves.append(curve)
+
+ return contained_curves
+
+
+def find_contained_forms(
+ formula: PdfFormula, page: Page, paragraph_xobj_id: int | None = None
+) -> list[PdfForm]:
+ """Find all forms that are contained within the given formula.
+
+ Args:
+ formula: The formula to check for contained forms
+ page: The page containing the forms
+ paragraph_xobj_id: The xobj_id of the paragraph containing the formula.
+ If provided, only forms with matching xobj_id will be returned.
+
+ Returns:
+ List of forms that are contained within the formula
+ """
+ if not formula.box or not page.pdf_form:
+ return []
+
+ contained_forms = []
+ for form in page.pdf_form:
+ if form.box and is_element_contained_in_formula(form.box, formula.box):
+ # If paragraph_xobj_id is specified, only include forms with matching xobj_id
+ if paragraph_xobj_id is not None and form.xobj_id != paragraph_xobj_id:
+ continue
+ contained_forms.append(form)
+
+ return contained_forms
+
+
+def find_all_contained_elements(
+ formula: PdfFormula, page: Page, paragraph_xobj_id: int | None = None
+) -> tuple[list[PdfCurve], list[PdfForm]]:
+ """Find all curves and forms that are contained within the given formula.
+
+ Args:
+ formula: The formula to check for contained elements
+ page: The page containing the elements
+ paragraph_xobj_id: The xobj_id of the paragraph containing the formula.
+ If provided, only elements with matching xobj_id will be returned.
+
+ Returns:
+ Tuple of (contained_curves, contained_forms)
+ """
+ contained_curves = find_contained_curves(formula, page, paragraph_xobj_id)
+ contained_forms = find_contained_forms(formula, page, paragraph_xobj_id)
+ return contained_curves, contained_forms
+
+
+def calculate_translation_and_scale(
+ old_box: Box, new_box: Box
+) -> tuple[float, float, float]:
+ """Calculate translation and scale factors between two boxes.
+
+ Args:
+ old_box: The original bounding box
+ new_box: The new bounding box
+
+ Returns:
+ Tuple of (translation_x, translation_y, scale_factor)
+ """
+ if old_box is None or new_box is None:
+ return 0.0, 0.0, 1.0
+
+ # Calculate translation (difference in top-left corners)
+ translation_x = new_box.x - old_box.x
+ translation_y = new_box.y - old_box.y
+
+ # Calculate scale factor (using width ratio, fallback to height if needed)
+ old_width = old_box.x2 - old_box.x
+ new_width = new_box.x2 - new_box.x
+
+ if old_width > 0:
+ scale_factor = new_width / old_width
+ else:
+ old_height = old_box.y2 - old_box.y
+ new_height = new_box.y2 - new_box.y
+ scale_factor = new_height / old_height if old_height > 0 else 1.0
+
+ return translation_x, translation_y, scale_factor
diff --git a/babeldoc/format/pdf/document_il/utils/style_helper.py b/babeldoc/format/pdf/document_il/utils/style_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..0e9c4a283fcc6e220b04588f8ed34e173ff3ced2
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/utils/style_helper.py
@@ -0,0 +1,94 @@
+from babeldoc.format.pdf.document_il import il_version_1
+
+
+def create_pdf_style(r, g, b, font_id="base", font_size=6):
+ """
+ Create a PdfStyle object from RGB values.
+
+ Args:
+ r: Red component in range 0-255
+ g: Green component in range 0-255
+ b: Blue component in range 0-255
+ font_id: Font identifier
+ font_size: Font size
+
+ Returns:
+ PdfStyle object with the specified color
+ """
+ r, g, b = [x / 255.0 for x in (r, g, b)]
+ return il_version_1.PdfStyle(
+ font_id=font_id,
+ font_size=font_size,
+ graphic_state=il_version_1.GraphicState(
+ passthrough_per_char_instruction=f"{r:.10f} {g:.10f} {b:.10f} rg",
+ ),
+ )
+
+
+BLACK = il_version_1.GraphicState(passthrough_per_char_instruction="0 g 0 G")
+
+WHITE = il_version_1.GraphicState(passthrough_per_char_instruction="1 g 1 G")
+
+GRAY80 = il_version_1.GraphicState(passthrough_per_char_instruction="0.80 g 0.80 G")
+GRAY67 = il_version_1.GraphicState(passthrough_per_char_instruction="0.67 g 0.67 G")
+GRAY33 = il_version_1.GraphicState(passthrough_per_char_instruction="0.33 g 0.33 G")
+
+# Generate all color styles
+RED = il_version_1.GraphicState(
+ passthrough_per_char_instruction="1.0000000000 0.2313725490 0.1882352941 rg "
+ "1.0000000000 0.2313725490 0.1882352941 RG",
+)
+
+ORANGE = il_version_1.GraphicState(
+ passthrough_per_char_instruction="1.0000000000 0.5843137255 0.0000000000 rg "
+ "1.0000000000 0.5843137255 0.0000000000 RG",
+)
+YELLOW = il_version_1.GraphicState(
+ passthrough_per_char_instruction="1.0000000000 0.8000000000 0.0000000000 rg "
+ "1.0000000000 0.8000000000 0.0000000000 RG",
+)
+
+GREEN = il_version_1.GraphicState(
+ passthrough_per_char_instruction="0.2039215686 0.7803921569 0.3490196078 rg "
+ "0.2039215686 0.7803921569 0.3490196078 RG",
+)
+
+MINT = il_version_1.GraphicState(
+ passthrough_per_char_instruction="0.0000000000 0.7803921569 0.7450980392 rg "
+ "0.0000000000 0.7803921569 0.7450980392 RG",
+)
+
+TEAL = il_version_1.GraphicState(
+ passthrough_per_char_instruction="0.1882352941 0.6901960784 0.7803921569 rg "
+ "0.1882352941 0.6901960784 0.7803921569 RG",
+)
+
+CYAN = il_version_1.GraphicState(
+ passthrough_per_char_instruction="0.1960784314 0.6784313725 0.9019607843 rg "
+ "0.1960784314 0.6784313725 0.9019607843 RG",
+)
+
+BLUE = il_version_1.GraphicState(
+ passthrough_per_char_instruction="0.0000000000 0.4784313725 1.0000000000 rg "
+ "0.0000000000 0.4784313725 1.0000000000 RG",
+)
+
+INDIGO = il_version_1.GraphicState(
+ passthrough_per_char_instruction="0.3450980392 0.3372549020 0.8392156863 rg "
+ "0.3450980392 0.3372549020 0.8392156863 RG",
+)
+
+PURPLE = il_version_1.GraphicState(
+ passthrough_per_char_instruction="0.6862745098 0.3215686275 0.8705882353 rg "
+ "0.6862745098 0.3215686275 0.8705882353 RG",
+)
+
+PINK = il_version_1.GraphicState(
+ passthrough_per_char_instruction="1.0000000000 0.1764705882 0.3333333333 rg "
+ "1.0000000000 0.1764705882 0.3333333333 RG",
+)
+
+BROWN = il_version_1.GraphicState(
+ passthrough_per_char_instruction="0.6352941176 0.5176470588 0.3686274510 rg "
+ "0.6352941176 0.5176470588 0.3686274510 RG",
+)
diff --git a/babeldoc/format/pdf/document_il/utils/zstd_helper.py b/babeldoc/format/pdf/document_il/utils/zstd_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd514360c83c96c8a60ab31becc248a58e6df8b0
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/utils/zstd_helper.py
@@ -0,0 +1,21 @@
+import base64
+
+import pyzstd
+
+
+def zstd_compress(data) -> str:
+ if isinstance(data, str):
+ data = data.encode()
+ if not isinstance(data, bytes):
+ raise TypeError(f"data must be str or bytes, not {type(data)}")
+
+ return base64.b85encode(pyzstd.compress(data)).decode()
+
+
+def zstd_decompress(data) -> str:
+ if isinstance(data, str):
+ data = data.encode()
+ if not isinstance(data, bytes):
+ raise TypeError(f"data must be str or bytes, not {type(data)}")
+
+ return pyzstd.decompress(base64.b85decode(data)).decode()
diff --git a/babeldoc/format/pdf/document_il/xml_converter.py b/babeldoc/format/pdf/document_il/xml_converter.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ab4fe98f42e11b41976d2725221d92f8753d685
--- /dev/null
+++ b/babeldoc/format/pdf/document_il/xml_converter.py
@@ -0,0 +1,152 @@
+import copy
+import json
+import threading
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import orjson
+from xsdata.formats.dataclass.context import XmlContext
+from xsdata.formats.dataclass.parsers import XmlParser
+from xsdata.formats.dataclass.serializers import XmlSerializer
+from xsdata.formats.dataclass.serializers.config import SerializerConfig
+
+from babeldoc.format.pdf.document_il import il_version_1
+
+
+class XMLConverter:
+ def __init__(self):
+ self.parser = XmlParser()
+ config = SerializerConfig(indent=" ")
+ context = XmlContext()
+ self.serializer = XmlSerializer(context=context, config=config)
+
+ # Internal state (not related to file paths)
+ self._lock = threading.Lock()
+ self.step_counter = 0
+ self.current_stage = None
+
+ # ==================== XML / JSON CONVERSION ====================
+
+ def write_xml(self, document: il_version_1.Document, path: str):
+ with Path(path).open("w", encoding="utf-8") as f:
+ f.write(self.to_xml(document))
+
+ def read_xml(self, path: str) -> il_version_1.Document:
+ with Path(path).open(encoding="utf-8") as f:
+ return self.from_xml(f.read())
+
+ def to_xml(self, document: il_version_1.Document) -> str:
+ return self.serializer.render(document)
+
+ def from_xml(self, xml: str) -> il_version_1.Document:
+ return self.parser.from_string(xml, il_version_1.Document)
+
+ def deepcopy(self, document: il_version_1.Document) -> il_version_1.Document:
+ return copy.deepcopy(document)
+
+ def to_json(self, document: il_version_1.Document) -> str:
+ return orjson.dumps(
+ document,
+ option=orjson.OPT_APPEND_NEWLINE
+ | orjson.OPT_INDENT_2
+ | orjson.OPT_SORT_KEYS,
+ ).decode()
+
+ def write_json(self, document: il_version_1.Document, path: str):
+ with Path(path).open("w", encoding="utf-8") as f:
+ f.write(self.to_json(document))
+
+ # ==================== TXT LOGGING METHODS ====================
+
+ def _safe_write_txt(self, path: Path, text: str):
+ """Thread-safe write to text file."""
+ try:
+ with self._lock:
+ with path.open("a", encoding="utf-8", errors="replace") as f:
+ f.write(text)
+ except Exception as e:
+ print(f"⚠️ Logging failed: {e}")
+
+ def _write_txt_header(self, path: Path):
+ """Write log header."""
+ header = (
+ "=" * 100 + "\n"
+ "PDF TRANSLATION DETAILED LOG\n"
+ f"Started at: {datetime.now().isoformat()}\n"
+ + "=" * 100 + "\n\n"
+ )
+ self._safe_write_txt(path, header)
+
+ def _write_txt_footer(self, path: Path):
+ """Write log footer."""
+ footer = (
+ "\n" + "=" * 100 + "\n"
+ f"Completed at: {datetime.now().isoformat()}\n"
+ + "=" * 100 + "\n"
+ )
+ self._safe_write_txt(path, footer)
+
+ def start_txt_stage(self, path: str, stage_name: str):
+ """Start a new stage in logging."""
+ path_obj = Path(path)
+ path_obj.parent.mkdir(parents=True, exist_ok=True)
+
+ # Start of new log — write header if file doesn't exist yet
+ if not path_obj.exists() or path_obj.stat().st_size == 0:
+ self._write_txt_header(path_obj)
+
+ self.current_stage = stage_name
+ self.step_counter = 0
+ self._safe_write_txt(
+ path_obj,
+ f"\n{'=' * 100}\nSTAGE: {stage_name}\n{'=' * 100}\n\n"
+ )
+
+ def end_txt_stage(self, path: str, stage_name: str):
+ """End a stage."""
+ path_obj = Path(path)
+ self._safe_write_txt(path_obj, f"\n--- End of {stage_name} ---\n\n")
+
+ def log_txt_step(self, path: str, step_name: str, details: str = "", data: Any = None):
+ """Log a single step."""
+ path_obj = Path(path)
+ self.step_counter += 1
+
+ lines = [f"\n[Step {self.step_counter}] {step_name}\n", "-" * 80 + "\n"]
+
+ if details:
+ lines.append(f"Details: {details}\n")
+
+ if data is not None:
+ lines.append("Data:\n")
+ if isinstance(data, (dict, list)):
+ json_data = json.dumps(data, indent=2, ensure_ascii=False)
+ truncated = json_data[:5000]
+ lines.append(truncated + "\n")
+ if len(json_data) > 5000:
+ lines.append("... [truncated for brevity]\n")
+ else:
+ text_data = str(data)
+ truncated = text_data[:5000]
+ lines.append(truncated + "\n")
+ if len(text_data) > 5000:
+ lines.append("... [truncated for brevity]\n")
+
+ lines.append("-" * 80 + "\n")
+ self._safe_write_txt(path_obj, "".join(lines))
+
+ def log_txt_paragraph(self, path: str, paragraph_data: dict):
+ """Log paragraph information."""
+ text = (
+ f"\n Paragraph:\n"
+ f" Text: {paragraph_data.get('text', '')[:200]}\n"
+ f" Layout: {paragraph_data.get('layout_label', 'N/A')}\n"
+ f" Bounding box: {paragraph_data.get('box', 'N/A')}\n"
+ f" Character count: {paragraph_data.get('char_count', 0)}\n"
+ )
+ self._safe_write_txt(Path(path), text)
+
+ def finalize_txt_log(self, path: str):
+ """Write footer and finalize."""
+ self._write_txt_footer(Path(path))
diff --git a/babeldoc/format/pdf/high_level.py b/babeldoc/format/pdf/high_level.py
new file mode 100644
index 0000000000000000000000000000000000000000..9f853e38a0c62100fa66e41c7b020eb78f049c97
--- /dev/null
+++ b/babeldoc/format/pdf/high_level.py
@@ -0,0 +1,1363 @@
+import asyncio
+import copy
+import hashlib
+import io
+import logging
+import pathlib
+import re
+import shutil
+import threading
+import time
+from asyncio import CancelledError
+from pathlib import Path
+from typing import Any
+from typing import BinaryIO
+
+import pymupdf
+from pymupdf import Document
+from pymupdf import Font
+
+from babeldoc import asynchronize
+from babeldoc.assets.assets import warmup
+from babeldoc.babeldoc_exception.BabelDOCException import ExtractTextError
+from babeldoc.babeldoc_exception.BabelDOCException import (
+ InputFileGeneratedByBabelDOCError,
+)
+from babeldoc.const import CACHE_FOLDER
+from babeldoc.const import WATERMARK_VERSION
+from babeldoc.const import close_process_pool
+from babeldoc.format.pdf.converter import TranslateConverter
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.backend.pdf_creater import SAVE_PDF_STAGE_NAME
+from babeldoc.format.pdf.document_il.backend.pdf_creater import SUBSET_FONT_STAGE_NAME
+from babeldoc.format.pdf.document_il.backend.pdf_creater import PDFCreater
+from babeldoc.format.pdf.document_il.backend.pdf_creater import reproduce_cmap
+from babeldoc.format.pdf.document_il.frontend.il_creater import ILCreater
+from babeldoc.format.pdf.document_il.midend.add_debug_information import (
+ AddDebugInformation,
+)
+from babeldoc.format.pdf.document_il.midend.automatic_term_extractor import (
+ AutomaticTermExtractor,
+)
+from babeldoc.format.pdf.document_il.midend.detect_scanned_file import DetectScannedFile
+from babeldoc.format.pdf.document_il.midend.il_translator import ILTranslator
+from babeldoc.format.pdf.document_il.midend.il_translator_llm_only import (
+ ILTranslatorLLMOnly,
+)
+from babeldoc.format.pdf.document_il.midend.layout_parser import LayoutParser
+from babeldoc.format.pdf.document_il.midend.paragraph_finder import ParagraphFinder
+from babeldoc.format.pdf.document_il.midend.styles_and_formulas import StylesAndFormulas
+from babeldoc.format.pdf.document_il.midend.table_parser import TableParser
+from babeldoc.format.pdf.document_il.midend.typesetting import Typesetting
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.xml_converter import XMLConverter
+from babeldoc.format.pdf.pdfinterp import PDFPageInterpreterEx
+from babeldoc.format.pdf.result_merger import ResultMerger
+from babeldoc.format.pdf.split_manager import SplitManager
+from babeldoc.format.pdf.translation_config import TranslateResult
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from babeldoc.format.pdf.translation_config import WatermarkOutputMode
+from babeldoc.pdfminer.pdfdocument import PDFDocument
+from babeldoc.pdfminer.pdfinterp import PDFResourceManager
+from babeldoc.pdfminer.pdfpage import PDFPage
+from babeldoc.pdfminer.pdfparser import PDFParser
+from babeldoc.progress_monitor import ProgressMonitor
+from babeldoc.utils import memory
+from babeldoc.detailed_logger import DetailedLogger, init_detailed_logger
+
+logger = logging.getLogger(__name__)
+
+TRANSLATE_STAGES = [
+ (ILCreater.stage_name, 14.12), # Parse PDF and Create IR
+ (DetectScannedFile.stage_name, 2.45), # DetectScannedFile
+ (LayoutParser.stage_name, 14.03), # Parse Page Layout
+ (TableParser.stage_name, 1.0), # Parse Table
+ (ParagraphFinder.stage_name, 6.26), # Parse Paragraphs
+ (StylesAndFormulas.stage_name, 1.66), # Parse Formulas and Styles
+ # (RemoveDescent.stage_name, 0.15), # Remove Char Descent
+ (AutomaticTermExtractor.stage_name, 30.0), # Extract Terms
+ (ILTranslator.stage_name, 46.96), # Translate Paragraphs
+ (Typesetting.stage_name, 4.71), # Typesetting
+ (FontMapper.stage_name, 0.61), # Add Fonts
+ (PDFCreater.stage_name, 1.96), # Generate drawing instructions
+ (SUBSET_FONT_STAGE_NAME, 0.92), # Subset font
+ (SAVE_PDF_STAGE_NAME, 6.34), # Save PDF
+]
+
+resfont_map = {
+ "zh-cn": "china-ss",
+ "zh-tw": "china-ts",
+ "zh-hans": "china-ss",
+ "zh-hant": "china-ts",
+ "zh": "china-ss",
+ "ja": "japan-s",
+ "ko": "korea-s",
+}
+
+
+def safe_save(doc, *args, **kwargs):
+ try:
+ # first try, saving without options
+ doc.save(*args, **kwargs)
+ except Exception:
+ # second try, saving with 'garbage=3' for object missing
+ doc.ez_save(*args, **kwargs)
+
+
+def check_metadata(pdf: Document):
+ meta = pdf.metadata
+ if not meta:
+ return
+ producer = meta.get("producer", None)
+ if (
+ producer
+ and "BabelDOC" in producer
+ and "Translation_generated_by_AI,please_carefully_discern" in producer
+ ):
+ raise InputFileGeneratedByBabelDOCError(
+ "Input file is generated by BabelDOC, Cannot translate files that have already been translated."
+ )
+
+
+def add_metadata(
+ translate_result: TranslateResult, translate_config: TranslationConfig
+):
+ processed = []
+ for attr in (
+ "mono_pdf_path",
+ "dual_pdf_path",
+ "no_watermark_mono_pdf_path",
+ "no_watermark_dual_pdf_path",
+ ):
+ path = getattr(translate_result, attr)
+ if not path or path in processed:
+ continue
+ processed.append(path)
+
+ temp_path = translate_config.get_working_file_path(f"{path.stem}.cmap.pdf")
+ pdf = pymupdf.open(path)
+ meta = pdf.metadata
+ if not meta:
+ meta = {}
+ creator = meta.get("creator", None)
+ producer = meta.get("producer", None)
+ if producer:
+ if not creator:
+ creator = producer
+ else:
+ creator += f", {producer}"
+
+ translated_by = f"BabelDOC{WATERMARK_VERSION}_{time.time()}_Translation_generated_by_AI,please_carefully_discern"
+ if translate_config.metadata_extra_data:
+ translated_by += f"_{translate_config.metadata_extra_data}"
+ meta["producer"] = translated_by
+ meta["creator"] = creator
+
+ for k, v in meta.items():
+ if v:
+ # 使用正则替换掉 surrogate 范围内的字符
+ meta[k] = re.sub(r"[\uD800-\uDFFF]", "", v)
+
+ pdf.set_metadata(meta)
+ safe_save(pdf, temp_path)
+ shutil.move(temp_path, path)
+
+
+def fix_cmap(translate_result: TranslateResult, translate_config: TranslationConfig):
+ processed = []
+ for attr in (
+ "mono_pdf_path",
+ "dual_pdf_path",
+ "no_watermark_mono_pdf_path",
+ "no_watermark_dual_pdf_path",
+ ):
+ path = getattr(translate_result, attr)
+ if not path or path in processed:
+ continue
+ processed.append(path)
+
+ temp_path = translate_config.get_working_file_path(f"{path.stem}.cmap.pdf")
+ pdf = pymupdf.open(path)
+ reproduce_cmap(pdf)
+ safe_save(pdf, temp_path)
+ shutil.move(temp_path, path)
+
+
+def verify_file_hash(file_path: str, expected_hash: str) -> bool:
+ """Verify the SHA256 hash of a file."""
+ sha256_hash = hashlib.sha256()
+ with Path(file_path).open("rb") as f:
+ # Read the file in chunks to handle large files efficiently
+ for byte_block in iter(lambda: f.read(4096), b""):
+ sha256_hash.update(byte_block)
+ return sha256_hash.hexdigest() == expected_hash
+
+
+def translator_supports_llm(translator) -> bool:
+ if not translator or not hasattr(translator, "do_llm_translate"):
+ return False
+ try:
+ translator.do_llm_translate(None)
+ return True
+ except NotImplementedError:
+ return False
+ except Exception as exc: # pragma: no cover - defensive logging
+ logger.debug("translator %s failed llm detection: %s", translator, exc)
+ return False
+
+
+def start_parse_il(
+ inf: BinaryIO,
+ pages: list[int] | None = None,
+ vfont: str = "",
+ vchar: str = "",
+ thread: int = 0,
+ doc_zh: Document = None,
+ lang_in: str = "",
+ lang_out: str = "",
+ service: str = "",
+ resfont: str = "",
+ noto: Font = None,
+ cancellation_event: asyncio.Event = None,
+ il_creater: ILCreater = None,
+ translation_config: TranslationConfig = None,
+ **kwarg: Any,
+) -> None:
+ rsrcmgr = PDFResourceManager()
+ layout = {}
+ device = TranslateConverter(
+ rsrcmgr,
+ vfont,
+ vchar,
+ thread,
+ layout,
+ lang_in,
+ lang_out,
+ service,
+ resfont,
+ noto,
+ kwarg.get("envs", {}),
+ kwarg.get("prompt", []),
+ il_creater=il_creater,
+ )
+ # model = DocLayoutModel.load_available()
+
+ assert device is not None
+ assert il_creater is not None
+ assert translation_config is not None
+ obj_patch = {}
+ interpreter = PDFPageInterpreterEx(rsrcmgr, device, obj_patch, il_creater)
+ if pages:
+ total_pages = len(pages)
+ else:
+ total_pages = doc_zh.page_count
+
+ il_creater.on_total_pages(total_pages)
+
+ parser = PDFParser(inf)
+ doc = PDFDocument(parser)
+
+ for pageno, page in enumerate(PDFPage.create_pages(doc)):
+ if cancellation_event and cancellation_event.is_set():
+ raise CancelledError("task cancelled")
+ if pages and (pageno not in pages):
+ continue
+ page.pageno = pageno
+
+ if not translation_config.should_translate_page(pageno + 1):
+ continue
+
+ height, width = (
+ page.cropbox[3] - page.cropbox[1],
+ page.cropbox[2] - page.cropbox[0],
+ )
+ if height > 1200 or width > 2000:
+ logger.warning(f"page {pageno + 1} is too large, maybe unable to translate")
+ # continue
+
+ translation_config.raise_if_cancelled()
+ # The current program no longer relies on
+ # the following layout recognition results,
+ # but in order to facilitate the migration of pdf2zh,
+ # the relevant code is temporarily retained.
+ # pix = doc_zh[page.pageno].get_pixmap()
+ # image = np.frombuffer(pix.samples, np.uint8).reshape(
+ # pix.height, pix.width, 3
+ # )[:, :, ::-1]
+ # page_layout = model.predict(
+ # image, imgsz=int(pix.height / 32) * 32)[0]
+ # # kdtree 是不可能 kdtree 的，不如直接渲染成图片，用空间换时间
+ # box = np.ones((pix.height, pix.width))
+ # h, w = box.shape
+ # vcls = ["abandon", "figure", "table",
+ # "isolate_formula", "formula_caption"]
+ # for i, d in enumerate(page_layout.boxes):
+ # if page_layout.names[int(d.cls)] not in vcls:
+ # x0, y0, x1, y1 = d.xyxy.squeeze()
+ # x0, y0, x1, y1 = (
+ # np.clip(int(x0 - 1), 0, w - 1),
+ # np.clip(int(h - y1 - 1), 0, h - 1),
+ # np.clip(int(x1 + 1), 0, w - 1),
+ # np.clip(int(h - y0 + 1), 0, h - 1),
+ # )
+ # box[y0:y1, x0:x1] = i + 2
+ # for i, d in enumerate(page_layout.boxes):
+ # if page_layout.names[int(d.cls)] in vcls:
+ # x0, y0, x1, y1 = d.xyxy.squeeze()
+ # x0, y0, x1, y1 = (
+ # np.clip(int(x0 - 1), 0, w - 1),
+ # np.clip(int(h - y1 - 1), 0, h - 1),
+ # np.clip(int(x1 + 1), 0, w - 1),
+ # np.clip(int(h - y0 + 1), 0, h - 1),
+ # )
+ # box[y0:y1, x0:x1] = 0
+ # layout[page.pageno] = box
+ # 新建一个 xref 存放新指令流
+ # page.page_xref = doc_zh.get_new_xref() # hack 插入页面的新 xref
+ # doc_zh.update_object(page.page_xref, "<<>>")
+ # doc_zh.update_stream(page.page_xref, b"")
+ # doc_zh[page.pageno].set_contents(page.page_xref)
+ ops_base = interpreter.process_page(page)
+ il_creater.on_page_base_operation(ops_base)
+ il_creater.on_page_end()
+ il_creater.on_finish()
+ device.close()
+
+
+def translate(translation_config: TranslationConfig) -> TranslateResult:
+ with ProgressMonitor(get_translation_stage(translation_config)) as pm:
+ return do_translate(pm, translation_config)
+
+
+def get_translation_stage(
+ translation_config: TranslationConfig,
+) -> list[tuple[str, float]]:
+ result = copy.deepcopy(TRANSLATE_STAGES)
+ should_remove = []
+
+ # If only parsing and generating PDF, skip all translation-related stages
+ if translation_config.only_parse_generate_pdf:
+ should_remove.extend(
+ [
+ DetectScannedFile.stage_name,
+ LayoutParser.stage_name,
+ TableParser.stage_name,
+ ParagraphFinder.stage_name,
+ StylesAndFormulas.stage_name,
+ AutomaticTermExtractor.stage_name,
+ ILTranslator.stage_name,
+ Typesetting.stage_name,
+ ]
+ )
+ else:
+ # Original logic for selective removal
+ if not translation_config.table_model:
+ should_remove.append(TableParser.stage_name)
+ if translation_config.skip_scanned_detection:
+ should_remove.append(DetectScannedFile.stage_name)
+ if not translation_config.auto_extract_glossary:
+ should_remove.append(AutomaticTermExtractor.stage_name)
+ if translation_config.skip_translation:
+ should_remove.append(ILTranslator.stage_name)
+
+ result = [x for x in result if x[0] not in should_remove]
+ return result
+
+
+async def async_translate(translation_config: TranslationConfig):
+ """Asynchronously translate a PDF file with real-time progress reporting.
+
+ This function yields progress events that can be used to update progress bars
+ or other UI elements. The events are dictionaries with the following structure:
+
+ - progress_start: {
+ "type": "progress_start",
+ "stage": str, # Stage name
+ "stage_progress": float, # Always 0.0
+ "stage_current": int, # Current count (0)
+ "stage_total": int # Total items in stage
+ }
+ - progress_update: {
+ "type": "progress_update",
+ "stage": str, # Stage name
+ "stage_progress": float, # Stage progress (0-100)
+ "stage_current": int, # Current items processed
+ "stage_total": int, # Total items in stage
+ "overall_progress": float # Overall progress (0-100)
+ }
+ - progress_end: {
+ "type": "progress_end",
+ "stage": str, # Stage name
+ "stage_progress": float, # Always 100.0
+ "stage_current": int, # Equal to stage_total
+ "stage_total": int, # Total items processed
+ "overall_progress": float # Overall progress (0-100)
+ }
+ - finish: {
+ "type": "finish",
+ "translate_result": TranslateResult
+ }
+ - error: {
+ "type": "error",
+ "error": str
+ }
+
+ Args:
+ translation_config: Configuration for the translation process
+
+ Yields:
+ dict: Progress events during translation
+
+ Raises:
+ CancelledError: If the translation is cancelled
+ Exception: Any other errors during translation
+ """
+ loop = asyncio.get_running_loop()
+ callback = asynchronize.AsyncCallback()
+
+ finish_event = asyncio.Event()
+ cancel_event = threading.Event()
+ with ProgressMonitor(
+ get_translation_stage(translation_config),
+ progress_change_callback=callback.step_callback,
+ finish_callback=callback.finished_callback,
+ finish_event=finish_event,
+ cancel_event=cancel_event,
+ loop=loop,
+ report_interval=translation_config.report_interval,
+ ) as pm:
+ future = loop.run_in_executor(None, do_translate, pm, translation_config)
+ try:
+ async for event in callback:
+ event = event.kwargs
+ yield event
+ if event["type"] == "error":
+ break
+ except CancelledError:
+ cancel_event.set()
+ except KeyboardInterrupt:
+ logger.info("Translation cancelled by user through keyboard interrupt")
+ cancel_event.set()
+ if cancel_event.is_set():
+ future.cancel()
+ logger.info("Waiting for translation to finish...")
+ await finish_event.wait()
+
+
+class MemoryMonitor:
+ """Monitor memory usage of current process and all child processes."""
+
+ def __init__(self, interval=0.1):
+ """Initialize memory monitor.
+
+ Args:
+ interval: Monitoring interval in seconds, defaults to 0.1s (100ms)
+ """
+ self.interval = interval
+ self.peak_memory_usage = 0
+ self.monitor_thread = None
+ self.stop_event = None
+ self.last_pss_check_time = None
+
+ def __enter__(self):
+ """Start memory monitoring."""
+ self.stop_event = threading.Event()
+ self.monitor_thread = threading.Thread(
+ target=self._monitor_memory_usage, daemon=True
+ )
+ self.monitor_thread.start()
+ logger.debug("Memory monitoring started")
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ """Stop monitoring and log peak memory usage."""
+ if not self.monitor_thread:
+ return
+
+ self.stop_event.set()
+ self.monitor_thread.join(timeout=2.0)
+ logger.info(f"Peak memory usage: {self.peak_memory_usage:.2f} MB")
+
+ def _monitor_memory_usage(self):
+ """Background thread that periodically checks memory usage."""
+ while not self.stop_event.is_set():
+ try:
+ # Use throttled memory check with 2-second PSS throttle
+ total_memory, self.last_pss_check_time = (
+ memory.get_memory_usage_with_throttle(
+ include_children=True,
+ prefer_pss=True,
+ last_pss_check_time=self.last_pss_check_time,
+ pss_throttle_seconds=2.0,
+ )
+ )
+
+ # Convert to MB for better readability
+ total_memory_mb = total_memory / (1024 * 1024)
+ if total_memory_mb > self.peak_memory_usage:
+ self.peak_memory_usage = total_memory_mb
+ except Exception as e:
+ logger.warning(f"Error monitoring memory: {e}")
+
+ time.sleep(self.interval)
+
+ def get_peek_memory_psutil(self):
+ """Get peak memory usage using psutil (for backwards compatibility)."""
+ return memory.get_memory_usage_bytes(include_children=True, prefer_pss=True)
+
+
+def fix_null_page_content(doc: Document) -> list[int]:
+ invalid_page = []
+ for x in range(len(doc)):
+ xref = doc[x].xref
+ if doc.xref_object(xref) == "null":
+ invalid_page.append(x)
+ for x in invalid_page:
+ doc.delete_page(x)
+ doc.insert_page(x)
+ return invalid_page
+
+
+def fix_null_xref(doc: Document) -> None:
+ """Fix null xref in PDF file by replacing them with empty arrays.
+
+ Args:
+ doc: PyMuPDF Document object to fix
+ """
+ for i in range(1, doc.xref_length()):
+ try:
+ obj = doc.xref_object(i)
+ if obj == "null":
+ doc.update_object(i, "[]")
+ elif obj and "/ASCII85Decode" in obj: # make pdfminer happy
+ data = doc.xref_stream(i)
+ doc.update_stream(i, data)
+ elif obj and "/LZWDecode" in obj:
+ data = doc.xref_stream(i)
+ doc.update_stream(i, data)
+ elif obj and "/Annots" in obj:
+ doc.xref_set_key(i, "Annots", "null")
+ except Exception:
+ doc.update_object(i, "[]")
+
+
+def fix_filter(doc):
+ page_contents = []
+ for page in doc:
+ page_contents.extend(page.get_contents())
+ for page_piece in page_contents:
+ f = doc.xref_get_key(page_piece, "Filter")
+ if f[0] == "xref":
+ data = doc.xref_stream(page_piece)
+ doc.update_stream(page_piece, data)
+ for page in doc:
+ contents = page.get_contents()
+ if len(contents) > 1:
+ page_streams = [doc.xref_stream(i) for i in contents]
+ r = doc.get_new_xref()
+ doc.update_object(r, "<<>>")
+ doc.update_stream(r, b" ".join(page_streams))
+ doc.xref_set_key(page.xref, "Contents", f"{r} 0 R")
+ return
+ # skip rotate for now
+ for page in doc:
+ contents = page.get_contents()
+ t, v = doc.xref_get_key(page.xref, "Rotate")
+ rotate = -int(v) if t == "int" else 0
+ if len(contents) > 1 or rotate:
+ page_streams = [doc.xref_stream(i) for i in contents]
+ r = doc.get_new_xref()
+ page_prefix = b""
+ page_suffix = b""
+ if rotate:
+ m0 = pymupdf.Matrix(rotate)
+ b0 = page.mediabox * m0
+ m1 = m0 * pymupdf.Matrix(1, 0, 0, 1, b0.x0, -b0.y0)
+ page_prefix = (
+ f" {m1.a} {m1.b} {m1.c} {m1.d} {m1.e} {m1.f} cm q ".encode()
+ )
+ page_suffix = b" Q "
+ update_page_bbox(doc, page, page.cropbox * m1, "CropBox")
+ update_page_bbox(doc, page, page.artbox * m1, "ArtBox")
+ update_page_bbox(doc, page, page.bleedbox * m1, "BleedBox")
+ update_page_bbox(doc, page, page.mediabox * m1, "MediaBox")
+ doc.xref_set_key(page.xref, "Rotate", "0")
+ doc.update_object(r, "<<>>")
+ doc.update_stream(r, page_prefix + b" ".join(page_streams) + page_suffix)
+ doc.xref_set_key(page.xref, "Contents", f"{r} 0 R")
+
+
+def update_page_bbox(doc, page, box, key):
+ if doc.xref_get_key(page.xref, key)[0] == "array":
+ doc.xref_set_key(page.xref, key, f"[{box.x0} {box.y0} {box.x1} {box.y1}]")
+
+def do_translate(
+ pm: ProgressMonitor, translation_config: TranslationConfig
+) -> TranslateResult:
+ try:
+ translation_config.progress_monitor = pm
+ original_pdf_path = translation_config.input_file
+ logger.info(f"start to translate: {original_pdf_path}")
+ try:
+ check_metadata(Document(original_pdf_path))
+ except InputFileGeneratedByBabelDOCError as e:
+ logger.error(
+ f"input file {original_pdf_path} is generated by BabelDOC, Cannot translate files that have already been translated."
+ )
+ raise e
+ except Exception as e:
+ logger.warning(f"Error in check metadata, continue: {e}")
+ start_time = time.time()
+ peak_memory_usage = 0
+ with MemoryMonitor() as memory_monitor:
+ # Check if split translation is enabled
+ if not translation_config.split_strategy:
+ print("\n\n\n\n\n\n\n\nSplit strategy not set, using single translation")
+ result = _do_translate_single(pm, translation_config)
+ else:
+ # Initialize split manager and determine split points
+ split_manager = SplitManager(translation_config)
+ split_points = split_manager.determine_split_points(translation_config)
+
+ if not split_points:
+ logger.warning(
+ "No split points determined, falling back to single translation"
+ )
+ result = _do_translate_single(pm, translation_config)
+ else:
+ logger.info(f"Split points determined: {len(split_points)} parts")
+
+ if len(split_points) == 1:
+ logger.info("Only one part, use single translation")
+ result = _do_translate_single(pm, translation_config)
+ else:
+ pm.total_parts = len(split_points)
+
+ # Process parts serially
+ results: dict[int, TranslateResult | None] = {}
+ original_watermark_mode = (
+ translation_config.watermark_output_mode
+ )
+ original_doc = Document(original_pdf_path)
+ for i, split_point in enumerate(split_points):
+ try:
+ # Create a copy of config for this part
+ part_config = copy.copy(translation_config)
+ part_config.skip_clean = True
+ should_translate_pages = []
+ for page in range(
+ split_point.start_page, split_point.end_page + 1
+ ):
+ if translation_config.should_translate_page(
+ page + 1
+ ):
+ should_translate_pages.append(
+ page - split_point.start_page + 1
+ )
+ part_config.pages = None
+ part_config.page_ranges = [
+ (x, x) for x in should_translate_pages
+ ]
+ if (
+ translation_config.only_include_translated_page
+ and not should_translate_pages
+ ):
+ results[i] = None
+ continue
+
+ # Only first part should do scanned detection if enabled
+ if i > 0:
+ part_config.skip_scanned_detection = True
+
+ part_config.working_dir = (
+ translation_config.get_part_working_dir(i)
+ )
+ part_config.output_dir = (
+ translation_config.get_part_output_dir(i)
+ )
+
+ assert id(
+ part_config.shared_context_cross_split_part
+ ) == id(
+ translation_config.shared_context_cross_split_part
+ ), "shared_context_cross_split_part must be the same"
+
+ part_temp_input_path = (
+ part_config.get_working_file_path(
+ f"input.part{i}.pdf"
+ )
+ )
+ part_config.input_file = part_temp_input_path
+
+ temp_doc = Document()
+ for x in range(
+ split_point.start_page, split_point.end_page + 1
+ ):
+ xref = original_doc[x].xref
+ if (
+ original_doc.xref_get_key(xref, "Annots")[0]
+ != "null"
+ ):
+ original_doc.xref_set_key(
+ xref, "Annots", "null"
+ )
+ temp_doc.insert_pdf(
+ original_doc,
+ from_page=split_point.start_page,
+ to_page=split_point.end_page,
+ )
+ safe_save(temp_doc, part_temp_input_path)
+ assert (
+ temp_doc.page_count
+ == split_point.end_page - split_point.start_page + 1
+ )
+
+ # Only first part should have watermark
+ if i > 0:
+ part_config.watermark_output_mode = (
+ WatermarkOutputMode.NoWatermark
+ )
+
+ # Create progress monitor for this part
+ part_monitor = pm.create_part_monitor(
+ i, len(split_points)
+ )
+
+ # Process this part
+ result = _do_translate_single(
+ part_monitor,
+ part_config,
+ )
+ results[i] = result
+
+ except Exception as e:
+ logger.error(f"Error in part {i}: {e}")
+ pm.translate_error(e)
+ raise
+ finally:
+ # Clean up part working directory
+ translation_config.cleanup_part_working_dir(i)
+
+ # Restore original watermark mode
+ translation_config.watermark_output_mode = (
+ original_watermark_mode
+ )
+
+ # Merge results
+ merger = ResultMerger(translation_config)
+ logger.info("start merge results")
+ result = merger.merge_results(results)
+ logger.info("finish merge results")
+ peak_memory_usage = memory_monitor.peak_memory_usage
+
+ finish_time = time.time()
+ result.total_seconds = finish_time - start_time
+
+ logger.info(
+ f"finish translate: {original_pdf_path}, cost: {finish_time - start_time} s",
+ )
+ result.original_pdf_path = translation_config.input_file
+ result.peak_memory_usage = peak_memory_usage
+
+ fix_cmap(result, translation_config)
+ add_metadata(result, translation_config)
+ try:
+ migrate_toc(translation_config, result)
+ except Exception as e:
+ logger.error(
+ f"Failed to migrate TOC from {translation_config.input_file}: {e}"
+ )
+ pm.translate_done(result)
+ return result
+
+ except Exception as e:
+ if translation_config.debug:
+ logger.exception("translate error:")
+ else:
+ logger.error(f"translate error: {e}")
+ pm.disable = False
+ pm.translate_error(e)
+ raise
+ finally:
+ logger.debug("do_translate finally")
+ pm.on_finish()
+ translation_config.cleanup_temp_files()
+
+
+def migrate_toc(
+ translation_config: TranslationConfig, translate_result: TranslateResult
+):
+ if translation_config.use_alternating_pages_dual:
+ logger.info('skipping TOC migration for "use_alternating_pages_dual" mode')
+ return
+ old_doc = Document(translation_config.input_file)
+ if not old_doc:
+ return
+ try:
+ fix_filter(old_doc)
+ fix_null_xref(old_doc)
+ except Exception:
+ logger.exception("auto fix failed, please check the pdf file")
+
+ toc_data = old_doc.get_toc()
+
+ if not toc_data:
+ logger.info("No TOC found in the original PDF, skipping migration.")
+ return
+
+ if translation_config.only_include_translated_page:
+ total_page = set(range(0, len(old_doc)))
+
+ pages_to_translate = {
+ i for i in len(old_doc) if translation_config.should_translate_page(i + 1)
+ }
+
+ should_removed_page = list(total_page - pages_to_translate)
+
+ files = {
+ translate_result.dual_pdf_path,
+ # translate_result.mono_pdf_path,
+ translate_result.no_watermark_dual_pdf_path,
+ # translate_result.no_watermark_mono_pdf_path
+ }
+
+ for f in files:
+ if not f:
+ continue
+ mig_toc_temp_input = translation_config.get_working_file_path(
+ "mig_toc_temp.pdf"
+ )
+ shutil.copy(f, mig_toc_temp_input)
+ new_doc = Document(mig_toc_temp_input.as_posix())
+ if not new_doc:
+ continue
+
+ new_doc.set_toc(toc_data)
+ PDFCreater.save_pdf_with_timeout(
+ new_doc,
+ f.as_posix(),
+ translation_config=translation_config,
+ clean=not translation_config.skip_clean,
+ tag="mig_toc",
+ )
+
+
+# mediabox -> '[0 nul 792]'
+def fix_media_box(doc: Document) -> None:
+ mediabox_data = {}
+ for x in range(1, doc.xref_length()):
+ t = doc.xref_get_key(x, "Type")
+ box_set = {}
+ if t[1] in ["/Pages", "/Page"]:
+ mediabox = doc.xref_get_key(x, "MediaBox")
+ if mediabox[0] == "array":
+ try:
+ _, _, x1, y1 = (
+ mediabox[1].replace("[", "").replace("]", "").split(" ")
+ )
+ doc.xref_set_key(x, "MediaBox", f"[0 0 {x1} {y1}]")
+ box_set["MediaBox"] = mediabox[1]
+ except Exception:
+ logger.warning(
+ "Attempt to fix media box failed; some pages may not have been processed correctly."
+ )
+ for k in ["CropBox", "BleedBox", "TrimBox", "ArtBox"]:
+ box = doc.xref_get_key(x, k)
+ if box[0] != "null":
+ box_set[k] = box[1]
+ doc.xref_set_key(x, k, "null")
+ if box_set:
+ mediabox_data[x] = box_set
+ return mediabox_data
+
+
+def check_cid_char(il: il_version_1.Document):
+ chars = []
+ for page in il.page:
+ chars.extend(page.pdf_character)
+
+ cid_count = 0
+ for char in chars:
+ if re.match(r"^\(cid:\d+\)$", char.char_unicode):
+ cid_count += 1
+
+ return cid_count > len(chars) * 0.8
+
+
+def _do_translate_single(
+ pm: ProgressMonitor,
+ translation_config: TranslationConfig,
+) -> TranslateResult:
+ """Original translation logic for a single document or part"""
+ translation_config.progress_monitor = pm
+ # Initialize detailed logger
+ detailed_log_path = translation_config.get_working_file_path("detailed_translation_log.txt")
+ detailed_logger = init_detailed_logger(str(detailed_log_path))
+
+ with detailed_logger:
+ detailed_logger.start_stage("Initialization")
+ detailed_logger.log_step(
+ "Configuration Setup",
+ f"Input file: {translation_config.input_file}\n"
+ f"Output language: {translation_config.lang_out}\n"
+ f"Debug mode: {translation_config.debug}\n"
+ f"OCR workaround: {translation_config.ocr_workaround}"
+ )
+ if translation_config.shared_context_cross_split_part.auto_enabled_ocr_workaround:
+ translation_config.ocr_workaround = True
+ translation_config.skip_scanned_detection = True
+
+ original_pdf_path = translation_config.input_file
+ if translation_config.debug:
+ doc_input = Document(original_pdf_path)
+ logger.debug("debug mode, save decompressed input pdf")
+ output_path = translation_config.get_working_file_path(
+ "input.decompressed.pdf",
+ )
+ # Fix null xref in PDF file
+ try:
+ _ = fix_null_page_content(doc_input)
+ fix_filter(doc_input)
+ fix_null_xref(doc_input)
+ except Exception:
+ logger.exception("auto fix failed, please check the pdf file")
+ safe_save(doc_input, output_path, expand=True, pretty=True)
+ del doc_input
+
+ # Continue with original processing
+ temp_pdf_path = translation_config.get_working_file_path("input.pdf")
+ doc_pdf2zh = Document(original_pdf_path)
+ safe_save(doc_pdf2zh, temp_pdf_path)
+
+ # Fix null xref in PDF file
+ invalid_pages = []
+ try:
+ invalid_pages = fix_null_page_content(doc_pdf2zh)
+ fix_filter(doc_pdf2zh)
+ fix_null_xref(doc_pdf2zh)
+ except Exception:
+ logger.exception("auto fix failed, please check the pdf file")
+
+ mediabox_data = fix_media_box(doc_pdf2zh)
+
+ # for page in doc_pdf2zh:
+ # page.insert_font(resfont, None)
+
+ resfont = None
+ safe_save(doc_pdf2zh, temp_pdf_path)
+
+ # if not translation_config.skip_scanned_detection and DetectScannedFile(
+ # translation_config
+ # ).fast_check(doc_pdf2zh):
+ # if translation_config.auto_enable_ocr_workaround:
+ # logger.warning(
+ # "Fast scanned check hit, Turning on OCR workaround.",
+ # )
+ # translation_config.shared_context_cross_split_part.auto_enabled_ocr_workaround = True
+ # translation_config.ocr_workaround = True
+ # translation_config.skip_scanned_detection = True
+ # else:
+ # logger.warning(
+ # "Fast scanned check hit, Please check the input PDF file.",
+ # )
+ # raise ScannedPDFError("Scanned PDF detected.")
+
+ # Before: il_creater = ILCreater(translation_config)
+ detailed_logger.start_stage("Parse PDF and Create Intermediate Representation")
+ detailed_logger.log_step(
+ "Starting PDF Parsing",
+ f"PDF path: {temp_pdf_path}\n"
+ f"Total pages: {doc_pdf2zh.page_count}"
+ )
+
+ il_creater = ILCreater(translation_config)
+ il_creater.mupdf = doc_pdf2zh
+ il_creater.detailed_logger = detailed_logger # Pass logger to ILCreater
+
+ il_creater = ILCreater(translation_config)
+ il_creater.mupdf = doc_pdf2zh
+ xml_converter = XMLConverter()
+ print("\n\n\n\n\n debug: start parse il \n\n\n\n\n")
+ logger.debug(f"start parse il from {temp_pdf_path}")
+ with Path(temp_pdf_path).open("rb") as f:
+ start_parse_il(
+ f,
+ doc_zh=doc_pdf2zh,
+ resfont=resfont,
+ il_creater=il_creater,
+ translation_config=translation_config,
+ )
+ logger.debug(f"finish parse il from {temp_pdf_path}")
+ docs = il_creater.create_il()
+ detailed_logger.log_step(
+ "PDF Parsing Complete",
+ f"Total pages processed: {len(docs.page)}\n"
+ f"Total characters extracted: {sum(len(page.pdf_character) for page in docs.page)}"
+ )
+ detailed_logger.end_stage("Parse PDF and Create Intermediate Representation")
+ logger.debug(f"finish create il from {temp_pdf_path}")
+ del il_creater
+ if translation_config.only_include_translated_page and not docs.page:
+ return None
+
+ # if translation_config.debug:
+ # print("debug mode, save il json")
+ # xml_converter.write_json(
+ # docs,
+ # translation_config.get_working_file_path("create_il.debug.json"),
+ # )
+
+
+ if check_cid_char(docs):
+ raise ExtractTextError("The document contains too many CID chars.")
+
+ # Skip all translation processing if only_parse_generate_pdf is enabled
+ if translation_config.only_parse_generate_pdf:
+ logger.debug("only_parse_generate_pdf enabled, skipping translation processing")
+ # Skip directly to PDF generation
+ pdf_creater = PDFCreater(temp_pdf_path, docs, translation_config, mediabox_data)
+ result = pdf_creater.write(translation_config)
+ result.original_pdf_path = translation_config.input_file
+ return result
+
+ # Rest of the original translation logic...
+ # [Previous implementation of do_translate continues here]
+
+ # 检测是否为扫描文件
+ # Replace existing detect scanned file section with:
+ if translation_config.skip_scanned_detection:
+ detailed_logger.start_stage("DetectScannedFile")
+ detailed_logger.log_step("Skipped", "Scanned file detection is disabled")
+ detailed_logger.end_stage("DetectScannedFile")
+ logger.debug("skipping scanned file detection")
+ else:
+ detailed_logger.start_stage("DetectScannedFile")
+ detailed_logger.log_step("Starting scanned file detection")
+ logger.debug("start detect scanned file")
+
+ detect_scanned = DetectScannedFile(translation_config)
+ detect_scanned.detailed_logger = detailed_logger
+ detect_scanned.process(docs, temp_pdf_path, mediabox_data)
+
+ detailed_logger.log_step("Scanned file detection complete")
+ detailed_logger.end_stage("DetectScannedFile")
+ logger.debug("finish detect scanned file")
+ # Generate layouts for all pages
+ # Replace layout parsing section:
+ detailed_logger.start_stage("Parse Page Layout")
+ detailed_logger.log_step("Starting layout generation")
+ logger.debug("start generating layouts")
+
+ layout_parser = LayoutParser(translation_config)
+ layout_parser.detailed_logger = detailed_logger
+ docs = layout_parser.process(docs, doc_pdf2zh)
+
+ detailed_logger.log_step(
+ "Layout generation complete",
+ f"Total layouts detected: {sum(len(page.pdf_layout_element) for page in docs.page if hasattr(page, 'pdf_layout_element'))}"
+ )
+ detailed_logger.end_stage("Parse Page Layout")
+ logger.debug("finish generating layouts")
+ close_process_pool()
+ if translation_config.debug:
+ xml_converter.write_json(
+ docs,
+ translation_config.get_working_file_path("layout_generator.json"),
+ )
+
+ if translation_config.table_model:
+ docs = TableParser(translation_config).process(docs, doc_pdf2zh)
+ logger.debug("finish table parser")
+ if translation_config.debug:
+ xml_converter.write_json(
+ docs,
+ translation_config.get_working_file_path("table_parser.json"),
+ )
+ # Before ParagraphFinder
+ detailed_logger.start_stage("Parse Paragraphs")
+ detailed_logger.log_step("Starting paragraph detection")
+
+ paragraph_finder = ParagraphFinder(translation_config)
+ paragraph_finder.detailed_logger = detailed_logger
+ paragraph_finder.process(docs)
+
+ total_paragraphs = sum(len(page.pdf_paragraph) for page in docs.page)
+ detailed_logger.log_step(
+ "Paragraph detection complete",
+ f"Total paragraphs found: {total_paragraphs}"
+ )
+
+ # Log sample paragraphs
+ for i, page in enumerate(docs.page[:3]): # First 3 pages
+ for j, para in enumerate(page.pdf_paragraph[:5]): # First 5 paragraphs per page
+ detailed_logger.log_paragraph({
+ 'text': para.unicode if hasattr(para, 'unicode') else '',
+ 'layout_label': para.layout_label if hasattr(para, 'layout_label') else 'N/A',
+ 'box': str(para.box) if hasattr(para, 'box') else 'N/A',
+ 'char_count': len(para.unicode) if hasattr(para, 'unicode') else 0
+ })
+
+ detailed_logger.end_stage("Parse Paragraphs")
+ if translation_config.debug:
+ xml_converter.write_json(
+ docs,
+ translation_config.get_working_file_path("paragraph_finder.json"),
+ )
+ # Before StylesAndFormulas
+ detailed_logger.start_stage("Parse Formulas and Styles")
+ detailed_logger.log_step("Starting formula and style detection")
+
+ styles_formulas = StylesAndFormulas(translation_config)
+ styles_formulas.detailed_logger = detailed_logger
+ styles_formulas.process(docs)
+
+ detailed_logger.log_step("Formula and style detection complete")
+ detailed_logger.end_stage("Parse Formulas and Styles")
+ if translation_config.debug:
+ xml_converter.write_json(
+ docs,
+ translation_config.get_working_file_path("styles_and_formulas.json"),
+ )
+
+ translate_engine = translation_config.translator
+ term_extraction_engine = translation_config.get_term_extraction_translator()
+
+ support_llm_translate = translator_supports_llm(translate_engine)
+ support_llm_term_extraction = translator_supports_llm(term_extraction_engine)
+
+ # Replace term extraction section:
+ if support_llm_term_extraction and translation_config.auto_extract_glossary:
+ detailed_logger.start_stage("Automatic Term Extraction")
+ detailed_logger.log_step("Starting automatic term extraction")
+
+ term_extractor = AutomaticTermExtractor(term_extraction_engine, translation_config)
+ term_extractor.detailed_logger = detailed_logger
+ term_extractor.procress(docs)
+
+ extracted_terms = translation_config.shared_context_cross_split_part.get_glossaries_for_translation(True)
+ detailed_logger.log_step(
+ "Term extraction complete",
+ f"Extracted terms: {len(extracted_terms)}"
+ )
+ detailed_logger.end_stage("Automatic Term Extraction")
+
+ # Replace translation section:
+ if not translation_config.skip_translation:
+ detailed_logger.start_stage("Translate Paragraphs")
+ detailed_logger.log_step(
+ "Starting translation",
+ f"Translation engine: {'LLM' if support_llm_translate else 'Standard'}"
+ )
+
+ if support_llm_translate:
+ il_translator = ILTranslatorLLMOnly(translate_engine, translation_config)
+ il_translator.detailed_logger = detailed_logger
+ logger.info("USING LLM ILTranslator")
+ else:
+ il_translator = ILTranslator(translate_engine, translation_config)
+ il_translator.detailed_logger = detailed_logger
+ logger.info("USING STANDARD ILTranslator")
+
+ il_translator.translate(docs)
+
+ detailed_logger.log_step("Translation complete")
+ detailed_logger.end_stage("Translate Paragraphs")
+
+ del il_translator
+ logger.debug(f"finish ILTranslator from {temp_pdf_path}")
+ else:
+ detailed_logger.start_stage("Translate Paragraphs")
+ detailed_logger.log_step("Translation skipped")
+ detailed_logger.end_stage("Translate Paragraphs")
+ logger.info("skip ILTranslator")
+
+ if translation_config.debug:
+ xml_converter.write_json(
+ docs,
+ translation_config.get_working_file_path("il_translated.json"),
+ )
+
+ if translation_config.debug:
+ AddDebugInformation(translation_config).process(docs)
+ xml_converter.write_json(
+ docs,
+ translation_config.get_working_file_path("add_debug_information.json"),
+ )
+ mono_watermark_first_page_doc_bytes = None
+ dual_watermark_first_page_doc_bytes = None
+ try:
+ if translation_config.watermark_output_mode == WatermarkOutputMode.Both:
+ mono_watermark_first_page_doc_bytes, dual_watermark_first_page_doc_bytes = (
+ generate_first_page_with_watermark(
+ doc_pdf2zh, translation_config, docs, mediabox_data
+ )
+ )
+ except Exception:
+ logger.warning(
+ "Failed to generate watermark for first page, using no watermark"
+ )
+ translation_config.watermark_output_mode = WatermarkOutputMode.NoWatermark
+ mono_watermark_first_page_doc_bytes = None
+ dual_watermark_first_page_doc_bytes = None
+
+ # Before Typesetting
+ detailed_logger.start_stage("Typesetting")
+ detailed_logger.log_step("Starting typesetting")
+
+ typesetter = Typesetting(translation_config)
+ typesetter.detailed_logger = detailed_logger
+ typesetter.typesetting_document(docs)
+
+ detailed_logger.log_step("Typesetting complete")
+ detailed_logger.end_stage("Typesetting")
+ logger.debug(f"finish typsetting from {temp_pdf_path}")
+ if translation_config.debug:
+ xml_converter.write_json(
+ docs,
+ translation_config.get_working_file_path("typsetting.json"),
+ )
+
+ # Before PDF creation
+ detailed_logger.start_stage("Generate Drawing Instructions and Save PDF")
+ detailed_logger.log_step("Creating PDF")
+
+ pdf_creater = PDFCreater(temp_pdf_path, docs, translation_config, mediabox_data)
+ pdf_creater.detailed_logger = detailed_logger
+ result = pdf_creater.write(translation_config)
+
+ detailed_logger.log_step(
+ "PDF creation complete",
+ f"Output path: {result.mono_pdf_path}"
+ )
+ detailed_logger.end_stage("Generate Drawing Instructions and Save PDF")
+ result = pdf_creater.write(translation_config)
+ try:
+ if mono_watermark_first_page_doc_bytes:
+ mono_watermark_pdf = merge_watermark_doc(
+ result.mono_pdf_path,
+ mono_watermark_first_page_doc_bytes,
+ translation_config,
+ )
+ result.mono_pdf_path = mono_watermark_pdf
+ except Exception:
+ result.mono_pdf_path = result.no_watermark_mono_pdf_path
+ try:
+ if dual_watermark_first_page_doc_bytes:
+ dual_watermark_pdf = merge_watermark_doc(
+ result.dual_pdf_path,
+ dual_watermark_first_page_doc_bytes,
+ translation_config,
+ )
+ result.dual_pdf_path = dual_watermark_pdf
+ except Exception:
+ result.dual_pdf_path = result.no_watermark_dual_pdf_path
+
+ result.original_pdf_path = translation_config.input_file
+
+ return result
+
+
+def generate_first_page_with_watermark(
+ mupdf: Document,
+ translation_config: TranslationConfig,
+ doc_il: il_version_1.Document,
+ mediabox_data: dict[int, Any] | None = None,
+) -> (io.BytesIO, io.BytesIO):
+ first_page_doc = Document()
+ first_page_doc.insert_pdf(mupdf, from_page=0, to_page=0)
+
+ il_only_first_page_doc = il_version_1.Document()
+ il_only_first_page_doc.total_pages = 1
+ il_only_first_page_doc.page = [copy.deepcopy(doc_il.page[0])]
+
+ watermarked_config = copy.copy(translation_config)
+ watermarked_config.watermark_output_mode = WatermarkOutputMode.Watermarked
+ try:
+ watermarked_config.progress_monitor.disable = True
+ watermarked_temp_pdf_path = watermarked_config.get_working_file_path(
+ "watermarked_temp_input.pdf"
+ )
+ safe_save(first_page_doc, watermarked_temp_pdf_path)
+
+ Typesetting(watermarked_config).typsetting_document(il_only_first_page_doc)
+ pdf_creater = PDFCreater(
+ watermarked_temp_pdf_path.as_posix(),
+ il_only_first_page_doc,
+ watermarked_config,
+ mediabox_data,
+ )
+ result = pdf_creater.write(watermarked_config)
+ mono_pdf_bytes = None
+ dual_pdf_bytes = None
+ if result.mono_pdf_path:
+ mono_pdf_bytes = io.BytesIO()
+ with Path(result.mono_pdf_path).open("rb") as f:
+ mono_pdf_bytes.write(f.read())
+ result.mono_pdf_path.unlink()
+ mono_pdf_bytes.seek(0)
+
+ if result.dual_pdf_path:
+ dual_pdf_bytes = io.BytesIO()
+ with Path(result.dual_pdf_path).open("rb") as f:
+ dual_pdf_bytes.write(f.read())
+ result.dual_pdf_path.unlink()
+ dual_pdf_bytes.seek(0)
+
+ return mono_pdf_bytes, dual_pdf_bytes
+ finally:
+ watermarked_config.progress_monitor.disable = False
+
+
+def merge_watermark_doc(
+ no_watermark_pdf_path: pathlib.PosixPath,
+ watermark_first_page_pdf_bytes: io.BytesIO,
+ translation_config: TranslationConfig,
+) -> pathlib.PosixPath:
+ if not no_watermark_pdf_path.exists():
+ raise FileNotFoundError(
+ f"no_watermark_pdf_path not found: {no_watermark_pdf_path}"
+ )
+ if not watermark_first_page_pdf_bytes:
+ raise FileNotFoundError(
+ f"watermark_first_page_pdf_bytes not found: {watermark_first_page_pdf_bytes}"
+ )
+
+ no_watermark_pdf = Document(no_watermark_pdf_path.as_posix())
+ no_watermark_pdf.delete_page(0)
+
+ watermark_first_page_pdf = Document("pdf", watermark_first_page_pdf_bytes)
+ no_watermark_pdf.insert_pdf(
+ watermark_first_page_pdf, from_page=0, to_page=0, start_at=0
+ )
+
+ new_save_path = no_watermark_pdf_path.with_name(
+ no_watermark_pdf_path.name.replace(".no_watermark", "")
+ )
+
+ PDFCreater.save_pdf_with_timeout(
+ no_watermark_pdf,
+ new_save_path.as_posix(),
+ translation_config=translation_config,
+ clean=not translation_config.skip_clean,
+ )
+ return new_save_path
+
+
+def download_font_assets():
+ warmup()
+
+
+def create_cache_folder():
+ try:
+ logger.debug(f"create cache folder at {CACHE_FOLDER}")
+ Path(CACHE_FOLDER).mkdir(parents=True, exist_ok=True)
+ except OSError:
+ logger.critical(
+ f"Failed to create cache folder at {CACHE_FOLDER}",
+ exc_info=True,
+ )
+ exit(1)
+
+
+def init():
+ create_cache_folder()
diff --git a/babeldoc/format/pdf/pdfinterp.py b/babeldoc/format/pdf/pdfinterp.py
new file mode 100644
index 0000000000000000000000000000000000000000..c5319899cb30bd6cea7ebfcaa6697c8300811827
--- /dev/null
+++ b/babeldoc/format/pdf/pdfinterp.py
@@ -0,0 +1,546 @@
+import logging
+from collections.abc import Sequence
+from typing import Any
+from typing import cast
+
+import numpy as np
+
+from babeldoc.format.pdf.babelpdf.utils import guarded_bbox
+from babeldoc.format.pdf.document_il.frontend.il_creater import ILCreater
+from babeldoc.pdfminer import settings
+from babeldoc.pdfminer.pdfcolor import PREDEFINED_COLORSPACE
+from babeldoc.pdfminer.pdfcolor import PDFColorSpace
+from babeldoc.pdfminer.pdfdevice import PDFDevice
+from babeldoc.pdfminer.pdfdevice import PDFTextSeq
+from babeldoc.pdfminer.pdffont import PDFFont
+from babeldoc.pdfminer.pdfinterp import LITERAL_FORM
+from babeldoc.pdfminer.pdfinterp import LITERAL_IMAGE
+from babeldoc.pdfminer.pdfinterp import Color
+from babeldoc.pdfminer.pdfinterp import PDFContentParser
+from babeldoc.pdfminer.pdfinterp import PDFInterpreterError
+from babeldoc.pdfminer.pdfinterp import PDFPageInterpreter
+from babeldoc.pdfminer.pdfinterp import PDFResourceManager
+from babeldoc.pdfminer.pdfinterp import PDFStackT
+from babeldoc.pdfminer.pdfpage import PDFPage
+from babeldoc.pdfminer.pdftypes import LITERALS_ASCII85_DECODE
+from babeldoc.pdfminer.pdftypes import PDFObjRef
+from babeldoc.pdfminer.pdftypes import PDFStream
+from babeldoc.pdfminer.pdftypes import dict_value
+from babeldoc.pdfminer.pdftypes import list_value
+from babeldoc.pdfminer.pdftypes import resolve1
+from babeldoc.pdfminer.pdftypes import stream_value
+from babeldoc.pdfminer.psexceptions import PSEOF
+from babeldoc.pdfminer.psexceptions import PSTypeError
+from babeldoc.pdfminer.psparser import PSKeyword
+from babeldoc.pdfminer.psparser import PSLiteral
+from babeldoc.pdfminer.psparser import keyword_name
+from babeldoc.pdfminer.psparser import literal_name
+from babeldoc.pdfminer.utils import MATRIX_IDENTITY
+from babeldoc.pdfminer.utils import Matrix
+from babeldoc.pdfminer.utils import Rect
+from babeldoc.pdfminer.utils import apply_matrix_pt
+from babeldoc.pdfminer.utils import choplist
+from babeldoc.pdfminer.utils import mult_matrix
+
+log = logging.getLogger(__name__)
+
+
+def safe_float(o: Any) -> float | None:
+ try:
+ return float(o)
+ except (TypeError, ValueError):
+ return None
+
+
+class PDFContentParserEx(PDFContentParser):
+ def __init__(self, streams: Sequence[object]) -> None:
+ super().__init__(streams)
+
+ def do_keyword(self, pos: int, token: PSKeyword) -> None:
+ if token is self.KEYWORD_BI:
+ # inline image within a content stream
+ self.start_type(pos, "inline")
+ elif token is self.KEYWORD_ID:
+ try:
+ (_, objs) = self.end_type("inline")
+ if len(objs) % 2 != 0:
+ error_msg = f"Invalid dictionary construct: {objs!r}"
+ raise PSTypeError(error_msg)
+ d = {literal_name(k): resolve1(v) for (k, v) in choplist(2, objs)}
+ eos = b"EI"
+ filter_ = d.get("F", None)
+ if filter_:
+ if isinstance(filter_, PSLiteral):
+ filter_ = [filter_]
+ if filter_[0] in LITERALS_ASCII85_DECODE:
+ eos = b"~>"
+ (pos, data) = self.get_inline_data(pos + len(b"ID "), target=eos)
+ if eos != b"EI": # it may be necessary for decoding
+ data += eos
+ obj = PDFStream(d, data)
+ self.push((pos, obj))
+ if eos == b"EI": # otherwise it is still in the stream
+ self.push((pos, self.KEYWORD_EI))
+ except PSTypeError:
+ if settings.STRICT:
+ raise
+ else:
+ self.push((pos, token))
+
+
+class PDFPageInterpreterEx(PDFPageInterpreter):
+ """Processor for the content of a PDF page
+
+ Reference: PDF Reference, Appendix A, Operator Summary
+ """
+
+ def __init__(
+ self,
+ rsrcmgr: PDFResourceManager,
+ device: PDFDevice,
+ obj_patch,
+ il_creater: ILCreater,
+ ) -> None:
+ self.rsrcmgr = rsrcmgr
+ self.device = device
+ self.obj_patch = obj_patch
+ self.il_creater = il_creater
+
+ def dup(self) -> "PDFPageInterpreterEx":
+ return self.__class__(
+ self.rsrcmgr,
+ self.device,
+ self.obj_patch,
+ self.il_creater,
+ )
+
+ def init_resources(self, resources: dict[object, object]) -> None:
+ # 重载设置 fontid 和 descent
+ """Prepare the fonts and XObjects listed in the Resource attribute."""
+ self.resources = resources
+ self.fontmap: dict[object, PDFFont] = {}
+ self.fontid: dict[PDFFont, object] = {}
+ self.xobjmap = {}
+ self.csmap: dict[str, PDFColorSpace] = PREDEFINED_COLORSPACE.copy()
+ if not resources:
+ return
+
+ def get_colorspace(spec: object) -> PDFColorSpace | None:
+ if isinstance(spec, list):
+ name = literal_name(spec[0])
+ else:
+ name = literal_name(spec)
+ if name == "ICCBased" and isinstance(spec, list) and len(spec) >= 2:
+ val = stream_value(spec[1])
+ if "N" in val:
+ return PDFColorSpace(name, val["N"])
+ elif "Alternate" in val:
+ return PREDEFINED_COLORSPACE[val["Alternate"].name]
+ elif name == "DeviceN" and isinstance(spec, list) and len(spec) >= 2:
+ return PDFColorSpace(name, len(list_value(spec[1])))
+ else:
+ return PREDEFINED_COLORSPACE.get(name)
+
+ for k, v in dict_value(resources).items():
+ # log.debug("Resource: %r: %r", k, v)
+ if k == "Font":
+ for fontid, spec in dict_value(v).items():
+ objid = None
+ if isinstance(spec, PDFObjRef):
+ objid = spec.objid
+ spec = dict_value(spec)
+ font = self.rsrcmgr.get_font(objid, spec)
+ font.xobj_id = objid
+ self.il_creater.on_page_resource_font(font, objid, fontid)
+ self.fontmap[fontid] = font
+ self.fontmap[fontid].descent = 0 # hack fix descent
+ self.fontid[self.fontmap[fontid]] = fontid
+ elif k == "ColorSpace":
+ for csid, spec in dict_value(v).items():
+ colorspace = get_colorspace(resolve1(spec))
+ if colorspace is not None:
+ self.csmap[csid] = colorspace
+ elif k == "ProcSet":
+ self.rsrcmgr.get_procset(list_value(v))
+ elif k == "XObject":
+ for xobjid, xobjstrm in dict_value(v).items():
+ self.xobjmap[xobjid] = xobjstrm
+ pass
+
+ def do_CS(self, name: PDFStackT) -> None:
+ """Set color space for stroking operations
+
+ Introduced in PDF 1.1
+ """
+ try:
+ self.il_creater.on_stroking_color_space(literal_name(name))
+ self.scs = self.csmap[literal_name(name)]
+ except KeyError:
+ if settings.STRICT:
+ raise PDFInterpreterError(f"Undefined ColorSpace: {name!r}") from None
+ return
+
+ def do_cs(self, name: PDFStackT) -> None:
+ """Set color space for nonstroking operations"""
+ try:
+ self.il_creater.on_non_stroking_color_space(literal_name(name))
+ self.ncs = self.csmap[literal_name(name)]
+ except KeyError:
+ if settings.STRICT:
+ raise PDFInterpreterError(f"Undefined ColorSpace: {name!r}") from None
+ return
+
+ ############################################################
+ # 重载返回调用参数（SCN）
+ def do_SCN(self) -> None:
+ """Set color for stroking operations."""
+ if self.scs:
+ n = self.scs.ncomponents
+ else:
+ if settings.STRICT:
+ raise PDFInterpreterError("No colorspace specified!")
+ n = 1
+ n = len(self.argstack)
+ args = self.pop(n)
+ self.il_creater.on_passthrough_per_char("SCN", args)
+ self.graphicstate.scolor = cast(Color, args)
+ return args
+
+ def do_scn(self) -> None:
+ """Set color for nonstroking operations"""
+ if self.ncs:
+ n = self.ncs.ncomponents
+ else:
+ if settings.STRICT:
+ raise PDFInterpreterError("No colorspace specified!")
+ n = 1
+ n = len(self.argstack)
+ args = self.pop(n)
+ self.il_creater.on_passthrough_per_char("scn", args)
+ self.graphicstate.ncolor = cast(Color, args)
+ return args
+
+ def do_SC(self) -> None:
+ """Set color for stroking operations"""
+ args = self.do_SCN()
+ self.il_creater.remove_latest_passthrough_per_char_instruction()
+ self.il_creater.on_passthrough_per_char("SC", args)
+ return args
+
+ def do_sc(self) -> None:
+ """Set color for nonstroking operations"""
+ args = self.do_scn()
+ self.il_creater.remove_latest_passthrough_per_char_instruction()
+ self.il_creater.on_passthrough_per_char("sc", args)
+ return args
+
+ # Ensure bbox has four numbers, otherwise determine it as an illegal image
+ # For example, some Form's bbox is '[ null -.00487 1.00412 .99393 ]'
+ def do_Do(self, xobjid_arg: PDFStackT) -> None:
+ # 重载设置 xobj 的 obj_patch
+ """Invoke named XObject"""
+ xobjid = literal_name(xobjid_arg)
+ try:
+ xobj = stream_value(self.xobjmap[xobjid])
+ except KeyError:
+ if settings.STRICT:
+ raise PDFInterpreterError(f"Undefined xobject id: {xobjid!r}") from None
+ return
+ # log.debug("Processing xobj: %r", xobj)
+ subtype = xobj.get("Subtype")
+ if subtype is LITERAL_FORM and "BBox" in xobj:
+ interpreter = self.dup()
+
+ # In extremely rare cases, a none might be mixed in the bbox, for example
+ # /BBox [ 0 3.052 null 274.9 157.3 ]
+ bbox = list(
+ filter(lambda x: x is not None, cast(Rect, list_value(xobj["BBox"])))
+ )
+ if len(bbox) < 4:
+ return
+
+ matrix = cast(Matrix, list_value(xobj.get("Matrix", MATRIX_IDENTITY)))
+ # According to PDF reference 1.7 section 4.9.1, XObjects in
+ # earlier PDFs (prior to v1.2) use the page's Resources entry
+ # instead of having their own Resources entry.
+ xobjres = xobj.get("Resources")
+ if xobjres:
+ resources = dict_value(xobjres)
+ else:
+ resources = self.resources.copy()
+
+ self.il_creater.on_xobj_form(
+ self.ctm,
+ self.il_creater.xobj_id,
+ xobj.objid,
+ "form",
+ xobjid,
+ bbox,
+ matrix,
+ )
+
+ self.device.begin_figure(xobjid, bbox, matrix)
+ ctm = mult_matrix(matrix, self.ctm)
+ (x, y, x2, y2) = guarded_bbox(bbox)
+ (x, y) = apply_matrix_pt(ctm, (x, y))
+ (x2, y2) = apply_matrix_pt(ctm, (x2, y2))
+ x_id = self.il_creater.on_xobj_begin((x, y, x2, y2), xobj.objid)
+ try:
+ ctm_inv = np.linalg.inv(np.array(ctm[:4]).reshape(2, 2))
+ except Exception:
+ self.il_creater.on_xobj_end(x_id, " ")
+ return
+ np_version = np.__version__
+ if np_version.split(".")[0] >= "2":
+ pos_inv = -np.asmatrix(ctm[4:]) * ctm_inv
+ else:
+ pos_inv = -np.mat(ctm[4:]) * ctm_inv
+ a, b, c, d = ctm_inv.reshape(4).tolist()
+ e, f = pos_inv.tolist()[0]
+ ops_base = interpreter.render_contents(
+ resources,
+ [xobj],
+ ctm=ctm,
+ )
+ self.ncs = interpreter.ncs
+ self.scs = interpreter.scs
+ self.il_creater.on_xobj_end(
+ x_id,
+ # f"q {ops_base} Q {a} {b} {c} {d} {e} {f} cm ",
+ f"{a:.6f} {b:.6f} {c:.6f} {d:.6f} {e:.6f} {f:.6f} cm ",
+ )
+ try: # 有的时候 form 字体加不上这里会烂掉
+ self.device.fontid = interpreter.fontid
+ self.device.fontmap = interpreter.fontmap
+ ops_new = self.device.end_figure(xobjid)
+ ctm_inv = np.linalg.inv(np.array(ctm[:4]).reshape(2, 2))
+ np_version = np.__version__
+ if np_version.split(".")[0] >= "2":
+ pos_inv = -np.asmatrix(ctm[4:]) * ctm_inv
+ else:
+ pos_inv = -np.mat(ctm[4:]) * ctm_inv
+ a, b, c, d = ctm_inv.reshape(4).tolist()
+ e, f = pos_inv.tolist()[0]
+ self.obj_patch[self.xobjmap[xobjid].objid] = (
+ f"q {ops_base}Q {a:.6f} {b:.6f} {c:.6f} {d:.6f} {e:.6f} {f:.6f} cm {ops_new}"
+ )
+ except Exception:
+ pass
+ elif subtype is LITERAL_IMAGE and "Width" in xobj and "Height" in xobj:
+ self.il_creater.on_xobj_form(
+ self.ctm,
+ self.il_creater.xobj_id,
+ xobj.objid,
+ "image",
+ xobjid,
+ (0, 0, 1, 1),
+ MATRIX_IDENTITY,
+ )
+ self.device.begin_figure(xobjid, (0, 0, 1, 1), MATRIX_IDENTITY)
+ self.device.render_image(xobjid, xobj)
+ self.device.end_figure(xobjid)
+ else:
+ # unsupported xobject type.
+ pass
+
+ def do_W(self) -> None:
+ """Set clipping path using nonzero winding number rule"""
+ self.handle_w(False)
+
+ def do_W_a(self) -> None:
+ """Set clipping path using even-odd rule"""
+ self.handle_w(True)
+
+ def handle_w(self, evenodd: bool):
+ path = self.curpath
+ self.il_creater.on_pdf_clip_path(path, evenodd, self.ctm)
+
+ def process_page(self, page: PDFPage) -> None:
+ # 重载设置 page 的 obj_patch
+ # log.debug("Processing page: %r", page)
+ # print(page.mediabox,page.cropbox)
+ # (x0, y0, x1, y1) = page.mediabox
+ (x0, y0, x1, y1) = page.cropbox
+ if page.rotate == 90:
+ ctm = (0, -1, 1, 0, -y0, x1)
+ elif page.rotate == 180:
+ ctm = (-1, 0, 0, -1, x1, y1)
+ elif page.rotate == 270:
+ ctm = (0, 1, -1, 0, y1, -x0)
+ else:
+ ctm = (1, 0, 0, 1, -x0, -y0)
+ # ctm_for_ops = copy.copy(ctm)
+ ctm_for_ops = (1, 0, 0, 1, -x0, -y0)
+ ctm = (1, 0, 0, 1, -x0, -y0)
+ if page.rotate == 90 or page.rotate == 270:
+ (x0, y0, x1, y1) = (y0, x1, y1, x0)
+ self.il_creater.on_page_start()
+ self.il_creater.on_page_crop_box(x0, y0, x1, y1)
+ self.device.begin_page(page, ctm)
+ ops_base = self.render_contents(page.resources, page.contents, ctm=ctm)
+ self.device.fontid = self.fontid
+ self.device.fontmap = self.fontmap
+ _ops_new = self.device.end_page(page)
+ # 上面渲染的时候会根据 cropbox 减掉页面偏移得到真实坐标，这里输出的时候需要用 cm 把页面偏移加回来
+ # self.obj_patch[page.page_xref] = (
+ # # f"q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}" # ops_base 里可能有图，需要让 ops_new 里的文字覆盖在上面，使用 q/Q 重置位置矩阵
+ # ""
+ # )
+ # for obj in page.contents:
+ # self.obj_patch[obj.objid] = ""
+ return f"q {ops_base} Q {' '.join(f'{x:f}' for x in ctm_for_ops)} cm"
+ # return f"q {ops_base} Q 1 0 0 1 {x0} {y0} cm"
+
+ def render_contents(
+ self,
+ resources: dict[object, object],
+ streams: Sequence[object],
+ ctm: Matrix = MATRIX_IDENTITY,
+ ) -> None:
+ # 重载返回指令流
+ """Render the content streams.
+
+ This method may be called recursively.
+ """
+ # log.debug(
+ # "render_contents: resources=%r, streams=%r, ctm=%r",
+ # resources,
+ # streams,
+ # ctm,
+ # )
+ self.init_resources(resources)
+ self.init_state(ctm)
+ return self.execute(list_value(streams))
+
+ def do_q(self) -> None:
+ """Save graphics state"""
+ self.gstack.append(self.get_current_state())
+ self.il_creater.push_passthrough_per_char_instruction()
+ return
+
+ def do_Q(self) -> None:
+ """Restore graphics state"""
+ if self.gstack:
+ self.set_current_state(self.gstack.pop())
+ self.il_creater.pop_passthrough_per_char_instruction()
+ return
+
+ def do_TJ(self, seq: PDFStackT) -> None:
+ """Show text, allowing individual glyph positioning"""
+ if self.textstate.font is None:
+ if settings.STRICT:
+ raise PDFInterpreterError("No font specified!")
+ return
+ if isinstance(seq, PSLiteral):
+ return
+ assert self.ncs is not None
+ gs = self.graphicstate.copy()
+ gs.passthrough_instruction = (
+ self.il_creater.passthrough_per_char_instruction.copy()
+ )
+ if isinstance(seq, int) or isinstance(seq, float):
+ seq = [seq]
+ self.device.render_string(self.textstate, cast(PDFTextSeq, seq), self.ncs, gs)
+ return
+
+ def do_d(self, dash: PDFStackT, phase: PDFStackT) -> None:
+ """Set line dash pattern"""
+ self.graphicstate.dash = (dash, phase)
+ self.il_creater.on_line_dash(dash, phase)
+
+ def do_BI(self) -> None:
+ """Begin inline image object"""
+ self.il_creater.on_inline_image_begin()
+
+ def do_ID(self) -> None:
+ """Begin inline image data"""
+ pass # Handled by PDFContentParserEx
+
+ def do_EI(self, obj: PDFStackT) -> None:
+ """End inline image object"""
+ if isinstance(obj, PDFStream):
+ self.il_creater.on_inline_image_end(obj, self.ctm)
+
+ # Run PostScript commands
+ # The Do_xxx method is the method for executing corresponding postscript instructions
+ def execute(self, streams: Sequence[object]) -> None:
+ ops = ""
+ for stream in streams:
+ self.il_creater.on_new_stream()
+ # 重载返回指令流
+ try:
+ parser = PDFContentParserEx([stream])
+ except PSEOF:
+ # empty page
+ return
+ while True:
+ try:
+ (_, obj) = parser.nextobject()
+ except PSEOF:
+ break
+ if isinstance(obj, PSKeyword):
+ name = keyword_name(obj)
+ act_name = (
+ name.replace("*", "_a").replace('"', "_w").replace("'", "_q")
+ )
+ method = f"do_{act_name}"
+ if hasattr(self, method):
+ func = getattr(self, method)
+ nargs = func.__code__.co_argcount - 1
+ if nargs:
+ args = self.pop(nargs)
+ # log.debug("exec: %s %r", name, args)
+ if len(args) == nargs:
+ func(*args)
+ if self.il_creater.is_passthrough_per_char_operation(
+ name,
+ ):
+ self.il_creater.on_passthrough_per_char(name, args)
+ if self.il_creater.is_graphic_operation(name):
+ continue
+ elif name == "d":
+ arg0 = f"[{' '.join(f'{arg}' for arg in args[0])}]"
+ arg1 = args[1]
+ ops += f"{arg0} {arg1} {name} "
+ elif not (
+ name[0] == "T"
+ or name
+ in ['"', "'", "EI", "MP", "DP", "BMC", "BDC"]
+ ): # 过滤 T 系列文字指令，因为 EI 的参数是 obj 所以也需要过滤（只在少数文档中画横线时使用），过滤 marked 系列指令
+ p = " ".join(
+ [
+ (
+ f"{x:f}"
+ if isinstance(x, float)
+ else str(x).replace("'", "")
+ )
+ for x in args
+ ],
+ )
+ ops += f"{p} {name} "
+ else:
+ # log.debug("exec: %s", name)
+ targs = func()
+ if targs is None:
+ targs = []
+ if self.il_creater.is_graphic_operation(name):
+ continue
+ elif not (name[0] == "T" or name in ["BI", "ID", "EMC"]):
+ p = " ".join(
+ [
+ (
+ f"{x:f}"
+ if isinstance(x, float)
+ else str(x).replace("'", "")
+ )
+ for x in targs
+ ],
+ )
+ ops += f"{p} {name} "
+ elif settings.STRICT:
+ error_msg = f"Unknown operator: {name!r}"
+ raise PDFInterpreterError(error_msg)
+ else:
+ self.push(obj)
+ # print('REV DATA',ops)
+ return ops
diff --git a/babeldoc/format/pdf/result_merger.py b/babeldoc/format/pdf/result_merger.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8069c9d7aeca83282d7c40ea965491f930c54c2
--- /dev/null
+++ b/babeldoc/format/pdf/result_merger.py
@@ -0,0 +1,196 @@
+import logging
+from pathlib import Path
+
+from pymupdf import Document
+
+from babeldoc.format.pdf.document_il.backend.pdf_creater import PDFCreater
+from babeldoc.format.pdf.translation_config import TranslateResult
+from babeldoc.format.pdf.translation_config import TranslationConfig
+
+logger = logging.getLogger(__name__)
+
+
+class ResultMerger:
+ """Handles merging of split translation results"""
+
+ def __init__(self, translation_config: TranslationConfig):
+ self.config = translation_config
+
+ def merge_results(
+ self, results: dict[int, TranslateResult | None]
+ ) -> TranslateResult:
+ """Merge multiple translation results into one"""
+ logger.debug(f"merge_results called with type: {type(results)}")
+ logger.debug(f"results content: {results}")
+ if not results:
+ raise ValueError("No results to merge")
+
+ basename = Path(self.config.input_file).stem
+ debug_suffix = ".debug" if self.config.debug else ""
+
+ mono_file_name = f"{basename}{debug_suffix}.{self.config.lang_out}.mono.pdf"
+ dual_file_name = f"{basename}{debug_suffix}.{self.config.lang_out}.dual.pdf"
+
+ debug_suffix += ".no_watermark"
+
+ mono_file_name_no_watermark = (
+ f"{basename}{debug_suffix}.{self.config.lang_out}.mono.pdf"
+ )
+ dual_file_name_no_watermark = (
+ f"{basename}{debug_suffix}.{self.config.lang_out}.dual.pdf"
+ )
+ results = {k: v for k, v in results.items() if v is not None}
+ # Sort results by part index
+ sorted_results = dict(sorted(results.items()))
+ first_result = next(iter(sorted_results.values()))
+
+ # Initialize paths for merged files
+ merged_mono_path = None
+ merged_dual_path = None
+ merged_no_watermark_mono_path = None
+ merged_no_watermark_dual_path = None
+ try:
+ # Merge monolingual PDFs if they exist
+ if (
+ any(r.mono_pdf_path for r in results.values())
+ and not self.config.no_mono
+ ):
+ merged_mono_path = self._merge_pdfs(
+ [
+ r.mono_pdf_path
+ for r in sorted_results.values()
+ if r.mono_pdf_path
+ ],
+ mono_file_name,
+ tag="merged_mono",
+ )
+ except Exception as e:
+ logger.error(f"Error merging monolingual PDFs: {e}")
+ merged_mono_path = None
+
+ try:
+ # Merge dual-language PDFs if they exist
+ if (
+ any(r.dual_pdf_path for r in results.values())
+ and not self.config.no_dual
+ ):
+ merged_dual_path = self._merge_pdfs(
+ [
+ r.dual_pdf_path
+ for r in sorted_results.values()
+ if r.dual_pdf_path
+ ],
+ dual_file_name,
+ tag="merged_dual",
+ )
+ except Exception as e:
+ logger.error(f"Error merging dual-language PDFs: {e}")
+ merged_dual_path = None
+
+ if any(
+ r.dual_pdf_path != r.no_watermark_dual_pdf_path
+ or r.mono_pdf_path != r.no_watermark_mono_pdf_path
+ for r in results.values()
+ ):
+ try:
+ # Merge no-watermark PDFs if they exist
+ if (
+ any(r.no_watermark_mono_pdf_path for r in results.values())
+ and not self.config.no_mono
+ ):
+ merged_no_watermark_mono_path = self._merge_pdfs(
+ [
+ r.no_watermark_mono_pdf_path
+ for r in sorted_results.values()
+ if r.no_watermark_mono_pdf_path
+ ],
+ mono_file_name_no_watermark,
+ tag="merged_no_watermark_mono",
+ )
+ except Exception as e:
+ logger.error(f"Error merging no-watermark PDFs: {e}")
+ merged_no_watermark_mono_path = None
+
+ try:
+ if (
+ any(r.no_watermark_dual_pdf_path for r in results.values())
+ and not self.config.no_dual
+ ):
+ merged_no_watermark_dual_path = self._merge_pdfs(
+ [
+ r.no_watermark_dual_pdf_path
+ for r in sorted_results.values()
+ if r.no_watermark_dual_pdf_path
+ ],
+ "merged_no_watermark_dual.pdf",
+ tag="merged_no_watermark_dual",
+ )
+ except Exception as e:
+ logger.error(f"Error merging no-watermark PDFs: {e}")
+ merged_no_watermark_dual_path = None
+
+ auto_extracted_glossary_path = None
+ if (
+ self.config.save_auto_extracted_glossary
+ and self.config.shared_context_cross_split_part.auto_extracted_glossary
+ ):
+ auto_extracted_glossary_path = self.config.get_output_file_path(
+ f"{basename}{debug_suffix}.{self.config.lang_out}.glossary.csv"
+ )
+ with auto_extracted_glossary_path.open("w", encoding="utf-8") as f:
+ logger.info(
+ f"save auto extracted glossary to {auto_extracted_glossary_path}"
+ )
+ f.write(
+ self.config.shared_context_cross_split_part.auto_extracted_glossary.to_csv()
+ )
+
+ # Create merged result
+ merged_result = TranslateResult(
+ mono_pdf_path=merged_mono_path,
+ dual_pdf_path=merged_dual_path,
+ auto_extracted_glossary_path=auto_extracted_glossary_path,
+ )
+ merged_result.no_watermark_mono_pdf_path = merged_no_watermark_mono_path
+ merged_result.no_watermark_dual_pdf_path = merged_no_watermark_dual_path
+
+ if merged_result.no_watermark_mono_pdf_path is None:
+ merged_result.no_watermark_mono_pdf_path = merged_mono_path
+ elif merged_result.mono_pdf_path is None:
+ merged_result.mono_pdf_path = merged_no_watermark_mono_path
+
+ if merged_result.no_watermark_dual_pdf_path is None:
+ merged_result.no_watermark_dual_pdf_path = merged_dual_path
+ elif merged_result.dual_pdf_path is None:
+ merged_result.dual_pdf_path = merged_no_watermark_dual_path
+
+ # Calculate total time
+ total_time = sum(
+ r.total_seconds for r in results.values() if hasattr(r, "total_seconds")
+ )
+ merged_result.total_seconds = total_time
+
+ return merged_result
+
+ def _merge_pdfs(
+ self, pdf_paths: list[str | Path], output_name: str, tag: str
+ ) -> Path:
+ """Merge multiple PDFs into one"""
+ if not pdf_paths:
+ return None
+
+ output_path = self.config.get_output_file_path(output_name)
+ merged_doc = Document()
+
+ for pdf_path in pdf_paths:
+ doc = Document(str(pdf_path))
+ merged_doc.insert_pdf(doc)
+
+ merged_doc = PDFCreater.subset_fonts_in_subprocess(
+ merged_doc, self.config, tag=tag
+ )
+ PDFCreater.save_pdf_with_timeout(
+ merged_doc, str(output_path), translation_config=self.config
+ )
+
+ return output_path
diff --git a/babeldoc/format/pdf/split_manager.py b/babeldoc/format/pdf/split_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..b72179c714387b2478880f5a3133e7c2a8dcee69
--- /dev/null
+++ b/babeldoc/format/pdf/split_manager.py
@@ -0,0 +1,67 @@
+import logging
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class SplitPoint:
+ """Represents a point where the document should be split"""
+
+ start_page: int
+ end_page: int
+ estimated_complexity: float = 1.0
+ chapter_title: str | None = None
+
+
+class BaseSplitStrategy:
+ """Base class for split strategies"""
+
+ def determine_split_points(self, config) -> list[SplitPoint]:
+ raise NotImplementedError
+
+
+class PageCountStrategy(BaseSplitStrategy):
+ """Split document based on page count"""
+
+ def __init__(self, max_pages_per_part: int = 20):
+ self.max_pages_per_part = max_pages_per_part
+
+ def determine_split_points(self, config) -> list[SplitPoint]:
+ from pymupdf import Document
+
+ doc = Document(str(config.input_file))
+ total_pages = doc.page_count
+
+ split_points = []
+ current_page = 0
+
+ while current_page < total_pages:
+ end_page = min(current_page + self.max_pages_per_part, total_pages)
+ split_points.append(
+ SplitPoint(
+ start_page=current_page,
+ end_page=end_page - 1, # end_page is inclusive
+ )
+ )
+ current_page = end_page
+
+ return split_points
+
+
+class SplitManager:
+ """Manages document splitting process"""
+
+ def __init__(self, config=None):
+ self.strategy = config.split_strategy
+
+ def determine_split_points(self, config) -> list[SplitPoint]:
+ """Determine where to split the document"""
+ return self.strategy.determine_split_points(config)
+
+ def estimate_part_complexity(self, split_point: SplitPoint) -> float:
+ """Estimate the complexity of a document part"""
+ # Simple estimation based on page count for now
+ return (
+ split_point.end_page - split_point.start_page + 1
+ ) * split_point.estimated_complexity
diff --git a/babeldoc/format/pdf/translation_config.py b/babeldoc/format/pdf/translation_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..82e07052b3ecdd23b6f7185f9e3f66c32188ccb7
--- /dev/null
+++ b/babeldoc/format/pdf/translation_config.py
@@ -0,0 +1,530 @@
+import enum
+import logging
+import shutil
+import tempfile
+import threading
+from collections import Counter
+from pathlib import Path
+
+from babeldoc.const import CACHE_FOLDER
+from babeldoc.format.pdf.split_manager import BaseSplitStrategy
+from babeldoc.format.pdf.split_manager import PageCountStrategy
+from babeldoc.glossary import Glossary
+from babeldoc.glossary import GlossaryEntry
+from babeldoc.progress_monitor import ProgressMonitor
+from babeldoc.translator.translator import BaseTranslator
+
+logger = logging.getLogger(__name__)
+
+
+class WatermarkOutputMode(enum.Enum):
+ Watermarked = "watermarked"
+ NoWatermark = "no_watermark"
+ Both = "both"
+
+
+class SharedContextCrossSplitPart:
+ def __init__(self):
+ self.first_paragraph = None
+ self.recent_title_paragraph = None
+ self._lock = threading.Lock()
+ self.user_glossaries: list[Glossary] = []
+ self.auto_extracted_glossary: Glossary | None = None
+ self.raw_extracted_terms: list[tuple[str, str]] = []
+ self.auto_enabled_ocr_workaround = False
+
+ def initialize_glossaries(self, initial_glossaries: list[Glossary] | None):
+ with self._lock:
+ self.user_glossaries = (
+ list(initial_glossaries) if initial_glossaries else []
+ )
+ self.auto_extracted_glossary = None
+ self.raw_extracted_terms = []
+ self.unique_name = self._generate_unique_auto_glossary_name()
+ self.norm_terms = set()
+ for g in self.user_glossaries:
+ for entity in g.normalized_lookup:
+ self.norm_terms.add(entity)
+
+ def add_raw_extracted_term_pair(self, src: str, tgt: str):
+ with self._lock:
+ self.raw_extracted_terms.append((src, tgt))
+
+ def _generate_unique_auto_glossary_name(self) -> str:
+ base_name = "auto_extracted_glossary"
+ current_name = base_name
+ suffix = 0
+ existing_names = {g.name for g in self.user_glossaries}
+ if (
+ self.auto_extracted_glossary
+ and self.auto_extracted_glossary.name == current_name
+ ):
+ pass
+
+ while current_name in existing_names:
+ suffix += 1
+ current_name = f"{base_name}#{suffix}"
+ return current_name
+
+ def contains_term(self, term: str) -> bool:
+ pass
+
+ def finalize_auto_extracted_glossary(self):
+ with self._lock:
+ self.auto_extracted_glossary = None
+
+ if not self.raw_extracted_terms:
+ self.raw_extracted_terms = []
+ return
+
+ term_translations: dict[str, list[str]] = {}
+ for src, tgt in self.raw_extracted_terms:
+ term_translations.setdefault(src, []).append(tgt)
+
+ final_entries: list[GlossaryEntry] = []
+ for src, tgts in term_translations.items():
+ if not tgts:
+ continue
+ most_common_tgt = Counter(tgts).most_common(1)[0][0]
+ final_entries.append(GlossaryEntry(src, most_common_tgt))
+
+ if final_entries:
+ self.auto_extracted_glossary = Glossary(
+ name=self.unique_name, entries=final_entries
+ )
+
+ def get_glossaries(self) -> list[Glossary]:
+ with self._lock:
+ all_glossaries = list(self.user_glossaries)
+ if self.auto_extracted_glossary:
+ all_glossaries.append(self.auto_extracted_glossary)
+ return all_glossaries
+
+ def get_glossaries_for_translation(
+ self, auto_extract_enabled: bool
+ ) -> list[Glossary]:
+ with self._lock:
+ if auto_extract_enabled and self.auto_extracted_glossary:
+ return [self.auto_extracted_glossary]
+ else:
+ all_glossaries = list(self.user_glossaries)
+ if self.auto_extracted_glossary:
+ all_glossaries.append(self.auto_extracted_glossary)
+ return all_glossaries
+
+
+class TranslationConfig:
+ @staticmethod
+ def create_max_pages_per_part_split_strategy(max_pages_per_part: int):
+ return PageCountStrategy(max_pages_per_part)
+
+ def __init__(
+ self,
+ translator: BaseTranslator,
+ input_file: str | Path,
+ lang_in: str,
+ lang_out: str,
+ doc_layout_model, # DocLayoutModel
+ # for backward compatibility
+ font: str | Path | None = None,
+ pages: str | None = None,
+ output_dir: str | Path | None = None,
+ debug: bool = False,
+ working_dir: str | Path | None = None,
+ no_dual: bool = False,
+ no_mono: bool = False,
+ formular_font_pattern: str | None = None,
+ formular_char_pattern: str | None = None,
+ qps: int = 1,
+ split_short_lines: bool = False,
+ short_line_split_factor: float = 0.8,
+ use_rich_pbar: bool = True,
+ progress_monitor: ProgressMonitor | None = None,
+ skip_clean: bool = False,
+ dual_translate_first: bool = False,
+ disable_rich_text_translate: bool = False,
+ enhance_compatibility: bool = False,
+ report_interval: float = 0.1,
+ min_text_length: int = 5,
+ use_side_by_side_dual: bool = True, # Deprecated: 是否使用拼版式双语 PDF（并排显示原文和译文）向下兼容选项，已停用。
+ use_alternating_pages_dual: bool = False,
+ watermark_output_mode: WatermarkOutputMode = WatermarkOutputMode.Watermarked,
+ # Add split-related parameters
+ split_strategy: BaseSplitStrategy | None = None,
+ table_model=None,
+ show_char_box: bool = False,
+ skip_scanned_detection: bool = False,
+ ocr_workaround: bool = False,
+ custom_system_prompt: str | None = None,
+ add_formula_placehold_hint: bool = False,
+ glossaries: list[Glossary] | None = None,
+ pool_max_workers: int | None = None,
+ auto_extract_glossary: bool = True,
+ auto_enable_ocr_workaround: bool = False,
+ primary_font_family: str | None = None,
+ only_include_translated_page: bool | None = False,
+ save_auto_extracted_glossary: bool = True,
+ enable_graphic_element_process: bool = True,
+ merge_alternating_line_numbers: bool = True,
+ skip_translation: bool = False,
+ skip_form_render: bool = False,
+ skip_curve_render: bool = False,
+ only_parse_generate_pdf: bool = False,
+ remove_non_formula_lines: bool = False,
+ non_formula_line_iou_threshold: float = 0.9,
+ figure_table_protection_threshold: float = 0.9,
+ skip_formula_offset_calculation: bool = False,
+ term_extraction_translator: BaseTranslator | None = None,
+ metadata_extra_data: str | None = None,
+ ):
+ self.translator = translator
+ self.term_extraction_translator = term_extraction_translator or translator
+ initial_user_glossaries = list(glossaries) if glossaries else []
+
+ self.input_file = input_file
+ self.lang_in = lang_in
+ self.lang_out = lang_out
+ # just ignore font
+ self.font = None
+
+ self.pages = pages
+ self.page_ranges = self.parse_pages(pages) if pages else None
+ self.debug = debug
+ self.watermark_output_mode = watermark_output_mode
+
+ self.output_dir = output_dir
+ self.working_dir = working_dir
+ self.no_dual = no_dual
+ self.no_mono = no_mono
+
+ self.formular_font_pattern = formular_font_pattern
+ self.formular_char_pattern = formular_char_pattern
+ self.qps = qps
+ # Set pool_max_workers with default value from qps
+ self.pool_max_workers = (
+ pool_max_workers if pool_max_workers is not None else qps
+ )
+ self.split_short_lines = split_short_lines
+
+ self.short_line_split_factor = short_line_split_factor
+ self.use_rich_pbar = use_rich_pbar
+ self.progress_monitor = progress_monitor
+ self.doc_layout_model = doc_layout_model
+
+ self.skip_clean = skip_clean or enhance_compatibility
+ self.skip_scanned_detection = skip_scanned_detection
+
+ self.dual_translate_first = dual_translate_first or enhance_compatibility
+ self.disable_rich_text_translate = (
+ disable_rich_text_translate or enhance_compatibility
+ )
+
+ self.report_interval = report_interval
+ self.min_text_length = min_text_length
+ self.use_alternating_pages_dual = use_alternating_pages_dual
+ self.ocr_workaround = ocr_workaround
+ self.merge_alternating_line_numbers = merge_alternating_line_numbers
+
+ if self.ocr_workaround:
+ self.skip_scanned_detection = True
+ self.disable_rich_text_translate = True
+
+ # for backward compatibility
+ if use_side_by_side_dual is False and use_alternating_pages_dual is False:
+ self.use_alternating_pages_dual = True
+
+ if progress_monitor and progress_monitor.cancel_event is None:
+ progress_monitor.cancel_event = threading.Event()
+
+ if working_dir is None:
+ if debug:
+ working_dir = Path(CACHE_FOLDER) / "working" / Path(input_file).stem
+ self._is_temp_dir = False
+ else:
+ working_dir = tempfile.mkdtemp()
+ self._is_temp_dir = True
+ else:
+ working_dir = Path(working_dir) / Path(input_file).stem
+ self._is_temp_dir = False
+
+ self.working_dir = working_dir
+
+ Path(working_dir).mkdir(parents=True, exist_ok=True)
+
+ if output_dir is None:
+ output_dir = Path.cwd()
+ self.output_dir = output_dir
+
+ Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+ if not doc_layout_model:
+ from babeldoc.docvision.doclayout import DocLayoutModel
+
+ doc_layout_model = DocLayoutModel.load_available()
+ self.doc_layout_model = doc_layout_model
+
+ self.shared_context_cross_split_part = SharedContextCrossSplitPart()
+ self.shared_context_cross_split_part.initialize_glossaries(
+ initial_user_glossaries
+ )
+
+ # Initialize split-related attributes
+ self.split_strategy = split_strategy
+
+ # Create a unique working directory for each part
+ self._part_working_dirs: dict[int, Path] = {}
+ self._part_output_dirs: dict[int, Path] = {}
+
+ self.table_model = table_model
+ self.show_char_box = show_char_box
+ self.custom_system_prompt = custom_system_prompt
+ self.add_formula_placehold_hint = add_formula_placehold_hint
+ self.auto_extract_glossary = auto_extract_glossary
+ self.auto_enable_ocr_workaround = auto_enable_ocr_workaround
+ self.skip_translation = skip_translation
+ self.only_parse_generate_pdf = only_parse_generate_pdf
+
+ if self.skip_translation or self.only_parse_generate_pdf:
+ self.auto_extract_glossary = False
+
+ if auto_enable_ocr_workaround:
+ self.ocr_workaround = False
+ self.skip_scanned_detection = False
+
+ assert primary_font_family in [
+ None,
+ "serif",
+ "sans-serif",
+ "script",
+ ]
+ self.primary_font_family = primary_font_family
+
+ if only_include_translated_page is None:
+ only_include_translated_page = False
+
+ self.only_include_translated_page = only_include_translated_page
+
+ self.save_auto_extracted_glossary = save_auto_extracted_glossary
+
+ # force disable table translate until the new model is ready
+ self.table_model = None
+ self.enable_graphic_element_process = enable_graphic_element_process
+ self.skip_form_render = skip_form_render
+ self.skip_curve_render = skip_curve_render
+ self.remove_non_formula_lines = remove_non_formula_lines
+ self.non_formula_line_iou_threshold = non_formula_line_iou_threshold
+ self.figure_table_protection_threshold = figure_table_protection_threshold
+ self.skip_formula_offset_calculation = skip_formula_offset_calculation
+
+ self.metadata_extra_data = metadata_extra_data
+
+ self.term_extraction_token_usage: dict[str, int] = {
+ "total_tokens": 0,
+ "prompt_tokens": 0,
+ "completion_tokens": 0,
+ "cache_hit_prompt_tokens": 0,
+ }
+
+ if self.ocr_workaround:
+ self.remove_non_formula_lines = False
+
+ def parse_pages(self, pages_str: str | None) -> list[tuple[int, int]] | None:
+ """解析页码字符串，返回页码范围列表
+
+ Args:
+ pages_str: 形如 "1-,2,-3,4" 的页码字符串
+
+ Returns:
+ 包含 (start, end) 元组的列表，其中 -1 表示无限制
+ """
+ if not pages_str:
+ return None
+
+ ranges: list[tuple[int, int]] = []
+ for part in pages_str.split(","):
+ part = part.strip()
+ if "-" in part:
+ start, end = part.split("-")
+ start_as_int = int(start) if start else 1
+ end_as_int = int(end) if end else -1
+ ranges.append((start_as_int, end_as_int))
+ else:
+ page = int(part)
+ ranges.append((page, page))
+ return ranges
+
+ def should_translate_page(self, page_number: int) -> bool:
+ """判断指定页码是否需要翻译
+ Args:
+ page_number: 页码
+ Returns:
+ 是否需要翻译该页
+ """
+ if isinstance(self.page_ranges, list) and len(self.page_ranges) == 0:
+ return False
+ if not self.page_ranges:
+ return True
+
+ for start, end in self.page_ranges:
+ if start <= page_number and (end == -1 or page_number <= end):
+ return True
+ return False
+
+ def get_output_file_path(self, filename: str) -> Path:
+ return Path(self.output_dir) / filename
+
+ def get_working_file_path(self, filename: str) -> Path:
+ return Path(self.working_dir) / filename
+
+ def get_part_working_dir(self, part_index: int) -> Path:
+ """Get working directory for a specific part"""
+ if part_index not in self._part_working_dirs:
+ if self.working_dir:
+ part_dir = Path(self.working_dir) / f"part_{part_index}"
+ else:
+ part_dir = Path(tempfile.mkdtemp()) / f"part_{part_index}"
+ part_dir.mkdir(parents=True, exist_ok=True)
+ self._part_working_dirs[part_index] = part_dir
+ return self._part_working_dirs[part_index]
+
+ def get_part_output_dir(self, part_index: int) -> Path:
+ """Get output directory for a specific part"""
+ if part_index not in self._part_output_dirs:
+ part_dir = Path(self.working_dir) / f"part_{part_index}_output"
+ part_dir.mkdir(parents=True, exist_ok=True)
+ self._part_output_dirs[part_index] = part_dir
+ return self._part_output_dirs[part_index]
+
+ def cleanup_part_output_dir(self, part_index: int):
+ """Clean up output directory for a specific part"""
+ if part_index in self._part_output_dirs:
+ part_dir = self._part_output_dirs[part_index]
+ if part_dir.exists():
+ shutil.rmtree(part_dir)
+ del self._part_output_dirs[part_index]
+
+ def cleanup_part_working_dir(self, part_index: int):
+ """Clean up working directory for a specific part"""
+ if part_index in self._part_working_dirs:
+ part_dir = self._part_working_dirs[part_index]
+ if part_dir.exists():
+ shutil.rmtree(part_dir, ignore_errors=True)
+ del self._part_working_dirs[part_index]
+
+ def cleanup_temp_files(self):
+ """Clean up all temporary files including part working directories"""
+ try:
+ for part_index in list(self._part_working_dirs.keys()):
+ self.cleanup_part_working_dir(part_index)
+ if self._is_temp_dir:
+ logger.info(f"cleanup temp files: {self.working_dir}")
+ shutil.rmtree(self.working_dir, ignore_errors=True)
+ except Exception:
+ logger.exception("Error cleaning up temporary files")
+
+ def raise_if_cancelled(self):
+ if self.progress_monitor is not None:
+ self.progress_monitor.raise_if_cancelled()
+
+ def cancel_translation(self):
+ if self.progress_monitor is not None:
+ self.progress_monitor.cancel()
+
+ def get_term_extraction_translator(self) -> BaseTranslator:
+ """Return the translator to use for automatic term extraction."""
+ return self.term_extraction_translator
+
+ def record_term_extraction_usage(
+ self,
+ total_tokens: int,
+ prompt_tokens: int,
+ completion_tokens: int,
+ cache_hit_prompt_tokens: int,
+ ) -> None:
+ """Accumulate token usage for automatic term extraction."""
+ if total_tokens > 0:
+ self.term_extraction_token_usage["total_tokens"] += total_tokens
+ if prompt_tokens > 0:
+ self.term_extraction_token_usage["prompt_tokens"] += prompt_tokens
+ if completion_tokens > 0:
+ self.term_extraction_token_usage["completion_tokens"] += completion_tokens
+ if cache_hit_prompt_tokens > 0:
+ self.term_extraction_token_usage["cache_hit_prompt_tokens"] += (
+ cache_hit_prompt_tokens
+ )
+
+
+class TranslateResult:
+ original_pdf_path: str
+ total_seconds: float
+ mono_pdf_path: Path | None
+ dual_pdf_path: Path | None
+ no_watermark_mono_pdf_path: Path | None
+ no_watermark_dual_pdf_path: Path | None
+ peak_memory_usage: int | None
+ auto_extracted_glossary_path: Path | None
+
+ def __init__(
+ self,
+ mono_pdf_path: Path | None,
+ dual_pdf_path: Path | None,
+ auto_extracted_glossary_path: Path | None = None,
+ ):
+ self.mono_pdf_path = mono_pdf_path
+ self.dual_pdf_path = dual_pdf_path
+
+ # For compatibility considerations, if only a non-watermarked PDF is generated,
+ # the values of mono_pdf_path and no_watermark_mono_pdf_path are the same.
+ self.no_watermark_mono_pdf_path = mono_pdf_path
+ self.no_watermark_dual_pdf_path = dual_pdf_path
+
+ self.auto_extracted_glossary_path = auto_extracted_glossary_path
+
+ def __str__(self):
+ """Return a human-readable string representation of the translation result."""
+ result = []
+ if hasattr(self, "original_pdf_path") and self.original_pdf_path:
+ result.append(f"\tOriginal PDF: {self.original_pdf_path}")
+
+ if hasattr(self, "total_seconds") and self.total_seconds:
+ result.append(f"\tTotal time: {self.total_seconds:.2f} seconds")
+
+ if self.mono_pdf_path:
+ result.append(f"\tMonolingual PDF: {self.mono_pdf_path}")
+
+ if self.dual_pdf_path:
+ result.append(f"\tDual-language PDF: {self.dual_pdf_path}")
+
+ if (
+ hasattr(self, "no_watermark_mono_pdf_path")
+ and self.no_watermark_mono_pdf_path
+ and self.no_watermark_mono_pdf_path != self.mono_pdf_path
+ ):
+ result.append(
+ f"\tNo-watermark Monolingual PDF: {self.no_watermark_mono_pdf_path}"
+ )
+
+ if (
+ hasattr(self, "no_watermark_dual_pdf_path")
+ and self.no_watermark_dual_pdf_path
+ and self.no_watermark_dual_pdf_path != self.dual_pdf_path
+ ):
+ result.append(
+ f"\tNo-watermark Dual-language PDF: {self.no_watermark_dual_pdf_path}"
+ )
+
+ if (
+ hasattr(self, "auto_extracted_glossary_path")
+ and self.auto_extracted_glossary_path
+ ):
+ result.append(
+ f"\tAuto-extracted glossary: {self.auto_extracted_glossary_path}"
+ )
+
+ if hasattr(self, "peak_memory_usage") and self.peak_memory_usage:
+ result.append(f"\tPeak memory usage: {self.peak_memory_usage} MB")
+
+ if result:
+ result.insert(0, "Translation results:")
+
+ return "\n".join(result) if result else "No translation results available"
diff --git a/babeldoc/glossary.py b/babeldoc/glossary.py
new file mode 100644
index 0000000000000000000000000000000000000000..f8ef88a5a5fdf614b7b22841b299ff92a6621f34
--- /dev/null
+++ b/babeldoc/glossary.py
@@ -0,0 +1,214 @@
+import csv
+import io
+import itertools
+import logging
+import re
+import time
+from pathlib import Path
+
+import chardet
+import hyperscan
+import regex
+
+logger = logging.getLogger(__name__)
+
+
+class GlossaryEntry:
+ def __init__(self, source: str, target: str, target_language: str | None = None):
+ self.source = source
+ self.target = target
+ self.target_language = target_language
+
+ def __repr__(self):
+ return f"GlossaryEntry(source='{self.source}', target='{self.target}', target_language='{self.target_language}')"
+
+
+def batched(iterable, n, *, strict=False):
+ # batched('ABCDEFG', 3) → ABC DEF G
+ if n < 1:
+ raise ValueError("n must be at least one")
+ iterator = iter(iterable)
+ while batch := tuple(itertools.islice(iterator, n)):
+ if strict and len(batch) != n:
+ raise ValueError("batched(): incomplete batch")
+ yield batch
+
+
+TERM_NORM_PATTERN = re.compile(r"\s+", regex.UNICODE)
+
+
+class Glossary:
+ def __init__(self, name: str, entries: list[GlossaryEntry]):
+ self.name = name
+
+ # Deduplicate entries based on normalized source
+ unique_entries = []
+ seen_normalized_sources = set()
+ for entry in entries:
+ normalized_source = self.normalize_source(entry.source)
+ if normalized_source not in seen_normalized_sources:
+ unique_entries.append(entry)
+ seen_normalized_sources.add(normalized_source)
+ self.entries = unique_entries
+
+ self.normalized_lookup: dict[str, tuple[str, str]] = {}
+ self.id_lookup: list[tuple[str, str]] = []
+ self.hs_dbs: list[hyperscan.Database] | None = None
+ self._build_regex_and_lookup()
+
+ @staticmethod
+ def normalize_source(source_term: str) -> str:
+ """Normalizes a source term by lowercasing and standardizing whitespace."""
+ term = source_term.lower()
+ term = TERM_NORM_PATTERN.sub(
+ " ", term
+ ) # Replace multiple whitespace with single space
+ return term.strip()
+
+ def _build_regex_and_lookup(self):
+ logger.debug(
+ f"start build regex for glossary {self.name} with {len(self.entries)} entries"
+ )
+ """
+ Builds a combined regex for all source terms and a lookup dictionary
+ from normalized source terms to (original_source, original_target).
+ Regex patterns are sorted by length in descending order to prioritize longer matches.
+ """
+ self.normalized_lookup = {}
+
+ if not self.entries:
+ self.source_terms_regex = None
+ return
+
+ self.hs_dbs = []
+ hs_pattern = []
+ start = time.time()
+ for idx, entry in enumerate(self.entries):
+ normalized_key = self.normalize_source(entry.source)
+ self.normalized_lookup[normalized_key] = (entry.source, entry.target)
+ self.id_lookup.append((entry.source, entry.target))
+
+ hs_pattern.append((re.escape(entry.source).encode("utf-8"), idx))
+
+ chunk_size = 20000
+ for i, pattern_chunk in enumerate(
+ batched(hs_pattern, chunk_size, strict=False)
+ ):
+ logger.debug(
+ f"building hs_db chunk {i + 1} / {len(self.entries) // chunk_size + 1}"
+ )
+ expressions, ids = zip(*pattern_chunk, strict=False)
+
+ hs_db = hyperscan.Database()
+ hs_db.compile(
+ expressions=expressions,
+ ids=ids,
+ elements=len(pattern_chunk),
+ flags=hyperscan.HS_FLAG_CASELESS | hyperscan.HS_FLAG_SINGLEMATCH,
+ # | hyperscan.HS_FLAG_UTF8
+ # | hyperscan.HS_FLAG_UCP,
+ )
+ self.hs_dbs.append(hs_db)
+
+ end = time.time()
+ logger.debug(
+ f"finished building regex for glossary {self.name} in {end - start:.2f} seconds"
+ )
+ logger.debug(
+ f"build hs database for glossary {self.name} with {len(self.entries)} entries, hs_info: {self.hs_dbs[0].info()}"
+ )
+ if not self.hs_dbs:
+ self.hs_dbs = None
+
+ @classmethod
+ def from_csv(cls, file_path: Path, target_lang_out: str) -> "Glossary":
+ """
+ Loads glossary entries from a CSV file.
+ CSV format: source,target,tgt_lng (tgt_lng is optional)
+ Filters entries based on tgt_lng matching target_lang_out.
+ The glossary name is derived from the CSV filename.
+ """
+ glossary_name = file_path.stem
+ loaded_entries: list[GlossaryEntry] = []
+
+ # Normalize target_lang_out once for comparison
+ normalized_target_lang_out = target_lang_out.lower().replace("-", "_")
+
+ try:
+ with file_path.open("rb") as f:
+ content = f.read()
+ encoding = chardet.detect(content)["encoding"]
+ buffer = io.StringIO(content.decode(encoding))
+ reader = csv.DictReader(buffer, doublequote=True)
+ if not all(col in reader.fieldnames for col in ["source", "target"]):
+ raise ValueError(
+ f"CSV file {file_path} must contain 'source' and 'target' columns."
+ )
+
+ for row in reader:
+ source = row["source"]
+ target = row["target"]
+ tgt_lng = row.get("tgt_lng", None) # Handle optional tgt_lng
+
+ if tgt_lng and tgt_lng.strip():
+ normalized_entry_tgt_lng = (
+ tgt_lng.strip().lower().replace("-", "_")
+ )
+ if normalized_entry_tgt_lng != normalized_target_lang_out:
+ continue # Skip if language doesn't match
+
+ loaded_entries.append(GlossaryEntry(source, target, tgt_lng))
+ except FileNotFoundError:
+ # Or handle as per your project's error strategy, e.g., log and return empty Glossary
+ raise
+ except Exception as e:
+ # Or handle as per your project's error strategy
+ raise ValueError(
+ f"Error reading or parsing CSV file {file_path}: {e}"
+ ) from e
+
+ return cls(name=glossary_name, entries=loaded_entries)
+
+ def to_csv(self) -> str:
+ """Exports the glossary entries to a CSV formatted string."""
+ dict_data = [
+ {
+ "source": x.source,
+ "target": x.target,
+ "tgt_lng": x.target_language if x.target_language else "",
+ }
+ for x in self.entries
+ ]
+ buffer = io.StringIO()
+ dict_writer = csv.DictWriter(
+ buffer, fieldnames=["source", "target", "tgt_lng"], doublequote=True
+ )
+ dict_writer.writeheader()
+ dict_writer.writerows(dict_data)
+ return buffer.getvalue()
+
+ def __repr__(self):
+ return f"Glossary(name='{self.name}', num_entries={len(self.entries)})"
+
+ def get_active_entries_for_text(self, text: str) -> list[tuple[str, str]]:
+ """Returns a list of (original_source, target_text) tuples for terms found in the given text."""
+ if not self.hs_dbs or not text:
+ return []
+
+ text = TERM_NORM_PATTERN.sub(" ", text) # Normalize whitespace in the text
+ if not text:
+ return []
+
+ active_entries = []
+
+ def on_match(
+ idx: int, _from: int, _to: int, _flags: int, _context=None
+ ) -> bool | None:
+ active_entries.append(self.id_lookup[idx])
+ return False
+
+ for hs_db in self.hs_dbs:
+ # Scan the text with the hyperscan database
+ scratch = hyperscan.Scratch(hs_db)
+ hs_db.scan(text.encode("utf-8"), on_match, scratch=scratch)
+ return active_entries
diff --git a/babeldoc/main.py b/babeldoc/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..813c0e177e5ef928cf608dac064c750f924d0f1e
--- /dev/null
+++ b/babeldoc/main.py
@@ -0,0 +1,909 @@
+import asyncio
+import logging
+import multiprocessing as mp
+import queue
+import random
+import sys
+from pathlib import Path
+from typing import Any
+import configargparse
+import tqdm
+from rich.progress import BarColumn
+from rich.progress import MofNCompleteColumn
+from rich.progress import Progress
+from rich.progress import TextColumn
+from rich.progress import TimeElapsedColumn
+from rich.progress import TimeRemainingColumn
+
+import babeldoc.assets.assets
+import babeldoc.format.pdf.high_level
+from babeldoc.const import enable_process_pool
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from babeldoc.format.pdf.translation_config import WatermarkOutputMode
+from babeldoc.glossary import Glossary
+from babeldoc.translator.translator import OpenAITranslator
+from babeldoc.translator.translator import set_translate_rate_limiter
+
+logger = logging.getLogger(__name__)
+__version__ = "0.5.16"
+
+
+def create_parser():
+ parser = configargparse.ArgParser(
+ config_file_parser_class=configargparse.TomlConfigParser(["babeldoc"]),
+ )
+ parser.add_argument(
+ "-c",
+ "--config",
+ is_config_file=True,
+ help="config file path",
+ )
+ parser.add_argument(
+ "--version",
+ action="version",
+ version=f"%(prog)s {__version__}",
+ )
+ parser.add_argument(
+ "--files",
+ action="append",
+ help="One or more paths to PDF files.",
+ )
+ parser.add_argument(
+ "--debug",
+ action="store_true",
+ help="Use debug logging level.",
+ )
+ parser.add_argument(
+ "--warmup",
+ action="store_true",
+ help="Only download and verify required assets then exit.",
+ )
+ parser.add_argument(
+ "--rpc-doclayout",
+ help="RPC service host address for document layout analysis",
+ )
+ parser.add_argument(
+ "--rpc-doclayout2",
+ help="RPC service host address for document layout analysis",
+ )
+ parser.add_argument(
+ "--rpc-doclayout3",
+ help="RPC service host address for document layout analysis",
+ )
+ parser.add_argument(
+ "--rpc-doclayout4",
+ help="RPC service host address for document layout analysis",
+ )
+ parser.add_argument(
+ "--rpc-doclayout5",
+ help="RPC service host address for document layout analysis",
+ )
+ parser.add_argument(
+ "--rpc-doclayout6",
+ help="RPC service host address for document layout analysis",
+ )
+ parser.add_argument(
+ "--rpc-doclayout7",
+ help="RPC service host address for document layout analysis",
+ )
+ parser.add_argument(
+ "--generate-offline-assets",
+ default=None,
+ help="Generate offline assets package in the specified directory",
+ )
+ parser.add_argument(
+ "--restore-offline-assets",
+ default=None,
+ help="Restore offline assets package from the specified file",
+ )
+ parser.add_argument(
+ "--working-dir",
+ default=None,
+ help="Working directory for translation. If not set, use temp directory.",
+ )
+ parser.add_argument(
+ "--metadata-extra-data",
+ default=None,
+ help="Extra data for metadata",
+ )
+ parser.add_argument(
+ "--enable-process-pool",
+ action="store_true",
+ help="DEBUG ONLY",
+ )
+ # translation option argument group
+ translation_group = parser.add_argument_group(
+ "Translation",
+ description="Used during translation",
+ )
+ translation_group.add_argument(
+ "--pages",
+ "-p",
+ help="Pages to translate. If not set, translate all pages. like: 1,2,1-,-3,3-5",
+ )
+ translation_group.add_argument(
+ "--min-text-length",
+ type=int,
+ default=5,
+ help="Minimum text length to translate (default: 5)",
+ )
+ translation_group.add_argument(
+ "--lang-in",
+ "-li",
+ default="en",
+ help="The code of source language.",
+ )
+ translation_group.add_argument(
+ "--lang-out",
+ "-lo",
+ default="en-ar",
+ help="The code of target language.",
+ )
+ translation_group.add_argument(
+ "--output",
+ "-o",
+ help="Output directory for files. if not set, use same as input.",
+ )
+ translation_group.add_argument(
+ "--qps",
+ "-q",
+ type=int,
+ default=4,
+ help="QPS limit of translation service",
+ )
+ translation_group.add_argument(
+ "--ignore-cache",
+ action="store_true",
+ help="Ignore translation cache.",
+ )
+ translation_group.add_argument(
+ "--no-dual",
+ action="store_true",
+ help="Do not output bilingual PDF files",
+ )
+ translation_group.add_argument(
+ "--no-mono",
+ action="store_true",
+ help="Do not output monolingual PDF files",
+ )
+ translation_group.add_argument(
+ "--formular-font-pattern",
+ help="Font pattern to identify formula text",
+ )
+ translation_group.add_argument(
+ "--formular-char-pattern",
+ help="Character pattern to identify formula text",
+ )
+ translation_group.add_argument(
+ "--split-short-lines",
+ action="store_true",
+ help="Force split short lines into different paragraphs (may cause poor typesetting & bugs)",
+ )
+ translation_group.add_argument(
+ "--short-line-split-factor",
+ type=float,
+ default=0.8,
+ help="Split threshold factor. The actual threshold is the median length of all lines on the current page * this factor",
+ )
+ translation_group.add_argument(
+ "--skip-clean",
+ action="store_true",
+ help="Skip PDF cleaning step",
+ )
+ translation_group.add_argument(
+ "--dual-translate-first",
+ action="store_true",
+ help="Put translated pages first in dual PDF mode",
+ )
+ translation_group.add_argument(
+ "--disable-rich-text-translate",
+ action="store_true",
+ help="Disable rich text translation (may help improve compatibility with some PDFs)",
+ )
+ translation_group.add_argument(
+ "--enhance-compatibility",
+ action="store_true",
+ help="Enable all compatibility enhancement options (equivalent to --skip-clean --dual-translate-first --disable-rich-text-translate)",
+ )
+ translation_group.add_argument(
+ "--use-alternating-pages-dual",
+ action="store_true",
+ help="Use alternating pages mode for dual PDF. When enabled, original and translated pages are arranged in alternate order.",
+ )
+ translation_group.add_argument(
+ "--watermark-output-mode",
+ type=str,
+ choices=["watermarked", "no_watermark", "both"],
+ default="watermarked",
+ help="Control watermark output mode: 'watermarked' (default) adds watermark to translated PDF, 'no_watermark' doesn't add watermark, 'both' outputs both versions.",
+ )
+ translation_group.add_argument(
+ "--max-pages-per-part",
+ type=int,
+ help="Maximum number of pages per part for split translation. If not set, no splitting will be performed.",
+ )
+ translation_group.add_argument(
+ "--no-watermark",
+ action="store_true",
+ help="[DEPRECATED] Use --watermark-output-mode=no_watermark instead. Do not add watermark to the translated PDF.",
+ )
+ translation_group.add_argument(
+ "--report-interval",
+ type=float,
+ default=0.1,
+ help="Progress report interval in seconds (default: 0.1)",
+ )
+ translation_group.add_argument(
+ "--translate-table-text",
+ action="store_true",
+ default=False,
+ help="Translate table text (experimental)",
+ )
+ translation_group.add_argument(
+ "--show-char-box",
+ action="store_true",
+ default=False,
+ help="Show character box (debug only)",
+ )
+ translation_group.add_argument(
+ "--skip-scanned-detection",
+ action="store_true",
+ default=False,
+ help="Skip scanned document detection (speeds up processing for non-scanned documents)",
+ )
+ translation_group.add_argument(
+ "--ocr-workaround",
+ action="store_true",
+ default=False,
+ help="Add text fill background (experimental)",
+ )
+ translation_group.add_argument(
+ "--custom-system-prompt",
+ help="Custom system prompt for translation.",
+ default=None,
+ )
+ translation_group.add_argument(
+ "--add-formula-placehold-hint",
+ action="store_true",
+ default=False,
+ help="Add formula placeholder hint for translation. (Currently not recommended, it may affect translation quality, default: False)",
+ )
+ translation_group.add_argument(
+ "--glossary-files",
+ type=str,
+ default=None,
+ help="Comma-separated paths to glossary CSV files.",
+ )
+ translation_group.add_argument(
+ "--pool-max-workers",
+ type=int,
+ help="Maximum number of worker threads for internal task processing pools. If not specified, defaults to QPS value. This parameter directly sets the worker count, replacing previous QPS-based dynamic calculations.",
+ )
+ translation_group.add_argument(
+ "--no-auto-extract-glossary",
+ action="store_false",
+ dest="auto_extract_glossary",
+ default=True,
+ help="Disable automatic term extraction. (Config file: set auto_extract_glossary = false)",
+ )
+ translation_group.add_argument(
+ "--auto-enable-ocr-workaround",
+ action="store_true",
+ default=False,
+ help="Enable automatic OCR workaround. If a document is detected as heavily scanned, this will attempt to enable OCR processing and skip further scan detection. Note: This option interacts with `--ocr-workaround` and `--skip-scanned-detection`. See documentation for details. (default: False)",
+ )
+ translation_group.add_argument(
+ "--primary-font-family",
+ type=str,
+ choices=["serif", "sans-serif", "script"],
+ default=None,
+ help="Override primary font family for translated text. Choices: 'serif' for serif fonts, 'sans-serif' for sans-serif fonts, 'script' for script/italic fonts. If not specified, uses automatic font selection based on original text properties.",
+ )
+ translation_group.add_argument(
+ "--only-include-translated-page",
+ action="store_true",
+ default=False,
+ help="Only include translated pages in the output PDF. Effective only when --pages is used.",
+ )
+ translation_group.add_argument(
+ "--save-auto-extracted-glossary",
+ action="store_true",
+ default=False,
+ help="Save automatically extracted glossary terms to a CSV file in the output directory.",
+ )
+ translation_group.add_argument(
+ "--disable-graphic-element-process",
+ action="store_true",
+ default=False,
+ help="Disable graphic element process. (default: False)",
+ )
+ translation_group.add_argument(
+ "--no-merge-alternating-line-numbers",
+ action="store_false",
+ dest="merge_alternating_line_numbers",
+ default=True,
+ help="Disable post-processing that merges alternating line-number layouts (by default this feature is enabled).",
+ )
+ translation_group.add_argument(
+ "--skip-translation",
+ action="store_true",
+ default=False,
+ help="Skip translation step. (default: False)",
+ )
+ translation_group.add_argument(
+ "--skip-form-render",
+ action="store_true",
+ default=False,
+ help="Skip form rendering. (default: False)",
+ )
+ translation_group.add_argument(
+ "--skip-curve-render",
+ action="store_true",
+ default=False,
+ help="Skip curve rendering. (default: False)",
+ )
+ translation_group.add_argument(
+ "--only-parse-generate-pdf",
+ action="store_true",
+ default=False,
+ help="Only parse PDF and generate output PDF without translation (default: False). This skips all translation-related processing including layout analysis, paragraph finding, style processing, and translation itself.",
+ )
+ translation_group.add_argument(
+ "--remove-non-formula-lines",
+ action="store_true",
+ default=False,
+ help="Remove non-formula lines from paragraph areas. This removes decorative lines that are not part of formulas, while protecting lines in figure/table areas. (default: False)",
+ )
+ translation_group.add_argument(
+ "--non-formula-line-iou-threshold",
+ type=float,
+ default=0.9,
+ help="IoU threshold for detecting paragraph overlap when removing non-formula lines. Higher values are more conservative. (default: 0.9)",
+ )
+ translation_group.add_argument(
+ "--figure-table-protection-threshold",
+ type=float,
+ default=0.9,
+ help="IoU threshold for protecting lines in figure/table areas when removing non-formula lines. Higher values provide more protection. (default: 0.9)",
+ )
+ translation_group.add_argument(
+ "--skip-formula-offset-calculation",
+ action="store_true",
+ default=False,
+ help="Skip formula offset calculation (default: False)",
+ )
+ # service option argument group
+ service_group = translation_group.add_mutually_exclusive_group()
+ service_group.add_argument(
+ "--openai",
+ action="store_true",
+ help="Use OpenAI translator.",
+ )
+ service_group = parser.add_argument_group(
+ "Translation - OpenAI Options",
+ description="OpenAI specific options",
+ )
+ service_group.add_argument(
+ "--openai-model",
+ default="gpt-4o-mini",
+ help="The OpenAI model to use for translation.",
+ )
+ service_group.add_argument(
+ "--openai-base-url",
+ help="The base URL for the OpenAI API.",
+ )
+ service_group.add_argument(
+ "--openai-api-key",
+ "-k",
+ help="The API key for the OpenAI API.",
+ )
+ service_group.add_argument(
+ "--openai-term-extraction-model",
+ default=None,
+ help="OpenAI model to use for automatic term extraction. Defaults to --openai-model when unset.",
+ )
+ service_group.add_argument(
+ "--openai-term-extraction-base-url",
+ default=None,
+ help="Base URL for the OpenAI API used during automatic term extraction. Falls back to --openai-base-url when unset.",
+ )
+ service_group.add_argument(
+ "--openai-term-extraction-api-key",
+ default=None,
+ help="API key for the OpenAI API used during automatic term extraction. Falls back to --openai-api-key when unset.",
+ )
+ service_group.add_argument(
+ "--enable-json-mode-if-requested",
+ action="store_true",
+ default=False,
+ help="Enable JSON mode for OpenAI requests.",
+ )
+ service_group.add_argument(
+ "--send-dashscope-header",
+ action="store_true",
+ default=False,
+ help="Send DashScope data inspection header to disable input/output inspection.",
+ )
+ service_group.add_argument(
+ "--no-send-temperature",
+ action="store_true",
+ default=False,
+ help="Do not send temperature parameter to OpenAI API (default: send temperature).",
+ )
+
+ return parser
+
+
+async def main():
+ parser = create_parser()
+ args: Any = parser.parse_args()
+
+ if args.debug:
+ logging.getLogger().setLevel(logging.DEBUG)
+
+ if args.generate_offline_assets:
+ babeldoc.assets.assets.generate_offline_assets_package(
+ Path(args.generate_offline_assets)
+ )
+ logger.info("Offline assets package generated, exiting...")
+ return
+
+ if args.restore_offline_assets:
+ babeldoc.assets.assets.restore_offline_assets_package(
+ Path(args.restore_offline_assets)
+ )
+ logger.info("Offline assets package restored, exiting...")
+ return
+
+ if args.warmup:
+ babeldoc.assets.assets.warmup()
+ logger.info("Warmup completed, exiting...")
+ return
+
+ # 验证翻译服务选择
+ if not args.openai:
+ parser.error("必须选择一个翻译服务：--openai")
+
+ # 验证 OpenAI 参数
+ if args.openai and not args.openai_api_key:
+ parser.error("使用 OpenAI 服务时必须提供 API key")
+
+ if args.enable_process_pool:
+ enable_process_pool()
+
+ # 实例化翻译器
+ if args.openai:
+ translator = OpenAITranslator(
+ lang_in=args.lang_in,
+ lang_out=args.lang_out,
+ model=args.openai_model,
+ base_url=args.openai_base_url,
+ api_key=args.openai_api_key,
+ ignore_cache=args.ignore_cache,
+ enable_json_mode_if_requested=args.enable_json_mode_if_requested,
+ send_dashscope_header=args.send_dashscope_header,
+ send_temperature=not args.no_send_temperature,
+ )
+ term_extraction_translator = translator
+ if (
+ args.openai_term_extraction_model
+ or args.openai_term_extraction_base_url
+ or args.openai_term_extraction_api_key
+ ):
+ term_extraction_translator = OpenAITranslator(
+ lang_in=args.lang_in,
+ lang_out=args.lang_out,
+ model=args.openai_term_extraction_model or args.openai_model,
+ base_url=(args.openai_term_extraction_base_url or args.openai_base_url),
+ api_key=args.openai_term_extraction_api_key or args.openai_api_key,
+ ignore_cache=args.ignore_cache,
+ enable_json_mode_if_requested=args.enable_json_mode_if_requested,
+ send_dashscope_header=args.send_dashscope_header,
+ send_temperature=not args.no_send_temperature,
+ )
+ else:
+ raise ValueError("Invalid translator type")
+
+ # 设置翻译速率限制
+ set_translate_rate_limiter(args.qps)
+ # 初始化文档布局模型
+ if args.rpc_doclayout:
+ from babeldoc.docvision.rpc_doclayout import RpcDocLayoutModel
+
+ doc_layout_model = RpcDocLayoutModel(host=args.rpc_doclayout)
+ elif args.rpc_doclayout2:
+ from babeldoc.docvision.rpc_doclayout2 import RpcDocLayoutModel
+
+ doc_layout_model = RpcDocLayoutModel(host=args.rpc_doclayout2)
+ elif args.rpc_doclayout3:
+ from babeldoc.docvision.rpc_doclayout3 import RpcDocLayoutModel
+
+ doc_layout_model = RpcDocLayoutModel(host=args.rpc_doclayout3)
+ elif args.rpc_doclayout4:
+ from babeldoc.docvision.rpc_doclayout4 import RpcDocLayoutModel
+
+ doc_layout_model = RpcDocLayoutModel(host=args.rpc_doclayout4)
+ elif args.rpc_doclayout5:
+ from babeldoc.docvision.rpc_doclayout5 import RpcDocLayoutModel
+
+ doc_layout_model = RpcDocLayoutModel(host=args.rpc_doclayout5)
+ elif args.rpc_doclayout6:
+ from babeldoc.docvision.rpc_doclayout6 import RpcDocLayoutModel
+
+ doc_layout_model = RpcDocLayoutModel(host=args.rpc_doclayout6)
+ elif args.rpc_doclayout7:
+ from babeldoc.docvision.rpc_doclayout7 import RpcDocLayoutModel
+
+ doc_layout_model = RpcDocLayoutModel(host=args.rpc_doclayout7)
+ else:
+ from babeldoc.docvision.doclayout import DocLayoutModel
+
+ doc_layout_model = DocLayoutModel.load_onnx()
+
+ if args.translate_table_text:
+ from babeldoc.docvision.table_detection.rapidocr import RapidOCRModel
+
+ table_model = RapidOCRModel()
+ else:
+ table_model = None
+
+ # Load glossaries
+ loaded_glossaries: list[Glossary] = []
+ if args.glossary_files:
+ paths_str = args.glossary_files.split(",")
+ for p_str in paths_str:
+ file_path = Path(p_str.strip())
+ if not file_path.exists():
+ logger.error(f"Glossary file not found: {file_path}")
+ continue
+ if not file_path.is_file():
+ logger.error(f"Glossary path is not a file: {file_path}")
+ continue
+ try:
+ glossary_obj = Glossary.from_csv(file_path, args.lang_out)
+ if glossary_obj.entries:
+ loaded_glossaries.append(glossary_obj)
+ logger.info(
+ f"Loaded glossary '{glossary_obj.name}' with {len(glossary_obj.entries)} entries."
+ )
+ else:
+ logger.info(
+ f"Glossary '{file_path.stem}' loaded with no applicable entries for lang_out '{args.lang_out}'."
+ )
+ except Exception as e:
+ logger.error(f"Failed to load glossary from {file_path}: {e}")
+
+ pending_files = []
+ for file in args.files:
+ # 清理文件路径，去除两端的引号
+ if file.startswith("--files="):
+ file = file[len("--files=") :]
+ file = file.lstrip("-").strip("\"'")
+ if not Path(file).exists():
+ logger.error(f"文件不存在：{file}")
+ exit(1)
+ if not file.lower().endswith(".pdf"):
+ logger.error(f"文件不是 PDF 文件：{file}")
+ exit(1)
+ pending_files.append(file)
+
+ if args.output:
+ if not Path(args.output).exists():
+ logger.info(f"输出目录不存在，创建：{args.output}")
+ try:
+ Path(args.output).mkdir(parents=True, exist_ok=True)
+ except OSError:
+ logger.critical(
+ f"Failed to create output folder at {args.output}",
+ exc_info=True,
+ )
+ exit(1)
+ else:
+ args.output = None
+
+ if args.working_dir:
+ working_dir = Path(args.working_dir)
+ if not working_dir.exists():
+ logger.info(f"工作目录不存在，创建：{working_dir}")
+ try:
+ working_dir.mkdir(parents=True, exist_ok=True)
+ except OSError:
+ logger.critical(
+ f"Failed to create working directory at {working_dir}",
+ exc_info=True,
+ )
+ exit(1)
+ else:
+ working_dir = None
+
+ watermark_output_mode = WatermarkOutputMode.Watermarked
+ if args.no_watermark:
+ watermark_output_mode = WatermarkOutputMode.NoWatermark
+ elif args.watermark_output_mode == "both":
+ watermark_output_mode = WatermarkOutputMode.Both
+ elif args.watermark_output_mode == "watermarked":
+ watermark_output_mode = WatermarkOutputMode.Watermarked
+ elif args.watermark_output_mode == "no_watermark":
+ watermark_output_mode = WatermarkOutputMode.NoWatermark
+
+ split_strategy = None
+ if args.max_pages_per_part:
+ split_strategy = TranslationConfig.create_max_pages_per_part_split_strategy(
+ args.max_pages_per_part
+ )
+
+ total_term_extraction_total_tokens = 0
+ total_term_extraction_prompt_tokens = 0
+ total_term_extraction_completion_tokens = 0
+ total_term_extraction_cache_hit_prompt_tokens = 0
+
+ for file in pending_files:
+ # 清理文件路径，去除两端的引号
+ file = file.strip("\"'")
+ # 创建配置对象
+ config = TranslationConfig(
+ input_file=file,
+ font=None,
+ pages=args.pages,
+ output_dir=args.output,
+ translator=translator,
+ term_extraction_translator=term_extraction_translator,
+ debug=args.debug,
+ lang_in=args.lang_in,
+ lang_out=args.lang_out,
+ no_dual=args.no_dual,
+ no_mono=args.no_mono,
+ qps=args.qps,
+ formular_font_pattern=args.formular_font_pattern,
+ formular_char_pattern=args.formular_char_pattern,
+ split_short_lines=args.split_short_lines,
+ short_line_split_factor=args.short_line_split_factor,
+ doc_layout_model=doc_layout_model,
+ skip_clean=args.skip_clean,
+ dual_translate_first=args.dual_translate_first,
+ disable_rich_text_translate=args.disable_rich_text_translate,
+ enhance_compatibility=args.enhance_compatibility,
+ use_alternating_pages_dual=args.use_alternating_pages_dual,
+ report_interval=args.report_interval,
+ min_text_length=args.min_text_length,
+ watermark_output_mode=watermark_output_mode,
+ split_strategy=split_strategy,
+ table_model=table_model,
+ show_char_box=args.show_char_box,
+ skip_scanned_detection=args.skip_scanned_detection,
+ ocr_workaround=args.ocr_workaround,
+ custom_system_prompt=args.custom_system_prompt,
+ working_dir=working_dir,
+ add_formula_placehold_hint=args.add_formula_placehold_hint,
+ glossaries=loaded_glossaries,
+ pool_max_workers=args.pool_max_workers,
+ auto_extract_glossary=args.auto_extract_glossary,
+ auto_enable_ocr_workaround=args.auto_enable_ocr_workaround,
+ primary_font_family=args.primary_font_family,
+ only_include_translated_page=args.only_include_translated_page,
+ save_auto_extracted_glossary=args.save_auto_extracted_glossary,
+ enable_graphic_element_process=not args.disable_graphic_element_process,
+ merge_alternating_line_numbers=args.merge_alternating_line_numbers,
+ skip_translation=args.skip_translation,
+ skip_form_render=args.skip_form_render,
+ skip_curve_render=args.skip_curve_render,
+ only_parse_generate_pdf=args.only_parse_generate_pdf,
+ remove_non_formula_lines=args.remove_non_formula_lines,
+ non_formula_line_iou_threshold=args.non_formula_line_iou_threshold,
+ figure_table_protection_threshold=args.figure_table_protection_threshold,
+ skip_formula_offset_calculation=args.skip_formula_offset_calculation,
+ metadata_extra_data=args.metadata_extra_data,
+ )
+
+ def nop(_x):
+ pass
+
+ getattr(doc_layout_model, "init_font_mapper", nop)(config)
+ # Create progress handler
+ progress_context, progress_handler = create_progress_handler(
+ config, show_log=False
+ )
+
+ # 开始翻译
+ with progress_context:
+ async for event in babeldoc.format.pdf.high_level.async_translate(config):
+ progress_handler(event)
+ if config.debug:
+ logger.debug(event)
+ if event["type"] == "error":
+ logger.error(f"Error: {event['error']}")
+ break
+ if event["type"] == "finish":
+ result = event["translate_result"]
+ logger.info(str(result))
+ break
+ usage = config.term_extraction_token_usage
+ total_term_extraction_total_tokens += usage["total_tokens"]
+ total_term_extraction_prompt_tokens += usage["prompt_tokens"]
+ total_term_extraction_completion_tokens += usage["completion_tokens"]
+ total_term_extraction_cache_hit_prompt_tokens += usage[
+ "cache_hit_prompt_tokens"
+ ]
+ logger.info(f"Total tokens: {translator.token_count.value}")
+ logger.info(f"Prompt tokens: {translator.prompt_token_count.value}")
+ logger.info(f"Completion tokens: {translator.completion_token_count.value}")
+ logger.info(
+ f"Cache hit prompt tokens: {translator.cache_hit_prompt_token_count.value}"
+ )
+ logger.info(
+ "Term extraction tokens: total=%s prompt=%s completion=%s cache_hit_prompt=%s",
+ total_term_extraction_total_tokens,
+ total_term_extraction_prompt_tokens,
+ total_term_extraction_completion_tokens,
+ total_term_extraction_cache_hit_prompt_tokens,
+ )
+ if term_extraction_translator is not translator:
+ logger.info(
+ "Term extraction translator raw tokens: total=%s prompt=%s completion=%s cache_hit_prompt=%s",
+ term_extraction_translator.token_count.value,
+ term_extraction_translator.prompt_token_count.value,
+ term_extraction_translator.completion_token_count.value,
+ term_extraction_translator.cache_hit_prompt_token_count.value,
+ )
+
+
+def create_progress_handler(
+ translation_config: TranslationConfig, show_log: bool = False
+):
+ """Create a progress handler function based on the configuration.
+
+ Args:
+ translation_config: The translation configuration.
+
+ Returns:
+ A tuple of (progress_context, progress_handler), where progress_context is a context
+ manager that should be used to wrap the translation process, and progress_handler
+ is a function that will be called with progress events.
+ """
+ if translation_config.use_rich_pbar:
+ progress = Progress(
+ TextColumn("[progress.description]{task.description}"),
+ BarColumn(),
+ MofNCompleteColumn(),
+ TimeElapsedColumn(),
+ TimeRemainingColumn(),
+ )
+ translate_task_id = progress.add_task("translate", total=100)
+ stage_tasks = {}
+
+ def progress_handler(event):
+ if show_log and random.random() <= 0.1: # noqa: S311
+ logger.info(event)
+ if event["type"] == "progress_start":
+ if event["stage"] not in stage_tasks:
+ stage_tasks[event["stage"]] = progress.add_task(
+ f"{event['stage']} ({event['part_index']}/{event['total_parts']})",
+ total=event.get("stage_total", 100),
+ )
+ elif event["type"] == "progress_update":
+ stage = event["stage"]
+ if stage in stage_tasks:
+ progress.update(
+ stage_tasks[stage],
+ completed=event["stage_current"],
+ total=event["stage_total"],
+ description=f"{event['stage']} ({event['part_index']}/{event['total_parts']})",
+ refresh=True,
+ )
+ progress.update(
+ translate_task_id,
+ completed=event["overall_progress"],
+ refresh=True,
+ )
+ elif event["type"] == "progress_end":
+ stage = event["stage"]
+ if stage in stage_tasks:
+ progress.update(
+ stage_tasks[stage],
+ completed=event["stage_total"],
+ total=event["stage_total"],
+ description=f"{event['stage']} ({event['part_index']}/{event['total_parts']})",
+ refresh=True,
+ )
+ progress.update(
+ translate_task_id,
+ completed=event["overall_progress"],
+ refresh=True,
+ )
+ progress.refresh()
+
+ return progress, progress_handler
+ else:
+ pbar = tqdm.tqdm(total=100, desc="translate")
+
+ def progress_handler(event):
+ if event["type"] == "progress_update":
+ pbar.update(event["overall_progress"] - pbar.n)
+ pbar.set_description(
+ f"{event['stage']} ({event['stage_current']}/{event['stage_total']})",
+ )
+ elif event["type"] == "progress_end":
+ pbar.set_description(f"{event['stage']} (Complete)")
+ pbar.refresh()
+
+ return pbar, progress_handler
+
+
+# for backward compatibility
+def create_cache_folder():
+ return babeldoc.format.pdf.high_level.create_cache_folder()
+
+
+# for backward compatibility
+def download_font_assets():
+ return babeldoc.format.pdf.high_level.download_font_assets()
+
+
+class EvictQueue(queue.Queue):
+ def __init__(self, maxsize):
+ self.discarded = 0
+ super().__init__(maxsize)
+
+ def put(self, item, block=False, timeout=None):
+ while True:
+ try:
+ super().put(item, block=False)
+ break
+ except queue.Full:
+ try:
+ self.get_nowait()
+ self.discarded += 1
+ except queue.Empty:
+ pass
+
+
+def speed_up_logs():
+ import logging.handlers
+
+ root_logger = logging.getLogger()
+ log_que = EvictQueue(1000)
+ queue_handler = logging.handlers.QueueHandler(log_que)
+ queue_listener = logging.handlers.QueueListener(log_que, *root_logger.handlers)
+ queue_listener.start()
+ root_logger.handlers = [queue_handler]
+
+
+def cli():
+ """Command line interface entry point."""
+ from rich.logging import RichHandler
+
+ logging.basicConfig(level=logging.INFO, handlers=[RichHandler()])
+
+ logging.getLogger("httpx").setLevel("CRITICAL")
+ logging.getLogger("httpx").propagate = False
+ logging.getLogger("openai").setLevel("CRITICAL")
+ logging.getLogger("openai").propagate = False
+ logging.getLogger("httpcore").setLevel("CRITICAL")
+ logging.getLogger("httpcore").propagate = False
+ logging.getLogger("http11").setLevel("CRITICAL")
+ logging.getLogger("http11").propagate = False
+ for v in logging.Logger.manager.loggerDict.values():
+ if getattr(v, "name", None) is None:
+ continue
+ if (
+ v.name.startswith("pdfminer")
+ or v.name.startswith("peewee")
+ or v.name.startswith("httpx")
+ or "http11" in v.name
+ or "openai" in v.name
+ or "pdfminer" in v.name
+ ):
+ v.disabled = True
+ v.propagate = False
+
+ speed_up_logs()
+ babeldoc.format.pdf.high_level.init()
+ asyncio.run(main())
+
+
+if __name__ == "__main__":
+ if sys.platform == "darwin" or sys.platform == "win32":
+ mp.set_start_method("spawn")
+ else:
+ mp.set_start_method("forkserver")
+ cli()
diff --git a/babeldoc/pdfminer/LICENSE b/babeldoc/pdfminer/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..67f3786571bfd6df02e93539ddca41b527fef625
--- /dev/null
+++ b/babeldoc/pdfminer/LICENSE
@@ -0,0 +1,22 @@
+Copyright (c) 2004-2016 Yusuke Shinyama
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
diff --git a/babeldoc/pdfminer/__init__.py b/babeldoc/pdfminer/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d3f787d6d8c255fbf25e214519292bebde43062
--- /dev/null
+++ b/babeldoc/pdfminer/__init__.py
@@ -0,0 +1,11 @@
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
+
+try:
+ __version__ = version("pdfminer.six")
+except PackageNotFoundError:
+ # package is not installed, return default
+ __version__ = "0.0"
+
+if __name__ == "__main__":
+ print(__version__)
diff --git a/babeldoc/pdfminer/_saslprep.py b/babeldoc/pdfminer/_saslprep.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d441e476e83373830cc97ff08dcee5fcf7404de
--- /dev/null
+++ b/babeldoc/pdfminer/_saslprep.py
@@ -0,0 +1,101 @@
+# Copyright 2016-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Some changes copyright 2021-present Matthias Valvekens,
+# licensed under the license of the pyHanko project (see LICENSE file).
+
+
+"""An implementation of RFC4013 SASLprep."""
+
+__all__ = ["saslprep"]
+
+import stringprep
+import unicodedata
+from collections.abc import Callable
+
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+
+# RFC4013 section 2.3 prohibited output.
+_PROHIBITED: tuple[Callable[[str], bool], ...] = (
+ # A strict reading of RFC 4013 requires table c12 here, but
+ # characters from it are mapped to SPACE in the Map step. Can
+ # normalization reintroduce them somehow?
+ stringprep.in_table_c12,
+ stringprep.in_table_c21_c22,
+ stringprep.in_table_c3,
+ stringprep.in_table_c4,
+ stringprep.in_table_c5,
+ stringprep.in_table_c6,
+ stringprep.in_table_c7,
+ stringprep.in_table_c8,
+ stringprep.in_table_c9,
+)
+
+
+def saslprep(data: str, prohibit_unassigned_code_points: bool = True) -> str:
+ """An implementation of RFC4013 SASLprep.
+ :param data:
+ The string to SASLprep.
+ :param prohibit_unassigned_code_points:
+ RFC 3454 and RFCs for various SASL mechanisms distinguish between
+ `queries` (unassigned code points allowed) and
+ `stored strings` (unassigned code points prohibited). Defaults
+ to ``True`` (unassigned code points are prohibited).
+ :return: The SASLprep'ed version of `data`.
+ """
+ if prohibit_unassigned_code_points:
+ prohibited = _PROHIBITED + (stringprep.in_table_a1,)
+ else:
+ prohibited = _PROHIBITED
+
+ # RFC3454 section 2, step 1 - Map
+ # RFC4013 section 2.1 mappings
+ # Map Non-ASCII space characters to SPACE (U+0020). Map
+ # commonly mapped to nothing characters to, well, nothing.
+ in_table_c12 = stringprep.in_table_c12
+ in_table_b1 = stringprep.in_table_b1
+ data = "".join(
+ [
+ "\u0020" if in_table_c12(elt) else elt
+ for elt in data
+ if not in_table_b1(elt)
+ ],
+ )
+
+ # RFC3454 section 2, step 2 - Normalize
+ # RFC4013 section 2.2 normalization
+ data = unicodedata.ucd_3_2_0.normalize("NFKC", data)
+
+ in_table_d1 = stringprep.in_table_d1
+ if in_table_d1(data[0]):
+ if not in_table_d1(data[-1]):
+ # RFC3454, Section 6, #3. If a string contains any
+ # RandALCat character, the first and last characters
+ # MUST be RandALCat characters.
+ raise PDFValueError("SASLprep: failed bidirectional check")
+ # RFC3454, Section 6, #2. If a string contains any RandALCat
+ # character, it MUST NOT contain any LCat character.
+ prohibited = prohibited + (stringprep.in_table_d2,)
+ else:
+ # RFC3454, Section 6, #3. Following the logic of #3, if
+ # the first character is not a RandALCat, no other character
+ # can be either.
+ prohibited = prohibited + (in_table_d1,)
+
+ # RFC3454 section 2, step 3 and 4 - Prohibit and check bidi
+ for char in data:
+ if any(in_table(char) for in_table in prohibited):
+ raise PDFValueError("SASLprep: failed prohibited character check")
+
+ return data
diff --git a/babeldoc/pdfminer/arcfour.py b/babeldoc/pdfminer/arcfour.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0f62103abc30c4b99fdc26a6d16bd5c214ebe44
--- /dev/null
+++ b/babeldoc/pdfminer/arcfour.py
@@ -0,0 +1,35 @@
+"""Python implementation of Arcfour encryption algorithm.
+See https://en.wikipedia.org/wiki/RC4
+This code is in the public domain.
+
+"""
+
+from collections.abc import Sequence
+
+
+class Arcfour:
+ def __init__(self, key: Sequence[int]) -> None:
+ # because Py3 range is not indexable
+ s = [i for i in range(256)]
+ j = 0
+ klen = len(key)
+ for i in range(256):
+ j = (j + s[i] + key[i % klen]) % 256
+ (s[i], s[j]) = (s[j], s[i])
+ self.s = s
+ (self.i, self.j) = (0, 0)
+
+ def process(self, data: bytes) -> bytes:
+ (i, j) = (self.i, self.j)
+ s = self.s
+ r = b""
+ for c in iter(data):
+ i = (i + 1) % 256
+ j = (j + s[i]) % 256
+ (s[i], s[j]) = (s[j], s[i])
+ k = s[(s[i] + s[j]) % 256]
+ r += bytes((c ^ k,))
+ (self.i, self.j) = (i, j)
+ return r
+
+ encrypt = decrypt = process
diff --git a/babeldoc/pdfminer/ascii85.py b/babeldoc/pdfminer/ascii85.py
new file mode 100644
index 0000000000000000000000000000000000000000..719cb86eebec88db32b5c7a075388e5d7ffc46bf
--- /dev/null
+++ b/babeldoc/pdfminer/ascii85.py
@@ -0,0 +1,48 @@
+"""Python implementation of ASCII85/ASCIIHex decoder (Adobe version)."""
+
+import re
+from base64 import a85decode
+from binascii import unhexlify
+
+start_re = re.compile(rb"^\s*?\s*$")
+
+
+def ascii85decode(data: bytes) -> bytes:
+ """In ASCII85 encoding, every four bytes are encoded with five ASCII
+ letters, using 85 different types of characters (as 256**4 < 85**5).
+ When the length of the original bytes is not a multiple of 4, a special
+ rule is used for round up.
+
+ Adobe's ASCII85 implementation expects the input to be terminated
+ by `b"~>"`, and (though this is absent from the PDF spec) it can
+ also begin with `b"<~"`. We can't reliably expect this to be the
+ case, and there can be off-by-one errors in stream lengths which
+ mean we only see `~` at the end. Worse yet, `<` and `>` are
+ ASCII85 digits, so we can't strip them. We settle on a compromise
+ where we strip leading `<~` or `~` and trailing `~` or `~>`.
+ """
+ data = start_re.sub(b"", data)
+ data = end_re.sub(b"", data)
+ return a85decode(data)
+
+
+bws_re = re.compile(rb"\s")
+
+
+def asciihexdecode(data: bytes) -> bytes:
+ """ASCIIHexDecode filter: PDFReference v1.4 section 3.3.1
+ For each pair of ASCII hexadecimal digits (0-9 and A-F or a-f), the
+ ASCIIHexDecode filter produces one byte of binary data. All white-space
+ characters are ignored. A right angle bracket character (>) indicates
+ EOD. Any other characters will cause an error. If the filter encounters
+ the EOD marker after reading an odd number of hexadecimal digits, it
+ will behave as if a 0 followed the last digit.
+ """
+ data = bws_re.sub(b"", data)
+ idx = data.find(b">")
+ if idx != -1:
+ data = data[:idx]
+ if idx % 2 == 1:
+ data += b"0"
+ return unhexlify(data)
diff --git a/babeldoc/pdfminer/casting.py b/babeldoc/pdfminer/casting.py
new file mode 100644
index 0000000000000000000000000000000000000000..f70a9e695e14389353484046a4df81523fbee6ae
--- /dev/null
+++ b/babeldoc/pdfminer/casting.py
@@ -0,0 +1,92 @@
+import itertools
+from typing import Any
+
+from babeldoc.pdfminer.utils import Matrix
+from babeldoc.pdfminer.utils import Rect
+
+_FloatTriple = tuple[float, float, float]
+_FloatQuadruple = tuple[float, float, float, float]
+
+
+def safe_int(o: Any) -> int | None:
+ try:
+ return int(o)
+ except (TypeError, ValueError):
+ return None
+
+
+def safe_float(o: Any) -> float | None:
+ try:
+ return float(o)
+ except (TypeError, ValueError):
+ return None
+
+
+def safe_matrix(a: Any, b: Any, c: Any, d: Any, e: Any, f: Any) -> Matrix | None:
+ a_f = safe_float(a)
+ b_f = safe_float(b)
+ c_f = safe_float(c)
+ d_f = safe_float(d)
+ e_f = safe_float(e)
+ f_f = safe_float(f)
+
+ if (
+ a_f is None
+ or b_f is None
+ or c_f is None
+ or d_f is None
+ or e_f is None
+ or f_f is None
+ ):
+ return None
+
+ return a_f, b_f, c_f, d_f, e_f, f_f
+
+
+def safe_rgb(r: Any, g: Any, b: Any) -> tuple[float, float, float] | None:
+ return _safe_float_triple(r, g, b)
+
+
+def safe_cmyk(
+ c: Any, m: Any, y: Any, k: Any
+) -> tuple[float, float, float, float] | None:
+ return _safe_float_quadruple(c, m, y, k)
+
+
+def safe_rect_list(value: Any) -> Rect | None:
+ try:
+ values = list(itertools.islice(value, 4))
+ except TypeError:
+ return None
+
+ if len(values) != 4:
+ return None
+
+ return safe_rect(*values)
+
+
+def safe_rect(a: Any, b: Any, c: Any, d: Any) -> Rect | None:
+ return _safe_float_quadruple(a, b, c, d)
+
+
+def _safe_float_triple(a: Any, b: Any, c: Any) -> _FloatTriple | None:
+ a_f = safe_float(a)
+ b_f = safe_float(b)
+ c_f = safe_float(c)
+
+ if a_f is None or b_f is None or c_f is None:
+ return None
+
+ return a_f, b_f, c_f
+
+
+def _safe_float_quadruple(a: Any, b: Any, c: Any, d: Any) -> _FloatQuadruple | None:
+ a_f = safe_float(a)
+ b_f = safe_float(b)
+ c_f = safe_float(c)
+ d_f = safe_float(d)
+
+ if a_f is None or b_f is None or c_f is None or d_f is None:
+ return None
+
+ return a_f, b_f, c_f, d_f
diff --git a/babeldoc/pdfminer/ccitt.py b/babeldoc/pdfminer/ccitt.py
new file mode 100644
index 0000000000000000000000000000000000000000..29da4ac8000c03dce2154e7b68fd5f22d333ba5c
--- /dev/null
+++ b/babeldoc/pdfminer/ccitt.py
@@ -0,0 +1,609 @@
+# CCITT Fax decoder
+#
+# Bugs: uncompressed mode untested.
+#
+# cf.
+# ITU-T Recommendation T.4
+# "Standardization of Group 3 facsimile terminals
+# for document transmission"
+# ITU-T Recommendation T.6
+# "FACSIMILE CODING SCHEMES AND CODING CONTROL FUNCTIONS
+# FOR GROUP 4 FACSIMILE APPARATUS"
+
+
+import array
+from collections.abc import Callable
+from collections.abc import Iterator
+from collections.abc import MutableSequence
+from collections.abc import Sequence
+from typing import Any
+from typing import cast
+
+from babeldoc.pdfminer.pdfexceptions import PDFException
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+
+
+def get_bytes(data: bytes) -> Iterator[int]:
+ yield from data
+
+
+# Workaround https://github.com/python/mypy/issues/731
+BitParserState = MutableSequence[Any]
+# A better definition (not supported by mypy) would be:
+# BitParserState = MutableSequence[Union["BitParserState", int, str, None]]
+
+
+class BitParser:
+ _state: BitParserState
+
+ # _accept is declared Optional solely as a workaround for
+ # https://github.com/python/mypy/issues/708
+ _accept: Callable[[Any], BitParserState] | None
+
+ def __init__(self) -> None:
+ self._pos = 0
+
+ @classmethod
+ def add(cls, root: BitParserState, v: int | str, bits: str) -> None:
+ p: BitParserState = root
+ b = None
+ for i in range(len(bits)):
+ if i > 0:
+ assert b is not None
+ if p[b] is None:
+ p[b] = [None, None]
+ p = p[b]
+ if bits[i] == "1":
+ b = 1
+ else:
+ b = 0
+ assert b is not None
+ p[b] = v
+
+ def feedbytes(self, data: bytes) -> None:
+ for byte in get_bytes(data):
+ for m in (128, 64, 32, 16, 8, 4, 2, 1):
+ self._parse_bit(byte & m)
+
+ def _parse_bit(self, x: object) -> None:
+ if x:
+ v = self._state[1]
+ else:
+ v = self._state[0]
+ self._pos += 1
+ if isinstance(v, list):
+ self._state = v
+ else:
+ assert self._accept is not None
+ self._state = self._accept(v)
+
+
+class CCITTG4Parser(BitParser):
+ MODE = [None, None]
+ BitParser.add(MODE, 0, "1")
+ BitParser.add(MODE, +1, "011")
+ BitParser.add(MODE, -1, "010")
+ BitParser.add(MODE, "h", "001")
+ BitParser.add(MODE, "p", "0001")
+ BitParser.add(MODE, +2, "000011")
+ BitParser.add(MODE, -2, "000010")
+ BitParser.add(MODE, +3, "0000011")
+ BitParser.add(MODE, -3, "0000010")
+ BitParser.add(MODE, "u", "0000001111")
+ BitParser.add(MODE, "x1", "0000001000")
+ BitParser.add(MODE, "x2", "0000001001")
+ BitParser.add(MODE, "x3", "0000001010")
+ BitParser.add(MODE, "x4", "0000001011")
+ BitParser.add(MODE, "x5", "0000001100")
+ BitParser.add(MODE, "x6", "0000001101")
+ BitParser.add(MODE, "x7", "0000001110")
+ BitParser.add(MODE, "e", "000000000001000000000001")
+
+ WHITE = [None, None]
+ BitParser.add(WHITE, 0, "00110101")
+ BitParser.add(WHITE, 1, "000111")
+ BitParser.add(WHITE, 2, "0111")
+ BitParser.add(WHITE, 3, "1000")
+ BitParser.add(WHITE, 4, "1011")
+ BitParser.add(WHITE, 5, "1100")
+ BitParser.add(WHITE, 6, "1110")
+ BitParser.add(WHITE, 7, "1111")
+ BitParser.add(WHITE, 8, "10011")
+ BitParser.add(WHITE, 9, "10100")
+ BitParser.add(WHITE, 10, "00111")
+ BitParser.add(WHITE, 11, "01000")
+ BitParser.add(WHITE, 12, "001000")
+ BitParser.add(WHITE, 13, "000011")
+ BitParser.add(WHITE, 14, "110100")
+ BitParser.add(WHITE, 15, "110101")
+ BitParser.add(WHITE, 16, "101010")
+ BitParser.add(WHITE, 17, "101011")
+ BitParser.add(WHITE, 18, "0100111")
+ BitParser.add(WHITE, 19, "0001100")
+ BitParser.add(WHITE, 20, "0001000")
+ BitParser.add(WHITE, 21, "0010111")
+ BitParser.add(WHITE, 22, "0000011")
+ BitParser.add(WHITE, 23, "0000100")
+ BitParser.add(WHITE, 24, "0101000")
+ BitParser.add(WHITE, 25, "0101011")
+ BitParser.add(WHITE, 26, "0010011")
+ BitParser.add(WHITE, 27, "0100100")
+ BitParser.add(WHITE, 28, "0011000")
+ BitParser.add(WHITE, 29, "00000010")
+ BitParser.add(WHITE, 30, "00000011")
+ BitParser.add(WHITE, 31, "00011010")
+ BitParser.add(WHITE, 32, "00011011")
+ BitParser.add(WHITE, 33, "00010010")
+ BitParser.add(WHITE, 34, "00010011")
+ BitParser.add(WHITE, 35, "00010100")
+ BitParser.add(WHITE, 36, "00010101")
+ BitParser.add(WHITE, 37, "00010110")
+ BitParser.add(WHITE, 38, "00010111")
+ BitParser.add(WHITE, 39, "00101000")
+ BitParser.add(WHITE, 40, "00101001")
+ BitParser.add(WHITE, 41, "00101010")
+ BitParser.add(WHITE, 42, "00101011")
+ BitParser.add(WHITE, 43, "00101100")
+ BitParser.add(WHITE, 44, "00101101")
+ BitParser.add(WHITE, 45, "00000100")
+ BitParser.add(WHITE, 46, "00000101")
+ BitParser.add(WHITE, 47, "00001010")
+ BitParser.add(WHITE, 48, "00001011")
+ BitParser.add(WHITE, 49, "01010010")
+ BitParser.add(WHITE, 50, "01010011")
+ BitParser.add(WHITE, 51, "01010100")
+ BitParser.add(WHITE, 52, "01010101")
+ BitParser.add(WHITE, 53, "00100100")
+ BitParser.add(WHITE, 54, "00100101")
+ BitParser.add(WHITE, 55, "01011000")
+ BitParser.add(WHITE, 56, "01011001")
+ BitParser.add(WHITE, 57, "01011010")
+ BitParser.add(WHITE, 58, "01011011")
+ BitParser.add(WHITE, 59, "01001010")
+ BitParser.add(WHITE, 60, "01001011")
+ BitParser.add(WHITE, 61, "00110010")
+ BitParser.add(WHITE, 62, "00110011")
+ BitParser.add(WHITE, 63, "00110100")
+ BitParser.add(WHITE, 64, "11011")
+ BitParser.add(WHITE, 128, "10010")
+ BitParser.add(WHITE, 192, "010111")
+ BitParser.add(WHITE, 256, "0110111")
+ BitParser.add(WHITE, 320, "00110110")
+ BitParser.add(WHITE, 384, "00110111")
+ BitParser.add(WHITE, 448, "01100100")
+ BitParser.add(WHITE, 512, "01100101")
+ BitParser.add(WHITE, 576, "01101000")
+ BitParser.add(WHITE, 640, "01100111")
+ BitParser.add(WHITE, 704, "011001100")
+ BitParser.add(WHITE, 768, "011001101")
+ BitParser.add(WHITE, 832, "011010010")
+ BitParser.add(WHITE, 896, "011010011")
+ BitParser.add(WHITE, 960, "011010100")
+ BitParser.add(WHITE, 1024, "011010101")
+ BitParser.add(WHITE, 1088, "011010110")
+ BitParser.add(WHITE, 1152, "011010111")
+ BitParser.add(WHITE, 1216, "011011000")
+ BitParser.add(WHITE, 1280, "011011001")
+ BitParser.add(WHITE, 1344, "011011010")
+ BitParser.add(WHITE, 1408, "011011011")
+ BitParser.add(WHITE, 1472, "010011000")
+ BitParser.add(WHITE, 1536, "010011001")
+ BitParser.add(WHITE, 1600, "010011010")
+ BitParser.add(WHITE, 1664, "011000")
+ BitParser.add(WHITE, 1728, "010011011")
+ BitParser.add(WHITE, 1792, "00000001000")
+ BitParser.add(WHITE, 1856, "00000001100")
+ BitParser.add(WHITE, 1920, "00000001101")
+ BitParser.add(WHITE, 1984, "000000010010")
+ BitParser.add(WHITE, 2048, "000000010011")
+ BitParser.add(WHITE, 2112, "000000010100")
+ BitParser.add(WHITE, 2176, "000000010101")
+ BitParser.add(WHITE, 2240, "000000010110")
+ BitParser.add(WHITE, 2304, "000000010111")
+ BitParser.add(WHITE, 2368, "000000011100")
+ BitParser.add(WHITE, 2432, "000000011101")
+ BitParser.add(WHITE, 2496, "000000011110")
+ BitParser.add(WHITE, 2560, "000000011111")
+
+ BLACK = [None, None]
+ BitParser.add(BLACK, 0, "0000110111")
+ BitParser.add(BLACK, 1, "010")
+ BitParser.add(BLACK, 2, "11")
+ BitParser.add(BLACK, 3, "10")
+ BitParser.add(BLACK, 4, "011")
+ BitParser.add(BLACK, 5, "0011")
+ BitParser.add(BLACK, 6, "0010")
+ BitParser.add(BLACK, 7, "00011")
+ BitParser.add(BLACK, 8, "000101")
+ BitParser.add(BLACK, 9, "000100")
+ BitParser.add(BLACK, 10, "0000100")
+ BitParser.add(BLACK, 11, "0000101")
+ BitParser.add(BLACK, 12, "0000111")
+ BitParser.add(BLACK, 13, "00000100")
+ BitParser.add(BLACK, 14, "00000111")
+ BitParser.add(BLACK, 15, "000011000")
+ BitParser.add(BLACK, 16, "0000010111")
+ BitParser.add(BLACK, 17, "0000011000")
+ BitParser.add(BLACK, 18, "0000001000")
+ BitParser.add(BLACK, 19, "00001100111")
+ BitParser.add(BLACK, 20, "00001101000")
+ BitParser.add(BLACK, 21, "00001101100")
+ BitParser.add(BLACK, 22, "00000110111")
+ BitParser.add(BLACK, 23, "00000101000")
+ BitParser.add(BLACK, 24, "00000010111")
+ BitParser.add(BLACK, 25, "00000011000")
+ BitParser.add(BLACK, 26, "000011001010")
+ BitParser.add(BLACK, 27, "000011001011")
+ BitParser.add(BLACK, 28, "000011001100")
+ BitParser.add(BLACK, 29, "000011001101")
+ BitParser.add(BLACK, 30, "000001101000")
+ BitParser.add(BLACK, 31, "000001101001")
+ BitParser.add(BLACK, 32, "000001101010")
+ BitParser.add(BLACK, 33, "000001101011")
+ BitParser.add(BLACK, 34, "000011010010")
+ BitParser.add(BLACK, 35, "000011010011")
+ BitParser.add(BLACK, 36, "000011010100")
+ BitParser.add(BLACK, 37, "000011010101")
+ BitParser.add(BLACK, 38, "000011010110")
+ BitParser.add(BLACK, 39, "000011010111")
+ BitParser.add(BLACK, 40, "000001101100")
+ BitParser.add(BLACK, 41, "000001101101")
+ BitParser.add(BLACK, 42, "000011011010")
+ BitParser.add(BLACK, 43, "000011011011")
+ BitParser.add(BLACK, 44, "000001010100")
+ BitParser.add(BLACK, 45, "000001010101")
+ BitParser.add(BLACK, 46, "000001010110")
+ BitParser.add(BLACK, 47, "000001010111")
+ BitParser.add(BLACK, 48, "000001100100")
+ BitParser.add(BLACK, 49, "000001100101")
+ BitParser.add(BLACK, 50, "000001010010")
+ BitParser.add(BLACK, 51, "000001010011")
+ BitParser.add(BLACK, 52, "000000100100")
+ BitParser.add(BLACK, 53, "000000110111")
+ BitParser.add(BLACK, 54, "000000111000")
+ BitParser.add(BLACK, 55, "000000100111")
+ BitParser.add(BLACK, 56, "000000101000")
+ BitParser.add(BLACK, 57, "000001011000")
+ BitParser.add(BLACK, 58, "000001011001")
+ BitParser.add(BLACK, 59, "000000101011")
+ BitParser.add(BLACK, 60, "000000101100")
+ BitParser.add(BLACK, 61, "000001011010")
+ BitParser.add(BLACK, 62, "000001100110")
+ BitParser.add(BLACK, 63, "000001100111")
+ BitParser.add(BLACK, 64, "0000001111")
+ BitParser.add(BLACK, 128, "000011001000")
+ BitParser.add(BLACK, 192, "000011001001")
+ BitParser.add(BLACK, 256, "000001011011")
+ BitParser.add(BLACK, 320, "000000110011")
+ BitParser.add(BLACK, 384, "000000110100")
+ BitParser.add(BLACK, 448, "000000110101")
+ BitParser.add(BLACK, 512, "0000001101100")
+ BitParser.add(BLACK, 576, "0000001101101")
+ BitParser.add(BLACK, 640, "0000001001010")
+ BitParser.add(BLACK, 704, "0000001001011")
+ BitParser.add(BLACK, 768, "0000001001100")
+ BitParser.add(BLACK, 832, "0000001001101")
+ BitParser.add(BLACK, 896, "0000001110010")
+ BitParser.add(BLACK, 960, "0000001110011")
+ BitParser.add(BLACK, 1024, "0000001110100")
+ BitParser.add(BLACK, 1088, "0000001110101")
+ BitParser.add(BLACK, 1152, "0000001110110")
+ BitParser.add(BLACK, 1216, "0000001110111")
+ BitParser.add(BLACK, 1280, "0000001010010")
+ BitParser.add(BLACK, 1344, "0000001010011")
+ BitParser.add(BLACK, 1408, "0000001010100")
+ BitParser.add(BLACK, 1472, "0000001010101")
+ BitParser.add(BLACK, 1536, "0000001011010")
+ BitParser.add(BLACK, 1600, "0000001011011")
+ BitParser.add(BLACK, 1664, "0000001100100")
+ BitParser.add(BLACK, 1728, "0000001100101")
+ BitParser.add(BLACK, 1792, "00000001000")
+ BitParser.add(BLACK, 1856, "00000001100")
+ BitParser.add(BLACK, 1920, "00000001101")
+ BitParser.add(BLACK, 1984, "000000010010")
+ BitParser.add(BLACK, 2048, "000000010011")
+ BitParser.add(BLACK, 2112, "000000010100")
+ BitParser.add(BLACK, 2176, "000000010101")
+ BitParser.add(BLACK, 2240, "000000010110")
+ BitParser.add(BLACK, 2304, "000000010111")
+ BitParser.add(BLACK, 2368, "000000011100")
+ BitParser.add(BLACK, 2432, "000000011101")
+ BitParser.add(BLACK, 2496, "000000011110")
+ BitParser.add(BLACK, 2560, "000000011111")
+
+ UNCOMPRESSED = [None, None]
+ BitParser.add(UNCOMPRESSED, "1", "1")
+ BitParser.add(UNCOMPRESSED, "01", "01")
+ BitParser.add(UNCOMPRESSED, "001", "001")
+ BitParser.add(UNCOMPRESSED, "0001", "0001")
+ BitParser.add(UNCOMPRESSED, "00001", "00001")
+ BitParser.add(UNCOMPRESSED, "00000", "000001")
+ BitParser.add(UNCOMPRESSED, "T00", "00000011")
+ BitParser.add(UNCOMPRESSED, "T10", "00000010")
+ BitParser.add(UNCOMPRESSED, "T000", "000000011")
+ BitParser.add(UNCOMPRESSED, "T100", "000000010")
+ BitParser.add(UNCOMPRESSED, "T0000", "0000000011")
+ BitParser.add(UNCOMPRESSED, "T1000", "0000000010")
+ BitParser.add(UNCOMPRESSED, "T00000", "00000000011")
+ BitParser.add(UNCOMPRESSED, "T10000", "00000000010")
+
+ class CCITTException(PDFException):
+ pass
+
+ class EOFB(CCITTException):
+ pass
+
+ class InvalidData(CCITTException):
+ pass
+
+ class ByteSkip(CCITTException):
+ pass
+
+ _color: int
+
+ def __init__(self, width: int, bytealign: bool = False) -> None:
+ BitParser.__init__(self)
+ self.width = width
+ self.bytealign = bytealign
+ self.reset()
+
+ def feedbytes(self, data: bytes) -> None:
+ for byte in get_bytes(data):
+ try:
+ for m in (128, 64, 32, 16, 8, 4, 2, 1):
+ self._parse_bit(byte & m)
+ except self.ByteSkip:
+ self._accept = self._parse_mode
+ self._state = self.MODE
+ except self.EOFB:
+ break
+
+ def _parse_mode(self, mode: object) -> BitParserState:
+ if mode == "p":
+ self._do_pass()
+ self._flush_line()
+ return self.MODE
+ elif mode == "h":
+ self._n1 = 0
+ self._accept = self._parse_horiz1
+ if self._color:
+ return self.WHITE
+ else:
+ return self.BLACK
+ elif mode == "u":
+ self._accept = self._parse_uncompressed
+ return self.UNCOMPRESSED
+ elif mode == "e":
+ raise self.EOFB
+ elif isinstance(mode, int):
+ self._do_vertical(mode)
+ self._flush_line()
+ return self.MODE
+ else:
+ raise self.InvalidData(mode)
+
+ def _parse_horiz1(self, n: Any) -> BitParserState:
+ if n is None:
+ raise self.InvalidData
+ self._n1 += n
+ if n < 64:
+ self._n2 = 0
+ self._color = 1 - self._color
+ self._accept = self._parse_horiz2
+ if self._color:
+ return self.WHITE
+ else:
+ return self.BLACK
+
+ def _parse_horiz2(self, n: Any) -> BitParserState:
+ if n is None:
+ raise self.InvalidData
+ self._n2 += n
+ if n < 64:
+ self._color = 1 - self._color
+ self._accept = self._parse_mode
+ self._do_horizontal(self._n1, self._n2)
+ self._flush_line()
+ return self.MODE
+ elif self._color:
+ return self.WHITE
+ else:
+ return self.BLACK
+
+ def _parse_uncompressed(self, bits: str | None) -> BitParserState:
+ if not bits:
+ raise self.InvalidData
+ if bits.startswith("T"):
+ self._accept = self._parse_mode
+ self._color = int(bits[1])
+ self._do_uncompressed(bits[2:])
+ return self.MODE
+ else:
+ self._do_uncompressed(bits)
+ return self.UNCOMPRESSED
+
+ def _get_bits(self) -> str:
+ return "".join(str(b) for b in self._curline[: self._curpos])
+
+ def _get_refline(self, i: int) -> str:
+ if i < 0:
+ return "[]" + "".join(str(b) for b in self._refline)
+ elif len(self._refline) <= i:
+ return "".join(str(b) for b in self._refline) + "[]"
+ else:
+ return (
+ "".join(str(b) for b in self._refline[:i])
+ + "["
+ + str(self._refline[i])
+ + "]"
+ + "".join(str(b) for b in self._refline[i + 1 :])
+ )
+
+ def reset(self) -> None:
+ self._y = 0
+ self._curline = array.array("b", [1] * self.width)
+ self._reset_line()
+ self._accept = self._parse_mode
+ self._state = self.MODE
+
+ def output_line(self, y: int, bits: Sequence[int]) -> None:
+ print(y, "".join(str(b) for b in bits))
+
+ def _reset_line(self) -> None:
+ self._refline = self._curline
+ self._curline = array.array("b", [1] * self.width)
+ self._curpos = -1
+ self._color = 1
+
+ def _flush_line(self) -> None:
+ if self.width <= self._curpos:
+ self.output_line(self._y, self._curline)
+ self._y += 1
+ self._reset_line()
+ if self.bytealign:
+ raise self.ByteSkip
+
+ def _do_vertical(self, dx: int) -> None:
+ x1 = self._curpos + 1
+ while 1:
+ if x1 == 0:
+ if self._color == 1 and self._refline[x1] != self._color:
+ break
+ elif x1 == len(self._refline) or (
+ self._refline[x1 - 1] == self._color
+ and self._refline[x1] != self._color
+ ):
+ break
+ x1 += 1
+ x1 += dx
+ x0 = max(0, self._curpos)
+ x1 = max(0, min(self.width, x1))
+ if x1 < x0:
+ for x in range(x1, x0):
+ self._curline[x] = self._color
+ elif x0 < x1:
+ for x in range(x0, x1):
+ self._curline[x] = self._color
+ self._curpos = x1
+ self._color = 1 - self._color
+
+ def _do_pass(self) -> None:
+ x1 = self._curpos + 1
+ while 1:
+ if x1 == 0:
+ if self._color == 1 and self._refline[x1] != self._color:
+ break
+ elif x1 == len(self._refline) or (
+ self._refline[x1 - 1] == self._color
+ and self._refline[x1] != self._color
+ ):
+ break
+ x1 += 1
+ while 1:
+ if x1 == 0:
+ if self._color == 0 and self._refline[x1] == self._color:
+ break
+ elif x1 == len(self._refline) or (
+ self._refline[x1 - 1] != self._color
+ and self._refline[x1] == self._color
+ ):
+ break
+ x1 += 1
+ for x in range(self._curpos, x1):
+ self._curline[x] = self._color
+ self._curpos = x1
+
+ def _do_horizontal(self, n1: int, n2: int) -> None:
+ if self._curpos < 0:
+ self._curpos = 0
+ x = self._curpos
+ for _ in range(n1):
+ if len(self._curline) <= x:
+ break
+ self._curline[x] = self._color
+ x += 1
+ for _ in range(n2):
+ if len(self._curline) <= x:
+ break
+ self._curline[x] = 1 - self._color
+ x += 1
+ self._curpos = x
+
+ def _do_uncompressed(self, bits: str) -> None:
+ for c in bits:
+ self._curline[self._curpos] = int(c)
+ self._curpos += 1
+ self._flush_line()
+
+
+class CCITTFaxDecoder(CCITTG4Parser):
+ def __init__(
+ self,
+ width: int,
+ bytealign: bool = False,
+ reversed: bool = False,
+ ) -> None:
+ CCITTG4Parser.__init__(self, width, bytealign=bytealign)
+ self.reversed = reversed
+ self._buf = b""
+
+ def close(self) -> bytes:
+ return self._buf
+
+ def output_line(self, y: int, bits: Sequence[int]) -> None:
+ arr = array.array("B", [0] * ((len(bits) + 7) // 8))
+ if self.reversed:
+ bits = [1 - b for b in bits]
+ for i, b in enumerate(bits):
+ if b:
+ arr[i // 8] += (128, 64, 32, 16, 8, 4, 2, 1)[i % 8]
+ self._buf += arr.tobytes()
+
+
+def ccittfaxdecode(data: bytes, params: dict[str, object]) -> bytes:
+ K = params.get("K")
+ if K == -1:
+ cols = cast(int, params.get("Columns"))
+ bytealign = cast(bool, params.get("EncodedByteAlign"))
+ reversed = cast(bool, params.get("BlackIs1"))
+ parser = CCITTFaxDecoder(cols, bytealign=bytealign, reversed=reversed)
+ else:
+ raise PDFValueError(K)
+ parser.feedbytes(data)
+ return parser.close()
+
+
+# test
+def main(argv: list[str]) -> None:
+ if not argv[1:]:
+ import unittest
+
+ unittest.main()
+ return
+
+ class Parser(CCITTG4Parser):
+ def __init__(self, width: int, bytealign: bool = False) -> None:
+ import pygame # type: ignore[import]
+
+ CCITTG4Parser.__init__(self, width, bytealign=bytealign)
+ self.img = pygame.Surface((self.width, 1000))
+
+ def output_line(self, y: int, bits: Sequence[int]) -> None:
+ for x, b in enumerate(bits):
+ if b:
+ self.img.set_at((x, y), (255, 255, 255))
+ else:
+ self.img.set_at((x, y), (0, 0, 0))
+
+ def close(self) -> None:
+ import pygame
+
+ pygame.image.save(self.img, "out.bmp")
+
+ for path in argv[1:]:
+ fp = open(path, "rb")
+ (_, _, k, w, h, _) = path.split(".")
+ parser = Parser(int(w))
+ parser.feedbytes(fp.read())
+ parser.close()
+ fp.close()
diff --git a/babeldoc/pdfminer/cmap/78-EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/78-EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..4711ed382b33770e39c85f434877af890320c4a5
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/78-EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7e1f9a0941e66f56d06bdd4b2f108237a66b5501b7fa4b5e9a09a96475457e4
+size 20532
diff --git a/babeldoc/pdfminer/cmap/78-EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/78-EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..42087ff0b92416aba148a04af1c5e3be588d5714
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/78-EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a547e21cce698d2a814cbed4aee94a46523f9a8fe8f0ca2fd1450173df6e98ec
+size 20551
diff --git a/babeldoc/pdfminer/cmap/78-H.pickle.gz b/babeldoc/pdfminer/cmap/78-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..0858338ed05e5f85cff93018e6787c8e7b1491d5
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/78-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09635f0a38adecb373e750ae64f483f1b6d45d064373eb3417b94b65f248c5d6
+size 19882
diff --git a/babeldoc/pdfminer/cmap/78-RKSJ-H.pickle.gz b/babeldoc/pdfminer/cmap/78-RKSJ-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..bb2c889b19ebbd1ccf1a7636ed2c0bc4ae8bab46
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/78-RKSJ-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ac22384b98196f02605357927367b88a9afd537b1504eb52a7b816733163a00
+size 22969
diff --git a/babeldoc/pdfminer/cmap/78-RKSJ-V.pickle.gz b/babeldoc/pdfminer/cmap/78-RKSJ-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e50db318eab2200450f8a946b3639b1b39a47e39
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/78-RKSJ-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:109240b21a496ff060dd7ededca78da5e3803157c3c0fb06aba17347ca58f200
+size 22990
diff --git a/babeldoc/pdfminer/cmap/78-V.pickle.gz b/babeldoc/pdfminer/cmap/78-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..639c8287f4de3c1fef1be14c4bb6c441ab9f7c50
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/78-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef2bbeb29d9e937f282b9129279ee39be3d3c29d0b062c1bc71a69214c687640
+size 19883
diff --git a/babeldoc/pdfminer/cmap/78ms-RKSJ-H.pickle.gz b/babeldoc/pdfminer/cmap/78ms-RKSJ-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..c3a0a1b7eae0f9ee49ff8a6625b94efb64a1a926
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/78ms-RKSJ-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ce2ed9bbca6e39026182320ea2b0e18a902c89534ebc0b6aaaf8f585e1749c4
+size 25942
diff --git a/babeldoc/pdfminer/cmap/78ms-RKSJ-V.pickle.gz b/babeldoc/pdfminer/cmap/78ms-RKSJ-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..032fefc0ca71ab7d5def10cae071e684ae591eb1
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/78ms-RKSJ-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c377b0eebdcd6460d732c45ceb3ec00d0b408834678c129c0e9ac0d52f4dec67
+size 25964
diff --git a/babeldoc/pdfminer/cmap/83pv-RKSJ-H.pickle.gz b/babeldoc/pdfminer/cmap/83pv-RKSJ-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..4117d567dc0c5dbc09fb27ec8dc55a38cbeaf30e
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/83pv-RKSJ-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a8b5267890592b9185a84cd43408d56690b93c7101bd0a9bf5a26a1f39d8031
+size 26305
diff --git a/babeldoc/pdfminer/cmap/83pv-RKSJ-V.pickle.gz b/babeldoc/pdfminer/cmap/83pv-RKSJ-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..586c425cc2116b75f1c92e977644f147f35a447f
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/83pv-RKSJ-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd273ac3ca018083886994691d3453327a0ddd87ebfe774e9b573a691f175407
+size 26305
diff --git a/babeldoc/pdfminer/cmap/90ms-RKSJ-H.pickle.gz b/babeldoc/pdfminer/cmap/90ms-RKSJ-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..1c082399cda49badbe61f1a0d6f4273101ab0b61
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/90ms-RKSJ-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5402ca275ff5cdb810186727547e3874b31461857ab48c26986c4130b5c3d9cf
+size 25732
diff --git a/babeldoc/pdfminer/cmap/90ms-RKSJ-V.pickle.gz b/babeldoc/pdfminer/cmap/90ms-RKSJ-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..7e91b75f647f7a5cc22e94288c9ed8273030a7f7
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/90ms-RKSJ-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67ec3b0d5565445492b12f7b5eab3017b84fd46af7736f5c331e54264bd2fb49
+size 25757
diff --git a/babeldoc/pdfminer/cmap/90msp-RKSJ-H.pickle.gz b/babeldoc/pdfminer/cmap/90msp-RKSJ-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..1310c4f5ad0e5a5d537e79326b48501d86a385d8
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/90msp-RKSJ-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a85bc8750cef621467aa476cd999e6bc5e66c42aaf1c74a08632e6c0f99955b3
+size 25670
diff --git a/babeldoc/pdfminer/cmap/90msp-RKSJ-V.pickle.gz b/babeldoc/pdfminer/cmap/90msp-RKSJ-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..cbe7188c4aaa63a7b03dacac0013384f6c6090ba
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/90msp-RKSJ-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48eb9737b2bb036499f1dbbb6821e74f218f5fdbff05825f2e18de55174165ec
+size 25688
diff --git a/babeldoc/pdfminer/cmap/90pv-RKSJ-H.pickle.gz b/babeldoc/pdfminer/cmap/90pv-RKSJ-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..2b4afd90279b10b5e6c7aca2e69834296f78ae5b
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/90pv-RKSJ-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c6463e60b4b24ba711c844c1d2abaee0d3e5f0f9452198ebeee161281a88f1
+size 24226
diff --git a/babeldoc/pdfminer/cmap/90pv-RKSJ-V.pickle.gz b/babeldoc/pdfminer/cmap/90pv-RKSJ-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e6dba4812602cd31144dcd726ae266a73f197a4c
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/90pv-RKSJ-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b1c838def87b351b1c16663465ffb1800977a274d842b5932ec64ceead32020
+size 24021
diff --git a/babeldoc/pdfminer/cmap/Add-H.pickle.gz b/babeldoc/pdfminer/cmap/Add-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..b4a0ecf67f94de08a80d009eed39a0b5a1563f70
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Add-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b2023beece24c938e9c6cb4c355be88d19b5b8b0dbd4dee64ef069910463441
+size 21027
diff --git a/babeldoc/pdfminer/cmap/Add-RKSJ-H.pickle.gz b/babeldoc/pdfminer/cmap/Add-RKSJ-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..224d3f5f078f2bfe428118d4fe2f7ff4d342f1ba
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Add-RKSJ-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:378003e40c4506327f325a0b4debda46934ca9035edded63acca56bc5576444b
+size 24275
diff --git a/babeldoc/pdfminer/cmap/Add-RKSJ-V.pickle.gz b/babeldoc/pdfminer/cmap/Add-RKSJ-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..bece88bbef0247274fbdd228f5010344e627a5f6
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Add-RKSJ-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:558157deeb898de99fe33281fdfacda5c95e256513a4ba44579a8862fad4b6d5
+size 24079
diff --git a/babeldoc/pdfminer/cmap/Add-V.pickle.gz b/babeldoc/pdfminer/cmap/Add-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..f0cc1870041ae2d4e7ec2c53c522796e4e8da0d3
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Add-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a5ab752aa4cd4acb12cee31a9b5650b917fd10fbe0a8b17aa9a57bcf065bb86
+size 20874
diff --git a/babeldoc/pdfminer/cmap/B5-H.pickle.gz b/babeldoc/pdfminer/cmap/B5-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d1028482cd8673cb422072ddc69b27e0126e72c3
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/B5-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:990e52c7459b0d9c3a6f3491611d1589af5980398e06ee7ed9b2f0de39dbfef4
+size 42594
diff --git a/babeldoc/pdfminer/cmap/B5-V.pickle.gz b/babeldoc/pdfminer/cmap/B5-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..808b6062c112b0ae5fc32a4b97bce600b36e898d
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/B5-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c330ad420434c68e27b27b16c4af7503a0d94b437ef8c4554356bb3b41eaf8a
+size 42549
diff --git a/babeldoc/pdfminer/cmap/B5pc-H.pickle.gz b/babeldoc/pdfminer/cmap/B5pc-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..36577c428872b42e4eba3f94a6a99d38083e31ce
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/B5pc-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01b777e8d2f67cb99c204e8792e810c2a4dc39760416810371d1fb6761940e16
+size 42602
diff --git a/babeldoc/pdfminer/cmap/B5pc-V.pickle.gz b/babeldoc/pdfminer/cmap/B5pc-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..bb700f226e065046e5b604bd75ebfa95edb1fdd0
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/B5pc-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dbfc4d888ccd5756275e455994f1a287288c4d8dfe24c845bd5a5209481022f
+size 42557
diff --git a/babeldoc/pdfminer/cmap/CNS-EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/CNS-EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..25e46e1dd05123ec780d071577d817ba09f880d3
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/CNS-EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7ae970c305e97bbc5d9a12e396fcf5d3dd53563d41be41e4b69ecbbc793b9e5
+size 56990
diff --git a/babeldoc/pdfminer/cmap/CNS-EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/CNS-EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e82ced5ec5488fefcdf08543a7fd7fd0b2a3049e
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/CNS-EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02f3d2e82617d952599548fcc0ea9f6d71c65c4709dcb96715f40901aca94491
+size 56943
diff --git a/babeldoc/pdfminer/cmap/CNS1-H.pickle.gz b/babeldoc/pdfminer/cmap/CNS1-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..32128a198a40b0172f0756eb658bdc1b6e9c9afa
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/CNS1-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c525b152e96b97a958b87a9fdc347039ec9188892a8d5d78e29f4543afc843e9
+size 17615
diff --git a/babeldoc/pdfminer/cmap/CNS1-V.pickle.gz b/babeldoc/pdfminer/cmap/CNS1-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..05ac4f72c91358df6cb4daee47741288c1f9f039
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/CNS1-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd870d3e08bf4900fefd2db701ccf1b742ea45d34c7af95a690689cec66164a7
+size 17564
diff --git a/babeldoc/pdfminer/cmap/CNS2-H.pickle.gz b/babeldoc/pdfminer/cmap/CNS2-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..064407ae470df052caf5d8f9e2508db4f039c44f
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/CNS2-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15500f341d9958d0f357ccd95da936564c2f46ec3f8f913a40ce365def58c119
+size 21723
diff --git a/babeldoc/pdfminer/cmap/CNS2-V.pickle.gz b/babeldoc/pdfminer/cmap/CNS2-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d4e58d40a6b5e1ae443fe79caca2bca28d3b575b
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/CNS2-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cbc6eab5d03a1e63239c33d316afa3b9c5832b74b4488f7408492347e578a29
+size 21723
diff --git a/babeldoc/pdfminer/cmap/ETHK-B5-H.pickle.gz b/babeldoc/pdfminer/cmap/ETHK-B5-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..966b322f28a87ecee1ffb830dfbea1dcac41ddaa
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/ETHK-B5-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b86cbe3eb115bf7aae0912686e882fb02c05f700c511a6d713a8b0b1e60df94
+size 59548
diff --git a/babeldoc/pdfminer/cmap/ETHK-B5-V.pickle.gz b/babeldoc/pdfminer/cmap/ETHK-B5-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..c96777a7fc4a1df7dd10ff6e4021deec54b1f8f7
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/ETHK-B5-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc6ac1269d57d1455f7e79d76bfe2aabc361a17952f06c3cfd8ca9afebaa60c3
+size 59481
diff --git a/babeldoc/pdfminer/cmap/ETen-B5-H.pickle.gz b/babeldoc/pdfminer/cmap/ETen-B5-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..03828083f2f0ed9674f03bf60b1bf120755386fa
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/ETen-B5-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:175b8e6478d641e17793c0c331f0d6ef60c2875eeec4a739135c742723acac98
+size 43982
diff --git a/babeldoc/pdfminer/cmap/ETen-B5-V.pickle.gz b/babeldoc/pdfminer/cmap/ETen-B5-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a31b594d86ce401660beb5daac62855634dd1bbe
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/ETen-B5-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02a8c5eb726baf1b9bb9313338d477c8ca200c4c06b2992e43dca6cbe7d44a12
+size 43924
diff --git a/babeldoc/pdfminer/cmap/ETenms-B5-H.pickle.gz b/babeldoc/pdfminer/cmap/ETenms-B5-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..0936c4ff54fda57d1d318dbe5de324b2bde95511
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/ETenms-B5-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:605d10c0da8336efe2294c7dcf7ac31d25af57b952b3484acfa6d94cb8faf2d0
+size 320
diff --git a/babeldoc/pdfminer/cmap/ETenms-B5-V.pickle.gz b/babeldoc/pdfminer/cmap/ETenms-B5-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..cd520e95096faf931b83350032efcb842a812475
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/ETenms-B5-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f5b39cb67547c61564883bbde8a7522eb4a284af0cee3b89fdff408630d5d5d
+size 438
diff --git a/babeldoc/pdfminer/cmap/EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..8db8db4c2bcbfe59aef7ec6d0eb99d3a49d6ae47
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f04d25667260f5c1ac7cc3f58cd0aa8f8fe59bd2fde50ce0e2bc69d31f7ac3d2
+size 20429
diff --git a/babeldoc/pdfminer/cmap/EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..5d21ecb71aa41009eba8473aa3e77ea326226a59
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c02eae4e48a2cdfbb1ef38beb447412f7b210280183824c9d452ed189599257c
+size 20455
diff --git a/babeldoc/pdfminer/cmap/Ext-H.pickle.gz b/babeldoc/pdfminer/cmap/Ext-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d59484b6e23084b4e94ecda2377584db042c3359
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Ext-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:607ffe278436167617315afe6f7b2ed4ca9a151dda18e0b8f669e208bd6abc6d
+size 22272
diff --git a/babeldoc/pdfminer/cmap/Ext-RKSJ-H.pickle.gz b/babeldoc/pdfminer/cmap/Ext-RKSJ-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..be21f6890bc7010f92dc395cfdda636bf97655c8
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Ext-RKSJ-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27f2e877ca024cbde2bebafa39d3064ce2e82c9ba805f518e9cd0fb3c70ecf49
+size 25721
diff --git a/babeldoc/pdfminer/cmap/Ext-RKSJ-V.pickle.gz b/babeldoc/pdfminer/cmap/Ext-RKSJ-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..fbd02b8561456ab5e25fff83b843d36092a46ffb
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Ext-RKSJ-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f48ef5c0cd83e4d188f4780616a0fa5fd816f173888501804bcc511618853932
+size 25750
diff --git a/babeldoc/pdfminer/cmap/Ext-V.pickle.gz b/babeldoc/pdfminer/cmap/Ext-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..52c945d1af4275c6d0617410c266836f586795b9
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Ext-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7695e7bee7d24f6922f408851a6bf7cf8a31dd099b103feb9650e37647ba74fc
+size 22307
diff --git a/babeldoc/pdfminer/cmap/GB-EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/GB-EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..598379b4390670f2d2034eb683e5c966d2dc07c6
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GB-EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f921365e809fd33fc10711772741edd164a9a3ebd8a0bcabaf618bcb83d20f62
+size 22118
diff --git a/babeldoc/pdfminer/cmap/GB-EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/GB-EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..ca4baacd420c0354cd40c4ad3f7e1d8c92f38e2d
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GB-EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9fea3f484a09b8ec3df9eaee3cd0dcb6c9890a18effeedd525d50551ce1cfb7
+size 22111
diff --git a/babeldoc/pdfminer/cmap/GB-H.pickle.gz b/babeldoc/pdfminer/cmap/GB-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..2635c38e4e6385e5c4fd1fb619a2dd23b2a62e40
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GB-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5a495fef1298e43404bd7e981438ed78fe84f00480c119e554409c8a270a61c
+size 21699
diff --git a/babeldoc/pdfminer/cmap/GB-V.pickle.gz b/babeldoc/pdfminer/cmap/GB-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..1d4a47b00a8fb04050f3d36c18f526acf40934d0
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GB-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f937d193a912656ea76be2c4f4a655e8deb586cce4787c399c692357d0cbf0a
+size 21694
diff --git a/babeldoc/pdfminer/cmap/GBK-EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/GBK-EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..3d55292935649f7cbf308fbe6a4c42250641ddd5
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBK-EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a8641fefbd6cf36216e4196433384d35d8d4fb3f2d167b3c1a1481968f30349
+size 68254
diff --git a/babeldoc/pdfminer/cmap/GBK-EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/GBK-EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..21988a4be67b30875f32e44c73bfdcd179f7bcd4
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBK-EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ead3d321f45f5d401b32121a4c0053f452f624a923293e858334430e788d5331
+size 68199
diff --git a/babeldoc/pdfminer/cmap/GBK2K-H.pickle.gz b/babeldoc/pdfminer/cmap/GBK2K-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d939d5bad6122b20a1d710c9ff9d948ac4ff7d1a
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBK2K-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c01baa0c788f3daaaff556834d4d5e302141ebb7ec3f26d9d911866660b1902f
+size 89917
diff --git a/babeldoc/pdfminer/cmap/GBK2K-V.pickle.gz b/babeldoc/pdfminer/cmap/GBK2K-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..69695d7be93677087dcde2234575618c737aac5d
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBK2K-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6408bfd672c1ae6e7795c72e5bb9afeea0d2510cd9447a21c36e2fe807895115
+size 89872
diff --git a/babeldoc/pdfminer/cmap/GBKp-EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/GBKp-EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..320fcb2d2aa2d13d2ad142b5c6f8f5d5e8393405
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBKp-EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3460acae8a9e40611bbc1182e4483582a76b04d4545319bd3f0a2a5bd83fadcf
+size 68148
diff --git a/babeldoc/pdfminer/cmap/GBKp-EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/GBKp-EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..eb0a7531d15321ba120f7d8099afc0551312e0db
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBKp-EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e39baf790b0d93e8504fc4d49bef08e831840b44b2aba2bad221bd3e2fa7f0a8
+size 68102
diff --git a/babeldoc/pdfminer/cmap/GBT-EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/GBT-EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..f5c5542e6c40fee7b6ce380fe6bbdeae142e62c0
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBT-EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5edc55d60e14dcaaf092f97eb64fea126693b104f7fe82fd1b435f25f33a7f7a
+size 23815
diff --git a/babeldoc/pdfminer/cmap/GBT-EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/GBT-EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..8378c1db98c962672f112f73e7a18a25dba5628a
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBT-EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:575cad798504695fdd390159235a6c4f754c4bc1c77c05817a9bdde3f05c89e4
+size 23806
diff --git a/babeldoc/pdfminer/cmap/GBT-H.pickle.gz b/babeldoc/pdfminer/cmap/GBT-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..2be1d8a23b98d858f8c289867cc1f1cdbe403fa2
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBT-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cb25fe3e632be0bfef4ba3c17c32ff61008b8aa269819c72af2165f9f774cf8
+size 23339
diff --git a/babeldoc/pdfminer/cmap/GBT-V.pickle.gz b/babeldoc/pdfminer/cmap/GBT-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..f8ce49dbb7c7756eb7032bd8297c99cedc0bd802
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBT-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ba16fb8a2f36664a1d6c6271f85bd1bd9e8f5018accb5c04a608db576344189
+size 23322
diff --git a/babeldoc/pdfminer/cmap/GBTpc-EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/GBTpc-EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..27c331adb879b83cc7563fd00f02485ca2d55497
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBTpc-EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c825575cf38352884b2919543911fa8580977dbe2b01d66139b209d2744c67c
+size 23650
diff --git a/babeldoc/pdfminer/cmap/GBTpc-EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/GBTpc-EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..5b6271a1a38a1901f6753606632690fe6e0f2bd1
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBTpc-EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae293868c474ae87f7a31e24940f98bff892ea977e689eea86f4a10fec412abc
+size 23647
diff --git a/babeldoc/pdfminer/cmap/GBpc-EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/GBpc-EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..58fce770b542be9b190020e7ab2904baa8cbdb5c
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBpc-EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:664d42d0377e588451132409f0b10c4c99497af41dc9abd85e48794308d44386
+size 21945
diff --git a/babeldoc/pdfminer/cmap/GBpc-EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/GBpc-EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..5cea78ab768fbcaa7625045a33407eec2c022f23
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/GBpc-EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa7badbe40d515b1cfeaf60158bf508a45f80d50414855fd18d03f371ba8f3a3
+size 21956
diff --git a/babeldoc/pdfminer/cmap/H.pickle.gz b/babeldoc/pdfminer/cmap/H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..18cddfaf2674b797a2b329b8eb87ee549493943b
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:650ef8348f3e2bb2a9e92250d79f0aed02b7301edf4d379578ea4d67ac02ca46
+size 19781
diff --git a/babeldoc/pdfminer/cmap/HKdla-B5-H.pickle.gz b/babeldoc/pdfminer/cmap/HKdla-B5-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..7697e5d1412909ae46f89d2f356c016d7b066cff
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKdla-B5-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6354699fe8b433c2ce539e652360cc7586378622d1163e50782d58a5e3e88943
+size 45212
diff --git a/babeldoc/pdfminer/cmap/HKdla-B5-V.pickle.gz b/babeldoc/pdfminer/cmap/HKdla-B5-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..f23b387c9195e52962861c4e59d06735f28785ec
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKdla-B5-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8309e0b1156f58911aa96eca0ede7bc0abdb9a3e0674855dc51380eb52b283d8
+size 45167
diff --git a/babeldoc/pdfminer/cmap/HKdlb-B5-H.pickle.gz b/babeldoc/pdfminer/cmap/HKdlb-B5-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..07ae3c6808fce6678d52291b79ce6561f4c5b01d
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKdlb-B5-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a95567ce70f8560f06d4301139cec9f048f975df0706d6956f4d856921be6d98
+size 44853
diff --git a/babeldoc/pdfminer/cmap/HKdlb-B5-V.pickle.gz b/babeldoc/pdfminer/cmap/HKdlb-B5-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a4d3a213b9ec3f0db55b35043a89f76c24ead453
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKdlb-B5-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b14223f89de638c4b4da3aee90719f596239e0b49cdf44a95c2d49fc38177e54
+size 44816
diff --git a/babeldoc/pdfminer/cmap/HKgccs-B5-H.pickle.gz b/babeldoc/pdfminer/cmap/HKgccs-B5-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..da8536247b34e64c9c88cb57149b470b12933bf1
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKgccs-B5-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aacafc532de1fb9868d156a711de1d6ce57718fc7ee1625afdcf628ba6b279f
+size 53104
diff --git a/babeldoc/pdfminer/cmap/HKgccs-B5-V.pickle.gz b/babeldoc/pdfminer/cmap/HKgccs-B5-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..34c9fd4ea2c8201c8b7b9289c1a7385c82d7caef
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKgccs-B5-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b133531f905dadd8227cb0179fb9eba066151f8f6d14bc40c629a18d9a7f944f
+size 53050
diff --git a/babeldoc/pdfminer/cmap/HKm314-B5-H.pickle.gz b/babeldoc/pdfminer/cmap/HKm314-B5-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..1662db64d40f9f0494fb1b233750187715727c4e
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKm314-B5-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bd57f53cbb5ed1e8b246d8048a8b019ffeb9b9c2006d6f5684edc2f1e4a7910
+size 43667
diff --git a/babeldoc/pdfminer/cmap/HKm314-B5-V.pickle.gz b/babeldoc/pdfminer/cmap/HKm314-B5-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..421332c43884a5f22384bfedd4690149cb762537
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKm314-B5-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dff77fdfdfda8bdc23785f0b0da060e2b39ee7dac590360d07813f2be7eb9172
+size 43618
diff --git a/babeldoc/pdfminer/cmap/HKm471-B5-H.pickle.gz b/babeldoc/pdfminer/cmap/HKm471-B5-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..cd603bb6f4ba95416cc7b0a5176670ac08b02d13
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKm471-B5-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:208bc0e81043047c152dbadf811fbec3f52ef3d2b7f9a25b5e3e728eb1c51dbe
+size 44187
diff --git a/babeldoc/pdfminer/cmap/HKm471-B5-V.pickle.gz b/babeldoc/pdfminer/cmap/HKm471-B5-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..ce50c6eb9cb01b4b3e8cbc0b12092bc417581e08
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKm471-B5-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19c95ab9b328fa3bfcc80f9776b1e76b26c6bb22269611b69918a643aa859cf2
+size 44144
diff --git a/babeldoc/pdfminer/cmap/HKscs-B5-H.pickle.gz b/babeldoc/pdfminer/cmap/HKscs-B5-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..c49c922c0d448c7198a7f65ab03e7534a7a7e0ee
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKscs-B5-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d1a3463baaf3ee810905937aae63599d211f8d275b37dd899fbce375e32ff2e
+size 59508
diff --git a/babeldoc/pdfminer/cmap/HKscs-B5-V.pickle.gz b/babeldoc/pdfminer/cmap/HKscs-B5-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..4d8360cf6a13d5bec8f5638b8a33ed9e8d79d734
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/HKscs-B5-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3535201facf28bbe553d262baea2f4e011946345eef1dd3f549db3ddb141a76c
+size 59473
diff --git a/babeldoc/pdfminer/cmap/Hankaku-H.pickle.gz b/babeldoc/pdfminer/cmap/Hankaku-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d2abc8b9b1dd803d1569d2d5ea2e6440ecff9a0e
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Hankaku-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fb3f131e6afd98d103f2a313e21c04a51421a46fb33f430781113a4aca11b53
+size 840
diff --git a/babeldoc/pdfminer/cmap/Hankaku-V.pickle.gz b/babeldoc/pdfminer/cmap/Hankaku-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..50238d06a14ec242e9363f6f78384a48ad369e03
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Hankaku-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b76bed506a5593a0c6e715e2e659b8e1b157decba02c426bbc39165e6a00d2a8
+size 839
diff --git a/babeldoc/pdfminer/cmap/Hiragana-H.pickle.gz b/babeldoc/pdfminer/cmap/Hiragana-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..72c94a11531f64274e4750203bcbfe24886507fb
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Hiragana-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad583a37dee55420e42020e4ac3628cc028975419383af393cc1f004700d0c7f
+size 391
diff --git a/babeldoc/pdfminer/cmap/Hiragana-V.pickle.gz b/babeldoc/pdfminer/cmap/Hiragana-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..8d3d1edf17e563bbabb6d24f7d42f5e58282c31b
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Hiragana-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3071b58cbf5f60c0b288c6d45a303bb259c55af87723ebb67dcaf41cb64ea1f
+size 391
diff --git a/babeldoc/pdfminer/cmap/KSC-EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/KSC-EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..9da34d4ed83d45bbd959ac81071fc4f1da93d008
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSC-EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b992f8c87eacac9de8459e21148fda28f325df7d307cfb0ed960505c8006090b
+size 24040
diff --git a/babeldoc/pdfminer/cmap/KSC-EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/KSC-EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..b41d7e7aa635588041b838bb800d931180a464e4
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSC-EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbb3dfabe2dc653fd3654dd64b7fb92dbbaf8fe198ef53fcfc15b3b197fa6817
+size 24078
diff --git a/babeldoc/pdfminer/cmap/KSC-H.pickle.gz b/babeldoc/pdfminer/cmap/KSC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..bb3dc7a2e1216e45a0b4cb3bb3a512cc8cb0f7a5
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6861fbeac9b5e8f1f26f745fe9139c8acd139753ad7de4c0bb0e5a203eec5f25
+size 23563
diff --git a/babeldoc/pdfminer/cmap/KSC-Johab-H.pickle.gz b/babeldoc/pdfminer/cmap/KSC-Johab-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..8bb1c185a55e902be60887551a75949e5c25e3e6
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSC-Johab-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8fe203384cfb07ac8906ba74f0cd850396980b9699b2e1ff9c3b7218443feb6
+size 55016
diff --git a/babeldoc/pdfminer/cmap/KSC-Johab-V.pickle.gz b/babeldoc/pdfminer/cmap/KSC-Johab-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d27e77e24999b2772a869f50ee66f70a160d98d5
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSC-Johab-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be98bd71b56eb9069cbfd4da52c9d16307577940361ee07ffa78b5993410f992
+size 55041
diff --git a/babeldoc/pdfminer/cmap/KSC-V.pickle.gz b/babeldoc/pdfminer/cmap/KSC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..2a255e3550775475fef85b27c53c3f7ef278c9cb
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21c15a7836958dd2346dda0fa47649d6a25e8c58afed82acaa02504723a522be
+size 23644
diff --git a/babeldoc/pdfminer/cmap/KSCms-UHC-H.pickle.gz b/babeldoc/pdfminer/cmap/KSCms-UHC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..21194f8c183cba5e8331d224caa18efb3c911aec
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSCms-UHC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e965d4a27bd5931fc77b646a129f0459bf4f855017346cd39b6df7410655695
+size 51667
diff --git a/babeldoc/pdfminer/cmap/KSCms-UHC-HW-H.pickle.gz b/babeldoc/pdfminer/cmap/KSCms-UHC-HW-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..697d3481f0860c4d295af91cf2d1e69775182868
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSCms-UHC-HW-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b54853e5cd6c30f96f941954fc1a078ad8b628b6dd012fea15c3e99ece402c23
+size 51788
diff --git a/babeldoc/pdfminer/cmap/KSCms-UHC-HW-V.pickle.gz b/babeldoc/pdfminer/cmap/KSCms-UHC-HW-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d0a9384d93dc0cf153c9f89faa26c0c726ccbf64
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSCms-UHC-HW-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56dc639520f4e129e8b9eccb0ede9c3f30a7de49540256007199a554226c5717
+size 51821
diff --git a/babeldoc/pdfminer/cmap/KSCms-UHC-V.pickle.gz b/babeldoc/pdfminer/cmap/KSCms-UHC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..18f4db2768c4541a855629d0fcd09bf58cedd4ff
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSCms-UHC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e745b52cb1973599ff8d3e39c6df2a6a92413ccb3c9831ae3cfdb9136798c40e
+size 51698
diff --git a/babeldoc/pdfminer/cmap/KSCpc-EUC-H.pickle.gz b/babeldoc/pdfminer/cmap/KSCpc-EUC-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..551688b481dd5df07a1507a49748a89be6083456
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSCpc-EUC-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:103dc40d5cb38aed5e990cc3cfd894ba1593ab94cce2803f48c6d55364111789
+size 27769
diff --git a/babeldoc/pdfminer/cmap/KSCpc-EUC-V.pickle.gz b/babeldoc/pdfminer/cmap/KSCpc-EUC-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..139c33baec4b548ddbcf2c9ea38e937c137cfa54
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/KSCpc-EUC-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3c983d6c66ab0a1de779bbaa8b16e8280bcf75df54db4c41f4f3367bac5eeee
+size 27820
diff --git a/babeldoc/pdfminer/cmap/Katakana-H.pickle.gz b/babeldoc/pdfminer/cmap/Katakana-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..ddab9bcd0fd8ab1f4936711bf0c130387cf0d404
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Katakana-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0081a9501b40879c63d6a276f90bca4ec0d6b05cc284d876e9a42849b75f6de6
+size 404
diff --git a/babeldoc/pdfminer/cmap/Katakana-V.pickle.gz b/babeldoc/pdfminer/cmap/Katakana-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e44ba15bd097be0a0950f73853ca4609e71ced59
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Katakana-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1aeaa3820d5b9dd6af61bed21690d5edd2d8e68d3f32810a515d72c11ba97146
+size 404
diff --git a/babeldoc/pdfminer/cmap/NWP-H.pickle.gz b/babeldoc/pdfminer/cmap/NWP-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..9ea8f0edfa727bd2b951603932d5c489101fb2b1
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/NWP-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8b04990c741e36eea61eb5f8d13bc99f5c85e3a74250c24ca0a5717f6da48d0
+size 21708
diff --git a/babeldoc/pdfminer/cmap/NWP-V.pickle.gz b/babeldoc/pdfminer/cmap/NWP-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..6d7723c461ae680f3279f736dfbcc1fe3f5adc6d
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/NWP-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6c5fff860411fd8bd0e867f017cb30dfebbe43383a5d40856dea7273b3b61e4
+size 21779
diff --git a/babeldoc/pdfminer/cmap/RKSJ-H.pickle.gz b/babeldoc/pdfminer/cmap/RKSJ-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e0e42dc7391b6c3a43e303ebfa15dc772166f029
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/RKSJ-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04eebe180a76d4cbf9bb30766f3f6e6a34bd405e9f3d2489a4a224b6ea8b4183
+size 23030
diff --git a/babeldoc/pdfminer/cmap/RKSJ-V.pickle.gz b/babeldoc/pdfminer/cmap/RKSJ-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d145a07f8ba9ed656c70574315d5fcbfa1a57b0c
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/RKSJ-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e82162f5e17a530d30ba4286e8cfe5fd7b06c15bb24bd35c24fffb0c0bfe6fae
+size 23048
diff --git a/babeldoc/pdfminer/cmap/Roman-H.pickle.gz b/babeldoc/pdfminer/cmap/Roman-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..97bcd47f4e51da55fe727438ed4fd5ee4eedfc0b
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Roman-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c14560f556fddfb5ce63969202d0dadf945e1d299c83b4a2d0f63bc32b621ae5
+size 394
diff --git a/babeldoc/pdfminer/cmap/Roman-V.pickle.gz b/babeldoc/pdfminer/cmap/Roman-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..f6b5ddf1534927015340ba04ea58305871aa650a
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/Roman-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c384ecad95c1d84f8217fb6d45e691f2dd6d32868ea376e1cdab5b5cadd3473
+size 394
diff --git a/babeldoc/pdfminer/cmap/UniCNS-UCS2-H.pickle.gz b/babeldoc/pdfminer/cmap/UniCNS-UCS2-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..ac43c8c2d2c0afde7e6a7af8438e6187fd326628
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniCNS-UCS2-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bfcba21348cd2f285da716de8cb8caba8792eb7c0d9f9e2d58e188c6a4fc540
+size 67459
diff --git a/babeldoc/pdfminer/cmap/UniCNS-UCS2-V.pickle.gz b/babeldoc/pdfminer/cmap/UniCNS-UCS2-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d3e03707c116ef39db889d69d1890c3d8b86669a
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniCNS-UCS2-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2ff40a0f62dcb9fa0870b762a4ae3df9d17a5a934d647adcf60bc87b9b6a9ac
+size 67395
diff --git a/babeldoc/pdfminer/cmap/UniCNS-UTF16-H.pickle.gz b/babeldoc/pdfminer/cmap/UniCNS-UTF16-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..18b0720e5153bfffc3e1a68dc58cf35557bc4c29
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniCNS-UTF16-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d046fd093e6e1c2faacc55d76f7e10004e604dc5c0d563a72798af9880b178f
+size 87819
diff --git a/babeldoc/pdfminer/cmap/UniCNS-UTF16-V.pickle.gz b/babeldoc/pdfminer/cmap/UniCNS-UTF16-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..8c6f3af9e3d3b0078c55ef9d59893ef23222b5e9
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniCNS-UTF16-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbe9d89a138223615d0ab393a02b17a503e2e234a8e9459830918cc2b1ae3c95
+size 87751
diff --git a/babeldoc/pdfminer/cmap/UniCNS-UTF32-H.pickle.gz b/babeldoc/pdfminer/cmap/UniCNS-UTF32-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..80c2cb78f78d376ec8f8a5fd84d6f4a09daeffe4
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniCNS-UTF32-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:deb724b20dbd952b6bd815f5c34c49ea09213699051abf6763616e67efc38cc1
+size 87400
diff --git a/babeldoc/pdfminer/cmap/UniCNS-UTF32-V.pickle.gz b/babeldoc/pdfminer/cmap/UniCNS-UTF32-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..7b034ebdd6968ef6c55f9d939f97e802b64a0f9b
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniCNS-UTF32-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:771a07682b349a6e3bf7940ea76f167fce13f8d48e843e2ef56c83c36563bc13
+size 87327
diff --git a/babeldoc/pdfminer/cmap/UniCNS-UTF8-H.pickle.gz b/babeldoc/pdfminer/cmap/UniCNS-UTF8-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d191f62cf6abbca34153dbd9a08fdd5c6c835474
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniCNS-UTF8-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a23c0066b14500c5fd93d6a96fcf4d00757766caf710532f8c272c6064be5ab3
+size 82631
diff --git a/babeldoc/pdfminer/cmap/UniCNS-UTF8-V.pickle.gz b/babeldoc/pdfminer/cmap/UniCNS-UTF8-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..5ae4322bf213caef2a11547e389fe8bbc4c1e4e6
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniCNS-UTF8-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb8aa3e0f692167dd234aab1c598997d822533f3aea0e61474d466c71c100ad5
+size 82562
diff --git a/babeldoc/pdfminer/cmap/UniGB-UCS2-H.pickle.gz b/babeldoc/pdfminer/cmap/UniGB-UCS2-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..3ba0c06db1ae793e107b5544251aafa64ebb5c99
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniGB-UCS2-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e33a654702dd5648e88e108f69cebf93995f31dbebfc17d92adc1e85949dd99
+size 97445
diff --git a/babeldoc/pdfminer/cmap/UniGB-UCS2-V.pickle.gz b/babeldoc/pdfminer/cmap/UniGB-UCS2-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..286b14aa3d334c48f2aa6bd6a2658c156da9a665
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniGB-UCS2-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a68b1f20918f68c3f122da4ed1f6b1dc26e4e71d0c5665c8501d82e4928ad956
+size 97441
diff --git a/babeldoc/pdfminer/cmap/UniGB-UTF16-H.pickle.gz b/babeldoc/pdfminer/cmap/UniGB-UTF16-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..5da3a242f118049e0e09ebdd4ede4b27e659dfd0
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniGB-UTF16-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d896f38caffb620c3b649c481e2296ad7863cb5352cf9c5b050476bb95c408f
+size 101459
diff --git a/babeldoc/pdfminer/cmap/UniGB-UTF16-V.pickle.gz b/babeldoc/pdfminer/cmap/UniGB-UTF16-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..9abc0818ed96f33408150d26a50e21cfb15a9c83
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniGB-UTF16-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:233a16fc3fa5b9cb68dc878fa7fef4fb75ecf29ddedd26e6c4030c098ac150ad
+size 101331
diff --git a/babeldoc/pdfminer/cmap/UniGB-UTF32-H.pickle.gz b/babeldoc/pdfminer/cmap/UniGB-UTF32-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..4f3480c03bc7e30503ad1699e63617fb3287a560
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniGB-UTF32-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bfd4d0fd838ccb6e061733d14bb6bcdcfc6785f9eebbd1aa1b44b26751e5629
+size 101490
diff --git a/babeldoc/pdfminer/cmap/UniGB-UTF32-V.pickle.gz b/babeldoc/pdfminer/cmap/UniGB-UTF32-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..15485e28b10a1dc6e8f1c68fc916a885fa3e32c0
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniGB-UTF32-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd299c9335fc05622b5f4b81ebbd003a4c4e610a14423b64a97447c5ab02ae2c
+size 101357
diff --git a/babeldoc/pdfminer/cmap/UniGB-UTF8-H.pickle.gz b/babeldoc/pdfminer/cmap/UniGB-UTF8-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..64dfac06e0e300b2512d4483d8ca033639531a05
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniGB-UTF8-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:115e481de49dd6e1f28c883f8e6f84c8b44d2c41b4e34ce945d344537e05e36d
+size 90500
diff --git a/babeldoc/pdfminer/cmap/UniGB-UTF8-V.pickle.gz b/babeldoc/pdfminer/cmap/UniGB-UTF8-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..0f76418648623fb3d7c1529a3db80cdb6efb37ab
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniGB-UTF8-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a494d40f5c4696ebc4711337a5c9c75a29c5709a1bce273e74bcb96e60eafb8
+size 90368
diff --git a/babeldoc/pdfminer/cmap/UniJIS-UCS2-H.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS-UCS2-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..73d0038db74cceaf732b40e2e5adfe08a87524b1
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS-UCS2-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:690f47bb8b1088a90eb725aaec46d0ae4e49514d34631b39781b4e02092da10b
+size 35934
diff --git a/babeldoc/pdfminer/cmap/UniJIS-UCS2-HW-H.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS-UCS2-HW-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..ea4c491a17d27f1ba4b930fea924967f3cc3b577
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS-UCS2-HW-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a99b50e9af8664f1355a9c11e854a0688572c60ceda7c98a59bd41f186023484
+size 412
diff --git a/babeldoc/pdfminer/cmap/UniJIS-UCS2-HW-V.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS-UCS2-HW-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..aaa25697745840d91c34d068a4860c07af1141c3
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS-UCS2-HW-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:267cffbeb41b759d691dce7e9358d1a044be8d13c1b3a6f18a9f2f134751a12d
+size 1402
diff --git a/babeldoc/pdfminer/cmap/UniJIS-UCS2-V.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS-UCS2-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..c923fb3e9ff089b0ec23cd53f1cce7a07b5cc0a2
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS-UCS2-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fea3900be9885098c8f0ecffd6364b0ca74925d47e8ab8e2e0b403b01983afe8
+size 35852
diff --git a/babeldoc/pdfminer/cmap/UniJIS-UTF16-H.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS-UTF16-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..6e3ca82d0d61230d443f707645f2a7c9428e576a
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS-UTF16-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:082896892df1150bd18a57e2322279dcfa03b5feabb2363b55617ba6a4d96004
+size 58054
diff --git a/babeldoc/pdfminer/cmap/UniJIS-UTF16-V.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS-UTF16-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..b357ddf7ee3052fbd3c86c6365a4476bf1dbf86f
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS-UTF16-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:befaa6c0e17d9bc3319e3c57b10208b19dd9dbadc4b426eb7edb7004e5bf5a4e
+size 57928
diff --git a/babeldoc/pdfminer/cmap/UniJIS-UTF32-H.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS-UTF32-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..50fcd2ce626eb2524258a38c49c5df1996192794
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS-UTF32-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f101cb6b0ae66721a3092ad23f324b29d9c4e05ce114b7a2825facd807eb9cd
+size 57910
diff --git a/babeldoc/pdfminer/cmap/UniJIS-UTF32-V.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS-UTF32-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..15e023596e1f6e6622e6c40f24ae08ef1950adcb
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS-UTF32-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ea53885e8ecea681bf391555474ad88351bcb84c75bb3259ddf83f46830b840
+size 57780
diff --git a/babeldoc/pdfminer/cmap/UniJIS-UTF8-H.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS-UTF8-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..64b1ce5f40bfccc9777bff1a43759e22d4331eda
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS-UTF8-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab8d525e876ea6422a73ecb30c0d545e9356df96ac6a4d915954f8c0b0d38050
+size 54764
diff --git a/babeldoc/pdfminer/cmap/UniJIS-UTF8-V.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS-UTF8-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..be1481e1fbf8f5ce534b0a8222a7afb43d7f9aed
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS-UTF8-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56aa1dafdd67a1989dc060b3ff29f1f671e8d12a5027545cd0467ba52b093b2a
+size 54684
diff --git a/babeldoc/pdfminer/cmap/UniJIS2004-UTF16-H.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS2004-UTF16-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e0216ad766e5eb23c79a65995ddc37f0da60c12b
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS2004-UTF16-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a04fdfd8ebba642fdc5a55cd11c415a875676e2a6916079824b7ce94addf189
+size 58081
diff --git a/babeldoc/pdfminer/cmap/UniJIS2004-UTF16-V.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS2004-UTF16-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..54e6e6cffaf80367532299c7ff529a1015cb13e1
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS2004-UTF16-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88c5045b22659f6e0d7668c6c06b9a3092ef81e019a65c990dda5af40b6e476e
+size 57960
diff --git a/babeldoc/pdfminer/cmap/UniJIS2004-UTF32-H.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS2004-UTF32-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..7a801df6829942f9afd561916701d916fb955743
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS2004-UTF32-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:810e1c4181a525a24e8f1579bf634b8cbf7c5ced3d95807f62730a3acd51d3de
+size 57940
diff --git a/babeldoc/pdfminer/cmap/UniJIS2004-UTF32-V.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS2004-UTF32-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..b3c9db558521c67737cb2310ba44882fe94a828c
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS2004-UTF32-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dd4e1cead3842c72f812e5c1bdb52a1c3fcbbbc8eef0be40442d704ba8b29af
+size 57811
diff --git a/babeldoc/pdfminer/cmap/UniJIS2004-UTF8-H.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS2004-UTF8-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..0cf2f6fcd2b68a5cbc0a079a2ea0344f99d4ee4b
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS2004-UTF8-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdaa8d7268922da4ada59953bda61f68cd57db0514f8fffdcda4061da5afb0bc
+size 54829
diff --git a/babeldoc/pdfminer/cmap/UniJIS2004-UTF8-V.pickle.gz b/babeldoc/pdfminer/cmap/UniJIS2004-UTF8-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d078740a6020b7e3e09d3126fc5dff3c5e3f2477
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJIS2004-UTF8-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2325aac634a5eb34e7b0658688eeb9bd3eec6376847cdb2d960776a17988aae9
+size 54749
diff --git a/babeldoc/pdfminer/cmap/UniJISX0213-UTF32-H.pickle.gz b/babeldoc/pdfminer/cmap/UniJISX0213-UTF32-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..f813eef3f221eee5c31f600f49f5ad070653a2af
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJISX0213-UTF32-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c05c04521e3825be9ca57a9b146276ef1798fde28efa774d6c6bdc66114918c
+size 57903
diff --git a/babeldoc/pdfminer/cmap/UniJISX0213-UTF32-V.pickle.gz b/babeldoc/pdfminer/cmap/UniJISX0213-UTF32-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..8edeacf76b855fef39cffe3bbbd43c72600967c7
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJISX0213-UTF32-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c5ecec5f93f2d787e8dcd87982b1d69cbd01260a5233119a62753100a417c66
+size 57778
diff --git a/babeldoc/pdfminer/cmap/UniJISX02132004-UTF32-H.pickle.gz b/babeldoc/pdfminer/cmap/UniJISX02132004-UTF32-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e34f7ab479299be02d3562c4c960ff3f6a13a19d
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJISX02132004-UTF32-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9015d0f18abed4b7116c6f40a0b3cbd656fa3368a7ae0418cddb2ca8eaf8d30
+size 57930
diff --git a/babeldoc/pdfminer/cmap/UniJISX02132004-UTF32-V.pickle.gz b/babeldoc/pdfminer/cmap/UniJISX02132004-UTF32-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..26d6d1f9e55e1f97ba2f4c733c81659ec0953a35
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniJISX02132004-UTF32-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3e27416c17ad70e5baf8c9689232416ff9a97e00281ecda65fa095c5d444351
+size 57808
diff --git a/babeldoc/pdfminer/cmap/UniKS-UCS2-H.pickle.gz b/babeldoc/pdfminer/cmap/UniKS-UCS2-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..2f50bd27fd86d48d1c1e0741625707d1ac6b922e
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniKS-UCS2-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25b2f6225f6578a6efa887abc7c9692c0d10b0f7e80cfdf3b2fc0a3b2436eef1
+size 60683
diff --git a/babeldoc/pdfminer/cmap/UniKS-UCS2-V.pickle.gz b/babeldoc/pdfminer/cmap/UniKS-UCS2-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..2aa9e6a249f86ebc8e3a79790a3721f3893b4830
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniKS-UCS2-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84e14f40840c8f0f568503ea73ef9d6db710e70fa72615d1fda409697c683cbd
+size 60699
diff --git a/babeldoc/pdfminer/cmap/UniKS-UTF16-H.pickle.gz b/babeldoc/pdfminer/cmap/UniKS-UTF16-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..d8652637354566019155924c772cbf9ff4aa080d
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniKS-UTF16-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e48c092ea05b663d4450a1568a6d28963e3009609748a6fac225d00aaa8e338f
+size 61278
diff --git a/babeldoc/pdfminer/cmap/UniKS-UTF16-V.pickle.gz b/babeldoc/pdfminer/cmap/UniKS-UTF16-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..63a3fd0c03ca761129f545f4d8a1e572149633c8
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniKS-UTF16-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:522c012d97ad399f9e886d50269458f04dd880140898f318ad0824a09679a023
+size 61298
diff --git a/babeldoc/pdfminer/cmap/UniKS-UTF32-H.pickle.gz b/babeldoc/pdfminer/cmap/UniKS-UTF32-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a29d3e65edc2572fee339d7ab8fe96e4b74eb120
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniKS-UTF32-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a9ad44f1b07e802d6275320f6bcb948f74738114c0649d0e45e243c4196421f
+size 61286
diff --git a/babeldoc/pdfminer/cmap/UniKS-UTF32-V.pickle.gz b/babeldoc/pdfminer/cmap/UniKS-UTF32-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..48d054d0912d4f71eb41254c73599a456380efcf
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniKS-UTF32-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f67bdc3b7983f570d7cdfdbf8969f09673ee077a1a3ff0575764adc6d4a0575
+size 61309
diff --git a/babeldoc/pdfminer/cmap/UniKS-UTF8-H.pickle.gz b/babeldoc/pdfminer/cmap/UniKS-UTF8-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..baee723eccb1680570d03d2c8afae71aebc757b4
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniKS-UTF8-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:986e8ad693f4f6ad16180908f5c019a17ccbd83a77fd432de34ac8e49d6f5a45
+size 54151
diff --git a/babeldoc/pdfminer/cmap/UniKS-UTF8-V.pickle.gz b/babeldoc/pdfminer/cmap/UniKS-UTF8-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..453159e5cf2471ad864f7b93c09ed19f525959ca
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/UniKS-UTF8-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddb8953b9135dd43bf7fb4f7b30d71769870c1a3a2d9915e42da1a2150807357
+size 54172
diff --git a/babeldoc/pdfminer/cmap/V.pickle.gz b/babeldoc/pdfminer/cmap/V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..4f705c854dbb0ffd76cd206e121f2c04b2fad883
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97a9c3f8c875fb8a5a5951f469f425a902237314cea487a88e2943fb383cc4c4
+size 19826
diff --git a/babeldoc/pdfminer/cmap/WP-Symbol-H.pickle.gz b/babeldoc/pdfminer/cmap/WP-Symbol-H.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..57f8ba10e138fdbb76bd2bba869d52c6112f2069
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/WP-Symbol-H.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:452d3f14d51578b90a0475949f028751be09b6c0f781e0c0f3259c97b3cf9946
+size 505
diff --git a/babeldoc/pdfminer/cmap/WP-Symbol-V.pickle.gz b/babeldoc/pdfminer/cmap/WP-Symbol-V.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..58cf39a4fbf41c4a800d711e43b75d7e710d0301
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/WP-Symbol-V.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:060fde272d9aeb6b28a190675e5a42db2c3f1f58695ae00eeeced415903d9bc4
+size 505
diff --git a/babeldoc/pdfminer/cmap/to-unicode-Adobe-CNS1.pickle.gz b/babeldoc/pdfminer/cmap/to-unicode-Adobe-CNS1.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..90e2865eaa29cee939c430becc0798b52afd272d
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/to-unicode-Adobe-CNS1.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089f95c2fa56447e68bb0b5619bb492b491198fd2ec5ea6cace097694142de68
+size 138237
diff --git a/babeldoc/pdfminer/cmap/to-unicode-Adobe-GB1.pickle.gz b/babeldoc/pdfminer/cmap/to-unicode-Adobe-GB1.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..681a3ba3d61c11f7af62f9b5f6540411da7bd8f8
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/to-unicode-Adobe-GB1.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06b3f8be1fab4fb8e5dd48d330337fd4c31292729b0a244137982d2521e4d30d
+size 204425
diff --git a/babeldoc/pdfminer/cmap/to-unicode-Adobe-Japan1.pickle.gz b/babeldoc/pdfminer/cmap/to-unicode-Adobe-Japan1.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..21f82a581e4667079bfd96ce50a1bbcab084300d
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/to-unicode-Adobe-Japan1.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de4e3d1dbe4e220f6406f31270af731b7650e8db2a5631018e6efd092384d053
+size 112987
diff --git a/babeldoc/pdfminer/cmap/to-unicode-Adobe-Korea1.pickle.gz b/babeldoc/pdfminer/cmap/to-unicode-Adobe-Korea1.pickle.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a563a0d9ca8460c5a87b38d31f96225e16f111ec
--- /dev/null
+++ b/babeldoc/pdfminer/cmap/to-unicode-Adobe-Korea1.pickle.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d728c8f6a2cfa3644fe3658d2847781001efef47c8feb75ead4ff3f021f309e
+size 120859
diff --git a/babeldoc/pdfminer/cmapdb.py b/babeldoc/pdfminer/cmapdb.py
new file mode 100644
index 0000000000000000000000000000000000000000..a7794c33e9862f48e4cf75e66acd6d147ef7d29d
--- /dev/null
+++ b/babeldoc/pdfminer/cmapdb.py
@@ -0,0 +1,472 @@
+"""Adobe character mapping (CMap) support.
+
+CMaps provide the mapping between character codes and Unicode
+code-points to character ids (CIDs).
+
+More information is available on:
+
+ https://github.com/adobe-type-tools/cmap-resources
+
+"""
+
+import gzip
+import logging
+import os
+import os.path
+import pickle as pickle
+import struct
+import sys
+from collections.abc import Iterable
+from collections.abc import Iterator
+from collections.abc import MutableMapping
+from typing import Any
+from typing import BinaryIO
+from typing import TextIO
+from typing import cast
+
+from babeldoc.pdfminer.encodingdb import name2unicode
+from babeldoc.pdfminer.pdfexceptions import PDFException
+from babeldoc.pdfminer.pdfexceptions import PDFTypeError
+from babeldoc.pdfminer.psexceptions import PSEOF
+from babeldoc.pdfminer.psexceptions import PSSyntaxError
+from babeldoc.pdfminer.psparser import KWD
+from babeldoc.pdfminer.psparser import PSKeyword
+from babeldoc.pdfminer.psparser import PSLiteral
+from babeldoc.pdfminer.psparser import PSStackParser
+from babeldoc.pdfminer.psparser import literal_name
+from babeldoc.pdfminer.utils import choplist
+from babeldoc.pdfminer.utils import nunpack
+
+log = logging.getLogger(__name__)
+
+
+class CMapError(PDFException):
+ pass
+
+
+class CMapBase:
+ debug = 0
+
+ def __init__(self, **kwargs: object) -> None:
+ self.attrs: MutableMapping[str, object] = kwargs.copy()
+
+ def is_vertical(self) -> bool:
+ return self.attrs.get("WMode", 0) != 0
+
+ def set_attr(self, k: str, v: object) -> None:
+ self.attrs[k] = v
+
+ def add_code2cid(self, code: str, cid: int) -> None:
+ pass
+
+ def add_cid2unichr(self, cid: int, code: PSLiteral | bytes | int) -> None:
+ pass
+
+ def use_cmap(self, cmap: "CMapBase") -> None:
+ pass
+
+ def decode(self, code: bytes) -> Iterable[int]:
+ raise NotImplementedError
+
+
+class CMap(CMapBase):
+ def __init__(self, **kwargs: str | int) -> None:
+ CMapBase.__init__(self, **kwargs)
+ self.code2cid: dict[int, object] = {}
+
+ def __repr__(self) -> str:
+ return "" % self.attrs.get("CMapName")
+
+ def use_cmap(self, cmap: CMapBase) -> None:
+ assert isinstance(cmap, CMap), str(type(cmap))
+
+ def copy(dst: dict[int, object], src: dict[int, object]) -> None:
+ for k, v in src.items():
+ if isinstance(v, dict):
+ d: dict[int, object] = {}
+ dst[k] = d
+ copy(d, v)
+ else:
+ dst[k] = v
+
+ copy(self.code2cid, cmap.code2cid)
+
+ def decode(self, code: bytes) -> Iterator[int]:
+ log.debug("decode: %r, %r", self, code)
+ d = self.code2cid
+ for i in iter(code):
+ if i in d:
+ x = d[i]
+ if isinstance(x, int):
+ yield x
+ d = self.code2cid
+ else:
+ d = cast(dict[int, object], x)
+ else:
+ d = self.code2cid
+
+ def dump(
+ self,
+ out: TextIO = sys.stdout,
+ code2cid: dict[int, object] | None = None,
+ code: tuple[int, ...] = (),
+ ) -> None:
+ if code2cid is None:
+ code2cid = self.code2cid
+ code = ()
+ for k, v in sorted(code2cid.items()):
+ c = code + (k,)
+ if isinstance(v, int):
+ out.write("code %r = cid %d\n" % (c, v))
+ else:
+ self.dump(out=out, code2cid=cast(dict[int, object], v), code=c)
+
+
+class IdentityCMap(CMapBase):
+ def decode(self, code: bytes) -> tuple[int, ...]:
+ n = len(code) // 2
+ if n:
+ return struct.unpack_from(f">{n}H", code)
+ else:
+ return ()
+
+
+class IdentityCMapByte(IdentityCMap):
+ def decode(self, code: bytes) -> tuple[int, ...]:
+ n = len(code)
+ if n:
+ return struct.unpack(">%dB" % n, code)
+ else:
+ return ()
+
+
+class UnicodeMap(CMapBase):
+ def __init__(self, **kwargs: str | int) -> None:
+ CMapBase.__init__(self, **kwargs)
+ self.cid2unichr: dict[int, str] = {}
+
+ def __repr__(self) -> str:
+ return "" % self.attrs.get("CMapName")
+
+ def get_unichr(self, cid: int) -> str:
+ log.debug("get_unichr: %r, %r", self, cid)
+ return self.cid2unichr[cid]
+
+ def dump(self, out: TextIO = sys.stdout) -> None:
+ for k, v in sorted(self.cid2unichr.items()):
+ out.write("cid %d = unicode %r\n" % (k, v))
+
+
+class IdentityUnicodeMap(UnicodeMap):
+ def get_unichr(self, cid: int) -> str:
+ """Interpret character id as unicode codepoint"""
+ log.debug("get_unichr: %r, %r", self, cid)
+ return chr(cid)
+
+
+class FileCMap(CMap):
+ def add_code2cid(self, code: str, cid: int) -> None:
+ assert isinstance(code, str) and isinstance(cid, int), str(
+ (type(code), type(cid)),
+ )
+ d = self.code2cid
+ for c in code[:-1]:
+ ci = ord(c)
+ if ci in d:
+ d = cast(dict[int, object], d[ci])
+ else:
+ t: dict[int, object] = {}
+ d[ci] = t
+ d = t
+ ci = ord(code[-1])
+ d[ci] = cid
+
+
+class FileUnicodeMap(UnicodeMap):
+ def add_cid2unichr(self, cid: int, code: PSLiteral | bytes | int) -> None:
+ assert isinstance(cid, int), str(type(cid))
+ if isinstance(code, PSLiteral):
+ # Interpret as an Adobe glyph name.
+ assert isinstance(code.name, str)
+ unichr = name2unicode(code.name)
+ elif isinstance(code, bytes):
+ # Interpret as UTF-16BE.
+ unichr = code.decode("UTF-16BE", "ignore")
+ elif isinstance(code, int):
+ unichr = chr(code)
+ else:
+ raise PDFTypeError(code)
+
+ # A0 = non-breaking space, some weird fonts can have a collision on a cid here.
+ if unichr == "\u00a0" and self.cid2unichr.get(cid) == " ":
+ return
+ self.cid2unichr[cid] = unichr
+
+
+class PyCMap(CMap):
+ def __init__(self, name: str, module: Any) -> None:
+ super().__init__(CMapName=name)
+ self.code2cid = module.CODE2CID
+ if module.IS_VERTICAL:
+ self.attrs["WMode"] = 1
+
+
+class PyUnicodeMap(UnicodeMap):
+ def __init__(self, name: str, module: Any, vertical: bool) -> None:
+ super().__init__(CMapName=name)
+ if vertical:
+ self.cid2unichr = module.CID2UNICHR_V
+ self.attrs["WMode"] = 1
+ else:
+ self.cid2unichr = module.CID2UNICHR_H
+
+
+class CMapDB:
+ _cmap_cache: dict[str, PyCMap] = {}
+ _umap_cache: dict[str, list[PyUnicodeMap]] = {}
+
+ class CMapNotFound(CMapError):
+ pass
+
+ @classmethod
+ def _load_data(cls, name: str) -> Any:
+ name = name.replace("\0", "")
+ filename = "%s.pickle.gz" % name
+ log.debug("loading: %r", name)
+ cmap_paths = (
+ os.environ.get("CMAP_PATH", "/usr/share/pdfminer/"),
+ os.path.join(os.path.dirname(__file__), "cmap"),
+ )
+ for directory in cmap_paths:
+ path = os.path.join(directory, filename)
+ if os.path.exists(path):
+ gzfile = gzip.open(path)
+ try:
+ return type(str(name), (), pickle.loads(gzfile.read()))
+ finally:
+ gzfile.close()
+ raise CMapDB.CMapNotFound(name)
+
+ @classmethod
+ def get_cmap(cls, name: str) -> CMapBase:
+ if name == "Identity-H":
+ return IdentityCMap(WMode=0)
+ elif name == "Identity-V":
+ return IdentityCMap(WMode=1)
+ elif name == "OneByteIdentityH":
+ return IdentityCMapByte(WMode=0)
+ elif name == "OneByteIdentityV":
+ return IdentityCMapByte(WMode=1)
+ try:
+ return cls._cmap_cache[name]
+ except KeyError:
+ pass
+ data = cls._load_data(name)
+ cls._cmap_cache[name] = cmap = PyCMap(name, data)
+ return cmap
+
+ @classmethod
+ def get_unicode_map(cls, name: str, vertical: bool = False) -> UnicodeMap:
+ try:
+ return cls._umap_cache[name][vertical]
+ except KeyError:
+ pass
+ data = cls._load_data("to-unicode-%s" % name)
+ cls._umap_cache[name] = [PyUnicodeMap(name, data, v) for v in (False, True)]
+ return cls._umap_cache[name][vertical]
+
+
+class CMapParser(PSStackParser[PSKeyword]):
+ def __init__(self, cmap: CMapBase, fp: BinaryIO) -> None:
+ PSStackParser.__init__(self, fp)
+ self.cmap = cmap
+ # some ToUnicode maps don't have "begincmap" keyword.
+ self._in_cmap = True
+ self._warnings: set[str] = set()
+
+ def run(self) -> None:
+ try:
+ self.nextobject()
+ except PSEOF:
+ pass
+
+ KEYWORD_BEGINCMAP = KWD(b"begincmap")
+ KEYWORD_ENDCMAP = KWD(b"endcmap")
+ KEYWORD_USECMAP = KWD(b"usecmap")
+ KEYWORD_DEF = KWD(b"def")
+ KEYWORD_BEGINCODESPACERANGE = KWD(b"begincodespacerange")
+ KEYWORD_ENDCODESPACERANGE = KWD(b"endcodespacerange")
+ KEYWORD_BEGINCIDRANGE = KWD(b"begincidrange")
+ KEYWORD_ENDCIDRANGE = KWD(b"endcidrange")
+ KEYWORD_BEGINCIDCHAR = KWD(b"begincidchar")
+ KEYWORD_ENDCIDCHAR = KWD(b"endcidchar")
+ KEYWORD_BEGINBFRANGE = KWD(b"beginbfrange")
+ KEYWORD_ENDBFRANGE = KWD(b"endbfrange")
+ KEYWORD_BEGINBFCHAR = KWD(b"beginbfchar")
+ KEYWORD_ENDBFCHAR = KWD(b"endbfchar")
+ KEYWORD_BEGINNOTDEFRANGE = KWD(b"beginnotdefrange")
+ KEYWORD_ENDNOTDEFRANGE = KWD(b"endnotdefrange")
+
+ def do_keyword(self, pos: int, token: PSKeyword) -> None:
+ """ToUnicode CMaps
+
+ See Section 5.9.2 - ToUnicode CMaps of the PDF Reference.
+ """
+ if token is self.KEYWORD_BEGINCMAP:
+ self._in_cmap = True
+ self.popall()
+ return
+
+ elif token is self.KEYWORD_ENDCMAP:
+ self._in_cmap = False
+ return
+
+ if not self._in_cmap:
+ return
+
+ if token is self.KEYWORD_DEF:
+ try:
+ ((_, k), (_, v)) = self.pop(2)
+ self.cmap.set_attr(literal_name(k), v)
+ except PSSyntaxError:
+ pass
+ return
+
+ if token is self.KEYWORD_USECMAP:
+ try:
+ ((_, cmapname),) = self.pop(1)
+ self.cmap.use_cmap(CMapDB.get_cmap(literal_name(cmapname)))
+ except PSSyntaxError:
+ pass
+ except CMapDB.CMapNotFound:
+ pass
+ return
+
+ if token is self.KEYWORD_BEGINCODESPACERANGE:
+ self.popall()
+ return
+ if token is self.KEYWORD_ENDCODESPACERANGE:
+ self.popall()
+ return
+
+ if token is self.KEYWORD_BEGINCIDRANGE:
+ self.popall()
+ return
+
+ if token is self.KEYWORD_ENDCIDRANGE:
+ objs = [obj for (__, obj) in self.popall()]
+ for start_byte, end_byte, cid in choplist(3, objs):
+ if not isinstance(start_byte, bytes):
+ self._warn_once("The start object of begincidrange is not a byte.")
+ continue
+ if not isinstance(end_byte, bytes):
+ self._warn_once("The end object of begincidrange is not a byte.")
+ continue
+ if not isinstance(cid, int):
+ self._warn_once("The cid object of begincidrange is not a byte.")
+ continue
+ if len(start_byte) != len(end_byte):
+ self._warn_once(
+ "The start and end byte of begincidrange have "
+ "different lengths.",
+ )
+ continue
+ start_prefix = start_byte[:-4]
+ end_prefix = end_byte[:-4]
+ if start_prefix != end_prefix:
+ self._warn_once(
+ "The prefix of the start and end byte of "
+ "begincidrange are not the same.",
+ )
+ continue
+ svar = start_byte[-4:]
+ evar = end_byte[-4:]
+ start = nunpack(svar)
+ end = nunpack(evar)
+ vlen = len(svar)
+ for i in range(end - start + 1):
+ x = start_prefix + struct.pack(">L", start + i)[-vlen:]
+ self.cmap.add_cid2unichr(cid + i, x)
+ return
+
+ if token is self.KEYWORD_BEGINCIDCHAR:
+ self.popall()
+ return
+
+ if token is self.KEYWORD_ENDCIDCHAR:
+ objs = [obj for (__, obj) in self.popall()]
+ for cid, code in choplist(2, objs):
+ if isinstance(code, bytes) and isinstance(cid, int):
+ self.cmap.add_cid2unichr(cid, code)
+ return
+
+ if token is self.KEYWORD_BEGINBFRANGE:
+ self.popall()
+ return
+
+ if token is self.KEYWORD_ENDBFRANGE:
+ objs = [obj for (__, obj) in self.popall()]
+ for start_byte, end_byte, code in choplist(3, objs):
+ if not isinstance(start_byte, bytes):
+ self._warn_once("The start object is not a byte.")
+ continue
+ if not isinstance(end_byte, bytes):
+ self._warn_once("The end object is not a byte.")
+ continue
+ if len(start_byte) != len(end_byte):
+ self._warn_once("The start and end byte have different lengths.")
+ continue
+ start = nunpack(start_byte)
+ end = nunpack(end_byte)
+ if isinstance(code, list):
+ if len(code) != end - start + 1:
+ self._warn_once(
+ "The difference between the start and end "
+ "offsets does not match the code length.",
+ )
+ for cid, unicode_value in zip(
+ range(start, end + 1), code, strict=False
+ ):
+ self.cmap.add_cid2unichr(cid, unicode_value)
+ else:
+ assert isinstance(code, bytes)
+ var = code[-4:]
+ base = nunpack(var)
+ prefix = code[:-4]
+ vlen = len(var)
+ for i in range(end - start + 1):
+ x = prefix + struct.pack(">L", base + i)[-vlen:]
+ self.cmap.add_cid2unichr(start + i, x)
+ return
+
+ if token is self.KEYWORD_BEGINBFCHAR:
+ self.popall()
+ return
+
+ if token is self.KEYWORD_ENDBFCHAR:
+ objs = [obj for (__, obj) in self.popall()]
+ for cid, code in choplist(2, objs):
+ if isinstance(cid, bytes) and isinstance(code, bytes):
+ self.cmap.add_cid2unichr(nunpack(cid), code)
+ return
+
+ if token is self.KEYWORD_BEGINNOTDEFRANGE:
+ self.popall()
+ return
+
+ if token is self.KEYWORD_ENDNOTDEFRANGE:
+ self.popall()
+ return
+
+ self.push((pos, token))
+
+ def _warn_once(self, msg: str) -> None:
+ """Warn once for each unique message"""
+ if msg not in self._warnings:
+ self._warnings.add(msg)
+ base_msg = (
+ "Ignoring (part of) ToUnicode map because the PDF data "
+ "does not conform to the format. This could result in "
+ "(cid) values in the output. "
+ )
+ log.warning(base_msg + msg)
diff --git a/babeldoc/pdfminer/converter.py b/babeldoc/pdfminer/converter.py
new file mode 100644
index 0000000000000000000000000000000000000000..b66b8572ad34fb965a6c996d9048aceb1a04fba7
--- /dev/null
+++ b/babeldoc/pdfminer/converter.py
@@ -0,0 +1,1062 @@
+import io
+import logging
+import re
+from collections.abc import Sequence
+from typing import BinaryIO
+from typing import Generic
+from typing import TextIO
+from typing import TypeVar
+from typing import cast
+
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.pdfminer.image import ImageWriter
+from babeldoc.pdfminer.layout import LAParams
+from babeldoc.pdfminer.layout import LTAnno
+from babeldoc.pdfminer.layout import LTChar
+from babeldoc.pdfminer.layout import LTComponent
+from babeldoc.pdfminer.layout import LTContainer
+from babeldoc.pdfminer.layout import LTCurve
+from babeldoc.pdfminer.layout import LTFigure
+from babeldoc.pdfminer.layout import LTImage
+from babeldoc.pdfminer.layout import LTItem
+from babeldoc.pdfminer.layout import LTLayoutContainer
+from babeldoc.pdfminer.layout import LTLine
+from babeldoc.pdfminer.layout import LTPage
+from babeldoc.pdfminer.layout import LTRect
+from babeldoc.pdfminer.layout import LTText
+from babeldoc.pdfminer.layout import LTTextBox
+from babeldoc.pdfminer.layout import LTTextBoxVertical
+from babeldoc.pdfminer.layout import LTTextGroup
+from babeldoc.pdfminer.layout import LTTextLine
+from babeldoc.pdfminer.layout import TextGroupElement
+from babeldoc.pdfminer.pdfcolor import PDFColorSpace
+from babeldoc.pdfminer.pdfdevice import PDFTextDevice
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+from babeldoc.pdfminer.pdffont import PDFFont
+from babeldoc.pdfminer.pdffont import PDFUnicodeNotDefined
+from babeldoc.pdfminer.pdfinterp import PDFGraphicState
+from babeldoc.pdfminer.pdfinterp import PDFResourceManager
+from babeldoc.pdfminer.pdfpage import PDFPage
+from babeldoc.pdfminer.pdftypes import PDFStream
+from babeldoc.pdfminer.utils import AnyIO
+from babeldoc.pdfminer.utils import Matrix
+from babeldoc.pdfminer.utils import PathSegment
+from babeldoc.pdfminer.utils import Point
+from babeldoc.pdfminer.utils import Rect
+from babeldoc.pdfminer.utils import apply_matrix_pt
+from babeldoc.pdfminer.utils import bbox2str
+from babeldoc.pdfminer.utils import enc
+from babeldoc.pdfminer.utils import make_compat_str
+from babeldoc.pdfminer.utils import mult_matrix
+from babeldoc.pdfminer import utils
+
+log = logging.getLogger(__name__)
+
+
+class PDFLayoutAnalyzer(PDFTextDevice):
+ cur_item: LTLayoutContainer
+ ctm: Matrix
+
+ def __init__(
+ self,
+ rsrcmgr: PDFResourceManager,
+ pageno: int = 1,
+ laparams: LAParams | None = None,
+ ) -> None:
+ PDFTextDevice.__init__(self, rsrcmgr)
+ self.pageno = pageno
+ self.laparams = laparams
+ self._stack: list[LTLayoutContainer] = []
+
+ def begin_page(self, page: PDFPage, ctm: Matrix) -> None:
+ (x0, y0, x1, y1) = page.mediabox
+ (x0, y0) = apply_matrix_pt(ctm, (x0, y0))
+ (x1, y1) = apply_matrix_pt(ctm, (x1, y1))
+ mediabox = (0, 0, abs(x0 - x1), abs(y0 - y1))
+ self.cur_item = LTPage(self.pageno, mediabox)
+
+ def end_page(self, page: PDFPage) -> None:
+ assert not self._stack, str(len(self._stack))
+ assert isinstance(self.cur_item, LTPage), str(type(self.cur_item))
+ if self.laparams is not None:
+ self.cur_item.analyze(self.laparams)
+ self.pageno += 1
+ self.receive_layout(self.cur_item)
+
+ def begin_figure(self, name: str, bbox: Rect, matrix: Matrix) -> None:
+ self._stack.append(self.cur_item)
+ self.cur_item = LTFigure(name, bbox, mult_matrix(matrix, self.ctm))
+
+ def end_figure(self, _: str) -> None:
+ fig = self.cur_item
+ assert isinstance(self.cur_item, LTFigure), str(type(self.cur_item))
+ self.cur_item = self._stack.pop()
+ self.cur_item.add(fig)
+
+ def render_image(self, name: str, stream: PDFStream) -> None:
+ assert isinstance(self.cur_item, LTFigure), str(type(self.cur_item))
+ item = LTImage(
+ name,
+ stream,
+ (self.cur_item.x0, self.cur_item.y0, self.cur_item.x1, self.cur_item.y1),
+ )
+ self.cur_item.add(item)
+
+ def paint_path(
+ self,
+ gstate: PDFGraphicState,
+ stroke: bool,
+ fill: bool,
+ evenodd: bool,
+ path: Sequence[PathSegment],
+ ) -> None:
+ """Paint paths described in section 4.4 of the PDF reference manual"""
+ shape = "".join(x[0] for x in path)
+ current_clip_paths = self.il_creater.current_clip_paths.copy()
+ if shape[:1] != "m":
+ # Per PDF Reference Section 4.4.1, "path construction operators may
+ # be invoked in any sequence, but the first one invoked must be m
+ # or re to begin a new subpath." Since pdfminer.six already
+ # converts all `re` (rectangle) operators to their equivelent
+ # `mlllh` representation, paths ingested by `.paint_path(...)` that
+ # do not begin with the `m` operator are invalid.
+ pass
+
+ # elif shape.count("m") > 1:
+ # # recurse if there are multiple m's in this shape
+ # for m in re.finditer(r"m[^m]+", shape):
+ # subpath = path[m.start(0) : m.end(0)]
+ # self.paint_path(gstate, stroke, fill, evenodd, subpath)
+
+ else:
+ # Although the 'h' command does not not literally provide a
+ # point-position, its position is (by definition) equal to the
+ # subpath's starting point.
+ #
+ # And, per Section 4.4's Table 4.9, all other path commands place
+ # their point-position in their final two arguments. (Any preceding
+ # arguments represent control points on Bézier curves.)
+ raw_pts = [
+ cast(Point, p[-2:] if p[0] != "h" else path[0][-2:]) for p in path
+ ]
+ pts = [apply_matrix_pt(self.ctm, pt) for pt in raw_pts]
+
+ operators = [str(operation[0]) for operation in path]
+ transformed_points = [
+ [
+ apply_matrix_pt(self.ctm, (float(operand1), float(operand2)))
+ for operand1, operand2 in zip(
+ operation[1::2], operation[2::2], strict=False
+ )
+ ]
+ for operation in path
+ ]
+ transformed_path = [
+ cast(PathSegment, (o, *p))
+ for o, p in zip(operators, transformed_points, strict=False)
+ ]
+
+ # Drop a redundant "l" on a path closed with "h"
+ if len(shape) > 3 and shape[-2:] == "lh" and pts[-2] == pts[0]:
+ shape = shape[:-2] + "h"
+ pts.pop()
+
+ passthrough_instruction = (
+ self.il_creater.passthrough_per_char_instruction.copy()
+ )
+ xobj_id = self.il_creater.xobj_id
+ if shape in {"mlh", "ml"}:
+ # single line segment
+ #
+ # Note: 'ml', in conditional above, is a frequent anomaly
+ # that we want to support.
+ line = LTLine(
+ gstate.linewidth,
+ pts[0],
+ pts[1],
+ stroke,
+ fill,
+ evenodd,
+ gstate.scolor,
+ gstate.ncolor,
+ original_path=transformed_path,
+ dashing_style=gstate.dash,
+ )
+ line.passthrough_instruction = passthrough_instruction
+ line.xobj_id = xobj_id
+ line.render_order = self.il_creater.get_render_order_and_increase()
+ line.ctm = self.ctm
+ line.raw_path = path.copy()
+ line.clip_paths = current_clip_paths
+ self.cur_item.add(line)
+
+ elif shape in {"mlllh", "mllll"}:
+ (x0, y0), (x1, y1), (x2, y2), (x3, y3), _ = pts
+
+ is_closed_loop = pts[0] == pts[4]
+ has_square_coordinates = (
+ x0 == x1 and y1 == y2 and x2 == x3 and y3 == y0
+ ) or (y0 == y1 and x1 == x2 and y2 == y3 and x3 == x0)
+ if is_closed_loop and has_square_coordinates:
+ rect = LTRect(
+ gstate.linewidth,
+ (*pts[0], *pts[2]),
+ stroke,
+ fill,
+ evenodd,
+ gstate.scolor,
+ gstate.ncolor,
+ transformed_path,
+ gstate.dash,
+ )
+ rect.passthrough_instruction = passthrough_instruction
+ rect.xobj_id = xobj_id
+ rect.render_order = self.il_creater.get_render_order_and_increase()
+ rect.ctm = self.ctm
+ rect.raw_path = path.copy()
+ rect.clip_paths = current_clip_paths
+ self.cur_item.add(rect)
+ else:
+ curve = LTCurve(
+ gstate.linewidth,
+ pts,
+ stroke,
+ fill,
+ evenodd,
+ gstate.scolor,
+ gstate.ncolor,
+ transformed_path,
+ gstate.dash,
+ )
+ curve.passthrough_instruction = passthrough_instruction
+ curve.xobj_id = xobj_id
+ curve.render_order = self.il_creater.get_render_order_and_increase()
+ curve.ctm = self.ctm
+ curve.raw_path = path.copy()
+ curve.clip_paths = current_clip_paths
+ self.cur_item.add(curve)
+ else:
+ curve = LTCurve(
+ gstate.linewidth,
+ pts,
+ stroke,
+ fill,
+ evenodd,
+ gstate.scolor,
+ gstate.ncolor,
+ transformed_path,
+ gstate.dash,
+ )
+ curve.passthrough_instruction = passthrough_instruction
+ curve.xobj_id = xobj_id
+ curve.render_order = self.il_creater.get_render_order_and_increase()
+ curve.ctm = self.ctm
+ curve.raw_path = path.copy()
+ curve.clip_paths = current_clip_paths
+ self.cur_item.add(curve)
+
+ def render_char(
+ self,
+ matrix: Matrix,
+ font: PDFFont,
+ fontsize: float,
+ scaling: float,
+ rise: float,
+ cid: int,
+ ncs: PDFColorSpace,
+ graphicstate: PDFGraphicState,
+ ) -> float:
+ try:
+ text = font.to_unichr(cid)
+ assert isinstance(text, str), str(type(text))
+ except PDFUnicodeNotDefined:
+ text = self.handle_undefined_char(font, cid)
+ textwidth = font.char_width(cid)
+ textdisp = font.char_disp(cid)
+ item = LTChar(
+ matrix,
+ font,
+ fontsize,
+ scaling,
+ rise,
+ text,
+ textwidth,
+ textdisp,
+ ncs,
+ graphicstate,
+ )
+ self.cur_item.add(item)
+ return item.adv
+
+ def handle_undefined_char(self, font: PDFFont, cid: int) -> str:
+ log.debug("undefined: %r, %r", font, cid)
+ return "(cid:%d)" % cid
+
+ def receive_layout(self, ltpage: LTPage) -> None:
+ pass
+
+
+class PDFPageAggregator(PDFLayoutAnalyzer):
+ def __init__(
+ self,
+ rsrcmgr: PDFResourceManager,
+ pageno: int = 1,
+ laparams: LAParams | None = None,
+ ) -> None:
+ PDFLayoutAnalyzer.__init__(self, rsrcmgr, pageno=pageno, laparams=laparams)
+ self.result: LTPage | None = None
+
+ def receive_layout(self, ltpage: LTPage) -> None:
+ self.result = ltpage
+
+ def get_result(self) -> LTPage:
+ assert self.result is not None
+ return self.result
+
+
+# Some PDFConverter children support only binary I/O
+IOType = TypeVar("IOType", TextIO, BinaryIO, AnyIO)
+
+
+class PDFConverter(PDFLayoutAnalyzer, Generic[IOType]):
+ def __init__(
+ self,
+ rsrcmgr: PDFResourceManager,
+ outfp: IOType,
+ codec: str = "utf-8",
+ pageno: int = 1,
+ laparams: LAParams | None = None,
+ ) -> None:
+ PDFLayoutAnalyzer.__init__(self, rsrcmgr, pageno=pageno, laparams=laparams)
+ self.outfp: IOType = outfp
+ self.codec = codec
+ self.outfp_binary = self._is_binary_stream(self.outfp)
+
+ @staticmethod
+ def _is_binary_stream(outfp: AnyIO) -> bool:
+ """Test if an stream is binary or not"""
+ if "b" in getattr(outfp, "mode", ""):
+ return True
+ elif hasattr(outfp, "mode"):
+ # output stream has a mode, but it does not contain 'b'
+ return False
+ elif isinstance(outfp, io.BytesIO):
+ return True
+ elif isinstance(outfp, io.StringIO) or isinstance(outfp, io.TextIOBase):
+ return False
+
+ return True
+
+
+class TextConverter(PDFConverter[AnyIO]):
+ def __init__(
+ self,
+ rsrcmgr: PDFResourceManager,
+ outfp: AnyIO,
+ codec: str = "utf-8",
+ pageno: int = 1,
+ laparams: LAParams | None = None,
+ showpageno: bool = False,
+ imagewriter: ImageWriter | None = None,
+ ) -> None:
+ super().__init__(rsrcmgr, outfp, codec=codec, pageno=pageno, laparams=laparams)
+ self.showpageno = showpageno
+ self.imagewriter = imagewriter
+
+ def write_text(self, text: str) -> None:
+ text = utils.compatible_encode_method(text, self.codec, "ignore")
+ if self.outfp_binary:
+ cast(BinaryIO, self.outfp).write(text.encode())
+ else:
+ cast(TextIO, self.outfp).write(text)
+
+ def receive_layout(self, ltpage: LTPage) -> None:
+ def render(item: LTItem) -> None:
+ if isinstance(item, LTContainer):
+ for child in item:
+ render(child)
+ elif isinstance(item, LTText):
+ self.write_text(item.get_text())
+ if isinstance(item, LTTextBox):
+ self.write_text("\n")
+ elif isinstance(item, LTImage):
+ if self.imagewriter is not None:
+ self.imagewriter.export_image(item)
+
+ if self.showpageno:
+ self.write_text("Page %s\n" % ltpage.pageid)
+ render(ltpage)
+ self.write_text("\f")
+
+ # Some dummy functions to save memory/CPU when all that is wanted
+ # is text. This stops all the image and drawing output from being
+ # recorded and taking up RAM.
+ def render_image(self, name: str, stream: PDFStream) -> None:
+ if self.imagewriter is not None:
+ PDFConverter.render_image(self, name, stream)
+
+ def paint_path(
+ self,
+ gstate: PDFGraphicState,
+ stroke: bool,
+ fill: bool,
+ evenodd: bool,
+ path: Sequence[PathSegment],
+ ) -> None:
+ pass
+
+
+class HTMLConverter(PDFConverter[AnyIO]):
+ RECT_COLORS = {
+ "figure": "yellow",
+ "textline": "magenta",
+ "textbox": "cyan",
+ "textgroup": "red",
+ "curve": "black",
+ "page": "gray",
+ }
+
+ TEXT_COLORS = {
+ "textbox": "blue",
+ "char": "black",
+ }
+
+ def __init__(
+ self,
+ rsrcmgr: PDFResourceManager,
+ outfp: AnyIO,
+ codec: str = "utf-8",
+ pageno: int = 1,
+ laparams: LAParams | None = None,
+ scale: float = 1,
+ fontscale: float = 1.0,
+ layoutmode: str = "normal",
+ showpageno: bool = True,
+ pagemargin: int = 50,
+ imagewriter: ImageWriter | None = None,
+ debug: int = 0,
+ rect_colors: dict[str, str] | None = None,
+ text_colors: dict[str, str] | None = None,
+ ) -> None:
+ PDFConverter.__init__(
+ self,
+ rsrcmgr,
+ outfp,
+ codec=codec,
+ pageno=pageno,
+ laparams=laparams,
+ )
+
+ # write() assumes a codec for binary I/O, or no codec for text I/O.
+ if self.outfp_binary and not self.codec:
+ raise PDFValueError("Codec is required for a binary I/O output")
+ if not self.outfp_binary and self.codec:
+ raise PDFValueError("Codec must not be specified for a text I/O output")
+
+ if text_colors is None:
+ text_colors = {"char": "black"}
+ if rect_colors is None:
+ rect_colors = {"curve": "black", "page": "gray"}
+
+ self.scale = scale
+ self.fontscale = fontscale
+ self.layoutmode = layoutmode
+ self.showpageno = showpageno
+ self.pagemargin = pagemargin
+ self.imagewriter = imagewriter
+ self.rect_colors = rect_colors
+ self.text_colors = text_colors
+ if debug:
+ self.rect_colors.update(self.RECT_COLORS)
+ self.text_colors.update(self.TEXT_COLORS)
+ self._yoffset: float = self.pagemargin
+ self._font: tuple[str, float] | None = None
+ self._fontstack: list[tuple[str, float] | None] = []
+ self.write_header()
+
+ def write(self, text: str) -> None:
+ if self.codec:
+ cast(BinaryIO, self.outfp).write(text.encode(self.codec))
+ else:
+ cast(TextIO, self.outfp).write(text)
+
+ def write_header(self) -> None:
+ self.write("\n")
+ if self.codec:
+ s = (
+ '\n' % self.codec
+ )
+ else:
+ s = '\n'
+ self.write(s)
+ self.write("\n")
+
+ def write_footer(self) -> None:
+ page_links = [f'{i}' for i in range(1, self.pageno)]
+ s = 'Page: %s
\n' % ", ".join(
+ page_links,
+ )
+ self.write(s)
+ self.write("\n")
+
+ def write_text(self, text: str) -> None:
+ self.write(enc(text))
+
+ def place_rect(
+ self,
+ color: str,
+ borderwidth: int,
+ x: float,
+ y: float,
+ w: float,
+ h: float,
+ ) -> None:
+ color2 = self.rect_colors.get(color)
+ if color2 is not None:
+ s = (
+ '\n'
+ % (
+ color2,
+ borderwidth,
+ x * self.scale,
+ (self._yoffset - y) * self.scale,
+ w * self.scale,
+ h * self.scale,
+ )
+ )
+ self.write(s)
+
+ def place_border(self, color: str, borderwidth: int, item: LTComponent) -> None:
+ self.place_rect(color, borderwidth, item.x0, item.y1, item.width, item.height)
+
+ def place_image(
+ self,
+ item: LTImage,
+ borderwidth: int,
+ x: float,
+ y: float,
+ w: float,
+ h: float,
+ ) -> None:
+ if self.imagewriter is not None:
+ name = self.imagewriter.export_image(item)
+ s = (
+ '
\n'
+ % (
+ enc(name),
+ borderwidth,
+ x * self.scale,
+ (self._yoffset - y) * self.scale,
+ w * self.scale,
+ h * self.scale,
+ )
+ )
+ self.write(s)
+
+ def place_text(
+ self,
+ color: str,
+ text: str,
+ x: float,
+ y: float,
+ size: float,
+ ) -> None:
+ color2 = self.text_colors.get(color)
+ if color2 is not None:
+ s = (
+ ''
+ % (
+ color2,
+ x * self.scale,
+ (self._yoffset - y) * self.scale,
+ size * self.scale * self.fontscale,
+ )
+ )
+ self.write(s)
+ self.write_text(text)
+ self.write("\n")
+
+ def begin_div(
+ self,
+ color: str,
+ borderwidth: int,
+ x: float,
+ y: float,
+ w: float,
+ h: float,
+ writing_mode: str = "False",
+ ) -> None:
+ self._fontstack.append(self._font)
+ self._font = None
+ s = (
+ ''
+ % (
+ color,
+ borderwidth,
+ writing_mode,
+ x * self.scale,
+ (self._yoffset - y) * self.scale,
+ w * self.scale,
+ h * self.scale,
+ )
+ )
+ self.write(s)
+
+ def end_div(self, color: str) -> None:
+ if self._font is not None:
+ self.write("")
+ self._font = self._fontstack.pop()
+ self.write("
")
+
+ def put_text(self, text: str, fontname: str, fontsize: float) -> None:
+ font = (fontname, fontsize)
+ if font != self._font:
+ if self._font is not None:
+ self.write("")
+ # Remove subset tag from fontname, see PDF Reference 5.5.3
+ fontname_without_subset_tag = fontname.split("+")[-1]
+ self.write(
+ ''
+ % (fontname_without_subset_tag, fontsize * self.scale * self.fontscale),
+ )
+ self._font = font
+ self.write_text(text)
+
+ def put_newline(self) -> None:
+ self.write("
")
+
+ def receive_layout(self, ltpage: LTPage) -> None:
+ def show_group(item: LTTextGroup | TextGroupElement) -> None:
+ if isinstance(item, LTTextGroup):
+ self.place_border("textgroup", 1, item)
+ for child in item:
+ show_group(child)
+
+ def render(item: LTItem) -> None:
+ child: LTItem
+ if isinstance(item, LTPage):
+ self._yoffset += item.y1
+ self.place_border("page", 1, item)
+ if self.showpageno:
+ self.write(
+ '\n',
+ )
+ for child in item:
+ render(child)
+ if item.groups is not None:
+ for group in item.groups:
+ show_group(group)
+ elif isinstance(item, LTCurve):
+ self.place_border("curve", 1, item)
+ elif isinstance(item, LTFigure):
+ self.begin_div("figure", 1, item.x0, item.y1, item.width, item.height)
+ for child in item:
+ render(child)
+ self.end_div("figure")
+ elif isinstance(item, LTImage):
+ self.place_image(item, 1, item.x0, item.y1, item.width, item.height)
+ elif self.layoutmode == "exact":
+ if isinstance(item, LTTextLine):
+ self.place_border("textline", 1, item)
+ for child in item:
+ render(child)
+ elif isinstance(item, LTTextBox):
+ self.place_border("textbox", 1, item)
+ self.place_text(
+ "textbox",
+ str(item.index + 1),
+ item.x0,
+ item.y1,
+ 20,
+ )
+ for child in item:
+ render(child)
+ elif isinstance(item, LTChar):
+ self.place_border("char", 1, item)
+ self.place_text(
+ "char",
+ item.get_text(),
+ item.x0,
+ item.y1,
+ item.size,
+ )
+ elif isinstance(item, LTTextLine):
+ for child in item:
+ render(child)
+ if self.layoutmode != "loose":
+ self.put_newline()
+ elif isinstance(item, LTTextBox):
+ self.begin_div(
+ "textbox",
+ 1,
+ item.x0,
+ item.y1,
+ item.width,
+ item.height,
+ item.get_writing_mode(),
+ )
+ for child in item:
+ render(child)
+ self.end_div("textbox")
+ elif isinstance(item, LTChar):
+ fontname = make_compat_str(item.fontname)
+ self.put_text(item.get_text(), fontname, item.size)
+ elif isinstance(item, LTText):
+ self.write_text(item.get_text())
+
+ render(ltpage)
+ self._yoffset += self.pagemargin
+
+ def close(self) -> None:
+ self.write_footer()
+
+
+class XMLConverter(PDFConverter[AnyIO]):
+ CONTROL = re.compile("[\x00-\x08\x0b-\x0c\x0e-\x1f]")
+
+ def __init__(
+ self,
+ rsrcmgr: PDFResourceManager,
+ outfp: AnyIO,
+ codec: str = "utf-8",
+ pageno: int = 1,
+ laparams: LAParams | None = None,
+ imagewriter: ImageWriter | None = None,
+ stripcontrol: bool = False,
+ ) -> None:
+ PDFConverter.__init__(
+ self,
+ rsrcmgr,
+ outfp,
+ codec=codec,
+ pageno=pageno,
+ laparams=laparams,
+ )
+
+ # write() assumes a codec for binary I/O, or no codec for text I/O.
+ if self.outfp_binary == (not self.codec):
+ raise PDFValueError("Codec is required for a binary I/O output")
+
+ self.imagewriter = imagewriter
+ self.stripcontrol = stripcontrol
+ self.write_header()
+
+ def write(self, text: str) -> None:
+ if self.codec:
+ cast(BinaryIO, self.outfp).write(text.encode(self.codec))
+ else:
+ cast(TextIO, self.outfp).write(text)
+
+ def write_header(self) -> None:
+ if self.codec:
+ self.write('\n' % self.codec)
+ else:
+ self.write('\n')
+ self.write("\n")
+
+ def write_footer(self) -> None:
+ self.write("\n")
+
+ def write_text(self, text: str) -> None:
+ if self.stripcontrol:
+ text = self.CONTROL.sub("", text)
+ self.write(enc(text))
+
+ def receive_layout(self, ltpage: LTPage) -> None:
+ def show_group(item: LTItem) -> None:
+ if isinstance(item, LTTextBox):
+ self.write(
+ '\n'
+ % (item.index, bbox2str(item.bbox)),
+ )
+ elif isinstance(item, LTTextGroup):
+ self.write('\n' % bbox2str(item.bbox))
+ for child in item:
+ show_group(child)
+ self.write("\n")
+
+ def render(item: LTItem) -> None:
+ child: LTItem
+ if isinstance(item, LTPage):
+ s = '\n' % (
+ item.pageid,
+ bbox2str(item.bbox),
+ item.rotate,
+ )
+ self.write(s)
+ for child in item:
+ render(child)
+ if item.groups is not None:
+ self.write("\n")
+ for group in item.groups:
+ show_group(group)
+ self.write("\n")
+ self.write("\n")
+ elif isinstance(item, LTLine):
+ s = '\n' % (
+ item.linewidth,
+ bbox2str(item.bbox),
+ )
+ self.write(s)
+ elif isinstance(item, LTRect):
+ s = '\n' % (
+ item.linewidth,
+ bbox2str(item.bbox),
+ )
+ self.write(s)
+ elif isinstance(item, LTCurve):
+ s = '\n' % (
+ item.linewidth,
+ bbox2str(item.bbox),
+ item.get_pts(),
+ )
+ self.write(s)
+ elif isinstance(item, LTFigure):
+ s = f'\n'
+ self.write(s)
+ for child in item:
+ render(child)
+ self.write("\n")
+ elif isinstance(item, LTTextLine):
+ self.write('\n' % bbox2str(item.bbox))
+ for child in item:
+ render(child)
+ self.write("\n")
+ elif isinstance(item, LTTextBox):
+ wmode = ""
+ if isinstance(item, LTTextBoxVertical):
+ wmode = ' wmode="vertical"'
+ s = '\n' % (
+ item.index,
+ bbox2str(item.bbox),
+ wmode,
+ )
+ self.write(s)
+ for child in item:
+ render(child)
+ self.write("\n")
+ elif isinstance(item, LTChar):
+ s = (
+ ''
+ % (
+ enc(item.fontname),
+ bbox2str(item.bbox),
+ item.ncs.name,
+ item.graphicstate.ncolor,
+ item.size,
+ )
+ )
+ self.write(s)
+ self.write_text(item.get_text())
+ self.write("\n")
+ elif isinstance(item, LTText):
+ self.write("%s\n" % item.get_text())
+ elif isinstance(item, LTImage):
+ if self.imagewriter is not None:
+ name = self.imagewriter.export_image(item)
+ self.write(
+ '\n'
+ % (enc(name), item.width, item.height),
+ )
+ else:
+ self.write(
+ '\n'
+ % (item.width, item.height),
+ )
+ else:
+ assert False, str(("Unhandled", item))
+
+ render(ltpage)
+
+ def close(self) -> None:
+ self.write_footer()
+
+
+class HOCRConverter(PDFConverter[AnyIO]):
+ """Extract an hOCR representation from explicit text information within a PDF."""
+
+ # Where text is being extracted from a variety of types of PDF within a
+ # business process, those PDFs where the text is only present in image
+ # form will need to be analysed using an OCR tool which will typically
+ # output hOCR. This converter extracts the explicit text information from
+ # those PDFs that do have it and uses it to genxerate a basic hOCR
+ # representation that is designed to be used in conjunction with the image
+ # of the PDF in the same way as genuine OCR output would be, but without the
+ # inevitable OCR errors.
+
+ # The converter does not handle images, diagrams or text colors.
+
+ # In the examples processed by the contributor it was necessary to set
+ # LAParams.all_texts to True.
+
+ CONTROL = re.compile(r"[\x00-\x08\x0b-\x0c\x0e-\x1f]")
+
+ def __init__(
+ self,
+ rsrcmgr: PDFResourceManager,
+ outfp: AnyIO,
+ codec: str = "utf8",
+ pageno: int = 1,
+ laparams: LAParams | None = None,
+ stripcontrol: bool = False,
+ ):
+ PDFConverter.__init__(
+ self,
+ rsrcmgr,
+ outfp,
+ codec=codec,
+ pageno=pageno,
+ laparams=laparams,
+ )
+ self.stripcontrol = stripcontrol
+ self.within_chars = False
+ self.write_header()
+
+ def bbox_repr(self, bbox: Rect) -> str:
+ (in_x0, in_y0, in_x1, in_y1) = bbox
+ # PDF y-coordinates are the other way round from hOCR coordinates
+ out_x0 = int(in_x0)
+ out_y0 = int(self.page_bbox[3] - in_y1)
+ out_x1 = int(in_x1)
+ out_y1 = int(self.page_bbox[3] - in_y0)
+ return f"bbox {out_x0} {out_y0} {out_x1} {out_y1}"
+
+ def write(self, text: str) -> None:
+ if self.codec:
+ encoded_text = text.encode(self.codec)
+ cast(BinaryIO, self.outfp).write(encoded_text)
+ else:
+ cast(TextIO, self.outfp).write(text)
+
+ def write_header(self) -> None:
+ if self.codec:
+ self.write(
+ "\n" % self.codec,
+ )
+ else:
+ self.write(
+ "\n",
+ )
+ self.write("\n")
+ self.write("\n")
+ self.write(
+ "\n",
+ )
+ self.write(
+ "\n",
+ )
+ self.write(
+ " \n",
+ )
+ self.write("\n")
+ self.write("\n")
+
+ def write_footer(self) -> None:
+ self.write("\n")
+ self.write(
+ "\n",
+ )
+
+ def write_text(self, text: str) -> None:
+ if self.stripcontrol:
+ text = self.CONTROL.sub("", text)
+ self.write(text)
+
+ def write_word(self) -> None:
+ if len(self.working_text) > 0:
+ bold_and_italic_styles = ""
+ if "Italic" in self.working_font:
+ bold_and_italic_styles = "font-style: italic; "
+ if "Bold" in self.working_font:
+ bold_and_italic_styles += "font-weight: bold; "
+ self.write(
+ "%s"
+ % (
+ (
+ self.working_font,
+ self.working_size,
+ bold_and_italic_styles,
+ self.bbox_repr(self.working_bbox),
+ self.working_font,
+ self.working_size,
+ self.working_text.strip(),
+ )
+ ),
+ )
+ self.within_chars = False
+
+ def receive_layout(self, ltpage: LTPage) -> None:
+ def render(item: LTItem) -> None:
+ if self.within_chars and isinstance(item, LTAnno):
+ self.write_word()
+ if isinstance(item, LTPage):
+ self.page_bbox = item.bbox
+ self.write(
+ "\n"
+ % (item.pageid, self.bbox_repr(item.bbox)),
+ )
+ for child in item:
+ render(child)
+ self.write("
\n")
+ elif isinstance(item, LTTextLine):
+ self.write(
+ "" % (self.bbox_repr(item.bbox)),
+ )
+ for child_line in item:
+ render(child_line)
+ self.write("\n")
+ elif isinstance(item, LTTextBox):
+ self.write(
+ "\n"
+ % (item.index, self.bbox_repr(item.bbox)),
+ )
+ for child in item:
+ render(child)
+ self.write("
\n")
+ elif isinstance(item, LTChar):
+ if not self.within_chars:
+ self.within_chars = True
+ self.working_text = item.get_text()
+ self.working_bbox = item.bbox
+ self.working_font = item.fontname
+ self.working_size = item.size
+ elif len(item.get_text().strip()) == 0:
+ self.write_word()
+ self.write(item.get_text())
+ else:
+ if (
+ self.working_bbox[1] != item.bbox[1]
+ or self.working_font != item.fontname
+ or self.working_size != item.size
+ ):
+ self.write_word()
+ self.working_bbox = item.bbox
+ self.working_font = item.fontname
+ self.working_size = item.size
+ self.working_text += item.get_text()
+ self.working_bbox = (
+ self.working_bbox[0],
+ self.working_bbox[1],
+ item.bbox[2],
+ self.working_bbox[3],
+ )
+
+ render(ltpage)
+
+ def close(self) -> None:
+ self.write_footer()
diff --git a/babeldoc/pdfminer/data_structures.py b/babeldoc/pdfminer/data_structures.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a6a5509e95948b399ed483c45a2bca91ca60d0a
--- /dev/null
+++ b/babeldoc/pdfminer/data_structures.py
@@ -0,0 +1,55 @@
+from collections.abc import Iterable
+from typing import Any
+
+from babeldoc.pdfminer.pdfparser import PDFSyntaxError
+from babeldoc.pdfminer.pdftypes import dict_value
+from babeldoc.pdfminer.pdftypes import int_value
+from babeldoc.pdfminer.pdftypes import list_value
+from babeldoc.pdfminer.utils import choplist
+from babeldoc.pdfminer import settings
+
+
+class NumberTree:
+ """A PDF number tree.
+
+ See Section 3.8.6 of the PDF Reference.
+ """
+
+ def __init__(self, obj: Any):
+ self._obj = dict_value(obj)
+ self.nums: Iterable[Any] | None = None
+ self.kids: Iterable[Any] | None = None
+ self.limits: Iterable[Any] | None = None
+
+ if "Nums" in self._obj:
+ self.nums = list_value(self._obj["Nums"])
+ if "Kids" in self._obj:
+ self.kids = list_value(self._obj["Kids"])
+ if "Limits" in self._obj:
+ self.limits = list_value(self._obj["Limits"])
+
+ def _parse(self) -> list[tuple[int, Any]]:
+ items = []
+ if self.nums: # Leaf node
+ for k, v in choplist(2, self.nums):
+ items.append((int_value(k), v))
+
+ if self.kids: # Root or intermediate node
+ for child_ref in self.kids:
+ items += NumberTree(child_ref)._parse()
+
+ return items
+
+ values: list[tuple[int, Any]] # workaround decorators unsupported by mypy
+
+ @property # type: ignore[no-redef,misc]
+ def values(self) -> list[tuple[int, Any]]:
+ values = self._parse()
+
+ if settings.STRICT:
+ if not all(a[0] <= b[0] for a, b in zip(values, values[1:], strict=False)):
+ raise PDFSyntaxError("Number tree elements are out of order")
+ else:
+ values.sort(key=lambda t: t[0])
+
+ return values
diff --git a/babeldoc/pdfminer/encodingdb.py b/babeldoc/pdfminer/encodingdb.py
new file mode 100644
index 0000000000000000000000000000000000000000..965aeda96e1a2cd488a8170a738df37da2e9fe58
--- /dev/null
+++ b/babeldoc/pdfminer/encodingdb.py
@@ -0,0 +1,127 @@
+import logging
+import re
+from collections.abc import Iterable
+from typing import cast
+
+from babeldoc.pdfminer.glyphlist import glyphname2unicode
+from babeldoc.pdfminer.latin_enc import ENCODING
+from babeldoc.pdfminer.pdfexceptions import PDFKeyError
+from babeldoc.pdfminer.psparser import PSLiteral
+
+HEXADECIMAL = re.compile(r"[0-9a-fA-F]+")
+
+log = logging.getLogger(__name__)
+
+
+def name2unicode(name: str) -> str:
+ """Converts Adobe glyph names to Unicode numbers.
+
+ In contrast to the specification, this raises a KeyError instead of return
+ an empty string when the key is unknown.
+ This way the caller must explicitly define what to do
+ when there is not a match.
+
+ Reference:
+ https://github.com/adobe-type-tools/agl-specification#2-the-mapping
+
+ :returns unicode character if name resembles something,
+ otherwise a KeyError
+ """
+ if not isinstance(name, str):
+ raise PDFKeyError(
+ 'Could not convert unicode name "%s" to character because '
+ "it should be of type str but is of type %s" % (name, type(name)),
+ )
+
+ name = name.split(".")[0]
+ components = name.split("_")
+
+ if len(components) > 1:
+ return "".join(map(name2unicode, components))
+
+ elif name in glyphname2unicode:
+ return glyphname2unicode[name]
+
+ elif name.startswith("uni"):
+ name_without_uni = name.strip("uni")
+
+ if HEXADECIMAL.match(name_without_uni) and len(name_without_uni) % 4 == 0:
+ unicode_digits = [
+ int(name_without_uni[i : i + 4], base=16)
+ for i in range(0, len(name_without_uni), 4)
+ ]
+ for digit in unicode_digits:
+ raise_key_error_for_invalid_unicode(digit)
+ characters = map(chr, unicode_digits)
+ return "".join(characters)
+
+ elif name.startswith("u"):
+ name_without_u = name.strip("u")
+
+ if HEXADECIMAL.match(name_without_u) and 4 <= len(name_without_u) <= 6:
+ unicode_digit = int(name_without_u, base=16)
+ raise_key_error_for_invalid_unicode(unicode_digit)
+ return chr(unicode_digit)
+
+ raise PDFKeyError(
+ 'Could not convert unicode name "%s" to character because '
+ "it does not match specification" % name,
+ )
+
+
+def raise_key_error_for_invalid_unicode(unicode_digit: int) -> None:
+ """Unicode values should not be in the range D800 through DFFF because
+ that is used for surrogate pairs in UTF-16
+
+ :raises KeyError if unicode digit is invalid
+ """
+ if 55295 < unicode_digit < 57344:
+ raise PDFKeyError(
+ "Unicode digit %d is invalid because "
+ "it is in the range D800 through DFFF" % unicode_digit,
+ )
+
+
+class EncodingDB:
+ std2unicode: dict[int, str] = {}
+ mac2unicode: dict[int, str] = {}
+ win2unicode: dict[int, str] = {}
+ pdf2unicode: dict[int, str] = {}
+ for name, std, mac, win, pdf in ENCODING:
+ c = name2unicode(name)
+ if std:
+ std2unicode[std] = c
+ if mac:
+ mac2unicode[mac] = c
+ if win:
+ win2unicode[win] = c
+ if pdf:
+ pdf2unicode[pdf] = c
+
+ encodings = {
+ "StandardEncoding": std2unicode,
+ "MacRomanEncoding": mac2unicode,
+ "WinAnsiEncoding": win2unicode,
+ "PDFDocEncoding": pdf2unicode,
+ }
+
+ @classmethod
+ def get_encoding(
+ cls,
+ name: str,
+ diff: Iterable[object] | None = None,
+ ) -> dict[int, str]:
+ cid2unicode = cls.encodings.get(name, cls.std2unicode)
+ if diff:
+ cid2unicode = cid2unicode.copy()
+ cid = 0
+ for x in diff:
+ if isinstance(x, int):
+ cid = x
+ elif isinstance(x, PSLiteral):
+ try:
+ cid2unicode[cid] = name2unicode(cast(str, x.name))
+ except (KeyError, ValueError) as e:
+ log.debug(str(e))
+ cid += 1
+ return cid2unicode
diff --git a/babeldoc/pdfminer/fontmetrics.py b/babeldoc/pdfminer/fontmetrics.py
new file mode 100644
index 0000000000000000000000000000000000000000..b6780b963a5599f9ba3cc6269b34b5bd42fa3c79
--- /dev/null
+++ b/babeldoc/pdfminer/fontmetrics.py
@@ -0,0 +1,4464 @@
+"""Font metrics for the Adobe core 14 fonts.
+
+Font metrics are used to compute the boundary of each character
+written with a proportional font.
+
+The following data were extracted from the AFM files:
+
+ http://www.ctan.org/tex-archive/fonts/adobe/afm/
+
+"""
+
+### BEGIN Verbatim copy of the license part
+
+#
+# Adobe Core 35 AFM Files with 314 Glyph Entries - ReadMe
+#
+# This file and the 35 PostScript(R) AFM files it accompanies may be
+# used, copied, and distributed for any purpose and without charge,
+# with or without modification, provided that all copyright notices
+# are retained; that the AFM files are not distributed without this
+# file; that all modifications to this file or any of the AFM files
+# are prominently noted in the modified file(s); and that this
+# paragraph is not modified. Adobe Systems has no responsibility or
+# obligation to support the use of the AFM files.
+#
+
+### END Verbatim copy of the license part
+
+# flake8: noqa
+from typing import Dict
+
+
+def convert_font_metrics(path: str) -> None:
+ """Convert an AFM file to a mapping of font metrics.
+
+ See below for the output.
+ """
+ fonts = {}
+ with open(path) as fileinput:
+ for line in fileinput.readlines():
+ f = line.strip().split(" ")
+ if not f:
+ continue
+ k = f[0]
+ if k == "FontName":
+ fontname = f[1]
+ props = {"FontName": fontname, "Flags": 0}
+ chars: Dict[int, int] = {}
+ fonts[fontname] = (props, chars)
+ elif k == "C":
+ cid = int(f[1])
+ if 0 <= cid and cid <= 255:
+ width = int(f[4])
+ chars[cid] = width
+ elif k in ("CapHeight", "XHeight", "ItalicAngle", "Ascender", "Descender"):
+ k = {"Ascender": "Ascent", "Descender": "Descent"}.get(k, k)
+ props[k] = float(f[1])
+ elif k in ("FontName", "FamilyName", "Weight"):
+ k = {"FamilyName": "FontFamily", "Weight": "FontWeight"}.get(k, k)
+ props[k] = f[1]
+ elif k == "IsFixedPitch":
+ if f[1].lower() == "true":
+ props["Flags"] = 64
+ elif k == "FontBBox":
+ props[k] = tuple(map(float, f[1:5]))
+ print("# -*- python -*-")
+ print("FONT_METRICS = {")
+ for fontname, (props, chars) in fonts.items():
+ print(f" {fontname!r}: {(props, chars)!r},")
+ print("}")
+
+
+FONT_METRICS = {
+ "Courier": (
+ {
+ "FontName": "Courier",
+ "Descent": -194.0,
+ "FontBBox": (-6.0, -249.0, 639.0, 803.0),
+ "FontWeight": "Medium",
+ "CapHeight": 572.0,
+ "FontFamily": "Courier",
+ "Flags": 64,
+ "XHeight": 434.0,
+ "ItalicAngle": 0.0,
+ "Ascent": 627.0,
+ },
+ {
+ " ": 600,
+ "!": 600,
+ '"': 600,
+ "#": 600,
+ "$": 600,
+ "%": 600,
+ "&": 600,
+ "'": 600,
+ "(": 600,
+ ")": 600,
+ "*": 600,
+ "+": 600,
+ ",": 600,
+ "-": 600,
+ ".": 600,
+ "/": 600,
+ "0": 600,
+ "1": 600,
+ "2": 600,
+ "3": 600,
+ "4": 600,
+ "5": 600,
+ "6": 600,
+ "7": 600,
+ "8": 600,
+ "9": 600,
+ ":": 600,
+ ";": 600,
+ "<": 600,
+ "=": 600,
+ ">": 600,
+ "?": 600,
+ "@": 600,
+ "A": 600,
+ "B": 600,
+ "C": 600,
+ "D": 600,
+ "E": 600,
+ "F": 600,
+ "G": 600,
+ "H": 600,
+ "I": 600,
+ "J": 600,
+ "K": 600,
+ "L": 600,
+ "M": 600,
+ "N": 600,
+ "O": 600,
+ "P": 600,
+ "Q": 600,
+ "R": 600,
+ "S": 600,
+ "T": 600,
+ "U": 600,
+ "V": 600,
+ "W": 600,
+ "X": 600,
+ "Y": 600,
+ "Z": 600,
+ "[": 600,
+ "\\": 600,
+ "]": 600,
+ "^": 600,
+ "_": 600,
+ "`": 600,
+ "a": 600,
+ "b": 600,
+ "c": 600,
+ "d": 600,
+ "e": 600,
+ "f": 600,
+ "g": 600,
+ "h": 600,
+ "i": 600,
+ "j": 600,
+ "k": 600,
+ "l": 600,
+ "m": 600,
+ "n": 600,
+ "o": 600,
+ "p": 600,
+ "q": 600,
+ "r": 600,
+ "s": 600,
+ "t": 600,
+ "u": 600,
+ "v": 600,
+ "w": 600,
+ "x": 600,
+ "y": 600,
+ "z": 600,
+ "{": 600,
+ "|": 600,
+ "}": 600,
+ "~": 600,
+ "\xa1": 600,
+ "\xa2": 600,
+ "\xa3": 600,
+ "\xa4": 600,
+ "\xa5": 600,
+ "\xa6": 600,
+ "\xa7": 600,
+ "\xa8": 600,
+ "\xa9": 600,
+ "\xaa": 600,
+ "\xab": 600,
+ "\xac": 600,
+ "\xae": 600,
+ "\xaf": 600,
+ "\xb0": 600,
+ "\xb1": 600,
+ "\xb2": 600,
+ "\xb3": 600,
+ "\xb4": 600,
+ "\xb5": 600,
+ "\xb6": 600,
+ "\xb7": 600,
+ "\xb8": 600,
+ "\xb9": 600,
+ "\xba": 600,
+ "\xbb": 600,
+ "\xbc": 600,
+ "\xbd": 600,
+ "\xbe": 600,
+ "\xbf": 600,
+ "\xc0": 600,
+ "\xc1": 600,
+ "\xc2": 600,
+ "\xc3": 600,
+ "\xc4": 600,
+ "\xc5": 600,
+ "\xc6": 600,
+ "\xc7": 600,
+ "\xc8": 600,
+ "\xc9": 600,
+ "\xca": 600,
+ "\xcb": 600,
+ "\xcc": 600,
+ "\xcd": 600,
+ "\xce": 600,
+ "\xcf": 600,
+ "\xd0": 600,
+ "\xd1": 600,
+ "\xd2": 600,
+ "\xd3": 600,
+ "\xd4": 600,
+ "\xd5": 600,
+ "\xd6": 600,
+ "\xd7": 600,
+ "\xd8": 600,
+ "\xd9": 600,
+ "\xda": 600,
+ "\xdb": 600,
+ "\xdc": 600,
+ "\xdd": 600,
+ "\xde": 600,
+ "\xdf": 600,
+ "\xe0": 600,
+ "\xe1": 600,
+ "\xe2": 600,
+ "\xe3": 600,
+ "\xe4": 600,
+ "\xe5": 600,
+ "\xe6": 600,
+ "\xe7": 600,
+ "\xe8": 600,
+ "\xe9": 600,
+ "\xea": 600,
+ "\xeb": 600,
+ "\xec": 600,
+ "\xed": 600,
+ "\xee": 600,
+ "\xef": 600,
+ "\xf0": 600,
+ "\xf1": 600,
+ "\xf2": 600,
+ "\xf3": 600,
+ "\xf4": 600,
+ "\xf5": 600,
+ "\xf6": 600,
+ "\xf7": 600,
+ "\xf8": 600,
+ "\xf9": 600,
+ "\xfa": 600,
+ "\xfb": 600,
+ "\xfc": 600,
+ "\xfd": 600,
+ "\xfe": 600,
+ "\xff": 600,
+ "\u0100": 600,
+ "\u0101": 600,
+ "\u0102": 600,
+ "\u0103": 600,
+ "\u0104": 600,
+ "\u0105": 600,
+ "\u0106": 600,
+ "\u0107": 600,
+ "\u010c": 600,
+ "\u010d": 600,
+ "\u010e": 600,
+ "\u010f": 600,
+ "\u0110": 600,
+ "\u0111": 600,
+ "\u0112": 600,
+ "\u0113": 600,
+ "\u0116": 600,
+ "\u0117": 600,
+ "\u0118": 600,
+ "\u0119": 600,
+ "\u011a": 600,
+ "\u011b": 600,
+ "\u011e": 600,
+ "\u011f": 600,
+ "\u0122": 600,
+ "\u0123": 600,
+ "\u012a": 600,
+ "\u012b": 600,
+ "\u012e": 600,
+ "\u012f": 600,
+ "\u0130": 600,
+ "\u0131": 600,
+ "\u0136": 600,
+ "\u0137": 600,
+ "\u0139": 600,
+ "\u013a": 600,
+ "\u013b": 600,
+ "\u013c": 600,
+ "\u013d": 600,
+ "\u013e": 600,
+ "\u0141": 600,
+ "\u0142": 600,
+ "\u0143": 600,
+ "\u0144": 600,
+ "\u0145": 600,
+ "\u0146": 600,
+ "\u0147": 600,
+ "\u0148": 600,
+ "\u014c": 600,
+ "\u014d": 600,
+ "\u0150": 600,
+ "\u0151": 600,
+ "\u0152": 600,
+ "\u0153": 600,
+ "\u0154": 600,
+ "\u0155": 600,
+ "\u0156": 600,
+ "\u0157": 600,
+ "\u0158": 600,
+ "\u0159": 600,
+ "\u015a": 600,
+ "\u015b": 600,
+ "\u015e": 600,
+ "\u015f": 600,
+ "\u0160": 600,
+ "\u0161": 600,
+ "\u0162": 600,
+ "\u0163": 600,
+ "\u0164": 600,
+ "\u0165": 600,
+ "\u016a": 600,
+ "\u016b": 600,
+ "\u016e": 600,
+ "\u016f": 600,
+ "\u0170": 600,
+ "\u0171": 600,
+ "\u0172": 600,
+ "\u0173": 600,
+ "\u0178": 600,
+ "\u0179": 600,
+ "\u017a": 600,
+ "\u017b": 600,
+ "\u017c": 600,
+ "\u017d": 600,
+ "\u017e": 600,
+ "\u0192": 600,
+ "\u0218": 600,
+ "\u0219": 600,
+ "\u02c6": 600,
+ "\u02c7": 600,
+ "\u02d8": 600,
+ "\u02d9": 600,
+ "\u02da": 600,
+ "\u02db": 600,
+ "\u02dc": 600,
+ "\u02dd": 600,
+ "\u2013": 600,
+ "\u2014": 600,
+ "\u2018": 600,
+ "\u2019": 600,
+ "\u201a": 600,
+ "\u201c": 600,
+ "\u201d": 600,
+ "\u201e": 600,
+ "\u2020": 600,
+ "\u2021": 600,
+ "\u2022": 600,
+ "\u2026": 600,
+ "\u2030": 600,
+ "\u2039": 600,
+ "\u203a": 600,
+ "\u2044": 600,
+ "\u2122": 600,
+ "\u2202": 600,
+ "\u2206": 600,
+ "\u2211": 600,
+ "\u2212": 600,
+ "\u221a": 600,
+ "\u2260": 600,
+ "\u2264": 600,
+ "\u2265": 600,
+ "\u25ca": 600,
+ "\uf6c3": 600,
+ "\ufb01": 600,
+ "\ufb02": 600,
+ },
+ ),
+ "Courier-Bold": (
+ {
+ "FontName": "Courier-Bold",
+ "Descent": -194.0,
+ "FontBBox": (-88.0, -249.0, 697.0, 811.0),
+ "FontWeight": "Bold",
+ "CapHeight": 572.0,
+ "FontFamily": "Courier",
+ "Flags": 64,
+ "XHeight": 434.0,
+ "ItalicAngle": 0.0,
+ "Ascent": 627.0,
+ },
+ {
+ " ": 600,
+ "!": 600,
+ '"': 600,
+ "#": 600,
+ "$": 600,
+ "%": 600,
+ "&": 600,
+ "'": 600,
+ "(": 600,
+ ")": 600,
+ "*": 600,
+ "+": 600,
+ ",": 600,
+ "-": 600,
+ ".": 600,
+ "/": 600,
+ "0": 600,
+ "1": 600,
+ "2": 600,
+ "3": 600,
+ "4": 600,
+ "5": 600,
+ "6": 600,
+ "7": 600,
+ "8": 600,
+ "9": 600,
+ ":": 600,
+ ";": 600,
+ "<": 600,
+ "=": 600,
+ ">": 600,
+ "?": 600,
+ "@": 600,
+ "A": 600,
+ "B": 600,
+ "C": 600,
+ "D": 600,
+ "E": 600,
+ "F": 600,
+ "G": 600,
+ "H": 600,
+ "I": 600,
+ "J": 600,
+ "K": 600,
+ "L": 600,
+ "M": 600,
+ "N": 600,
+ "O": 600,
+ "P": 600,
+ "Q": 600,
+ "R": 600,
+ "S": 600,
+ "T": 600,
+ "U": 600,
+ "V": 600,
+ "W": 600,
+ "X": 600,
+ "Y": 600,
+ "Z": 600,
+ "[": 600,
+ "\\": 600,
+ "]": 600,
+ "^": 600,
+ "_": 600,
+ "`": 600,
+ "a": 600,
+ "b": 600,
+ "c": 600,
+ "d": 600,
+ "e": 600,
+ "f": 600,
+ "g": 600,
+ "h": 600,
+ "i": 600,
+ "j": 600,
+ "k": 600,
+ "l": 600,
+ "m": 600,
+ "n": 600,
+ "o": 600,
+ "p": 600,
+ "q": 600,
+ "r": 600,
+ "s": 600,
+ "t": 600,
+ "u": 600,
+ "v": 600,
+ "w": 600,
+ "x": 600,
+ "y": 600,
+ "z": 600,
+ "{": 600,
+ "|": 600,
+ "}": 600,
+ "~": 600,
+ "\xa1": 600,
+ "\xa2": 600,
+ "\xa3": 600,
+ "\xa4": 600,
+ "\xa5": 600,
+ "\xa6": 600,
+ "\xa7": 600,
+ "\xa8": 600,
+ "\xa9": 600,
+ "\xaa": 600,
+ "\xab": 600,
+ "\xac": 600,
+ "\xae": 600,
+ "\xaf": 600,
+ "\xb0": 600,
+ "\xb1": 600,
+ "\xb2": 600,
+ "\xb3": 600,
+ "\xb4": 600,
+ "\xb5": 600,
+ "\xb6": 600,
+ "\xb7": 600,
+ "\xb8": 600,
+ "\xb9": 600,
+ "\xba": 600,
+ "\xbb": 600,
+ "\xbc": 600,
+ "\xbd": 600,
+ "\xbe": 600,
+ "\xbf": 600,
+ "\xc0": 600,
+ "\xc1": 600,
+ "\xc2": 600,
+ "\xc3": 600,
+ "\xc4": 600,
+ "\xc5": 600,
+ "\xc6": 600,
+ "\xc7": 600,
+ "\xc8": 600,
+ "\xc9": 600,
+ "\xca": 600,
+ "\xcb": 600,
+ "\xcc": 600,
+ "\xcd": 600,
+ "\xce": 600,
+ "\xcf": 600,
+ "\xd0": 600,
+ "\xd1": 600,
+ "\xd2": 600,
+ "\xd3": 600,
+ "\xd4": 600,
+ "\xd5": 600,
+ "\xd6": 600,
+ "\xd7": 600,
+ "\xd8": 600,
+ "\xd9": 600,
+ "\xda": 600,
+ "\xdb": 600,
+ "\xdc": 600,
+ "\xdd": 600,
+ "\xde": 600,
+ "\xdf": 600,
+ "\xe0": 600,
+ "\xe1": 600,
+ "\xe2": 600,
+ "\xe3": 600,
+ "\xe4": 600,
+ "\xe5": 600,
+ "\xe6": 600,
+ "\xe7": 600,
+ "\xe8": 600,
+ "\xe9": 600,
+ "\xea": 600,
+ "\xeb": 600,
+ "\xec": 600,
+ "\xed": 600,
+ "\xee": 600,
+ "\xef": 600,
+ "\xf0": 600,
+ "\xf1": 600,
+ "\xf2": 600,
+ "\xf3": 600,
+ "\xf4": 600,
+ "\xf5": 600,
+ "\xf6": 600,
+ "\xf7": 600,
+ "\xf8": 600,
+ "\xf9": 600,
+ "\xfa": 600,
+ "\xfb": 600,
+ "\xfc": 600,
+ "\xfd": 600,
+ "\xfe": 600,
+ "\xff": 600,
+ "\u0100": 600,
+ "\u0101": 600,
+ "\u0102": 600,
+ "\u0103": 600,
+ "\u0104": 600,
+ "\u0105": 600,
+ "\u0106": 600,
+ "\u0107": 600,
+ "\u010c": 600,
+ "\u010d": 600,
+ "\u010e": 600,
+ "\u010f": 600,
+ "\u0110": 600,
+ "\u0111": 600,
+ "\u0112": 600,
+ "\u0113": 600,
+ "\u0116": 600,
+ "\u0117": 600,
+ "\u0118": 600,
+ "\u0119": 600,
+ "\u011a": 600,
+ "\u011b": 600,
+ "\u011e": 600,
+ "\u011f": 600,
+ "\u0122": 600,
+ "\u0123": 600,
+ "\u012a": 600,
+ "\u012b": 600,
+ "\u012e": 600,
+ "\u012f": 600,
+ "\u0130": 600,
+ "\u0131": 600,
+ "\u0136": 600,
+ "\u0137": 600,
+ "\u0139": 600,
+ "\u013a": 600,
+ "\u013b": 600,
+ "\u013c": 600,
+ "\u013d": 600,
+ "\u013e": 600,
+ "\u0141": 600,
+ "\u0142": 600,
+ "\u0143": 600,
+ "\u0144": 600,
+ "\u0145": 600,
+ "\u0146": 600,
+ "\u0147": 600,
+ "\u0148": 600,
+ "\u014c": 600,
+ "\u014d": 600,
+ "\u0150": 600,
+ "\u0151": 600,
+ "\u0152": 600,
+ "\u0153": 600,
+ "\u0154": 600,
+ "\u0155": 600,
+ "\u0156": 600,
+ "\u0157": 600,
+ "\u0158": 600,
+ "\u0159": 600,
+ "\u015a": 600,
+ "\u015b": 600,
+ "\u015e": 600,
+ "\u015f": 600,
+ "\u0160": 600,
+ "\u0161": 600,
+ "\u0162": 600,
+ "\u0163": 600,
+ "\u0164": 600,
+ "\u0165": 600,
+ "\u016a": 600,
+ "\u016b": 600,
+ "\u016e": 600,
+ "\u016f": 600,
+ "\u0170": 600,
+ "\u0171": 600,
+ "\u0172": 600,
+ "\u0173": 600,
+ "\u0178": 600,
+ "\u0179": 600,
+ "\u017a": 600,
+ "\u017b": 600,
+ "\u017c": 600,
+ "\u017d": 600,
+ "\u017e": 600,
+ "\u0192": 600,
+ "\u0218": 600,
+ "\u0219": 600,
+ "\u02c6": 600,
+ "\u02c7": 600,
+ "\u02d8": 600,
+ "\u02d9": 600,
+ "\u02da": 600,
+ "\u02db": 600,
+ "\u02dc": 600,
+ "\u02dd": 600,
+ "\u2013": 600,
+ "\u2014": 600,
+ "\u2018": 600,
+ "\u2019": 600,
+ "\u201a": 600,
+ "\u201c": 600,
+ "\u201d": 600,
+ "\u201e": 600,
+ "\u2020": 600,
+ "\u2021": 600,
+ "\u2022": 600,
+ "\u2026": 600,
+ "\u2030": 600,
+ "\u2039": 600,
+ "\u203a": 600,
+ "\u2044": 600,
+ "\u2122": 600,
+ "\u2202": 600,
+ "\u2206": 600,
+ "\u2211": 600,
+ "\u2212": 600,
+ "\u221a": 600,
+ "\u2260": 600,
+ "\u2264": 600,
+ "\u2265": 600,
+ "\u25ca": 600,
+ "\uf6c3": 600,
+ "\ufb01": 600,
+ "\ufb02": 600,
+ },
+ ),
+ "Courier-BoldOblique": (
+ {
+ "FontName": "Courier-BoldOblique",
+ "Descent": -194.0,
+ "FontBBox": (-49.0, -249.0, 758.0, 811.0),
+ "FontWeight": "Bold",
+ "CapHeight": 572.0,
+ "FontFamily": "Courier",
+ "Flags": 64,
+ "XHeight": 434.0,
+ "ItalicAngle": -11.0,
+ "Ascent": 627.0,
+ },
+ {
+ " ": 600,
+ "!": 600,
+ '"': 600,
+ "#": 600,
+ "$": 600,
+ "%": 600,
+ "&": 600,
+ "'": 600,
+ "(": 600,
+ ")": 600,
+ "*": 600,
+ "+": 600,
+ ",": 600,
+ "-": 600,
+ ".": 600,
+ "/": 600,
+ "0": 600,
+ "1": 600,
+ "2": 600,
+ "3": 600,
+ "4": 600,
+ "5": 600,
+ "6": 600,
+ "7": 600,
+ "8": 600,
+ "9": 600,
+ ":": 600,
+ ";": 600,
+ "<": 600,
+ "=": 600,
+ ">": 600,
+ "?": 600,
+ "@": 600,
+ "A": 600,
+ "B": 600,
+ "C": 600,
+ "D": 600,
+ "E": 600,
+ "F": 600,
+ "G": 600,
+ "H": 600,
+ "I": 600,
+ "J": 600,
+ "K": 600,
+ "L": 600,
+ "M": 600,
+ "N": 600,
+ "O": 600,
+ "P": 600,
+ "Q": 600,
+ "R": 600,
+ "S": 600,
+ "T": 600,
+ "U": 600,
+ "V": 600,
+ "W": 600,
+ "X": 600,
+ "Y": 600,
+ "Z": 600,
+ "[": 600,
+ "\\": 600,
+ "]": 600,
+ "^": 600,
+ "_": 600,
+ "`": 600,
+ "a": 600,
+ "b": 600,
+ "c": 600,
+ "d": 600,
+ "e": 600,
+ "f": 600,
+ "g": 600,
+ "h": 600,
+ "i": 600,
+ "j": 600,
+ "k": 600,
+ "l": 600,
+ "m": 600,
+ "n": 600,
+ "o": 600,
+ "p": 600,
+ "q": 600,
+ "r": 600,
+ "s": 600,
+ "t": 600,
+ "u": 600,
+ "v": 600,
+ "w": 600,
+ "x": 600,
+ "y": 600,
+ "z": 600,
+ "{": 600,
+ "|": 600,
+ "}": 600,
+ "~": 600,
+ "\xa1": 600,
+ "\xa2": 600,
+ "\xa3": 600,
+ "\xa4": 600,
+ "\xa5": 600,
+ "\xa6": 600,
+ "\xa7": 600,
+ "\xa8": 600,
+ "\xa9": 600,
+ "\xaa": 600,
+ "\xab": 600,
+ "\xac": 600,
+ "\xae": 600,
+ "\xaf": 600,
+ "\xb0": 600,
+ "\xb1": 600,
+ "\xb2": 600,
+ "\xb3": 600,
+ "\xb4": 600,
+ "\xb5": 600,
+ "\xb6": 600,
+ "\xb7": 600,
+ "\xb8": 600,
+ "\xb9": 600,
+ "\xba": 600,
+ "\xbb": 600,
+ "\xbc": 600,
+ "\xbd": 600,
+ "\xbe": 600,
+ "\xbf": 600,
+ "\xc0": 600,
+ "\xc1": 600,
+ "\xc2": 600,
+ "\xc3": 600,
+ "\xc4": 600,
+ "\xc5": 600,
+ "\xc6": 600,
+ "\xc7": 600,
+ "\xc8": 600,
+ "\xc9": 600,
+ "\xca": 600,
+ "\xcb": 600,
+ "\xcc": 600,
+ "\xcd": 600,
+ "\xce": 600,
+ "\xcf": 600,
+ "\xd0": 600,
+ "\xd1": 600,
+ "\xd2": 600,
+ "\xd3": 600,
+ "\xd4": 600,
+ "\xd5": 600,
+ "\xd6": 600,
+ "\xd7": 600,
+ "\xd8": 600,
+ "\xd9": 600,
+ "\xda": 600,
+ "\xdb": 600,
+ "\xdc": 600,
+ "\xdd": 600,
+ "\xde": 600,
+ "\xdf": 600,
+ "\xe0": 600,
+ "\xe1": 600,
+ "\xe2": 600,
+ "\xe3": 600,
+ "\xe4": 600,
+ "\xe5": 600,
+ "\xe6": 600,
+ "\xe7": 600,
+ "\xe8": 600,
+ "\xe9": 600,
+ "\xea": 600,
+ "\xeb": 600,
+ "\xec": 600,
+ "\xed": 600,
+ "\xee": 600,
+ "\xef": 600,
+ "\xf0": 600,
+ "\xf1": 600,
+ "\xf2": 600,
+ "\xf3": 600,
+ "\xf4": 600,
+ "\xf5": 600,
+ "\xf6": 600,
+ "\xf7": 600,
+ "\xf8": 600,
+ "\xf9": 600,
+ "\xfa": 600,
+ "\xfb": 600,
+ "\xfc": 600,
+ "\xfd": 600,
+ "\xfe": 600,
+ "\xff": 600,
+ "\u0100": 600,
+ "\u0101": 600,
+ "\u0102": 600,
+ "\u0103": 600,
+ "\u0104": 600,
+ "\u0105": 600,
+ "\u0106": 600,
+ "\u0107": 600,
+ "\u010c": 600,
+ "\u010d": 600,
+ "\u010e": 600,
+ "\u010f": 600,
+ "\u0110": 600,
+ "\u0111": 600,
+ "\u0112": 600,
+ "\u0113": 600,
+ "\u0116": 600,
+ "\u0117": 600,
+ "\u0118": 600,
+ "\u0119": 600,
+ "\u011a": 600,
+ "\u011b": 600,
+ "\u011e": 600,
+ "\u011f": 600,
+ "\u0122": 600,
+ "\u0123": 600,
+ "\u012a": 600,
+ "\u012b": 600,
+ "\u012e": 600,
+ "\u012f": 600,
+ "\u0130": 600,
+ "\u0131": 600,
+ "\u0136": 600,
+ "\u0137": 600,
+ "\u0139": 600,
+ "\u013a": 600,
+ "\u013b": 600,
+ "\u013c": 600,
+ "\u013d": 600,
+ "\u013e": 600,
+ "\u0141": 600,
+ "\u0142": 600,
+ "\u0143": 600,
+ "\u0144": 600,
+ "\u0145": 600,
+ "\u0146": 600,
+ "\u0147": 600,
+ "\u0148": 600,
+ "\u014c": 600,
+ "\u014d": 600,
+ "\u0150": 600,
+ "\u0151": 600,
+ "\u0152": 600,
+ "\u0153": 600,
+ "\u0154": 600,
+ "\u0155": 600,
+ "\u0156": 600,
+ "\u0157": 600,
+ "\u0158": 600,
+ "\u0159": 600,
+ "\u015a": 600,
+ "\u015b": 600,
+ "\u015e": 600,
+ "\u015f": 600,
+ "\u0160": 600,
+ "\u0161": 600,
+ "\u0162": 600,
+ "\u0163": 600,
+ "\u0164": 600,
+ "\u0165": 600,
+ "\u016a": 600,
+ "\u016b": 600,
+ "\u016e": 600,
+ "\u016f": 600,
+ "\u0170": 600,
+ "\u0171": 600,
+ "\u0172": 600,
+ "\u0173": 600,
+ "\u0178": 600,
+ "\u0179": 600,
+ "\u017a": 600,
+ "\u017b": 600,
+ "\u017c": 600,
+ "\u017d": 600,
+ "\u017e": 600,
+ "\u0192": 600,
+ "\u0218": 600,
+ "\u0219": 600,
+ "\u02c6": 600,
+ "\u02c7": 600,
+ "\u02d8": 600,
+ "\u02d9": 600,
+ "\u02da": 600,
+ "\u02db": 600,
+ "\u02dc": 600,
+ "\u02dd": 600,
+ "\u2013": 600,
+ "\u2014": 600,
+ "\u2018": 600,
+ "\u2019": 600,
+ "\u201a": 600,
+ "\u201c": 600,
+ "\u201d": 600,
+ "\u201e": 600,
+ "\u2020": 600,
+ "\u2021": 600,
+ "\u2022": 600,
+ "\u2026": 600,
+ "\u2030": 600,
+ "\u2039": 600,
+ "\u203a": 600,
+ "\u2044": 600,
+ "\u2122": 600,
+ "\u2202": 600,
+ "\u2206": 600,
+ "\u2211": 600,
+ "\u2212": 600,
+ "\u221a": 600,
+ "\u2260": 600,
+ "\u2264": 600,
+ "\u2265": 600,
+ "\u25ca": 600,
+ "\uf6c3": 600,
+ "\ufb01": 600,
+ "\ufb02": 600,
+ },
+ ),
+ "Courier-Oblique": (
+ {
+ "FontName": "Courier-Oblique",
+ "Descent": -194.0,
+ "FontBBox": (-49.0, -249.0, 749.0, 803.0),
+ "FontWeight": "Medium",
+ "CapHeight": 572.0,
+ "FontFamily": "Courier",
+ "Flags": 64,
+ "XHeight": 434.0,
+ "ItalicAngle": -11.0,
+ "Ascent": 627.0,
+ },
+ {
+ " ": 600,
+ "!": 600,
+ '"': 600,
+ "#": 600,
+ "$": 600,
+ "%": 600,
+ "&": 600,
+ "'": 600,
+ "(": 600,
+ ")": 600,
+ "*": 600,
+ "+": 600,
+ ",": 600,
+ "-": 600,
+ ".": 600,
+ "/": 600,
+ "0": 600,
+ "1": 600,
+ "2": 600,
+ "3": 600,
+ "4": 600,
+ "5": 600,
+ "6": 600,
+ "7": 600,
+ "8": 600,
+ "9": 600,
+ ":": 600,
+ ";": 600,
+ "<": 600,
+ "=": 600,
+ ">": 600,
+ "?": 600,
+ "@": 600,
+ "A": 600,
+ "B": 600,
+ "C": 600,
+ "D": 600,
+ "E": 600,
+ "F": 600,
+ "G": 600,
+ "H": 600,
+ "I": 600,
+ "J": 600,
+ "K": 600,
+ "L": 600,
+ "M": 600,
+ "N": 600,
+ "O": 600,
+ "P": 600,
+ "Q": 600,
+ "R": 600,
+ "S": 600,
+ "T": 600,
+ "U": 600,
+ "V": 600,
+ "W": 600,
+ "X": 600,
+ "Y": 600,
+ "Z": 600,
+ "[": 600,
+ "\\": 600,
+ "]": 600,
+ "^": 600,
+ "_": 600,
+ "`": 600,
+ "a": 600,
+ "b": 600,
+ "c": 600,
+ "d": 600,
+ "e": 600,
+ "f": 600,
+ "g": 600,
+ "h": 600,
+ "i": 600,
+ "j": 600,
+ "k": 600,
+ "l": 600,
+ "m": 600,
+ "n": 600,
+ "o": 600,
+ "p": 600,
+ "q": 600,
+ "r": 600,
+ "s": 600,
+ "t": 600,
+ "u": 600,
+ "v": 600,
+ "w": 600,
+ "x": 600,
+ "y": 600,
+ "z": 600,
+ "{": 600,
+ "|": 600,
+ "}": 600,
+ "~": 600,
+ "\xa1": 600,
+ "\xa2": 600,
+ "\xa3": 600,
+ "\xa4": 600,
+ "\xa5": 600,
+ "\xa6": 600,
+ "\xa7": 600,
+ "\xa8": 600,
+ "\xa9": 600,
+ "\xaa": 600,
+ "\xab": 600,
+ "\xac": 600,
+ "\xae": 600,
+ "\xaf": 600,
+ "\xb0": 600,
+ "\xb1": 600,
+ "\xb2": 600,
+ "\xb3": 600,
+ "\xb4": 600,
+ "\xb5": 600,
+ "\xb6": 600,
+ "\xb7": 600,
+ "\xb8": 600,
+ "\xb9": 600,
+ "\xba": 600,
+ "\xbb": 600,
+ "\xbc": 600,
+ "\xbd": 600,
+ "\xbe": 600,
+ "\xbf": 600,
+ "\xc0": 600,
+ "\xc1": 600,
+ "\xc2": 600,
+ "\xc3": 600,
+ "\xc4": 600,
+ "\xc5": 600,
+ "\xc6": 600,
+ "\xc7": 600,
+ "\xc8": 600,
+ "\xc9": 600,
+ "\xca": 600,
+ "\xcb": 600,
+ "\xcc": 600,
+ "\xcd": 600,
+ "\xce": 600,
+ "\xcf": 600,
+ "\xd0": 600,
+ "\xd1": 600,
+ "\xd2": 600,
+ "\xd3": 600,
+ "\xd4": 600,
+ "\xd5": 600,
+ "\xd6": 600,
+ "\xd7": 600,
+ "\xd8": 600,
+ "\xd9": 600,
+ "\xda": 600,
+ "\xdb": 600,
+ "\xdc": 600,
+ "\xdd": 600,
+ "\xde": 600,
+ "\xdf": 600,
+ "\xe0": 600,
+ "\xe1": 600,
+ "\xe2": 600,
+ "\xe3": 600,
+ "\xe4": 600,
+ "\xe5": 600,
+ "\xe6": 600,
+ "\xe7": 600,
+ "\xe8": 600,
+ "\xe9": 600,
+ "\xea": 600,
+ "\xeb": 600,
+ "\xec": 600,
+ "\xed": 600,
+ "\xee": 600,
+ "\xef": 600,
+ "\xf0": 600,
+ "\xf1": 600,
+ "\xf2": 600,
+ "\xf3": 600,
+ "\xf4": 600,
+ "\xf5": 600,
+ "\xf6": 600,
+ "\xf7": 600,
+ "\xf8": 600,
+ "\xf9": 600,
+ "\xfa": 600,
+ "\xfb": 600,
+ "\xfc": 600,
+ "\xfd": 600,
+ "\xfe": 600,
+ "\xff": 600,
+ "\u0100": 600,
+ "\u0101": 600,
+ "\u0102": 600,
+ "\u0103": 600,
+ "\u0104": 600,
+ "\u0105": 600,
+ "\u0106": 600,
+ "\u0107": 600,
+ "\u010c": 600,
+ "\u010d": 600,
+ "\u010e": 600,
+ "\u010f": 600,
+ "\u0110": 600,
+ "\u0111": 600,
+ "\u0112": 600,
+ "\u0113": 600,
+ "\u0116": 600,
+ "\u0117": 600,
+ "\u0118": 600,
+ "\u0119": 600,
+ "\u011a": 600,
+ "\u011b": 600,
+ "\u011e": 600,
+ "\u011f": 600,
+ "\u0122": 600,
+ "\u0123": 600,
+ "\u012a": 600,
+ "\u012b": 600,
+ "\u012e": 600,
+ "\u012f": 600,
+ "\u0130": 600,
+ "\u0131": 600,
+ "\u0136": 600,
+ "\u0137": 600,
+ "\u0139": 600,
+ "\u013a": 600,
+ "\u013b": 600,
+ "\u013c": 600,
+ "\u013d": 600,
+ "\u013e": 600,
+ "\u0141": 600,
+ "\u0142": 600,
+ "\u0143": 600,
+ "\u0144": 600,
+ "\u0145": 600,
+ "\u0146": 600,
+ "\u0147": 600,
+ "\u0148": 600,
+ "\u014c": 600,
+ "\u014d": 600,
+ "\u0150": 600,
+ "\u0151": 600,
+ "\u0152": 600,
+ "\u0153": 600,
+ "\u0154": 600,
+ "\u0155": 600,
+ "\u0156": 600,
+ "\u0157": 600,
+ "\u0158": 600,
+ "\u0159": 600,
+ "\u015a": 600,
+ "\u015b": 600,
+ "\u015e": 600,
+ "\u015f": 600,
+ "\u0160": 600,
+ "\u0161": 600,
+ "\u0162": 600,
+ "\u0163": 600,
+ "\u0164": 600,
+ "\u0165": 600,
+ "\u016a": 600,
+ "\u016b": 600,
+ "\u016e": 600,
+ "\u016f": 600,
+ "\u0170": 600,
+ "\u0171": 600,
+ "\u0172": 600,
+ "\u0173": 600,
+ "\u0178": 600,
+ "\u0179": 600,
+ "\u017a": 600,
+ "\u017b": 600,
+ "\u017c": 600,
+ "\u017d": 600,
+ "\u017e": 600,
+ "\u0192": 600,
+ "\u0218": 600,
+ "\u0219": 600,
+ "\u02c6": 600,
+ "\u02c7": 600,
+ "\u02d8": 600,
+ "\u02d9": 600,
+ "\u02da": 600,
+ "\u02db": 600,
+ "\u02dc": 600,
+ "\u02dd": 600,
+ "\u2013": 600,
+ "\u2014": 600,
+ "\u2018": 600,
+ "\u2019": 600,
+ "\u201a": 600,
+ "\u201c": 600,
+ "\u201d": 600,
+ "\u201e": 600,
+ "\u2020": 600,
+ "\u2021": 600,
+ "\u2022": 600,
+ "\u2026": 600,
+ "\u2030": 600,
+ "\u2039": 600,
+ "\u203a": 600,
+ "\u2044": 600,
+ "\u2122": 600,
+ "\u2202": 600,
+ "\u2206": 600,
+ "\u2211": 600,
+ "\u2212": 600,
+ "\u221a": 600,
+ "\u2260": 600,
+ "\u2264": 600,
+ "\u2265": 600,
+ "\u25ca": 600,
+ "\uf6c3": 600,
+ "\ufb01": 600,
+ "\ufb02": 600,
+ },
+ ),
+ "Helvetica": (
+ {
+ "FontName": "Helvetica",
+ "Descent": -207.0,
+ "FontBBox": (-166.0, -225.0, 1000.0, 931.0),
+ "FontWeight": "Medium",
+ "CapHeight": 718.0,
+ "FontFamily": "Helvetica",
+ "Flags": 0,
+ "XHeight": 523.0,
+ "ItalicAngle": 0.0,
+ "Ascent": 718.0,
+ },
+ {
+ " ": 278,
+ "!": 278,
+ '"': 355,
+ "#": 556,
+ "$": 556,
+ "%": 889,
+ "&": 667,
+ "'": 191,
+ "(": 333,
+ ")": 333,
+ "*": 389,
+ "+": 584,
+ ",": 278,
+ "-": 333,
+ ".": 278,
+ "/": 278,
+ "0": 556,
+ "1": 556,
+ "2": 556,
+ "3": 556,
+ "4": 556,
+ "5": 556,
+ "6": 556,
+ "7": 556,
+ "8": 556,
+ "9": 556,
+ ":": 278,
+ ";": 278,
+ "<": 584,
+ "=": 584,
+ ">": 584,
+ "?": 556,
+ "@": 1015,
+ "A": 667,
+ "B": 667,
+ "C": 722,
+ "D": 722,
+ "E": 667,
+ "F": 611,
+ "G": 778,
+ "H": 722,
+ "I": 278,
+ "J": 500,
+ "K": 667,
+ "L": 556,
+ "M": 833,
+ "N": 722,
+ "O": 778,
+ "P": 667,
+ "Q": 778,
+ "R": 722,
+ "S": 667,
+ "T": 611,
+ "U": 722,
+ "V": 667,
+ "W": 944,
+ "X": 667,
+ "Y": 667,
+ "Z": 611,
+ "[": 278,
+ "\\": 278,
+ "]": 278,
+ "^": 469,
+ "_": 556,
+ "`": 333,
+ "a": 556,
+ "b": 556,
+ "c": 500,
+ "d": 556,
+ "e": 556,
+ "f": 278,
+ "g": 556,
+ "h": 556,
+ "i": 222,
+ "j": 222,
+ "k": 500,
+ "l": 222,
+ "m": 833,
+ "n": 556,
+ "o": 556,
+ "p": 556,
+ "q": 556,
+ "r": 333,
+ "s": 500,
+ "t": 278,
+ "u": 556,
+ "v": 500,
+ "w": 722,
+ "x": 500,
+ "y": 500,
+ "z": 500,
+ "{": 334,
+ "|": 260,
+ "}": 334,
+ "~": 584,
+ "\xa1": 333,
+ "\xa2": 556,
+ "\xa3": 556,
+ "\xa4": 556,
+ "\xa5": 556,
+ "\xa6": 260,
+ "\xa7": 556,
+ "\xa8": 333,
+ "\xa9": 737,
+ "\xaa": 370,
+ "\xab": 556,
+ "\xac": 584,
+ "\xae": 737,
+ "\xaf": 333,
+ "\xb0": 400,
+ "\xb1": 584,
+ "\xb2": 333,
+ "\xb3": 333,
+ "\xb4": 333,
+ "\xb5": 556,
+ "\xb6": 537,
+ "\xb7": 278,
+ "\xb8": 333,
+ "\xb9": 333,
+ "\xba": 365,
+ "\xbb": 556,
+ "\xbc": 834,
+ "\xbd": 834,
+ "\xbe": 834,
+ "\xbf": 611,
+ "\xc0": 667,
+ "\xc1": 667,
+ "\xc2": 667,
+ "\xc3": 667,
+ "\xc4": 667,
+ "\xc5": 667,
+ "\xc6": 1000,
+ "\xc7": 722,
+ "\xc8": 667,
+ "\xc9": 667,
+ "\xca": 667,
+ "\xcb": 667,
+ "\xcc": 278,
+ "\xcd": 278,
+ "\xce": 278,
+ "\xcf": 278,
+ "\xd0": 722,
+ "\xd1": 722,
+ "\xd2": 778,
+ "\xd3": 778,
+ "\xd4": 778,
+ "\xd5": 778,
+ "\xd6": 778,
+ "\xd7": 584,
+ "\xd8": 778,
+ "\xd9": 722,
+ "\xda": 722,
+ "\xdb": 722,
+ "\xdc": 722,
+ "\xdd": 667,
+ "\xde": 667,
+ "\xdf": 611,
+ "\xe0": 556,
+ "\xe1": 556,
+ "\xe2": 556,
+ "\xe3": 556,
+ "\xe4": 556,
+ "\xe5": 556,
+ "\xe6": 889,
+ "\xe7": 500,
+ "\xe8": 556,
+ "\xe9": 556,
+ "\xea": 556,
+ "\xeb": 556,
+ "\xec": 278,
+ "\xed": 278,
+ "\xee": 278,
+ "\xef": 278,
+ "\xf0": 556,
+ "\xf1": 556,
+ "\xf2": 556,
+ "\xf3": 556,
+ "\xf4": 556,
+ "\xf5": 556,
+ "\xf6": 556,
+ "\xf7": 584,
+ "\xf8": 611,
+ "\xf9": 556,
+ "\xfa": 556,
+ "\xfb": 556,
+ "\xfc": 556,
+ "\xfd": 500,
+ "\xfe": 556,
+ "\xff": 500,
+ "\u0100": 667,
+ "\u0101": 556,
+ "\u0102": 667,
+ "\u0103": 556,
+ "\u0104": 667,
+ "\u0105": 556,
+ "\u0106": 722,
+ "\u0107": 500,
+ "\u010c": 722,
+ "\u010d": 500,
+ "\u010e": 722,
+ "\u010f": 643,
+ "\u0110": 722,
+ "\u0111": 556,
+ "\u0112": 667,
+ "\u0113": 556,
+ "\u0116": 667,
+ "\u0117": 556,
+ "\u0118": 667,
+ "\u0119": 556,
+ "\u011a": 667,
+ "\u011b": 556,
+ "\u011e": 778,
+ "\u011f": 556,
+ "\u0122": 778,
+ "\u0123": 556,
+ "\u012a": 278,
+ "\u012b": 278,
+ "\u012e": 278,
+ "\u012f": 222,
+ "\u0130": 278,
+ "\u0131": 278,
+ "\u0136": 667,
+ "\u0137": 500,
+ "\u0139": 556,
+ "\u013a": 222,
+ "\u013b": 556,
+ "\u013c": 222,
+ "\u013d": 556,
+ "\u013e": 299,
+ "\u0141": 556,
+ "\u0142": 222,
+ "\u0143": 722,
+ "\u0144": 556,
+ "\u0145": 722,
+ "\u0146": 556,
+ "\u0147": 722,
+ "\u0148": 556,
+ "\u014c": 778,
+ "\u014d": 556,
+ "\u0150": 778,
+ "\u0151": 556,
+ "\u0152": 1000,
+ "\u0153": 944,
+ "\u0154": 722,
+ "\u0155": 333,
+ "\u0156": 722,
+ "\u0157": 333,
+ "\u0158": 722,
+ "\u0159": 333,
+ "\u015a": 667,
+ "\u015b": 500,
+ "\u015e": 667,
+ "\u015f": 500,
+ "\u0160": 667,
+ "\u0161": 500,
+ "\u0162": 611,
+ "\u0163": 278,
+ "\u0164": 611,
+ "\u0165": 317,
+ "\u016a": 722,
+ "\u016b": 556,
+ "\u016e": 722,
+ "\u016f": 556,
+ "\u0170": 722,
+ "\u0171": 556,
+ "\u0172": 722,
+ "\u0173": 556,
+ "\u0178": 667,
+ "\u0179": 611,
+ "\u017a": 500,
+ "\u017b": 611,
+ "\u017c": 500,
+ "\u017d": 611,
+ "\u017e": 500,
+ "\u0192": 556,
+ "\u0218": 667,
+ "\u0219": 500,
+ "\u02c6": 333,
+ "\u02c7": 333,
+ "\u02d8": 333,
+ "\u02d9": 333,
+ "\u02da": 333,
+ "\u02db": 333,
+ "\u02dc": 333,
+ "\u02dd": 333,
+ "\u2013": 556,
+ "\u2014": 1000,
+ "\u2018": 222,
+ "\u2019": 222,
+ "\u201a": 222,
+ "\u201c": 333,
+ "\u201d": 333,
+ "\u201e": 333,
+ "\u2020": 556,
+ "\u2021": 556,
+ "\u2022": 350,
+ "\u2026": 1000,
+ "\u2030": 1000,
+ "\u2039": 333,
+ "\u203a": 333,
+ "\u2044": 167,
+ "\u2122": 1000,
+ "\u2202": 476,
+ "\u2206": 612,
+ "\u2211": 600,
+ "\u2212": 584,
+ "\u221a": 453,
+ "\u2260": 549,
+ "\u2264": 549,
+ "\u2265": 549,
+ "\u25ca": 471,
+ "\uf6c3": 250,
+ "\ufb01": 500,
+ "\ufb02": 500,
+ },
+ ),
+ "Helvetica-Bold": (
+ {
+ "FontName": "Helvetica-Bold",
+ "Descent": -207.0,
+ "FontBBox": (-170.0, -228.0, 1003.0, 962.0),
+ "FontWeight": "Bold",
+ "CapHeight": 718.0,
+ "FontFamily": "Helvetica",
+ "Flags": 0,
+ "XHeight": 532.0,
+ "ItalicAngle": 0.0,
+ "Ascent": 718.0,
+ },
+ {
+ " ": 278,
+ "!": 333,
+ '"': 474,
+ "#": 556,
+ "$": 556,
+ "%": 889,
+ "&": 722,
+ "'": 238,
+ "(": 333,
+ ")": 333,
+ "*": 389,
+ "+": 584,
+ ",": 278,
+ "-": 333,
+ ".": 278,
+ "/": 278,
+ "0": 556,
+ "1": 556,
+ "2": 556,
+ "3": 556,
+ "4": 556,
+ "5": 556,
+ "6": 556,
+ "7": 556,
+ "8": 556,
+ "9": 556,
+ ":": 333,
+ ";": 333,
+ "<": 584,
+ "=": 584,
+ ">": 584,
+ "?": 611,
+ "@": 975,
+ "A": 722,
+ "B": 722,
+ "C": 722,
+ "D": 722,
+ "E": 667,
+ "F": 611,
+ "G": 778,
+ "H": 722,
+ "I": 278,
+ "J": 556,
+ "K": 722,
+ "L": 611,
+ "M": 833,
+ "N": 722,
+ "O": 778,
+ "P": 667,
+ "Q": 778,
+ "R": 722,
+ "S": 667,
+ "T": 611,
+ "U": 722,
+ "V": 667,
+ "W": 944,
+ "X": 667,
+ "Y": 667,
+ "Z": 611,
+ "[": 333,
+ "\\": 278,
+ "]": 333,
+ "^": 584,
+ "_": 556,
+ "`": 333,
+ "a": 556,
+ "b": 611,
+ "c": 556,
+ "d": 611,
+ "e": 556,
+ "f": 333,
+ "g": 611,
+ "h": 611,
+ "i": 278,
+ "j": 278,
+ "k": 556,
+ "l": 278,
+ "m": 889,
+ "n": 611,
+ "o": 611,
+ "p": 611,
+ "q": 611,
+ "r": 389,
+ "s": 556,
+ "t": 333,
+ "u": 611,
+ "v": 556,
+ "w": 778,
+ "x": 556,
+ "y": 556,
+ "z": 500,
+ "{": 389,
+ "|": 280,
+ "}": 389,
+ "~": 584,
+ "\xa1": 333,
+ "\xa2": 556,
+ "\xa3": 556,
+ "\xa4": 556,
+ "\xa5": 556,
+ "\xa6": 280,
+ "\xa7": 556,
+ "\xa8": 333,
+ "\xa9": 737,
+ "\xaa": 370,
+ "\xab": 556,
+ "\xac": 584,
+ "\xae": 737,
+ "\xaf": 333,
+ "\xb0": 400,
+ "\xb1": 584,
+ "\xb2": 333,
+ "\xb3": 333,
+ "\xb4": 333,
+ "\xb5": 611,
+ "\xb6": 556,
+ "\xb7": 278,
+ "\xb8": 333,
+ "\xb9": 333,
+ "\xba": 365,
+ "\xbb": 556,
+ "\xbc": 834,
+ "\xbd": 834,
+ "\xbe": 834,
+ "\xbf": 611,
+ "\xc0": 722,
+ "\xc1": 722,
+ "\xc2": 722,
+ "\xc3": 722,
+ "\xc4": 722,
+ "\xc5": 722,
+ "\xc6": 1000,
+ "\xc7": 722,
+ "\xc8": 667,
+ "\xc9": 667,
+ "\xca": 667,
+ "\xcb": 667,
+ "\xcc": 278,
+ "\xcd": 278,
+ "\xce": 278,
+ "\xcf": 278,
+ "\xd0": 722,
+ "\xd1": 722,
+ "\xd2": 778,
+ "\xd3": 778,
+ "\xd4": 778,
+ "\xd5": 778,
+ "\xd6": 778,
+ "\xd7": 584,
+ "\xd8": 778,
+ "\xd9": 722,
+ "\xda": 722,
+ "\xdb": 722,
+ "\xdc": 722,
+ "\xdd": 667,
+ "\xde": 667,
+ "\xdf": 611,
+ "\xe0": 556,
+ "\xe1": 556,
+ "\xe2": 556,
+ "\xe3": 556,
+ "\xe4": 556,
+ "\xe5": 556,
+ "\xe6": 889,
+ "\xe7": 556,
+ "\xe8": 556,
+ "\xe9": 556,
+ "\xea": 556,
+ "\xeb": 556,
+ "\xec": 278,
+ "\xed": 278,
+ "\xee": 278,
+ "\xef": 278,
+ "\xf0": 611,
+ "\xf1": 611,
+ "\xf2": 611,
+ "\xf3": 611,
+ "\xf4": 611,
+ "\xf5": 611,
+ "\xf6": 611,
+ "\xf7": 584,
+ "\xf8": 611,
+ "\xf9": 611,
+ "\xfa": 611,
+ "\xfb": 611,
+ "\xfc": 611,
+ "\xfd": 556,
+ "\xfe": 611,
+ "\xff": 556,
+ "\u0100": 722,
+ "\u0101": 556,
+ "\u0102": 722,
+ "\u0103": 556,
+ "\u0104": 722,
+ "\u0105": 556,
+ "\u0106": 722,
+ "\u0107": 556,
+ "\u010c": 722,
+ "\u010d": 556,
+ "\u010e": 722,
+ "\u010f": 743,
+ "\u0110": 722,
+ "\u0111": 611,
+ "\u0112": 667,
+ "\u0113": 556,
+ "\u0116": 667,
+ "\u0117": 556,
+ "\u0118": 667,
+ "\u0119": 556,
+ "\u011a": 667,
+ "\u011b": 556,
+ "\u011e": 778,
+ "\u011f": 611,
+ "\u0122": 778,
+ "\u0123": 611,
+ "\u012a": 278,
+ "\u012b": 278,
+ "\u012e": 278,
+ "\u012f": 278,
+ "\u0130": 278,
+ "\u0131": 278,
+ "\u0136": 722,
+ "\u0137": 556,
+ "\u0139": 611,
+ "\u013a": 278,
+ "\u013b": 611,
+ "\u013c": 278,
+ "\u013d": 611,
+ "\u013e": 400,
+ "\u0141": 611,
+ "\u0142": 278,
+ "\u0143": 722,
+ "\u0144": 611,
+ "\u0145": 722,
+ "\u0146": 611,
+ "\u0147": 722,
+ "\u0148": 611,
+ "\u014c": 778,
+ "\u014d": 611,
+ "\u0150": 778,
+ "\u0151": 611,
+ "\u0152": 1000,
+ "\u0153": 944,
+ "\u0154": 722,
+ "\u0155": 389,
+ "\u0156": 722,
+ "\u0157": 389,
+ "\u0158": 722,
+ "\u0159": 389,
+ "\u015a": 667,
+ "\u015b": 556,
+ "\u015e": 667,
+ "\u015f": 556,
+ "\u0160": 667,
+ "\u0161": 556,
+ "\u0162": 611,
+ "\u0163": 333,
+ "\u0164": 611,
+ "\u0165": 389,
+ "\u016a": 722,
+ "\u016b": 611,
+ "\u016e": 722,
+ "\u016f": 611,
+ "\u0170": 722,
+ "\u0171": 611,
+ "\u0172": 722,
+ "\u0173": 611,
+ "\u0178": 667,
+ "\u0179": 611,
+ "\u017a": 500,
+ "\u017b": 611,
+ "\u017c": 500,
+ "\u017d": 611,
+ "\u017e": 500,
+ "\u0192": 556,
+ "\u0218": 667,
+ "\u0219": 556,
+ "\u02c6": 333,
+ "\u02c7": 333,
+ "\u02d8": 333,
+ "\u02d9": 333,
+ "\u02da": 333,
+ "\u02db": 333,
+ "\u02dc": 333,
+ "\u02dd": 333,
+ "\u2013": 556,
+ "\u2014": 1000,
+ "\u2018": 278,
+ "\u2019": 278,
+ "\u201a": 278,
+ "\u201c": 500,
+ "\u201d": 500,
+ "\u201e": 500,
+ "\u2020": 556,
+ "\u2021": 556,
+ "\u2022": 350,
+ "\u2026": 1000,
+ "\u2030": 1000,
+ "\u2039": 333,
+ "\u203a": 333,
+ "\u2044": 167,
+ "\u2122": 1000,
+ "\u2202": 494,
+ "\u2206": 612,
+ "\u2211": 600,
+ "\u2212": 584,
+ "\u221a": 549,
+ "\u2260": 549,
+ "\u2264": 549,
+ "\u2265": 549,
+ "\u25ca": 494,
+ "\uf6c3": 250,
+ "\ufb01": 611,
+ "\ufb02": 611,
+ },
+ ),
+ "Helvetica-BoldOblique": (
+ {
+ "FontName": "Helvetica-BoldOblique",
+ "Descent": -207.0,
+ "FontBBox": (-175.0, -228.0, 1114.0, 962.0),
+ "FontWeight": "Bold",
+ "CapHeight": 718.0,
+ "FontFamily": "Helvetica",
+ "Flags": 0,
+ "XHeight": 532.0,
+ "ItalicAngle": -12.0,
+ "Ascent": 718.0,
+ },
+ {
+ " ": 278,
+ "!": 333,
+ '"': 474,
+ "#": 556,
+ "$": 556,
+ "%": 889,
+ "&": 722,
+ "'": 238,
+ "(": 333,
+ ")": 333,
+ "*": 389,
+ "+": 584,
+ ",": 278,
+ "-": 333,
+ ".": 278,
+ "/": 278,
+ "0": 556,
+ "1": 556,
+ "2": 556,
+ "3": 556,
+ "4": 556,
+ "5": 556,
+ "6": 556,
+ "7": 556,
+ "8": 556,
+ "9": 556,
+ ":": 333,
+ ";": 333,
+ "<": 584,
+ "=": 584,
+ ">": 584,
+ "?": 611,
+ "@": 975,
+ "A": 722,
+ "B": 722,
+ "C": 722,
+ "D": 722,
+ "E": 667,
+ "F": 611,
+ "G": 778,
+ "H": 722,
+ "I": 278,
+ "J": 556,
+ "K": 722,
+ "L": 611,
+ "M": 833,
+ "N": 722,
+ "O": 778,
+ "P": 667,
+ "Q": 778,
+ "R": 722,
+ "S": 667,
+ "T": 611,
+ "U": 722,
+ "V": 667,
+ "W": 944,
+ "X": 667,
+ "Y": 667,
+ "Z": 611,
+ "[": 333,
+ "\\": 278,
+ "]": 333,
+ "^": 584,
+ "_": 556,
+ "`": 333,
+ "a": 556,
+ "b": 611,
+ "c": 556,
+ "d": 611,
+ "e": 556,
+ "f": 333,
+ "g": 611,
+ "h": 611,
+ "i": 278,
+ "j": 278,
+ "k": 556,
+ "l": 278,
+ "m": 889,
+ "n": 611,
+ "o": 611,
+ "p": 611,
+ "q": 611,
+ "r": 389,
+ "s": 556,
+ "t": 333,
+ "u": 611,
+ "v": 556,
+ "w": 778,
+ "x": 556,
+ "y": 556,
+ "z": 500,
+ "{": 389,
+ "|": 280,
+ "}": 389,
+ "~": 584,
+ "\xa1": 333,
+ "\xa2": 556,
+ "\xa3": 556,
+ "\xa4": 556,
+ "\xa5": 556,
+ "\xa6": 280,
+ "\xa7": 556,
+ "\xa8": 333,
+ "\xa9": 737,
+ "\xaa": 370,
+ "\xab": 556,
+ "\xac": 584,
+ "\xae": 737,
+ "\xaf": 333,
+ "\xb0": 400,
+ "\xb1": 584,
+ "\xb2": 333,
+ "\xb3": 333,
+ "\xb4": 333,
+ "\xb5": 611,
+ "\xb6": 556,
+ "\xb7": 278,
+ "\xb8": 333,
+ "\xb9": 333,
+ "\xba": 365,
+ "\xbb": 556,
+ "\xbc": 834,
+ "\xbd": 834,
+ "\xbe": 834,
+ "\xbf": 611,
+ "\xc0": 722,
+ "\xc1": 722,
+ "\xc2": 722,
+ "\xc3": 722,
+ "\xc4": 722,
+ "\xc5": 722,
+ "\xc6": 1000,
+ "\xc7": 722,
+ "\xc8": 667,
+ "\xc9": 667,
+ "\xca": 667,
+ "\xcb": 667,
+ "\xcc": 278,
+ "\xcd": 278,
+ "\xce": 278,
+ "\xcf": 278,
+ "\xd0": 722,
+ "\xd1": 722,
+ "\xd2": 778,
+ "\xd3": 778,
+ "\xd4": 778,
+ "\xd5": 778,
+ "\xd6": 778,
+ "\xd7": 584,
+ "\xd8": 778,
+ "\xd9": 722,
+ "\xda": 722,
+ "\xdb": 722,
+ "\xdc": 722,
+ "\xdd": 667,
+ "\xde": 667,
+ "\xdf": 611,
+ "\xe0": 556,
+ "\xe1": 556,
+ "\xe2": 556,
+ "\xe3": 556,
+ "\xe4": 556,
+ "\xe5": 556,
+ "\xe6": 889,
+ "\xe7": 556,
+ "\xe8": 556,
+ "\xe9": 556,
+ "\xea": 556,
+ "\xeb": 556,
+ "\xec": 278,
+ "\xed": 278,
+ "\xee": 278,
+ "\xef": 278,
+ "\xf0": 611,
+ "\xf1": 611,
+ "\xf2": 611,
+ "\xf3": 611,
+ "\xf4": 611,
+ "\xf5": 611,
+ "\xf6": 611,
+ "\xf7": 584,
+ "\xf8": 611,
+ "\xf9": 611,
+ "\xfa": 611,
+ "\xfb": 611,
+ "\xfc": 611,
+ "\xfd": 556,
+ "\xfe": 611,
+ "\xff": 556,
+ "\u0100": 722,
+ "\u0101": 556,
+ "\u0102": 722,
+ "\u0103": 556,
+ "\u0104": 722,
+ "\u0105": 556,
+ "\u0106": 722,
+ "\u0107": 556,
+ "\u010c": 722,
+ "\u010d": 556,
+ "\u010e": 722,
+ "\u010f": 743,
+ "\u0110": 722,
+ "\u0111": 611,
+ "\u0112": 667,
+ "\u0113": 556,
+ "\u0116": 667,
+ "\u0117": 556,
+ "\u0118": 667,
+ "\u0119": 556,
+ "\u011a": 667,
+ "\u011b": 556,
+ "\u011e": 778,
+ "\u011f": 611,
+ "\u0122": 778,
+ "\u0123": 611,
+ "\u012a": 278,
+ "\u012b": 278,
+ "\u012e": 278,
+ "\u012f": 278,
+ "\u0130": 278,
+ "\u0131": 278,
+ "\u0136": 722,
+ "\u0137": 556,
+ "\u0139": 611,
+ "\u013a": 278,
+ "\u013b": 611,
+ "\u013c": 278,
+ "\u013d": 611,
+ "\u013e": 400,
+ "\u0141": 611,
+ "\u0142": 278,
+ "\u0143": 722,
+ "\u0144": 611,
+ "\u0145": 722,
+ "\u0146": 611,
+ "\u0147": 722,
+ "\u0148": 611,
+ "\u014c": 778,
+ "\u014d": 611,
+ "\u0150": 778,
+ "\u0151": 611,
+ "\u0152": 1000,
+ "\u0153": 944,
+ "\u0154": 722,
+ "\u0155": 389,
+ "\u0156": 722,
+ "\u0157": 389,
+ "\u0158": 722,
+ "\u0159": 389,
+ "\u015a": 667,
+ "\u015b": 556,
+ "\u015e": 667,
+ "\u015f": 556,
+ "\u0160": 667,
+ "\u0161": 556,
+ "\u0162": 611,
+ "\u0163": 333,
+ "\u0164": 611,
+ "\u0165": 389,
+ "\u016a": 722,
+ "\u016b": 611,
+ "\u016e": 722,
+ "\u016f": 611,
+ "\u0170": 722,
+ "\u0171": 611,
+ "\u0172": 722,
+ "\u0173": 611,
+ "\u0178": 667,
+ "\u0179": 611,
+ "\u017a": 500,
+ "\u017b": 611,
+ "\u017c": 500,
+ "\u017d": 611,
+ "\u017e": 500,
+ "\u0192": 556,
+ "\u0218": 667,
+ "\u0219": 556,
+ "\u02c6": 333,
+ "\u02c7": 333,
+ "\u02d8": 333,
+ "\u02d9": 333,
+ "\u02da": 333,
+ "\u02db": 333,
+ "\u02dc": 333,
+ "\u02dd": 333,
+ "\u2013": 556,
+ "\u2014": 1000,
+ "\u2018": 278,
+ "\u2019": 278,
+ "\u201a": 278,
+ "\u201c": 500,
+ "\u201d": 500,
+ "\u201e": 500,
+ "\u2020": 556,
+ "\u2021": 556,
+ "\u2022": 350,
+ "\u2026": 1000,
+ "\u2030": 1000,
+ "\u2039": 333,
+ "\u203a": 333,
+ "\u2044": 167,
+ "\u2122": 1000,
+ "\u2202": 494,
+ "\u2206": 612,
+ "\u2211": 600,
+ "\u2212": 584,
+ "\u221a": 549,
+ "\u2260": 549,
+ "\u2264": 549,
+ "\u2265": 549,
+ "\u25ca": 494,
+ "\uf6c3": 250,
+ "\ufb01": 611,
+ "\ufb02": 611,
+ },
+ ),
+ "Helvetica-Oblique": (
+ {
+ "FontName": "Helvetica-Oblique",
+ "Descent": -207.0,
+ "FontBBox": (-171.0, -225.0, 1116.0, 931.0),
+ "FontWeight": "Medium",
+ "CapHeight": 718.0,
+ "FontFamily": "Helvetica",
+ "Flags": 0,
+ "XHeight": 523.0,
+ "ItalicAngle": -12.0,
+ "Ascent": 718.0,
+ },
+ {
+ " ": 278,
+ "!": 278,
+ '"': 355,
+ "#": 556,
+ "$": 556,
+ "%": 889,
+ "&": 667,
+ "'": 191,
+ "(": 333,
+ ")": 333,
+ "*": 389,
+ "+": 584,
+ ",": 278,
+ "-": 333,
+ ".": 278,
+ "/": 278,
+ "0": 556,
+ "1": 556,
+ "2": 556,
+ "3": 556,
+ "4": 556,
+ "5": 556,
+ "6": 556,
+ "7": 556,
+ "8": 556,
+ "9": 556,
+ ":": 278,
+ ";": 278,
+ "<": 584,
+ "=": 584,
+ ">": 584,
+ "?": 556,
+ "@": 1015,
+ "A": 667,
+ "B": 667,
+ "C": 722,
+ "D": 722,
+ "E": 667,
+ "F": 611,
+ "G": 778,
+ "H": 722,
+ "I": 278,
+ "J": 500,
+ "K": 667,
+ "L": 556,
+ "M": 833,
+ "N": 722,
+ "O": 778,
+ "P": 667,
+ "Q": 778,
+ "R": 722,
+ "S": 667,
+ "T": 611,
+ "U": 722,
+ "V": 667,
+ "W": 944,
+ "X": 667,
+ "Y": 667,
+ "Z": 611,
+ "[": 278,
+ "\\": 278,
+ "]": 278,
+ "^": 469,
+ "_": 556,
+ "`": 333,
+ "a": 556,
+ "b": 556,
+ "c": 500,
+ "d": 556,
+ "e": 556,
+ "f": 278,
+ "g": 556,
+ "h": 556,
+ "i": 222,
+ "j": 222,
+ "k": 500,
+ "l": 222,
+ "m": 833,
+ "n": 556,
+ "o": 556,
+ "p": 556,
+ "q": 556,
+ "r": 333,
+ "s": 500,
+ "t": 278,
+ "u": 556,
+ "v": 500,
+ "w": 722,
+ "x": 500,
+ "y": 500,
+ "z": 500,
+ "{": 334,
+ "|": 260,
+ "}": 334,
+ "~": 584,
+ "\xa1": 333,
+ "\xa2": 556,
+ "\xa3": 556,
+ "\xa4": 556,
+ "\xa5": 556,
+ "\xa6": 260,
+ "\xa7": 556,
+ "\xa8": 333,
+ "\xa9": 737,
+ "\xaa": 370,
+ "\xab": 556,
+ "\xac": 584,
+ "\xae": 737,
+ "\xaf": 333,
+ "\xb0": 400,
+ "\xb1": 584,
+ "\xb2": 333,
+ "\xb3": 333,
+ "\xb4": 333,
+ "\xb5": 556,
+ "\xb6": 537,
+ "\xb7": 278,
+ "\xb8": 333,
+ "\xb9": 333,
+ "\xba": 365,
+ "\xbb": 556,
+ "\xbc": 834,
+ "\xbd": 834,
+ "\xbe": 834,
+ "\xbf": 611,
+ "\xc0": 667,
+ "\xc1": 667,
+ "\xc2": 667,
+ "\xc3": 667,
+ "\xc4": 667,
+ "\xc5": 667,
+ "\xc6": 1000,
+ "\xc7": 722,
+ "\xc8": 667,
+ "\xc9": 667,
+ "\xca": 667,
+ "\xcb": 667,
+ "\xcc": 278,
+ "\xcd": 278,
+ "\xce": 278,
+ "\xcf": 278,
+ "\xd0": 722,
+ "\xd1": 722,
+ "\xd2": 778,
+ "\xd3": 778,
+ "\xd4": 778,
+ "\xd5": 778,
+ "\xd6": 778,
+ "\xd7": 584,
+ "\xd8": 778,
+ "\xd9": 722,
+ "\xda": 722,
+ "\xdb": 722,
+ "\xdc": 722,
+ "\xdd": 667,
+ "\xde": 667,
+ "\xdf": 611,
+ "\xe0": 556,
+ "\xe1": 556,
+ "\xe2": 556,
+ "\xe3": 556,
+ "\xe4": 556,
+ "\xe5": 556,
+ "\xe6": 889,
+ "\xe7": 500,
+ "\xe8": 556,
+ "\xe9": 556,
+ "\xea": 556,
+ "\xeb": 556,
+ "\xec": 278,
+ "\xed": 278,
+ "\xee": 278,
+ "\xef": 278,
+ "\xf0": 556,
+ "\xf1": 556,
+ "\xf2": 556,
+ "\xf3": 556,
+ "\xf4": 556,
+ "\xf5": 556,
+ "\xf6": 556,
+ "\xf7": 584,
+ "\xf8": 611,
+ "\xf9": 556,
+ "\xfa": 556,
+ "\xfb": 556,
+ "\xfc": 556,
+ "\xfd": 500,
+ "\xfe": 556,
+ "\xff": 500,
+ "\u0100": 667,
+ "\u0101": 556,
+ "\u0102": 667,
+ "\u0103": 556,
+ "\u0104": 667,
+ "\u0105": 556,
+ "\u0106": 722,
+ "\u0107": 500,
+ "\u010c": 722,
+ "\u010d": 500,
+ "\u010e": 722,
+ "\u010f": 643,
+ "\u0110": 722,
+ "\u0111": 556,
+ "\u0112": 667,
+ "\u0113": 556,
+ "\u0116": 667,
+ "\u0117": 556,
+ "\u0118": 667,
+ "\u0119": 556,
+ "\u011a": 667,
+ "\u011b": 556,
+ "\u011e": 778,
+ "\u011f": 556,
+ "\u0122": 778,
+ "\u0123": 556,
+ "\u012a": 278,
+ "\u012b": 278,
+ "\u012e": 278,
+ "\u012f": 222,
+ "\u0130": 278,
+ "\u0131": 278,
+ "\u0136": 667,
+ "\u0137": 500,
+ "\u0139": 556,
+ "\u013a": 222,
+ "\u013b": 556,
+ "\u013c": 222,
+ "\u013d": 556,
+ "\u013e": 299,
+ "\u0141": 556,
+ "\u0142": 222,
+ "\u0143": 722,
+ "\u0144": 556,
+ "\u0145": 722,
+ "\u0146": 556,
+ "\u0147": 722,
+ "\u0148": 556,
+ "\u014c": 778,
+ "\u014d": 556,
+ "\u0150": 778,
+ "\u0151": 556,
+ "\u0152": 1000,
+ "\u0153": 944,
+ "\u0154": 722,
+ "\u0155": 333,
+ "\u0156": 722,
+ "\u0157": 333,
+ "\u0158": 722,
+ "\u0159": 333,
+ "\u015a": 667,
+ "\u015b": 500,
+ "\u015e": 667,
+ "\u015f": 500,
+ "\u0160": 667,
+ "\u0161": 500,
+ "\u0162": 611,
+ "\u0163": 278,
+ "\u0164": 611,
+ "\u0165": 317,
+ "\u016a": 722,
+ "\u016b": 556,
+ "\u016e": 722,
+ "\u016f": 556,
+ "\u0170": 722,
+ "\u0171": 556,
+ "\u0172": 722,
+ "\u0173": 556,
+ "\u0178": 667,
+ "\u0179": 611,
+ "\u017a": 500,
+ "\u017b": 611,
+ "\u017c": 500,
+ "\u017d": 611,
+ "\u017e": 500,
+ "\u0192": 556,
+ "\u0218": 667,
+ "\u0219": 500,
+ "\u02c6": 333,
+ "\u02c7": 333,
+ "\u02d8": 333,
+ "\u02d9": 333,
+ "\u02da": 333,
+ "\u02db": 333,
+ "\u02dc": 333,
+ "\u02dd": 333,
+ "\u2013": 556,
+ "\u2014": 1000,
+ "\u2018": 222,
+ "\u2019": 222,
+ "\u201a": 222,
+ "\u201c": 333,
+ "\u201d": 333,
+ "\u201e": 333,
+ "\u2020": 556,
+ "\u2021": 556,
+ "\u2022": 350,
+ "\u2026": 1000,
+ "\u2030": 1000,
+ "\u2039": 333,
+ "\u203a": 333,
+ "\u2044": 167,
+ "\u2122": 1000,
+ "\u2202": 476,
+ "\u2206": 612,
+ "\u2211": 600,
+ "\u2212": 584,
+ "\u221a": 453,
+ "\u2260": 549,
+ "\u2264": 549,
+ "\u2265": 549,
+ "\u25ca": 471,
+ "\uf6c3": 250,
+ "\ufb01": 500,
+ "\ufb02": 500,
+ },
+ ),
+ "Symbol": (
+ {
+ "FontName": "Symbol",
+ "FontBBox": (-180.0, -293.0, 1090.0, 1010.0),
+ "FontWeight": "Medium",
+ "FontFamily": "Symbol",
+ "Flags": 0,
+ "ItalicAngle": 0.0,
+ },
+ {
+ " ": 250,
+ "!": 333,
+ "#": 500,
+ "%": 833,
+ "&": 778,
+ "(": 333,
+ ")": 333,
+ "+": 549,
+ ",": 250,
+ ".": 250,
+ "/": 278,
+ "0": 500,
+ "1": 500,
+ "2": 500,
+ "3": 500,
+ "4": 500,
+ "5": 500,
+ "6": 500,
+ "7": 500,
+ "8": 500,
+ "9": 500,
+ ":": 278,
+ ";": 278,
+ "<": 549,
+ "=": 549,
+ ">": 549,
+ "?": 444,
+ "[": 333,
+ "]": 333,
+ "_": 500,
+ "{": 480,
+ "|": 200,
+ "}": 480,
+ "\xac": 713,
+ "\xb0": 400,
+ "\xb1": 549,
+ "\xb5": 576,
+ "\xd7": 549,
+ "\xf7": 549,
+ "\u0192": 500,
+ "\u0391": 722,
+ "\u0392": 667,
+ "\u0393": 603,
+ "\u0395": 611,
+ "\u0396": 611,
+ "\u0397": 722,
+ "\u0398": 741,
+ "\u0399": 333,
+ "\u039a": 722,
+ "\u039b": 686,
+ "\u039c": 889,
+ "\u039d": 722,
+ "\u039e": 645,
+ "\u039f": 722,
+ "\u03a0": 768,
+ "\u03a1": 556,
+ "\u03a3": 592,
+ "\u03a4": 611,
+ "\u03a5": 690,
+ "\u03a6": 763,
+ "\u03a7": 722,
+ "\u03a8": 795,
+ "\u03b1": 631,
+ "\u03b2": 549,
+ "\u03b3": 411,
+ "\u03b4": 494,
+ "\u03b5": 439,
+ "\u03b6": 494,
+ "\u03b7": 603,
+ "\u03b8": 521,
+ "\u03b9": 329,
+ "\u03ba": 549,
+ "\u03bb": 549,
+ "\u03bd": 521,
+ "\u03be": 493,
+ "\u03bf": 549,
+ "\u03c0": 549,
+ "\u03c1": 549,
+ "\u03c2": 439,
+ "\u03c3": 603,
+ "\u03c4": 439,
+ "\u03c5": 576,
+ "\u03c6": 521,
+ "\u03c7": 549,
+ "\u03c8": 686,
+ "\u03c9": 686,
+ "\u03d1": 631,
+ "\u03d2": 620,
+ "\u03d5": 603,
+ "\u03d6": 713,
+ "\u2022": 460,
+ "\u2026": 1000,
+ "\u2032": 247,
+ "\u2033": 411,
+ "\u2044": 167,
+ "\u20ac": 750,
+ "\u2111": 686,
+ "\u2118": 987,
+ "\u211c": 795,
+ "\u2126": 768,
+ "\u2135": 823,
+ "\u2190": 987,
+ "\u2191": 603,
+ "\u2192": 987,
+ "\u2193": 603,
+ "\u2194": 1042,
+ "\u21b5": 658,
+ "\u21d0": 987,
+ "\u21d1": 603,
+ "\u21d2": 987,
+ "\u21d3": 603,
+ "\u21d4": 1042,
+ "\u2200": 713,
+ "\u2202": 494,
+ "\u2203": 549,
+ "\u2205": 823,
+ "\u2206": 612,
+ "\u2207": 713,
+ "\u2208": 713,
+ "\u2209": 713,
+ "\u220b": 439,
+ "\u220f": 823,
+ "\u2211": 713,
+ "\u2212": 549,
+ "\u2217": 500,
+ "\u221a": 549,
+ "\u221d": 713,
+ "\u221e": 713,
+ "\u2220": 768,
+ "\u2227": 603,
+ "\u2228": 603,
+ "\u2229": 768,
+ "\u222a": 768,
+ "\u222b": 274,
+ "\u2234": 863,
+ "\u223c": 549,
+ "\u2245": 549,
+ "\u2248": 549,
+ "\u2260": 549,
+ "\u2261": 549,
+ "\u2264": 549,
+ "\u2265": 549,
+ "\u2282": 713,
+ "\u2283": 713,
+ "\u2284": 713,
+ "\u2286": 713,
+ "\u2287": 713,
+ "\u2295": 768,
+ "\u2297": 768,
+ "\u22a5": 658,
+ "\u22c5": 250,
+ "\u2320": 686,
+ "\u2321": 686,
+ "\u2329": 329,
+ "\u232a": 329,
+ "\u25ca": 494,
+ "\u2660": 753,
+ "\u2663": 753,
+ "\u2665": 753,
+ "\u2666": 753,
+ "\uf6d9": 790,
+ "\uf6da": 790,
+ "\uf6db": 890,
+ "\uf8e5": 500,
+ "\uf8e6": 603,
+ "\uf8e7": 1000,
+ "\uf8e8": 790,
+ "\uf8e9": 790,
+ "\uf8ea": 786,
+ "\uf8eb": 384,
+ "\uf8ec": 384,
+ "\uf8ed": 384,
+ "\uf8ee": 384,
+ "\uf8ef": 384,
+ "\uf8f0": 384,
+ "\uf8f1": 494,
+ "\uf8f2": 494,
+ "\uf8f3": 494,
+ "\uf8f4": 494,
+ "\uf8f5": 686,
+ "\uf8f6": 384,
+ "\uf8f7": 384,
+ "\uf8f8": 384,
+ "\uf8f9": 384,
+ "\uf8fa": 384,
+ "\uf8fb": 384,
+ "\uf8fc": 494,
+ "\uf8fd": 494,
+ "\uf8fe": 494,
+ "\uf8ff": 790,
+ },
+ ),
+ "Times-Bold": (
+ {
+ "FontName": "Times-Bold",
+ "Descent": -217.0,
+ "FontBBox": (-168.0, -218.0, 1000.0, 935.0),
+ "FontWeight": "Bold",
+ "CapHeight": 676.0,
+ "FontFamily": "Times",
+ "Flags": 0,
+ "XHeight": 461.0,
+ "ItalicAngle": 0.0,
+ "Ascent": 683.0,
+ },
+ {
+ " ": 250,
+ "!": 333,
+ '"': 555,
+ "#": 500,
+ "$": 500,
+ "%": 1000,
+ "&": 833,
+ "'": 278,
+ "(": 333,
+ ")": 333,
+ "*": 500,
+ "+": 570,
+ ",": 250,
+ "-": 333,
+ ".": 250,
+ "/": 278,
+ "0": 500,
+ "1": 500,
+ "2": 500,
+ "3": 500,
+ "4": 500,
+ "5": 500,
+ "6": 500,
+ "7": 500,
+ "8": 500,
+ "9": 500,
+ ":": 333,
+ ";": 333,
+ "<": 570,
+ "=": 570,
+ ">": 570,
+ "?": 500,
+ "@": 930,
+ "A": 722,
+ "B": 667,
+ "C": 722,
+ "D": 722,
+ "E": 667,
+ "F": 611,
+ "G": 778,
+ "H": 778,
+ "I": 389,
+ "J": 500,
+ "K": 778,
+ "L": 667,
+ "M": 944,
+ "N": 722,
+ "O": 778,
+ "P": 611,
+ "Q": 778,
+ "R": 722,
+ "S": 556,
+ "T": 667,
+ "U": 722,
+ "V": 722,
+ "W": 1000,
+ "X": 722,
+ "Y": 722,
+ "Z": 667,
+ "[": 333,
+ "\\": 278,
+ "]": 333,
+ "^": 581,
+ "_": 500,
+ "`": 333,
+ "a": 500,
+ "b": 556,
+ "c": 444,
+ "d": 556,
+ "e": 444,
+ "f": 333,
+ "g": 500,
+ "h": 556,
+ "i": 278,
+ "j": 333,
+ "k": 556,
+ "l": 278,
+ "m": 833,
+ "n": 556,
+ "o": 500,
+ "p": 556,
+ "q": 556,
+ "r": 444,
+ "s": 389,
+ "t": 333,
+ "u": 556,
+ "v": 500,
+ "w": 722,
+ "x": 500,
+ "y": 500,
+ "z": 444,
+ "{": 394,
+ "|": 220,
+ "}": 394,
+ "~": 520,
+ "\xa1": 333,
+ "\xa2": 500,
+ "\xa3": 500,
+ "\xa4": 500,
+ "\xa5": 500,
+ "\xa6": 220,
+ "\xa7": 500,
+ "\xa8": 333,
+ "\xa9": 747,
+ "\xaa": 300,
+ "\xab": 500,
+ "\xac": 570,
+ "\xae": 747,
+ "\xaf": 333,
+ "\xb0": 400,
+ "\xb1": 570,
+ "\xb2": 300,
+ "\xb3": 300,
+ "\xb4": 333,
+ "\xb5": 556,
+ "\xb6": 540,
+ "\xb7": 250,
+ "\xb8": 333,
+ "\xb9": 300,
+ "\xba": 330,
+ "\xbb": 500,
+ "\xbc": 750,
+ "\xbd": 750,
+ "\xbe": 750,
+ "\xbf": 500,
+ "\xc0": 722,
+ "\xc1": 722,
+ "\xc2": 722,
+ "\xc3": 722,
+ "\xc4": 722,
+ "\xc5": 722,
+ "\xc6": 1000,
+ "\xc7": 722,
+ "\xc8": 667,
+ "\xc9": 667,
+ "\xca": 667,
+ "\xcb": 667,
+ "\xcc": 389,
+ "\xcd": 389,
+ "\xce": 389,
+ "\xcf": 389,
+ "\xd0": 722,
+ "\xd1": 722,
+ "\xd2": 778,
+ "\xd3": 778,
+ "\xd4": 778,
+ "\xd5": 778,
+ "\xd6": 778,
+ "\xd7": 570,
+ "\xd8": 778,
+ "\xd9": 722,
+ "\xda": 722,
+ "\xdb": 722,
+ "\xdc": 722,
+ "\xdd": 722,
+ "\xde": 611,
+ "\xdf": 556,
+ "\xe0": 500,
+ "\xe1": 500,
+ "\xe2": 500,
+ "\xe3": 500,
+ "\xe4": 500,
+ "\xe5": 500,
+ "\xe6": 722,
+ "\xe7": 444,
+ "\xe8": 444,
+ "\xe9": 444,
+ "\xea": 444,
+ "\xeb": 444,
+ "\xec": 278,
+ "\xed": 278,
+ "\xee": 278,
+ "\xef": 278,
+ "\xf0": 500,
+ "\xf1": 556,
+ "\xf2": 500,
+ "\xf3": 500,
+ "\xf4": 500,
+ "\xf5": 500,
+ "\xf6": 500,
+ "\xf7": 570,
+ "\xf8": 500,
+ "\xf9": 556,
+ "\xfa": 556,
+ "\xfb": 556,
+ "\xfc": 556,
+ "\xfd": 500,
+ "\xfe": 556,
+ "\xff": 500,
+ "\u0100": 722,
+ "\u0101": 500,
+ "\u0102": 722,
+ "\u0103": 500,
+ "\u0104": 722,
+ "\u0105": 500,
+ "\u0106": 722,
+ "\u0107": 444,
+ "\u010c": 722,
+ "\u010d": 444,
+ "\u010e": 722,
+ "\u010f": 672,
+ "\u0110": 722,
+ "\u0111": 556,
+ "\u0112": 667,
+ "\u0113": 444,
+ "\u0116": 667,
+ "\u0117": 444,
+ "\u0118": 667,
+ "\u0119": 444,
+ "\u011a": 667,
+ "\u011b": 444,
+ "\u011e": 778,
+ "\u011f": 500,
+ "\u0122": 778,
+ "\u0123": 500,
+ "\u012a": 389,
+ "\u012b": 278,
+ "\u012e": 389,
+ "\u012f": 278,
+ "\u0130": 389,
+ "\u0131": 278,
+ "\u0136": 778,
+ "\u0137": 556,
+ "\u0139": 667,
+ "\u013a": 278,
+ "\u013b": 667,
+ "\u013c": 278,
+ "\u013d": 667,
+ "\u013e": 394,
+ "\u0141": 667,
+ "\u0142": 278,
+ "\u0143": 722,
+ "\u0144": 556,
+ "\u0145": 722,
+ "\u0146": 556,
+ "\u0147": 722,
+ "\u0148": 556,
+ "\u014c": 778,
+ "\u014d": 500,
+ "\u0150": 778,
+ "\u0151": 500,
+ "\u0152": 1000,
+ "\u0153": 722,
+ "\u0154": 722,
+ "\u0155": 444,
+ "\u0156": 722,
+ "\u0157": 444,
+ "\u0158": 722,
+ "\u0159": 444,
+ "\u015a": 556,
+ "\u015b": 389,
+ "\u015e": 556,
+ "\u015f": 389,
+ "\u0160": 556,
+ "\u0161": 389,
+ "\u0162": 667,
+ "\u0163": 333,
+ "\u0164": 667,
+ "\u0165": 416,
+ "\u016a": 722,
+ "\u016b": 556,
+ "\u016e": 722,
+ "\u016f": 556,
+ "\u0170": 722,
+ "\u0171": 556,
+ "\u0172": 722,
+ "\u0173": 556,
+ "\u0178": 722,
+ "\u0179": 667,
+ "\u017a": 444,
+ "\u017b": 667,
+ "\u017c": 444,
+ "\u017d": 667,
+ "\u017e": 444,
+ "\u0192": 500,
+ "\u0218": 556,
+ "\u0219": 389,
+ "\u02c6": 333,
+ "\u02c7": 333,
+ "\u02d8": 333,
+ "\u02d9": 333,
+ "\u02da": 333,
+ "\u02db": 333,
+ "\u02dc": 333,
+ "\u02dd": 333,
+ "\u2013": 500,
+ "\u2014": 1000,
+ "\u2018": 333,
+ "\u2019": 333,
+ "\u201a": 333,
+ "\u201c": 500,
+ "\u201d": 500,
+ "\u201e": 500,
+ "\u2020": 500,
+ "\u2021": 500,
+ "\u2022": 350,
+ "\u2026": 1000,
+ "\u2030": 1000,
+ "\u2039": 333,
+ "\u203a": 333,
+ "\u2044": 167,
+ "\u2122": 1000,
+ "\u2202": 494,
+ "\u2206": 612,
+ "\u2211": 600,
+ "\u2212": 570,
+ "\u221a": 549,
+ "\u2260": 549,
+ "\u2264": 549,
+ "\u2265": 549,
+ "\u25ca": 494,
+ "\uf6c3": 250,
+ "\ufb01": 556,
+ "\ufb02": 556,
+ },
+ ),
+ "Times-BoldItalic": (
+ {
+ "FontName": "Times-BoldItalic",
+ "Descent": -217.0,
+ "FontBBox": (-200.0, -218.0, 996.0, 921.0),
+ "FontWeight": "Bold",
+ "CapHeight": 669.0,
+ "FontFamily": "Times",
+ "Flags": 0,
+ "XHeight": 462.0,
+ "ItalicAngle": -15.0,
+ "Ascent": 683.0,
+ },
+ {
+ " ": 250,
+ "!": 389,
+ '"': 555,
+ "#": 500,
+ "$": 500,
+ "%": 833,
+ "&": 778,
+ "'": 278,
+ "(": 333,
+ ")": 333,
+ "*": 500,
+ "+": 570,
+ ",": 250,
+ "-": 333,
+ ".": 250,
+ "/": 278,
+ "0": 500,
+ "1": 500,
+ "2": 500,
+ "3": 500,
+ "4": 500,
+ "5": 500,
+ "6": 500,
+ "7": 500,
+ "8": 500,
+ "9": 500,
+ ":": 333,
+ ";": 333,
+ "<": 570,
+ "=": 570,
+ ">": 570,
+ "?": 500,
+ "@": 832,
+ "A": 667,
+ "B": 667,
+ "C": 667,
+ "D": 722,
+ "E": 667,
+ "F": 667,
+ "G": 722,
+ "H": 778,
+ "I": 389,
+ "J": 500,
+ "K": 667,
+ "L": 611,
+ "M": 889,
+ "N": 722,
+ "O": 722,
+ "P": 611,
+ "Q": 722,
+ "R": 667,
+ "S": 556,
+ "T": 611,
+ "U": 722,
+ "V": 667,
+ "W": 889,
+ "X": 667,
+ "Y": 611,
+ "Z": 611,
+ "[": 333,
+ "\\": 278,
+ "]": 333,
+ "^": 570,
+ "_": 500,
+ "`": 333,
+ "a": 500,
+ "b": 500,
+ "c": 444,
+ "d": 500,
+ "e": 444,
+ "f": 333,
+ "g": 500,
+ "h": 556,
+ "i": 278,
+ "j": 278,
+ "k": 500,
+ "l": 278,
+ "m": 778,
+ "n": 556,
+ "o": 500,
+ "p": 500,
+ "q": 500,
+ "r": 389,
+ "s": 389,
+ "t": 278,
+ "u": 556,
+ "v": 444,
+ "w": 667,
+ "x": 500,
+ "y": 444,
+ "z": 389,
+ "{": 348,
+ "|": 220,
+ "}": 348,
+ "~": 570,
+ "\xa1": 389,
+ "\xa2": 500,
+ "\xa3": 500,
+ "\xa4": 500,
+ "\xa5": 500,
+ "\xa6": 220,
+ "\xa7": 500,
+ "\xa8": 333,
+ "\xa9": 747,
+ "\xaa": 266,
+ "\xab": 500,
+ "\xac": 606,
+ "\xae": 747,
+ "\xaf": 333,
+ "\xb0": 400,
+ "\xb1": 570,
+ "\xb2": 300,
+ "\xb3": 300,
+ "\xb4": 333,
+ "\xb5": 576,
+ "\xb6": 500,
+ "\xb7": 250,
+ "\xb8": 333,
+ "\xb9": 300,
+ "\xba": 300,
+ "\xbb": 500,
+ "\xbc": 750,
+ "\xbd": 750,
+ "\xbe": 750,
+ "\xbf": 500,
+ "\xc0": 667,
+ "\xc1": 667,
+ "\xc2": 667,
+ "\xc3": 667,
+ "\xc4": 667,
+ "\xc5": 667,
+ "\xc6": 944,
+ "\xc7": 667,
+ "\xc8": 667,
+ "\xc9": 667,
+ "\xca": 667,
+ "\xcb": 667,
+ "\xcc": 389,
+ "\xcd": 389,
+ "\xce": 389,
+ "\xcf": 389,
+ "\xd0": 722,
+ "\xd1": 722,
+ "\xd2": 722,
+ "\xd3": 722,
+ "\xd4": 722,
+ "\xd5": 722,
+ "\xd6": 722,
+ "\xd7": 570,
+ "\xd8": 722,
+ "\xd9": 722,
+ "\xda": 722,
+ "\xdb": 722,
+ "\xdc": 722,
+ "\xdd": 611,
+ "\xde": 611,
+ "\xdf": 500,
+ "\xe0": 500,
+ "\xe1": 500,
+ "\xe2": 500,
+ "\xe3": 500,
+ "\xe4": 500,
+ "\xe5": 500,
+ "\xe6": 722,
+ "\xe7": 444,
+ "\xe8": 444,
+ "\xe9": 444,
+ "\xea": 444,
+ "\xeb": 444,
+ "\xec": 278,
+ "\xed": 278,
+ "\xee": 278,
+ "\xef": 278,
+ "\xf0": 500,
+ "\xf1": 556,
+ "\xf2": 500,
+ "\xf3": 500,
+ "\xf4": 500,
+ "\xf5": 500,
+ "\xf6": 500,
+ "\xf7": 570,
+ "\xf8": 500,
+ "\xf9": 556,
+ "\xfa": 556,
+ "\xfb": 556,
+ "\xfc": 556,
+ "\xfd": 444,
+ "\xfe": 500,
+ "\xff": 444,
+ "\u0100": 667,
+ "\u0101": 500,
+ "\u0102": 667,
+ "\u0103": 500,
+ "\u0104": 667,
+ "\u0105": 500,
+ "\u0106": 667,
+ "\u0107": 444,
+ "\u010c": 667,
+ "\u010d": 444,
+ "\u010e": 722,
+ "\u010f": 608,
+ "\u0110": 722,
+ "\u0111": 500,
+ "\u0112": 667,
+ "\u0113": 444,
+ "\u0116": 667,
+ "\u0117": 444,
+ "\u0118": 667,
+ "\u0119": 444,
+ "\u011a": 667,
+ "\u011b": 444,
+ "\u011e": 722,
+ "\u011f": 500,
+ "\u0122": 722,
+ "\u0123": 500,
+ "\u012a": 389,
+ "\u012b": 278,
+ "\u012e": 389,
+ "\u012f": 278,
+ "\u0130": 389,
+ "\u0131": 278,
+ "\u0136": 667,
+ "\u0137": 500,
+ "\u0139": 611,
+ "\u013a": 278,
+ "\u013b": 611,
+ "\u013c": 278,
+ "\u013d": 611,
+ "\u013e": 382,
+ "\u0141": 611,
+ "\u0142": 278,
+ "\u0143": 722,
+ "\u0144": 556,
+ "\u0145": 722,
+ "\u0146": 556,
+ "\u0147": 722,
+ "\u0148": 556,
+ "\u014c": 722,
+ "\u014d": 500,
+ "\u0150": 722,
+ "\u0151": 500,
+ "\u0152": 944,
+ "\u0153": 722,
+ "\u0154": 667,
+ "\u0155": 389,
+ "\u0156": 667,
+ "\u0157": 389,
+ "\u0158": 667,
+ "\u0159": 389,
+ "\u015a": 556,
+ "\u015b": 389,
+ "\u015e": 556,
+ "\u015f": 389,
+ "\u0160": 556,
+ "\u0161": 389,
+ "\u0162": 611,
+ "\u0163": 278,
+ "\u0164": 611,
+ "\u0165": 366,
+ "\u016a": 722,
+ "\u016b": 556,
+ "\u016e": 722,
+ "\u016f": 556,
+ "\u0170": 722,
+ "\u0171": 556,
+ "\u0172": 722,
+ "\u0173": 556,
+ "\u0178": 611,
+ "\u0179": 611,
+ "\u017a": 389,
+ "\u017b": 611,
+ "\u017c": 389,
+ "\u017d": 611,
+ "\u017e": 389,
+ "\u0192": 500,
+ "\u0218": 556,
+ "\u0219": 389,
+ "\u02c6": 333,
+ "\u02c7": 333,
+ "\u02d8": 333,
+ "\u02d9": 333,
+ "\u02da": 333,
+ "\u02db": 333,
+ "\u02dc": 333,
+ "\u02dd": 333,
+ "\u2013": 500,
+ "\u2014": 1000,
+ "\u2018": 333,
+ "\u2019": 333,
+ "\u201a": 333,
+ "\u201c": 500,
+ "\u201d": 500,
+ "\u201e": 500,
+ "\u2020": 500,
+ "\u2021": 500,
+ "\u2022": 350,
+ "\u2026": 1000,
+ "\u2030": 1000,
+ "\u2039": 333,
+ "\u203a": 333,
+ "\u2044": 167,
+ "\u2122": 1000,
+ "\u2202": 494,
+ "\u2206": 612,
+ "\u2211": 600,
+ "\u2212": 606,
+ "\u221a": 549,
+ "\u2260": 549,
+ "\u2264": 549,
+ "\u2265": 549,
+ "\u25ca": 494,
+ "\uf6c3": 250,
+ "\ufb01": 556,
+ "\ufb02": 556,
+ },
+ ),
+ "Times-Italic": (
+ {
+ "FontName": "Times-Italic",
+ "Descent": -217.0,
+ "FontBBox": (-169.0, -217.0, 1010.0, 883.0),
+ "FontWeight": "Medium",
+ "CapHeight": 653.0,
+ "FontFamily": "Times",
+ "Flags": 0,
+ "XHeight": 441.0,
+ "ItalicAngle": -15.5,
+ "Ascent": 683.0,
+ },
+ {
+ " ": 250,
+ "!": 333,
+ '"': 420,
+ "#": 500,
+ "$": 500,
+ "%": 833,
+ "&": 778,
+ "'": 214,
+ "(": 333,
+ ")": 333,
+ "*": 500,
+ "+": 675,
+ ",": 250,
+ "-": 333,
+ ".": 250,
+ "/": 278,
+ "0": 500,
+ "1": 500,
+ "2": 500,
+ "3": 500,
+ "4": 500,
+ "5": 500,
+ "6": 500,
+ "7": 500,
+ "8": 500,
+ "9": 500,
+ ":": 333,
+ ";": 333,
+ "<": 675,
+ "=": 675,
+ ">": 675,
+ "?": 500,
+ "@": 920,
+ "A": 611,
+ "B": 611,
+ "C": 667,
+ "D": 722,
+ "E": 611,
+ "F": 611,
+ "G": 722,
+ "H": 722,
+ "I": 333,
+ "J": 444,
+ "K": 667,
+ "L": 556,
+ "M": 833,
+ "N": 667,
+ "O": 722,
+ "P": 611,
+ "Q": 722,
+ "R": 611,
+ "S": 500,
+ "T": 556,
+ "U": 722,
+ "V": 611,
+ "W": 833,
+ "X": 611,
+ "Y": 556,
+ "Z": 556,
+ "[": 389,
+ "\\": 278,
+ "]": 389,
+ "^": 422,
+ "_": 500,
+ "`": 333,
+ "a": 500,
+ "b": 500,
+ "c": 444,
+ "d": 500,
+ "e": 444,
+ "f": 278,
+ "g": 500,
+ "h": 500,
+ "i": 278,
+ "j": 278,
+ "k": 444,
+ "l": 278,
+ "m": 722,
+ "n": 500,
+ "o": 500,
+ "p": 500,
+ "q": 500,
+ "r": 389,
+ "s": 389,
+ "t": 278,
+ "u": 500,
+ "v": 444,
+ "w": 667,
+ "x": 444,
+ "y": 444,
+ "z": 389,
+ "{": 400,
+ "|": 275,
+ "}": 400,
+ "~": 541,
+ "\xa1": 389,
+ "\xa2": 500,
+ "\xa3": 500,
+ "\xa4": 500,
+ "\xa5": 500,
+ "\xa6": 275,
+ "\xa7": 500,
+ "\xa8": 333,
+ "\xa9": 760,
+ "\xaa": 276,
+ "\xab": 500,
+ "\xac": 675,
+ "\xae": 760,
+ "\xaf": 333,
+ "\xb0": 400,
+ "\xb1": 675,
+ "\xb2": 300,
+ "\xb3": 300,
+ "\xb4": 333,
+ "\xb5": 500,
+ "\xb6": 523,
+ "\xb7": 250,
+ "\xb8": 333,
+ "\xb9": 300,
+ "\xba": 310,
+ "\xbb": 500,
+ "\xbc": 750,
+ "\xbd": 750,
+ "\xbe": 750,
+ "\xbf": 500,
+ "\xc0": 611,
+ "\xc1": 611,
+ "\xc2": 611,
+ "\xc3": 611,
+ "\xc4": 611,
+ "\xc5": 611,
+ "\xc6": 889,
+ "\xc7": 667,
+ "\xc8": 611,
+ "\xc9": 611,
+ "\xca": 611,
+ "\xcb": 611,
+ "\xcc": 333,
+ "\xcd": 333,
+ "\xce": 333,
+ "\xcf": 333,
+ "\xd0": 722,
+ "\xd1": 667,
+ "\xd2": 722,
+ "\xd3": 722,
+ "\xd4": 722,
+ "\xd5": 722,
+ "\xd6": 722,
+ "\xd7": 675,
+ "\xd8": 722,
+ "\xd9": 722,
+ "\xda": 722,
+ "\xdb": 722,
+ "\xdc": 722,
+ "\xdd": 556,
+ "\xde": 611,
+ "\xdf": 500,
+ "\xe0": 500,
+ "\xe1": 500,
+ "\xe2": 500,
+ "\xe3": 500,
+ "\xe4": 500,
+ "\xe5": 500,
+ "\xe6": 667,
+ "\xe7": 444,
+ "\xe8": 444,
+ "\xe9": 444,
+ "\xea": 444,
+ "\xeb": 444,
+ "\xec": 278,
+ "\xed": 278,
+ "\xee": 278,
+ "\xef": 278,
+ "\xf0": 500,
+ "\xf1": 500,
+ "\xf2": 500,
+ "\xf3": 500,
+ "\xf4": 500,
+ "\xf5": 500,
+ "\xf6": 500,
+ "\xf7": 675,
+ "\xf8": 500,
+ "\xf9": 500,
+ "\xfa": 500,
+ "\xfb": 500,
+ "\xfc": 500,
+ "\xfd": 444,
+ "\xfe": 500,
+ "\xff": 444,
+ "\u0100": 611,
+ "\u0101": 500,
+ "\u0102": 611,
+ "\u0103": 500,
+ "\u0104": 611,
+ "\u0105": 500,
+ "\u0106": 667,
+ "\u0107": 444,
+ "\u010c": 667,
+ "\u010d": 444,
+ "\u010e": 722,
+ "\u010f": 544,
+ "\u0110": 722,
+ "\u0111": 500,
+ "\u0112": 611,
+ "\u0113": 444,
+ "\u0116": 611,
+ "\u0117": 444,
+ "\u0118": 611,
+ "\u0119": 444,
+ "\u011a": 611,
+ "\u011b": 444,
+ "\u011e": 722,
+ "\u011f": 500,
+ "\u0122": 722,
+ "\u0123": 500,
+ "\u012a": 333,
+ "\u012b": 278,
+ "\u012e": 333,
+ "\u012f": 278,
+ "\u0130": 333,
+ "\u0131": 278,
+ "\u0136": 667,
+ "\u0137": 444,
+ "\u0139": 556,
+ "\u013a": 278,
+ "\u013b": 556,
+ "\u013c": 278,
+ "\u013d": 611,
+ "\u013e": 300,
+ "\u0141": 556,
+ "\u0142": 278,
+ "\u0143": 667,
+ "\u0144": 500,
+ "\u0145": 667,
+ "\u0146": 500,
+ "\u0147": 667,
+ "\u0148": 500,
+ "\u014c": 722,
+ "\u014d": 500,
+ "\u0150": 722,
+ "\u0151": 500,
+ "\u0152": 944,
+ "\u0153": 667,
+ "\u0154": 611,
+ "\u0155": 389,
+ "\u0156": 611,
+ "\u0157": 389,
+ "\u0158": 611,
+ "\u0159": 389,
+ "\u015a": 500,
+ "\u015b": 389,
+ "\u015e": 500,
+ "\u015f": 389,
+ "\u0160": 500,
+ "\u0161": 389,
+ "\u0162": 556,
+ "\u0163": 278,
+ "\u0164": 556,
+ "\u0165": 300,
+ "\u016a": 722,
+ "\u016b": 500,
+ "\u016e": 722,
+ "\u016f": 500,
+ "\u0170": 722,
+ "\u0171": 500,
+ "\u0172": 722,
+ "\u0173": 500,
+ "\u0178": 556,
+ "\u0179": 556,
+ "\u017a": 389,
+ "\u017b": 556,
+ "\u017c": 389,
+ "\u017d": 556,
+ "\u017e": 389,
+ "\u0192": 500,
+ "\u0218": 500,
+ "\u0219": 389,
+ "\u02c6": 333,
+ "\u02c7": 333,
+ "\u02d8": 333,
+ "\u02d9": 333,
+ "\u02da": 333,
+ "\u02db": 333,
+ "\u02dc": 333,
+ "\u02dd": 333,
+ "\u2013": 500,
+ "\u2014": 889,
+ "\u2018": 333,
+ "\u2019": 333,
+ "\u201a": 333,
+ "\u201c": 556,
+ "\u201d": 556,
+ "\u201e": 556,
+ "\u2020": 500,
+ "\u2021": 500,
+ "\u2022": 350,
+ "\u2026": 889,
+ "\u2030": 1000,
+ "\u2039": 333,
+ "\u203a": 333,
+ "\u2044": 167,
+ "\u2122": 980,
+ "\u2202": 476,
+ "\u2206": 612,
+ "\u2211": 600,
+ "\u2212": 675,
+ "\u221a": 453,
+ "\u2260": 549,
+ "\u2264": 549,
+ "\u2265": 549,
+ "\u25ca": 471,
+ "\uf6c3": 250,
+ "\ufb01": 500,
+ "\ufb02": 500,
+ },
+ ),
+ "Times-Roman": (
+ {
+ "FontName": "Times-Roman",
+ "Descent": -217.0,
+ "FontBBox": (-168.0, -218.0, 1000.0, 898.0),
+ "FontWeight": "Roman",
+ "CapHeight": 662.0,
+ "FontFamily": "Times",
+ "Flags": 0,
+ "XHeight": 450.0,
+ "ItalicAngle": 0.0,
+ "Ascent": 683.0,
+ },
+ {
+ " ": 250,
+ "!": 333,
+ '"': 408,
+ "#": 500,
+ "$": 500,
+ "%": 833,
+ "&": 778,
+ "'": 180,
+ "(": 333,
+ ")": 333,
+ "*": 500,
+ "+": 564,
+ ",": 250,
+ "-": 333,
+ ".": 250,
+ "/": 278,
+ "0": 500,
+ "1": 500,
+ "2": 500,
+ "3": 500,
+ "4": 500,
+ "5": 500,
+ "6": 500,
+ "7": 500,
+ "8": 500,
+ "9": 500,
+ ":": 278,
+ ";": 278,
+ "<": 564,
+ "=": 564,
+ ">": 564,
+ "?": 444,
+ "@": 921,
+ "A": 722,
+ "B": 667,
+ "C": 667,
+ "D": 722,
+ "E": 611,
+ "F": 556,
+ "G": 722,
+ "H": 722,
+ "I": 333,
+ "J": 389,
+ "K": 722,
+ "L": 611,
+ "M": 889,
+ "N": 722,
+ "O": 722,
+ "P": 556,
+ "Q": 722,
+ "R": 667,
+ "S": 556,
+ "T": 611,
+ "U": 722,
+ "V": 722,
+ "W": 944,
+ "X": 722,
+ "Y": 722,
+ "Z": 611,
+ "[": 333,
+ "\\": 278,
+ "]": 333,
+ "^": 469,
+ "_": 500,
+ "`": 333,
+ "a": 444,
+ "b": 500,
+ "c": 444,
+ "d": 500,
+ "e": 444,
+ "f": 333,
+ "g": 500,
+ "h": 500,
+ "i": 278,
+ "j": 278,
+ "k": 500,
+ "l": 278,
+ "m": 778,
+ "n": 500,
+ "o": 500,
+ "p": 500,
+ "q": 500,
+ "r": 333,
+ "s": 389,
+ "t": 278,
+ "u": 500,
+ "v": 500,
+ "w": 722,
+ "x": 500,
+ "y": 500,
+ "z": 444,
+ "{": 480,
+ "|": 200,
+ "}": 480,
+ "~": 541,
+ "\xa1": 333,
+ "\xa2": 500,
+ "\xa3": 500,
+ "\xa4": 500,
+ "\xa5": 500,
+ "\xa6": 200,
+ "\xa7": 500,
+ "\xa8": 333,
+ "\xa9": 760,
+ "\xaa": 276,
+ "\xab": 500,
+ "\xac": 564,
+ "\xae": 760,
+ "\xaf": 333,
+ "\xb0": 400,
+ "\xb1": 564,
+ "\xb2": 300,
+ "\xb3": 300,
+ "\xb4": 333,
+ "\xb5": 500,
+ "\xb6": 453,
+ "\xb7": 250,
+ "\xb8": 333,
+ "\xb9": 300,
+ "\xba": 310,
+ "\xbb": 500,
+ "\xbc": 750,
+ "\xbd": 750,
+ "\xbe": 750,
+ "\xbf": 444,
+ "\xc0": 722,
+ "\xc1": 722,
+ "\xc2": 722,
+ "\xc3": 722,
+ "\xc4": 722,
+ "\xc5": 722,
+ "\xc6": 889,
+ "\xc7": 667,
+ "\xc8": 611,
+ "\xc9": 611,
+ "\xca": 611,
+ "\xcb": 611,
+ "\xcc": 333,
+ "\xcd": 333,
+ "\xce": 333,
+ "\xcf": 333,
+ "\xd0": 722,
+ "\xd1": 722,
+ "\xd2": 722,
+ "\xd3": 722,
+ "\xd4": 722,
+ "\xd5": 722,
+ "\xd6": 722,
+ "\xd7": 564,
+ "\xd8": 722,
+ "\xd9": 722,
+ "\xda": 722,
+ "\xdb": 722,
+ "\xdc": 722,
+ "\xdd": 722,
+ "\xde": 556,
+ "\xdf": 500,
+ "\xe0": 444,
+ "\xe1": 444,
+ "\xe2": 444,
+ "\xe3": 444,
+ "\xe4": 444,
+ "\xe5": 444,
+ "\xe6": 667,
+ "\xe7": 444,
+ "\xe8": 444,
+ "\xe9": 444,
+ "\xea": 444,
+ "\xeb": 444,
+ "\xec": 278,
+ "\xed": 278,
+ "\xee": 278,
+ "\xef": 278,
+ "\xf0": 500,
+ "\xf1": 500,
+ "\xf2": 500,
+ "\xf3": 500,
+ "\xf4": 500,
+ "\xf5": 500,
+ "\xf6": 500,
+ "\xf7": 564,
+ "\xf8": 500,
+ "\xf9": 500,
+ "\xfa": 500,
+ "\xfb": 500,
+ "\xfc": 500,
+ "\xfd": 500,
+ "\xfe": 500,
+ "\xff": 500,
+ "\u0100": 722,
+ "\u0101": 444,
+ "\u0102": 722,
+ "\u0103": 444,
+ "\u0104": 722,
+ "\u0105": 444,
+ "\u0106": 667,
+ "\u0107": 444,
+ "\u010c": 667,
+ "\u010d": 444,
+ "\u010e": 722,
+ "\u010f": 588,
+ "\u0110": 722,
+ "\u0111": 500,
+ "\u0112": 611,
+ "\u0113": 444,
+ "\u0116": 611,
+ "\u0117": 444,
+ "\u0118": 611,
+ "\u0119": 444,
+ "\u011a": 611,
+ "\u011b": 444,
+ "\u011e": 722,
+ "\u011f": 500,
+ "\u0122": 722,
+ "\u0123": 500,
+ "\u012a": 333,
+ "\u012b": 278,
+ "\u012e": 333,
+ "\u012f": 278,
+ "\u0130": 333,
+ "\u0131": 278,
+ "\u0136": 722,
+ "\u0137": 500,
+ "\u0139": 611,
+ "\u013a": 278,
+ "\u013b": 611,
+ "\u013c": 278,
+ "\u013d": 611,
+ "\u013e": 344,
+ "\u0141": 611,
+ "\u0142": 278,
+ "\u0143": 722,
+ "\u0144": 500,
+ "\u0145": 722,
+ "\u0146": 500,
+ "\u0147": 722,
+ "\u0148": 500,
+ "\u014c": 722,
+ "\u014d": 500,
+ "\u0150": 722,
+ "\u0151": 500,
+ "\u0152": 889,
+ "\u0153": 722,
+ "\u0154": 667,
+ "\u0155": 333,
+ "\u0156": 667,
+ "\u0157": 333,
+ "\u0158": 667,
+ "\u0159": 333,
+ "\u015a": 556,
+ "\u015b": 389,
+ "\u015e": 556,
+ "\u015f": 389,
+ "\u0160": 556,
+ "\u0161": 389,
+ "\u0162": 611,
+ "\u0163": 278,
+ "\u0164": 611,
+ "\u0165": 326,
+ "\u016a": 722,
+ "\u016b": 500,
+ "\u016e": 722,
+ "\u016f": 500,
+ "\u0170": 722,
+ "\u0171": 500,
+ "\u0172": 722,
+ "\u0173": 500,
+ "\u0178": 722,
+ "\u0179": 611,
+ "\u017a": 444,
+ "\u017b": 611,
+ "\u017c": 444,
+ "\u017d": 611,
+ "\u017e": 444,
+ "\u0192": 500,
+ "\u0218": 556,
+ "\u0219": 389,
+ "\u02c6": 333,
+ "\u02c7": 333,
+ "\u02d8": 333,
+ "\u02d9": 333,
+ "\u02da": 333,
+ "\u02db": 333,
+ "\u02dc": 333,
+ "\u02dd": 333,
+ "\u2013": 500,
+ "\u2014": 1000,
+ "\u2018": 333,
+ "\u2019": 333,
+ "\u201a": 333,
+ "\u201c": 444,
+ "\u201d": 444,
+ "\u201e": 444,
+ "\u2020": 500,
+ "\u2021": 500,
+ "\u2022": 350,
+ "\u2026": 1000,
+ "\u2030": 1000,
+ "\u2039": 333,
+ "\u203a": 333,
+ "\u2044": 167,
+ "\u2122": 980,
+ "\u2202": 476,
+ "\u2206": 612,
+ "\u2211": 600,
+ "\u2212": 564,
+ "\u221a": 453,
+ "\u2260": 549,
+ "\u2264": 549,
+ "\u2265": 549,
+ "\u25ca": 471,
+ "\uf6c3": 250,
+ "\ufb01": 556,
+ "\ufb02": 556,
+ },
+ ),
+ "ZapfDingbats": (
+ {
+ "FontName": "ZapfDingbats",
+ "FontBBox": (-1.0, -143.0, 981.0, 820.0),
+ "FontWeight": "Medium",
+ "FontFamily": "ITC",
+ "Flags": 0,
+ "ItalicAngle": 0.0,
+ },
+ {
+ "\x01": 974,
+ "\x02": 961,
+ "\x03": 980,
+ "\x04": 719,
+ "\x05": 789,
+ "\x06": 494,
+ "\x07": 552,
+ "\x08": 537,
+ "\t": 577,
+ "\n": 692,
+ "\x0b": 960,
+ "\x0c": 939,
+ "\r": 549,
+ "\x0e": 855,
+ "\x0f": 911,
+ "\x10": 933,
+ "\x11": 945,
+ "\x12": 974,
+ "\x13": 755,
+ "\x14": 846,
+ "\x15": 762,
+ "\x16": 761,
+ "\x17": 571,
+ "\x18": 677,
+ "\x19": 763,
+ "\x1a": 760,
+ "\x1b": 759,
+ "\x1c": 754,
+ "\x1d": 786,
+ "\x1e": 788,
+ "\x1f": 788,
+ " ": 790,
+ "!": 793,
+ '"': 794,
+ "#": 816,
+ "$": 823,
+ "%": 789,
+ "&": 841,
+ "'": 823,
+ "(": 833,
+ ")": 816,
+ "*": 831,
+ "+": 923,
+ ",": 744,
+ "-": 723,
+ ".": 749,
+ "/": 790,
+ "0": 792,
+ "1": 695,
+ "2": 776,
+ "3": 768,
+ "4": 792,
+ "5": 759,
+ "6": 707,
+ "7": 708,
+ "8": 682,
+ "9": 701,
+ ":": 826,
+ ";": 815,
+ "<": 789,
+ "=": 789,
+ ">": 707,
+ "?": 687,
+ "@": 696,
+ "A": 689,
+ "B": 786,
+ "C": 787,
+ "D": 713,
+ "E": 791,
+ "F": 785,
+ "G": 791,
+ "H": 873,
+ "I": 761,
+ "J": 762,
+ "K": 759,
+ "L": 892,
+ "M": 892,
+ "N": 788,
+ "O": 784,
+ "Q": 438,
+ "R": 138,
+ "S": 277,
+ "T": 415,
+ "U": 509,
+ "V": 410,
+ "W": 234,
+ "X": 234,
+ "Y": 390,
+ "Z": 390,
+ "[": 276,
+ "\\": 276,
+ "]": 317,
+ "^": 317,
+ "_": 334,
+ "`": 334,
+ "a": 392,
+ "b": 392,
+ "c": 668,
+ "d": 668,
+ "e": 732,
+ "f": 544,
+ "g": 544,
+ "h": 910,
+ "i": 911,
+ "j": 667,
+ "k": 760,
+ "l": 760,
+ "m": 626,
+ "n": 694,
+ "o": 595,
+ "p": 776,
+ "u": 690,
+ "v": 791,
+ "w": 790,
+ "x": 788,
+ "y": 788,
+ "z": 788,
+ "{": 788,
+ "|": 788,
+ "}": 788,
+ "~": 788,
+ "\x7f": 788,
+ "\x80": 788,
+ "\x81": 788,
+ "\x82": 788,
+ "\x83": 788,
+ "\x84": 788,
+ "\x85": 788,
+ "\x86": 788,
+ "\x87": 788,
+ "\x88": 788,
+ "\x89": 788,
+ "\x8a": 788,
+ "\x8b": 788,
+ "\x8c": 788,
+ "\x8d": 788,
+ "\x8e": 788,
+ "\x8f": 788,
+ "\x90": 788,
+ "\x91": 788,
+ "\x92": 788,
+ "\x93": 788,
+ "\x94": 788,
+ "\x95": 788,
+ "\x96": 788,
+ "\x97": 788,
+ "\x98": 788,
+ "\x99": 788,
+ "\x9a": 788,
+ "\x9b": 788,
+ "\x9c": 788,
+ "\x9d": 788,
+ "\x9e": 788,
+ "\x9f": 788,
+ "\xa0": 894,
+ "\xa1": 838,
+ "\xa2": 924,
+ "\xa3": 1016,
+ "\xa4": 458,
+ "\xa5": 924,
+ "\xa6": 918,
+ "\xa7": 927,
+ "\xa8": 928,
+ "\xa9": 928,
+ "\xaa": 834,
+ "\xab": 873,
+ "\xac": 828,
+ "\xad": 924,
+ "\xae": 917,
+ "\xaf": 930,
+ "\xb0": 931,
+ "\xb1": 463,
+ "\xb2": 883,
+ "\xb3": 836,
+ "\xb4": 867,
+ "\xb5": 696,
+ "\xb6": 874,
+ "\xb7": 760,
+ "\xb8": 946,
+ "\xb9": 865,
+ "\xba": 967,
+ "\xbb": 831,
+ "\xbc": 873,
+ "\xbd": 927,
+ "\xbe": 970,
+ "\xbf": 918,
+ "\xc0": 748,
+ "\xc1": 836,
+ "\xc2": 771,
+ "\xc3": 888,
+ "\xc4": 748,
+ "\xc5": 771,
+ "\xc6": 888,
+ "\xc7": 867,
+ "\xc8": 696,
+ "\xc9": 874,
+ "\xca": 974,
+ "\xcb": 762,
+ "\xcc": 759,
+ "\xcd": 509,
+ "\xce": 410,
+ },
+ ),
+}
+
+# Aliases defined in implementation note 62 in Appecix H. related to section 5.5.1
+# (Type 1 Fonts) in the PDF Reference.
+FONT_METRICS["Arial"] = FONT_METRICS["Helvetica"]
+FONT_METRICS["Arial,Italic"] = FONT_METRICS["Helvetica-Oblique"]
+FONT_METRICS["Arial,Bold"] = FONT_METRICS["Helvetica-Bold"]
+FONT_METRICS["Arial,BoldItalic"] = FONT_METRICS["Helvetica-BoldOblique"]
+FONT_METRICS["CourierNew"] = FONT_METRICS["Courier"]
+FONT_METRICS["CourierNew,Italic"] = FONT_METRICS["Courier-Oblique"]
+FONT_METRICS["CourierNew,Bold"] = FONT_METRICS["Courier-Bold"]
+FONT_METRICS["CourierNew,BoldItalic"] = FONT_METRICS["Courier-BoldOblique"]
+FONT_METRICS["TimesNewRoman"] = FONT_METRICS["Times-Roman"]
+FONT_METRICS["TimesNewRoman,Italic"] = FONT_METRICS["Times-Italic"]
+FONT_METRICS["TimesNewRoman,Bold"] = FONT_METRICS["Times-Bold"]
+FONT_METRICS["TimesNewRoman,BoldItalic"] = FONT_METRICS["Times-BoldItalic"]
diff --git a/babeldoc/pdfminer/glyphlist.py b/babeldoc/pdfminer/glyphlist.py
new file mode 100644
index 0000000000000000000000000000000000000000..fdc6fa1b12c5f401f62571b96c18cac85ccff02b
--- /dev/null
+++ b/babeldoc/pdfminer/glyphlist.py
@@ -0,0 +1,4365 @@
+"""Mappings from Adobe glyph names to Unicode characters.
+
+In some CMap tables, Adobe glyph names are used for specifying
+Unicode characters instead of using decimal/hex character code.
+
+The following data was taken by
+
+ $ wget https://partners.adobe.com/public/developer/en/opentype/glyphlist.txt
+
+```python
+from babeldoc.pdfminer.glyphlist import convert_glyphlist
+
+convert_glyphlist("glyphlist.txt")"""
+
+# ###################################################################################
+# Copyright (c) 1997,1998,2002,2007 Adobe Systems Incorporated
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this documentation file to use, copy, publish, distribute,
+# sublicense, and/or sell copies of the documentation, and to permit
+# others to do the same, provided that:
+# - No modification, editing or other alteration of this document is
+# allowed; and
+# - The above copyright notice and this permission notice shall be
+# included in all copies of the documentation.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this documentation file, to create their own derivative works
+# from the content of this document to use, copy, publish, distribute,
+# sublicense, and/or sell the derivative works, and to permit others to do
+# the same, provided that the derived work is not represented as being a
+# copy or version of this document.
+#
+# Adobe shall not be liable to any party for any loss of revenue or profit
+# or for indirect, incidental, special, consequential, or other similar
+# damages, whether based on tort (including without limitation negligence
+# or strict liability), contract or other legal or equitable grounds even
+# if Adobe has been advised or had reason to know of the possibility of
+# such damages. The Adobe materials are provided on an "AS IS" basis.
+# Adobe specifically disclaims all express, statutory, or implied
+# warranties relating to the Adobe materials, including but not limited to
+# those concerning merchantability or fitness for a particular purpose or
+# non-infringement of any third party rights regarding the Adobe
+# materials.
+# ###################################################################################
+# Name: Adobe Glyph List
+# Table version: 2.0
+# Date: September 20, 2002
+#
+# See http://partners.adobe.com/asn/developer/typeforum/unicodegn.html
+#
+# Format: Semicolon-delimited fields:
+# (1) glyph name
+# (2) Unicode scalar value
+
+
+def convert_glyphlist(path: str) -> None:
+ """Convert a glyph list into a python representation.
+
+ See output below.
+ """
+ state = 0
+ with open(path) as fileinput:
+ for line in fileinput.readlines():
+ line = line.strip()
+ if not line or line.startswith("#"):
+ if state == 1:
+ state = 2
+ print("}\n")
+ print(line)
+ continue
+ if state == 0:
+ print("\nglyphname2unicode = {")
+ state = 1
+ (name, x) = line.split(";")
+ codes = x.split(" ")
+ print(
+ " {!r}: u'{}',".format(name, "".join("\\u%s" % code for code in codes)),
+ )
+
+
+glyphname2unicode = {
+ "A": "\u0041",
+ "AE": "\u00c6",
+ "AEacute": "\u01fc",
+ "AEmacron": "\u01e2",
+ "AEsmall": "\uf7e6",
+ "Aacute": "\u00c1",
+ "Aacutesmall": "\uf7e1",
+ "Abreve": "\u0102",
+ "Abreveacute": "\u1eae",
+ "Abrevecyrillic": "\u04d0",
+ "Abrevedotbelow": "\u1eb6",
+ "Abrevegrave": "\u1eb0",
+ "Abrevehookabove": "\u1eb2",
+ "Abrevetilde": "\u1eb4",
+ "Acaron": "\u01cd",
+ "Acircle": "\u24b6",
+ "Acircumflex": "\u00c2",
+ "Acircumflexacute": "\u1ea4",
+ "Acircumflexdotbelow": "\u1eac",
+ "Acircumflexgrave": "\u1ea6",
+ "Acircumflexhookabove": "\u1ea8",
+ "Acircumflexsmall": "\uf7e2",
+ "Acircumflextilde": "\u1eaa",
+ "Acute": "\uf6c9",
+ "Acutesmall": "\uf7b4",
+ "Acyrillic": "\u0410",
+ "Adblgrave": "\u0200",
+ "Adieresis": "\u00c4",
+ "Adieresiscyrillic": "\u04d2",
+ "Adieresismacron": "\u01de",
+ "Adieresissmall": "\uf7e4",
+ "Adotbelow": "\u1ea0",
+ "Adotmacron": "\u01e0",
+ "Agrave": "\u00c0",
+ "Agravesmall": "\uf7e0",
+ "Ahookabove": "\u1ea2",
+ "Aiecyrillic": "\u04d4",
+ "Ainvertedbreve": "\u0202",
+ "Alpha": "\u0391",
+ "Alphatonos": "\u0386",
+ "Amacron": "\u0100",
+ "Amonospace": "\uff21",
+ "Aogonek": "\u0104",
+ "Aring": "\u00c5",
+ "Aringacute": "\u01fa",
+ "Aringbelow": "\u1e00",
+ "Aringsmall": "\uf7e5",
+ "Asmall": "\uf761",
+ "Atilde": "\u00c3",
+ "Atildesmall": "\uf7e3",
+ "Aybarmenian": "\u0531",
+ "B": "\u0042",
+ "Bcircle": "\u24b7",
+ "Bdotaccent": "\u1e02",
+ "Bdotbelow": "\u1e04",
+ "Becyrillic": "\u0411",
+ "Benarmenian": "\u0532",
+ "Beta": "\u0392",
+ "Bhook": "\u0181",
+ "Blinebelow": "\u1e06",
+ "Bmonospace": "\uff22",
+ "Brevesmall": "\uf6f4",
+ "Bsmall": "\uf762",
+ "Btopbar": "\u0182",
+ "C": "\u0043",
+ "Caarmenian": "\u053e",
+ "Cacute": "\u0106",
+ "Caron": "\uf6ca",
+ "Caronsmall": "\uf6f5",
+ "Ccaron": "\u010c",
+ "Ccedilla": "\u00c7",
+ "Ccedillaacute": "\u1e08",
+ "Ccedillasmall": "\uf7e7",
+ "Ccircle": "\u24b8",
+ "Ccircumflex": "\u0108",
+ "Cdot": "\u010a",
+ "Cdotaccent": "\u010a",
+ "Cedillasmall": "\uf7b8",
+ "Chaarmenian": "\u0549",
+ "Cheabkhasiancyrillic": "\u04bc",
+ "Checyrillic": "\u0427",
+ "Chedescenderabkhasiancyrillic": "\u04be",
+ "Chedescendercyrillic": "\u04b6",
+ "Chedieresiscyrillic": "\u04f4",
+ "Cheharmenian": "\u0543",
+ "Chekhakassiancyrillic": "\u04cb",
+ "Cheverticalstrokecyrillic": "\u04b8",
+ "Chi": "\u03a7",
+ "Chook": "\u0187",
+ "Circumflexsmall": "\uf6f6",
+ "Cmonospace": "\uff23",
+ "Coarmenian": "\u0551",
+ "Csmall": "\uf763",
+ "D": "\u0044",
+ "DZ": "\u01f1",
+ "DZcaron": "\u01c4",
+ "Daarmenian": "\u0534",
+ "Dafrican": "\u0189",
+ "Dcaron": "\u010e",
+ "Dcedilla": "\u1e10",
+ "Dcircle": "\u24b9",
+ "Dcircumflexbelow": "\u1e12",
+ "Dcroat": "\u0110",
+ "Ddotaccent": "\u1e0a",
+ "Ddotbelow": "\u1e0c",
+ "Decyrillic": "\u0414",
+ "Deicoptic": "\u03ee",
+ "Delta": "\u2206",
+ "Deltagreek": "\u0394",
+ "Dhook": "\u018a",
+ "Dieresis": "\uf6cb",
+ "DieresisAcute": "\uf6cc",
+ "DieresisGrave": "\uf6cd",
+ "Dieresissmall": "\uf7a8",
+ "Digammagreek": "\u03dc",
+ "Djecyrillic": "\u0402",
+ "Dlinebelow": "\u1e0e",
+ "Dmonospace": "\uff24",
+ "Dotaccentsmall": "\uf6f7",
+ "Dslash": "\u0110",
+ "Dsmall": "\uf764",
+ "Dtopbar": "\u018b",
+ "Dz": "\u01f2",
+ "Dzcaron": "\u01c5",
+ "Dzeabkhasiancyrillic": "\u04e0",
+ "Dzecyrillic": "\u0405",
+ "Dzhecyrillic": "\u040f",
+ "E": "\u0045",
+ "Eacute": "\u00c9",
+ "Eacutesmall": "\uf7e9",
+ "Ebreve": "\u0114",
+ "Ecaron": "\u011a",
+ "Ecedillabreve": "\u1e1c",
+ "Echarmenian": "\u0535",
+ "Ecircle": "\u24ba",
+ "Ecircumflex": "\u00ca",
+ "Ecircumflexacute": "\u1ebe",
+ "Ecircumflexbelow": "\u1e18",
+ "Ecircumflexdotbelow": "\u1ec6",
+ "Ecircumflexgrave": "\u1ec0",
+ "Ecircumflexhookabove": "\u1ec2",
+ "Ecircumflexsmall": "\uf7ea",
+ "Ecircumflextilde": "\u1ec4",
+ "Ecyrillic": "\u0404",
+ "Edblgrave": "\u0204",
+ "Edieresis": "\u00cb",
+ "Edieresissmall": "\uf7eb",
+ "Edot": "\u0116",
+ "Edotaccent": "\u0116",
+ "Edotbelow": "\u1eb8",
+ "Efcyrillic": "\u0424",
+ "Egrave": "\u00c8",
+ "Egravesmall": "\uf7e8",
+ "Eharmenian": "\u0537",
+ "Ehookabove": "\u1eba",
+ "Eightroman": "\u2167",
+ "Einvertedbreve": "\u0206",
+ "Eiotifiedcyrillic": "\u0464",
+ "Elcyrillic": "\u041b",
+ "Elevenroman": "\u216a",
+ "Emacron": "\u0112",
+ "Emacronacute": "\u1e16",
+ "Emacrongrave": "\u1e14",
+ "Emcyrillic": "\u041c",
+ "Emonospace": "\uff25",
+ "Encyrillic": "\u041d",
+ "Endescendercyrillic": "\u04a2",
+ "Eng": "\u014a",
+ "Enghecyrillic": "\u04a4",
+ "Enhookcyrillic": "\u04c7",
+ "Eogonek": "\u0118",
+ "Eopen": "\u0190",
+ "Epsilon": "\u0395",
+ "Epsilontonos": "\u0388",
+ "Ercyrillic": "\u0420",
+ "Ereversed": "\u018e",
+ "Ereversedcyrillic": "\u042d",
+ "Escyrillic": "\u0421",
+ "Esdescendercyrillic": "\u04aa",
+ "Esh": "\u01a9",
+ "Esmall": "\uf765",
+ "Eta": "\u0397",
+ "Etarmenian": "\u0538",
+ "Etatonos": "\u0389",
+ "Eth": "\u00d0",
+ "Ethsmall": "\uf7f0",
+ "Etilde": "\u1ebc",
+ "Etildebelow": "\u1e1a",
+ "Euro": "\u20ac",
+ "Ezh": "\u01b7",
+ "Ezhcaron": "\u01ee",
+ "Ezhreversed": "\u01b8",
+ "F": "\u0046",
+ "Fcircle": "\u24bb",
+ "Fdotaccent": "\u1e1e",
+ "Feharmenian": "\u0556",
+ "Feicoptic": "\u03e4",
+ "Fhook": "\u0191",
+ "Fitacyrillic": "\u0472",
+ "Fiveroman": "\u2164",
+ "Fmonospace": "\uff26",
+ "Fourroman": "\u2163",
+ "Fsmall": "\uf766",
+ "G": "\u0047",
+ "GBsquare": "\u3387",
+ "Gacute": "\u01f4",
+ "Gamma": "\u0393",
+ "Gammaafrican": "\u0194",
+ "Gangiacoptic": "\u03ea",
+ "Gbreve": "\u011e",
+ "Gcaron": "\u01e6",
+ "Gcedilla": "\u0122",
+ "Gcircle": "\u24bc",
+ "Gcircumflex": "\u011c",
+ "Gcommaaccent": "\u0122",
+ "Gdot": "\u0120",
+ "Gdotaccent": "\u0120",
+ "Gecyrillic": "\u0413",
+ "Ghadarmenian": "\u0542",
+ "Ghemiddlehookcyrillic": "\u0494",
+ "Ghestrokecyrillic": "\u0492",
+ "Gheupturncyrillic": "\u0490",
+ "Ghook": "\u0193",
+ "Gimarmenian": "\u0533",
+ "Gjecyrillic": "\u0403",
+ "Gmacron": "\u1e20",
+ "Gmonospace": "\uff27",
+ "Grave": "\uf6ce",
+ "Gravesmall": "\uf760",
+ "Gsmall": "\uf767",
+ "Gsmallhook": "\u029b",
+ "Gstroke": "\u01e4",
+ "H": "\u0048",
+ "H18533": "\u25cf",
+ "H18543": "\u25aa",
+ "H18551": "\u25ab",
+ "H22073": "\u25a1",
+ "HPsquare": "\u33cb",
+ "Haabkhasiancyrillic": "\u04a8",
+ "Hadescendercyrillic": "\u04b2",
+ "Hardsigncyrillic": "\u042a",
+ "Hbar": "\u0126",
+ "Hbrevebelow": "\u1e2a",
+ "Hcedilla": "\u1e28",
+ "Hcircle": "\u24bd",
+ "Hcircumflex": "\u0124",
+ "Hdieresis": "\u1e26",
+ "Hdotaccent": "\u1e22",
+ "Hdotbelow": "\u1e24",
+ "Hmonospace": "\uff28",
+ "Hoarmenian": "\u0540",
+ "Horicoptic": "\u03e8",
+ "Hsmall": "\uf768",
+ "Hungarumlaut": "\uf6cf",
+ "Hungarumlautsmall": "\uf6f8",
+ "Hzsquare": "\u3390",
+ "I": "\u0049",
+ "IAcyrillic": "\u042f",
+ "IJ": "\u0132",
+ "IUcyrillic": "\u042e",
+ "Iacute": "\u00cd",
+ "Iacutesmall": "\uf7ed",
+ "Ibreve": "\u012c",
+ "Icaron": "\u01cf",
+ "Icircle": "\u24be",
+ "Icircumflex": "\u00ce",
+ "Icircumflexsmall": "\uf7ee",
+ "Icyrillic": "\u0406",
+ "Idblgrave": "\u0208",
+ "Idieresis": "\u00cf",
+ "Idieresisacute": "\u1e2e",
+ "Idieresiscyrillic": "\u04e4",
+ "Idieresissmall": "\uf7ef",
+ "Idot": "\u0130",
+ "Idotaccent": "\u0130",
+ "Idotbelow": "\u1eca",
+ "Iebrevecyrillic": "\u04d6",
+ "Iecyrillic": "\u0415",
+ "Ifraktur": "\u2111",
+ "Igrave": "\u00cc",
+ "Igravesmall": "\uf7ec",
+ "Ihookabove": "\u1ec8",
+ "Iicyrillic": "\u0418",
+ "Iinvertedbreve": "\u020a",
+ "Iishortcyrillic": "\u0419",
+ "Imacron": "\u012a",
+ "Imacroncyrillic": "\u04e2",
+ "Imonospace": "\uff29",
+ "Iniarmenian": "\u053b",
+ "Iocyrillic": "\u0401",
+ "Iogonek": "\u012e",
+ "Iota": "\u0399",
+ "Iotaafrican": "\u0196",
+ "Iotadieresis": "\u03aa",
+ "Iotatonos": "\u038a",
+ "Ismall": "\uf769",
+ "Istroke": "\u0197",
+ "Itilde": "\u0128",
+ "Itildebelow": "\u1e2c",
+ "Izhitsacyrillic": "\u0474",
+ "Izhitsadblgravecyrillic": "\u0476",
+ "J": "\u004a",
+ "Jaarmenian": "\u0541",
+ "Jcircle": "\u24bf",
+ "Jcircumflex": "\u0134",
+ "Jecyrillic": "\u0408",
+ "Jheharmenian": "\u054b",
+ "Jmonospace": "\uff2a",
+ "Jsmall": "\uf76a",
+ "K": "\u004b",
+ "KBsquare": "\u3385",
+ "KKsquare": "\u33cd",
+ "Kabashkircyrillic": "\u04a0",
+ "Kacute": "\u1e30",
+ "Kacyrillic": "\u041a",
+ "Kadescendercyrillic": "\u049a",
+ "Kahookcyrillic": "\u04c3",
+ "Kappa": "\u039a",
+ "Kastrokecyrillic": "\u049e",
+ "Kaverticalstrokecyrillic": "\u049c",
+ "Kcaron": "\u01e8",
+ "Kcedilla": "\u0136",
+ "Kcircle": "\u24c0",
+ "Kcommaaccent": "\u0136",
+ "Kdotbelow": "\u1e32",
+ "Keharmenian": "\u0554",
+ "Kenarmenian": "\u053f",
+ "Khacyrillic": "\u0425",
+ "Kheicoptic": "\u03e6",
+ "Khook": "\u0198",
+ "Kjecyrillic": "\u040c",
+ "Klinebelow": "\u1e34",
+ "Kmonospace": "\uff2b",
+ "Koppacyrillic": "\u0480",
+ "Koppagreek": "\u03de",
+ "Ksicyrillic": "\u046e",
+ "Ksmall": "\uf76b",
+ "L": "\u004c",
+ "LJ": "\u01c7",
+ "LL": "\uf6bf",
+ "Lacute": "\u0139",
+ "Lambda": "\u039b",
+ "Lcaron": "\u013d",
+ "Lcedilla": "\u013b",
+ "Lcircle": "\u24c1",
+ "Lcircumflexbelow": "\u1e3c",
+ "Lcommaaccent": "\u013b",
+ "Ldot": "\u013f",
+ "Ldotaccent": "\u013f",
+ "Ldotbelow": "\u1e36",
+ "Ldotbelowmacron": "\u1e38",
+ "Liwnarmenian": "\u053c",
+ "Lj": "\u01c8",
+ "Ljecyrillic": "\u0409",
+ "Llinebelow": "\u1e3a",
+ "Lmonospace": "\uff2c",
+ "Lslash": "\u0141",
+ "Lslashsmall": "\uf6f9",
+ "Lsmall": "\uf76c",
+ "M": "\u004d",
+ "MBsquare": "\u3386",
+ "Macron": "\uf6d0",
+ "Macronsmall": "\uf7af",
+ "Macute": "\u1e3e",
+ "Mcircle": "\u24c2",
+ "Mdotaccent": "\u1e40",
+ "Mdotbelow": "\u1e42",
+ "Menarmenian": "\u0544",
+ "Mmonospace": "\uff2d",
+ "Msmall": "\uf76d",
+ "Mturned": "\u019c",
+ "Mu": "\u039c",
+ "N": "\u004e",
+ "NJ": "\u01ca",
+ "Nacute": "\u0143",
+ "Ncaron": "\u0147",
+ "Ncedilla": "\u0145",
+ "Ncircle": "\u24c3",
+ "Ncircumflexbelow": "\u1e4a",
+ "Ncommaaccent": "\u0145",
+ "Ndotaccent": "\u1e44",
+ "Ndotbelow": "\u1e46",
+ "Nhookleft": "\u019d",
+ "Nineroman": "\u2168",
+ "Nj": "\u01cb",
+ "Njecyrillic": "\u040a",
+ "Nlinebelow": "\u1e48",
+ "Nmonospace": "\uff2e",
+ "Nowarmenian": "\u0546",
+ "Nsmall": "\uf76e",
+ "Ntilde": "\u00d1",
+ "Ntildesmall": "\uf7f1",
+ "Nu": "\u039d",
+ "O": "\u004f",
+ "OE": "\u0152",
+ "OEsmall": "\uf6fa",
+ "Oacute": "\u00d3",
+ "Oacutesmall": "\uf7f3",
+ "Obarredcyrillic": "\u04e8",
+ "Obarreddieresiscyrillic": "\u04ea",
+ "Obreve": "\u014e",
+ "Ocaron": "\u01d1",
+ "Ocenteredtilde": "\u019f",
+ "Ocircle": "\u24c4",
+ "Ocircumflex": "\u00d4",
+ "Ocircumflexacute": "\u1ed0",
+ "Ocircumflexdotbelow": "\u1ed8",
+ "Ocircumflexgrave": "\u1ed2",
+ "Ocircumflexhookabove": "\u1ed4",
+ "Ocircumflexsmall": "\uf7f4",
+ "Ocircumflextilde": "\u1ed6",
+ "Ocyrillic": "\u041e",
+ "Odblacute": "\u0150",
+ "Odblgrave": "\u020c",
+ "Odieresis": "\u00d6",
+ "Odieresiscyrillic": "\u04e6",
+ "Odieresissmall": "\uf7f6",
+ "Odotbelow": "\u1ecc",
+ "Ogoneksmall": "\uf6fb",
+ "Ograve": "\u00d2",
+ "Ogravesmall": "\uf7f2",
+ "Oharmenian": "\u0555",
+ "Ohm": "\u2126",
+ "Ohookabove": "\u1ece",
+ "Ohorn": "\u01a0",
+ "Ohornacute": "\u1eda",
+ "Ohorndotbelow": "\u1ee2",
+ "Ohorngrave": "\u1edc",
+ "Ohornhookabove": "\u1ede",
+ "Ohorntilde": "\u1ee0",
+ "Ohungarumlaut": "\u0150",
+ "Oi": "\u01a2",
+ "Oinvertedbreve": "\u020e",
+ "Omacron": "\u014c",
+ "Omacronacute": "\u1e52",
+ "Omacrongrave": "\u1e50",
+ "Omega": "\u2126",
+ "Omegacyrillic": "\u0460",
+ "Omegagreek": "\u03a9",
+ "Omegaroundcyrillic": "\u047a",
+ "Omegatitlocyrillic": "\u047c",
+ "Omegatonos": "\u038f",
+ "Omicron": "\u039f",
+ "Omicrontonos": "\u038c",
+ "Omonospace": "\uff2f",
+ "Oneroman": "\u2160",
+ "Oogonek": "\u01ea",
+ "Oogonekmacron": "\u01ec",
+ "Oopen": "\u0186",
+ "Oslash": "\u00d8",
+ "Oslashacute": "\u01fe",
+ "Oslashsmall": "\uf7f8",
+ "Osmall": "\uf76f",
+ "Ostrokeacute": "\u01fe",
+ "Otcyrillic": "\u047e",
+ "Otilde": "\u00d5",
+ "Otildeacute": "\u1e4c",
+ "Otildedieresis": "\u1e4e",
+ "Otildesmall": "\uf7f5",
+ "P": "\u0050",
+ "Pacute": "\u1e54",
+ "Pcircle": "\u24c5",
+ "Pdotaccent": "\u1e56",
+ "Pecyrillic": "\u041f",
+ "Peharmenian": "\u054a",
+ "Pemiddlehookcyrillic": "\u04a6",
+ "Phi": "\u03a6",
+ "Phook": "\u01a4",
+ "Pi": "\u03a0",
+ "Piwrarmenian": "\u0553",
+ "Pmonospace": "\uff30",
+ "Psi": "\u03a8",
+ "Psicyrillic": "\u0470",
+ "Psmall": "\uf770",
+ "Q": "\u0051",
+ "Qcircle": "\u24c6",
+ "Qmonospace": "\uff31",
+ "Qsmall": "\uf771",
+ "R": "\u0052",
+ "Raarmenian": "\u054c",
+ "Racute": "\u0154",
+ "Rcaron": "\u0158",
+ "Rcedilla": "\u0156",
+ "Rcircle": "\u24c7",
+ "Rcommaaccent": "\u0156",
+ "Rdblgrave": "\u0210",
+ "Rdotaccent": "\u1e58",
+ "Rdotbelow": "\u1e5a",
+ "Rdotbelowmacron": "\u1e5c",
+ "Reharmenian": "\u0550",
+ "Rfraktur": "\u211c",
+ "Rho": "\u03a1",
+ "Ringsmall": "\uf6fc",
+ "Rinvertedbreve": "\u0212",
+ "Rlinebelow": "\u1e5e",
+ "Rmonospace": "\uff32",
+ "Rsmall": "\uf772",
+ "Rsmallinverted": "\u0281",
+ "Rsmallinvertedsuperior": "\u02b6",
+ "S": "\u0053",
+ "SF010000": "\u250c",
+ "SF020000": "\u2514",
+ "SF030000": "\u2510",
+ "SF040000": "\u2518",
+ "SF050000": "\u253c",
+ "SF060000": "\u252c",
+ "SF070000": "\u2534",
+ "SF080000": "\u251c",
+ "SF090000": "\u2524",
+ "SF100000": "\u2500",
+ "SF110000": "\u2502",
+ "SF190000": "\u2561",
+ "SF200000": "\u2562",
+ "SF210000": "\u2556",
+ "SF220000": "\u2555",
+ "SF230000": "\u2563",
+ "SF240000": "\u2551",
+ "SF250000": "\u2557",
+ "SF260000": "\u255d",
+ "SF270000": "\u255c",
+ "SF280000": "\u255b",
+ "SF360000": "\u255e",
+ "SF370000": "\u255f",
+ "SF380000": "\u255a",
+ "SF390000": "\u2554",
+ "SF400000": "\u2569",
+ "SF410000": "\u2566",
+ "SF420000": "\u2560",
+ "SF430000": "\u2550",
+ "SF440000": "\u256c",
+ "SF450000": "\u2567",
+ "SF460000": "\u2568",
+ "SF470000": "\u2564",
+ "SF480000": "\u2565",
+ "SF490000": "\u2559",
+ "SF500000": "\u2558",
+ "SF510000": "\u2552",
+ "SF520000": "\u2553",
+ "SF530000": "\u256b",
+ "SF540000": "\u256a",
+ "Sacute": "\u015a",
+ "Sacutedotaccent": "\u1e64",
+ "Sampigreek": "\u03e0",
+ "Scaron": "\u0160",
+ "Scarondotaccent": "\u1e66",
+ "Scaronsmall": "\uf6fd",
+ "Scedilla": "\u015e",
+ "Schwa": "\u018f",
+ "Schwacyrillic": "\u04d8",
+ "Schwadieresiscyrillic": "\u04da",
+ "Scircle": "\u24c8",
+ "Scircumflex": "\u015c",
+ "Scommaaccent": "\u0218",
+ "Sdotaccent": "\u1e60",
+ "Sdotbelow": "\u1e62",
+ "Sdotbelowdotaccent": "\u1e68",
+ "Seharmenian": "\u054d",
+ "Sevenroman": "\u2166",
+ "Shaarmenian": "\u0547",
+ "Shacyrillic": "\u0428",
+ "Shchacyrillic": "\u0429",
+ "Sheicoptic": "\u03e2",
+ "Shhacyrillic": "\u04ba",
+ "Shimacoptic": "\u03ec",
+ "Sigma": "\u03a3",
+ "Sixroman": "\u2165",
+ "Smonospace": "\uff33",
+ "Softsigncyrillic": "\u042c",
+ "Ssmall": "\uf773",
+ "Stigmagreek": "\u03da",
+ "T": "\u0054",
+ "Tau": "\u03a4",
+ "Tbar": "\u0166",
+ "Tcaron": "\u0164",
+ "Tcedilla": "\u0162",
+ "Tcircle": "\u24c9",
+ "Tcircumflexbelow": "\u1e70",
+ "Tcommaaccent": "\u0162",
+ "Tdotaccent": "\u1e6a",
+ "Tdotbelow": "\u1e6c",
+ "Tecyrillic": "\u0422",
+ "Tedescendercyrillic": "\u04ac",
+ "Tenroman": "\u2169",
+ "Tetsecyrillic": "\u04b4",
+ "Theta": "\u0398",
+ "Thook": "\u01ac",
+ "Thorn": "\u00de",
+ "Thornsmall": "\uf7fe",
+ "Threeroman": "\u2162",
+ "Tildesmall": "\uf6fe",
+ "Tiwnarmenian": "\u054f",
+ "Tlinebelow": "\u1e6e",
+ "Tmonospace": "\uff34",
+ "Toarmenian": "\u0539",
+ "Tonefive": "\u01bc",
+ "Tonesix": "\u0184",
+ "Tonetwo": "\u01a7",
+ "Tretroflexhook": "\u01ae",
+ "Tsecyrillic": "\u0426",
+ "Tshecyrillic": "\u040b",
+ "Tsmall": "\uf774",
+ "Twelveroman": "\u216b",
+ "Tworoman": "\u2161",
+ "U": "\u0055",
+ "Uacute": "\u00da",
+ "Uacutesmall": "\uf7fa",
+ "Ubreve": "\u016c",
+ "Ucaron": "\u01d3",
+ "Ucircle": "\u24ca",
+ "Ucircumflex": "\u00db",
+ "Ucircumflexbelow": "\u1e76",
+ "Ucircumflexsmall": "\uf7fb",
+ "Ucyrillic": "\u0423",
+ "Udblacute": "\u0170",
+ "Udblgrave": "\u0214",
+ "Udieresis": "\u00dc",
+ "Udieresisacute": "\u01d7",
+ "Udieresisbelow": "\u1e72",
+ "Udieresiscaron": "\u01d9",
+ "Udieresiscyrillic": "\u04f0",
+ "Udieresisgrave": "\u01db",
+ "Udieresismacron": "\u01d5",
+ "Udieresissmall": "\uf7fc",
+ "Udotbelow": "\u1ee4",
+ "Ugrave": "\u00d9",
+ "Ugravesmall": "\uf7f9",
+ "Uhookabove": "\u1ee6",
+ "Uhorn": "\u01af",
+ "Uhornacute": "\u1ee8",
+ "Uhorndotbelow": "\u1ef0",
+ "Uhorngrave": "\u1eea",
+ "Uhornhookabove": "\u1eec",
+ "Uhorntilde": "\u1eee",
+ "Uhungarumlaut": "\u0170",
+ "Uhungarumlautcyrillic": "\u04f2",
+ "Uinvertedbreve": "\u0216",
+ "Ukcyrillic": "\u0478",
+ "Umacron": "\u016a",
+ "Umacroncyrillic": "\u04ee",
+ "Umacrondieresis": "\u1e7a",
+ "Umonospace": "\uff35",
+ "Uogonek": "\u0172",
+ "Upsilon": "\u03a5",
+ "Upsilon1": "\u03d2",
+ "Upsilonacutehooksymbolgreek": "\u03d3",
+ "Upsilonafrican": "\u01b1",
+ "Upsilondieresis": "\u03ab",
+ "Upsilondieresishooksymbolgreek": "\u03d4",
+ "Upsilonhooksymbol": "\u03d2",
+ "Upsilontonos": "\u038e",
+ "Uring": "\u016e",
+ "Ushortcyrillic": "\u040e",
+ "Usmall": "\uf775",
+ "Ustraightcyrillic": "\u04ae",
+ "Ustraightstrokecyrillic": "\u04b0",
+ "Utilde": "\u0168",
+ "Utildeacute": "\u1e78",
+ "Utildebelow": "\u1e74",
+ "V": "\u0056",
+ "Vcircle": "\u24cb",
+ "Vdotbelow": "\u1e7e",
+ "Vecyrillic": "\u0412",
+ "Vewarmenian": "\u054e",
+ "Vhook": "\u01b2",
+ "Vmonospace": "\uff36",
+ "Voarmenian": "\u0548",
+ "Vsmall": "\uf776",
+ "Vtilde": "\u1e7c",
+ "W": "\u0057",
+ "Wacute": "\u1e82",
+ "Wcircle": "\u24cc",
+ "Wcircumflex": "\u0174",
+ "Wdieresis": "\u1e84",
+ "Wdotaccent": "\u1e86",
+ "Wdotbelow": "\u1e88",
+ "Wgrave": "\u1e80",
+ "Wmonospace": "\uff37",
+ "Wsmall": "\uf777",
+ "X": "\u0058",
+ "Xcircle": "\u24cd",
+ "Xdieresis": "\u1e8c",
+ "Xdotaccent": "\u1e8a",
+ "Xeharmenian": "\u053d",
+ "Xi": "\u039e",
+ "Xmonospace": "\uff38",
+ "Xsmall": "\uf778",
+ "Y": "\u0059",
+ "Yacute": "\u00dd",
+ "Yacutesmall": "\uf7fd",
+ "Yatcyrillic": "\u0462",
+ "Ycircle": "\u24ce",
+ "Ycircumflex": "\u0176",
+ "Ydieresis": "\u0178",
+ "Ydieresissmall": "\uf7ff",
+ "Ydotaccent": "\u1e8e",
+ "Ydotbelow": "\u1ef4",
+ "Yericyrillic": "\u042b",
+ "Yerudieresiscyrillic": "\u04f8",
+ "Ygrave": "\u1ef2",
+ "Yhook": "\u01b3",
+ "Yhookabove": "\u1ef6",
+ "Yiarmenian": "\u0545",
+ "Yicyrillic": "\u0407",
+ "Yiwnarmenian": "\u0552",
+ "Ymonospace": "\uff39",
+ "Ysmall": "\uf779",
+ "Ytilde": "\u1ef8",
+ "Yusbigcyrillic": "\u046a",
+ "Yusbigiotifiedcyrillic": "\u046c",
+ "Yuslittlecyrillic": "\u0466",
+ "Yuslittleiotifiedcyrillic": "\u0468",
+ "Z": "\u005a",
+ "Zaarmenian": "\u0536",
+ "Zacute": "\u0179",
+ "Zcaron": "\u017d",
+ "Zcaronsmall": "\uf6ff",
+ "Zcircle": "\u24cf",
+ "Zcircumflex": "\u1e90",
+ "Zdot": "\u017b",
+ "Zdotaccent": "\u017b",
+ "Zdotbelow": "\u1e92",
+ "Zecyrillic": "\u0417",
+ "Zedescendercyrillic": "\u0498",
+ "Zedieresiscyrillic": "\u04de",
+ "Zeta": "\u0396",
+ "Zhearmenian": "\u053a",
+ "Zhebrevecyrillic": "\u04c1",
+ "Zhecyrillic": "\u0416",
+ "Zhedescendercyrillic": "\u0496",
+ "Zhedieresiscyrillic": "\u04dc",
+ "Zlinebelow": "\u1e94",
+ "Zmonospace": "\uff3a",
+ "Zsmall": "\uf77a",
+ "Zstroke": "\u01b5",
+ "a": "\u0061",
+ "aabengali": "\u0986",
+ "aacute": "\u00e1",
+ "aadeva": "\u0906",
+ "aagujarati": "\u0a86",
+ "aagurmukhi": "\u0a06",
+ "aamatragurmukhi": "\u0a3e",
+ "aarusquare": "\u3303",
+ "aavowelsignbengali": "\u09be",
+ "aavowelsigndeva": "\u093e",
+ "aavowelsigngujarati": "\u0abe",
+ "abbreviationmarkarmenian": "\u055f",
+ "abbreviationsigndeva": "\u0970",
+ "abengali": "\u0985",
+ "abopomofo": "\u311a",
+ "abreve": "\u0103",
+ "abreveacute": "\u1eaf",
+ "abrevecyrillic": "\u04d1",
+ "abrevedotbelow": "\u1eb7",
+ "abrevegrave": "\u1eb1",
+ "abrevehookabove": "\u1eb3",
+ "abrevetilde": "\u1eb5",
+ "acaron": "\u01ce",
+ "acircle": "\u24d0",
+ "acircumflex": "\u00e2",
+ "acircumflexacute": "\u1ea5",
+ "acircumflexdotbelow": "\u1ead",
+ "acircumflexgrave": "\u1ea7",
+ "acircumflexhookabove": "\u1ea9",
+ "acircumflextilde": "\u1eab",
+ "acute": "\u00b4",
+ "acutebelowcmb": "\u0317",
+ "acutecmb": "\u0301",
+ "acutecomb": "\u0301",
+ "acutedeva": "\u0954",
+ "acutelowmod": "\u02cf",
+ "acutetonecmb": "\u0341",
+ "acyrillic": "\u0430",
+ "adblgrave": "\u0201",
+ "addakgurmukhi": "\u0a71",
+ "adeva": "\u0905",
+ "adieresis": "\u00e4",
+ "adieresiscyrillic": "\u04d3",
+ "adieresismacron": "\u01df",
+ "adotbelow": "\u1ea1",
+ "adotmacron": "\u01e1",
+ "ae": "\u00e6",
+ "aeacute": "\u01fd",
+ "aekorean": "\u3150",
+ "aemacron": "\u01e3",
+ "afii00208": "\u2015",
+ "afii08941": "\u20a4",
+ "afii10017": "\u0410",
+ "afii10018": "\u0411",
+ "afii10019": "\u0412",
+ "afii10020": "\u0413",
+ "afii10021": "\u0414",
+ "afii10022": "\u0415",
+ "afii10023": "\u0401",
+ "afii10024": "\u0416",
+ "afii10025": "\u0417",
+ "afii10026": "\u0418",
+ "afii10027": "\u0419",
+ "afii10028": "\u041a",
+ "afii10029": "\u041b",
+ "afii10030": "\u041c",
+ "afii10031": "\u041d",
+ "afii10032": "\u041e",
+ "afii10033": "\u041f",
+ "afii10034": "\u0420",
+ "afii10035": "\u0421",
+ "afii10036": "\u0422",
+ "afii10037": "\u0423",
+ "afii10038": "\u0424",
+ "afii10039": "\u0425",
+ "afii10040": "\u0426",
+ "afii10041": "\u0427",
+ "afii10042": "\u0428",
+ "afii10043": "\u0429",
+ "afii10044": "\u042a",
+ "afii10045": "\u042b",
+ "afii10046": "\u042c",
+ "afii10047": "\u042d",
+ "afii10048": "\u042e",
+ "afii10049": "\u042f",
+ "afii10050": "\u0490",
+ "afii10051": "\u0402",
+ "afii10052": "\u0403",
+ "afii10053": "\u0404",
+ "afii10054": "\u0405",
+ "afii10055": "\u0406",
+ "afii10056": "\u0407",
+ "afii10057": "\u0408",
+ "afii10058": "\u0409",
+ "afii10059": "\u040a",
+ "afii10060": "\u040b",
+ "afii10061": "\u040c",
+ "afii10062": "\u040e",
+ "afii10063": "\uf6c4",
+ "afii10064": "\uf6c5",
+ "afii10065": "\u0430",
+ "afii10066": "\u0431",
+ "afii10067": "\u0432",
+ "afii10068": "\u0433",
+ "afii10069": "\u0434",
+ "afii10070": "\u0435",
+ "afii10071": "\u0451",
+ "afii10072": "\u0436",
+ "afii10073": "\u0437",
+ "afii10074": "\u0438",
+ "afii10075": "\u0439",
+ "afii10076": "\u043a",
+ "afii10077": "\u043b",
+ "afii10078": "\u043c",
+ "afii10079": "\u043d",
+ "afii10080": "\u043e",
+ "afii10081": "\u043f",
+ "afii10082": "\u0440",
+ "afii10083": "\u0441",
+ "afii10084": "\u0442",
+ "afii10085": "\u0443",
+ "afii10086": "\u0444",
+ "afii10087": "\u0445",
+ "afii10088": "\u0446",
+ "afii10089": "\u0447",
+ "afii10090": "\u0448",
+ "afii10091": "\u0449",
+ "afii10092": "\u044a",
+ "afii10093": "\u044b",
+ "afii10094": "\u044c",
+ "afii10095": "\u044d",
+ "afii10096": "\u044e",
+ "afii10097": "\u044f",
+ "afii10098": "\u0491",
+ "afii10099": "\u0452",
+ "afii10100": "\u0453",
+ "afii10101": "\u0454",
+ "afii10102": "\u0455",
+ "afii10103": "\u0456",
+ "afii10104": "\u0457",
+ "afii10105": "\u0458",
+ "afii10106": "\u0459",
+ "afii10107": "\u045a",
+ "afii10108": "\u045b",
+ "afii10109": "\u045c",
+ "afii10110": "\u045e",
+ "afii10145": "\u040f",
+ "afii10146": "\u0462",
+ "afii10147": "\u0472",
+ "afii10148": "\u0474",
+ "afii10192": "\uf6c6",
+ "afii10193": "\u045f",
+ "afii10194": "\u0463",
+ "afii10195": "\u0473",
+ "afii10196": "\u0475",
+ "afii10831": "\uf6c7",
+ "afii10832": "\uf6c8",
+ "afii10846": "\u04d9",
+ "afii299": "\u200e",
+ "afii300": "\u200f",
+ "afii301": "\u200d",
+ "afii57381": "\u066a",
+ "afii57388": "\u060c",
+ "afii57392": "\u0660",
+ "afii57393": "\u0661",
+ "afii57394": "\u0662",
+ "afii57395": "\u0663",
+ "afii57396": "\u0664",
+ "afii57397": "\u0665",
+ "afii57398": "\u0666",
+ "afii57399": "\u0667",
+ "afii57400": "\u0668",
+ "afii57401": "\u0669",
+ "afii57403": "\u061b",
+ "afii57407": "\u061f",
+ "afii57409": "\u0621",
+ "afii57410": "\u0622",
+ "afii57411": "\u0623",
+ "afii57412": "\u0624",
+ "afii57413": "\u0625",
+ "afii57414": "\u0626",
+ "afii57415": "\u0627",
+ "afii57416": "\u0628",
+ "afii57417": "\u0629",
+ "afii57418": "\u062a",
+ "afii57419": "\u062b",
+ "afii57420": "\u062c",
+ "afii57421": "\u062d",
+ "afii57422": "\u062e",
+ "afii57423": "\u062f",
+ "afii57424": "\u0630",
+ "afii57425": "\u0631",
+ "afii57426": "\u0632",
+ "afii57427": "\u0633",
+ "afii57428": "\u0634",
+ "afii57429": "\u0635",
+ "afii57430": "\u0636",
+ "afii57431": "\u0637",
+ "afii57432": "\u0638",
+ "afii57433": "\u0639",
+ "afii57434": "\u063a",
+ "afii57440": "\u0640",
+ "afii57441": "\u0641",
+ "afii57442": "\u0642",
+ "afii57443": "\u0643",
+ "afii57444": "\u0644",
+ "afii57445": "\u0645",
+ "afii57446": "\u0646",
+ "afii57448": "\u0648",
+ "afii57449": "\u0649",
+ "afii57450": "\u064a",
+ "afii57451": "\u064b",
+ "afii57452": "\u064c",
+ "afii57453": "\u064d",
+ "afii57454": "\u064e",
+ "afii57455": "\u064f",
+ "afii57456": "\u0650",
+ "afii57457": "\u0651",
+ "afii57458": "\u0652",
+ "afii57470": "\u0647",
+ "afii57505": "\u06a4",
+ "afii57506": "\u067e",
+ "afii57507": "\u0686",
+ "afii57508": "\u0698",
+ "afii57509": "\u06af",
+ "afii57511": "\u0679",
+ "afii57512": "\u0688",
+ "afii57513": "\u0691",
+ "afii57514": "\u06ba",
+ "afii57519": "\u06d2",
+ "afii57534": "\u06d5",
+ "afii57636": "\u20aa",
+ "afii57645": "\u05be",
+ "afii57658": "\u05c3",
+ "afii57664": "\u05d0",
+ "afii57665": "\u05d1",
+ "afii57666": "\u05d2",
+ "afii57667": "\u05d3",
+ "afii57668": "\u05d4",
+ "afii57669": "\u05d5",
+ "afii57670": "\u05d6",
+ "afii57671": "\u05d7",
+ "afii57672": "\u05d8",
+ "afii57673": "\u05d9",
+ "afii57674": "\u05da",
+ "afii57675": "\u05db",
+ "afii57676": "\u05dc",
+ "afii57677": "\u05dd",
+ "afii57678": "\u05de",
+ "afii57679": "\u05df",
+ "afii57680": "\u05e0",
+ "afii57681": "\u05e1",
+ "afii57682": "\u05e2",
+ "afii57683": "\u05e3",
+ "afii57684": "\u05e4",
+ "afii57685": "\u05e5",
+ "afii57686": "\u05e6",
+ "afii57687": "\u05e7",
+ "afii57688": "\u05e8",
+ "afii57689": "\u05e9",
+ "afii57690": "\u05ea",
+ "afii57694": "\ufb2a",
+ "afii57695": "\ufb2b",
+ "afii57700": "\ufb4b",
+ "afii57705": "\ufb1f",
+ "afii57716": "\u05f0",
+ "afii57717": "\u05f1",
+ "afii57718": "\u05f2",
+ "afii57723": "\ufb35",
+ "afii57793": "\u05b4",
+ "afii57794": "\u05b5",
+ "afii57795": "\u05b6",
+ "afii57796": "\u05bb",
+ "afii57797": "\u05b8",
+ "afii57798": "\u05b7",
+ "afii57799": "\u05b0",
+ "afii57800": "\u05b2",
+ "afii57801": "\u05b1",
+ "afii57802": "\u05b3",
+ "afii57803": "\u05c2",
+ "afii57804": "\u05c1",
+ "afii57806": "\u05b9",
+ "afii57807": "\u05bc",
+ "afii57839": "\u05bd",
+ "afii57841": "\u05bf",
+ "afii57842": "\u05c0",
+ "afii57929": "\u02bc",
+ "afii61248": "\u2105",
+ "afii61289": "\u2113",
+ "afii61352": "\u2116",
+ "afii61573": "\u202c",
+ "afii61574": "\u202d",
+ "afii61575": "\u202e",
+ "afii61664": "\u200c",
+ "afii63167": "\u066d",
+ "afii64937": "\u02bd",
+ "agrave": "\u00e0",
+ "agujarati": "\u0a85",
+ "agurmukhi": "\u0a05",
+ "ahiragana": "\u3042",
+ "ahookabove": "\u1ea3",
+ "aibengali": "\u0990",
+ "aibopomofo": "\u311e",
+ "aideva": "\u0910",
+ "aiecyrillic": "\u04d5",
+ "aigujarati": "\u0a90",
+ "aigurmukhi": "\u0a10",
+ "aimatragurmukhi": "\u0a48",
+ "ainarabic": "\u0639",
+ "ainfinalarabic": "\ufeca",
+ "aininitialarabic": "\ufecb",
+ "ainmedialarabic": "\ufecc",
+ "ainvertedbreve": "\u0203",
+ "aivowelsignbengali": "\u09c8",
+ "aivowelsigndeva": "\u0948",
+ "aivowelsigngujarati": "\u0ac8",
+ "akatakana": "\u30a2",
+ "akatakanahalfwidth": "\uff71",
+ "akorean": "\u314f",
+ "alef": "\u05d0",
+ "alefarabic": "\u0627",
+ "alefdageshhebrew": "\ufb30",
+ "aleffinalarabic": "\ufe8e",
+ "alefhamzaabovearabic": "\u0623",
+ "alefhamzaabovefinalarabic": "\ufe84",
+ "alefhamzabelowarabic": "\u0625",
+ "alefhamzabelowfinalarabic": "\ufe88",
+ "alefhebrew": "\u05d0",
+ "aleflamedhebrew": "\ufb4f",
+ "alefmaddaabovearabic": "\u0622",
+ "alefmaddaabovefinalarabic": "\ufe82",
+ "alefmaksuraarabic": "\u0649",
+ "alefmaksurafinalarabic": "\ufef0",
+ "alefmaksurainitialarabic": "\ufef3",
+ "alefmaksuramedialarabic": "\ufef4",
+ "alefpatahhebrew": "\ufb2e",
+ "alefqamatshebrew": "\ufb2f",
+ "aleph": "\u2135",
+ "allequal": "\u224c",
+ "alpha": "\u03b1",
+ "alphatonos": "\u03ac",
+ "amacron": "\u0101",
+ "amonospace": "\uff41",
+ "ampersand": "\u0026",
+ "ampersandmonospace": "\uff06",
+ "ampersandsmall": "\uf726",
+ "amsquare": "\u33c2",
+ "anbopomofo": "\u3122",
+ "angbopomofo": "\u3124",
+ "angkhankhuthai": "\u0e5a",
+ "angle": "\u2220",
+ "anglebracketleft": "\u3008",
+ "anglebracketleftvertical": "\ufe3f",
+ "anglebracketright": "\u3009",
+ "anglebracketrightvertical": "\ufe40",
+ "angleleft": "\u2329",
+ "angleright": "\u232a",
+ "angstrom": "\u212b",
+ "anoteleia": "\u0387",
+ "anudattadeva": "\u0952",
+ "anusvarabengali": "\u0982",
+ "anusvaradeva": "\u0902",
+ "anusvaragujarati": "\u0a82",
+ "aogonek": "\u0105",
+ "apaatosquare": "\u3300",
+ "aparen": "\u249c",
+ "apostrophearmenian": "\u055a",
+ "apostrophemod": "\u02bc",
+ "apple": "\uf8ff",
+ "approaches": "\u2250",
+ "approxequal": "\u2248",
+ "approxequalorimage": "\u2252",
+ "approximatelyequal": "\u2245",
+ "araeaekorean": "\u318e",
+ "araeakorean": "\u318d",
+ "arc": "\u2312",
+ "arighthalfring": "\u1e9a",
+ "aring": "\u00e5",
+ "aringacute": "\u01fb",
+ "aringbelow": "\u1e01",
+ "arrowboth": "\u2194",
+ "arrowdashdown": "\u21e3",
+ "arrowdashleft": "\u21e0",
+ "arrowdashright": "\u21e2",
+ "arrowdashup": "\u21e1",
+ "arrowdblboth": "\u21d4",
+ "arrowdbldown": "\u21d3",
+ "arrowdblleft": "\u21d0",
+ "arrowdblright": "\u21d2",
+ "arrowdblup": "\u21d1",
+ "arrowdown": "\u2193",
+ "arrowdownleft": "\u2199",
+ "arrowdownright": "\u2198",
+ "arrowdownwhite": "\u21e9",
+ "arrowheaddownmod": "\u02c5",
+ "arrowheadleftmod": "\u02c2",
+ "arrowheadrightmod": "\u02c3",
+ "arrowheadupmod": "\u02c4",
+ "arrowhorizex": "\uf8e7",
+ "arrowleft": "\u2190",
+ "arrowleftdbl": "\u21d0",
+ "arrowleftdblstroke": "\u21cd",
+ "arrowleftoverright": "\u21c6",
+ "arrowleftwhite": "\u21e6",
+ "arrowright": "\u2192",
+ "arrowrightdblstroke": "\u21cf",
+ "arrowrightheavy": "\u279e",
+ "arrowrightoverleft": "\u21c4",
+ "arrowrightwhite": "\u21e8",
+ "arrowtableft": "\u21e4",
+ "arrowtabright": "\u21e5",
+ "arrowup": "\u2191",
+ "arrowupdn": "\u2195",
+ "arrowupdnbse": "\u21a8",
+ "arrowupdownbase": "\u21a8",
+ "arrowupleft": "\u2196",
+ "arrowupleftofdown": "\u21c5",
+ "arrowupright": "\u2197",
+ "arrowupwhite": "\u21e7",
+ "arrowvertex": "\uf8e6",
+ "asciicircum": "\u005e",
+ "asciicircummonospace": "\uff3e",
+ "asciitilde": "\u007e",
+ "asciitildemonospace": "\uff5e",
+ "ascript": "\u0251",
+ "ascriptturned": "\u0252",
+ "asmallhiragana": "\u3041",
+ "asmallkatakana": "\u30a1",
+ "asmallkatakanahalfwidth": "\uff67",
+ "asterisk": "\u002a",
+ "asteriskaltonearabic": "\u066d",
+ "asteriskarabic": "\u066d",
+ "asteriskmath": "\u2217",
+ "asteriskmonospace": "\uff0a",
+ "asterisksmall": "\ufe61",
+ "asterism": "\u2042",
+ "asuperior": "\uf6e9",
+ "asymptoticallyequal": "\u2243",
+ "at": "\u0040",
+ "atilde": "\u00e3",
+ "atmonospace": "\uff20",
+ "atsmall": "\ufe6b",
+ "aturned": "\u0250",
+ "aubengali": "\u0994",
+ "aubopomofo": "\u3120",
+ "audeva": "\u0914",
+ "augujarati": "\u0a94",
+ "augurmukhi": "\u0a14",
+ "aulengthmarkbengali": "\u09d7",
+ "aumatragurmukhi": "\u0a4c",
+ "auvowelsignbengali": "\u09cc",
+ "auvowelsigndeva": "\u094c",
+ "auvowelsigngujarati": "\u0acc",
+ "avagrahadeva": "\u093d",
+ "aybarmenian": "\u0561",
+ "ayin": "\u05e2",
+ "ayinaltonehebrew": "\ufb20",
+ "ayinhebrew": "\u05e2",
+ "b": "\u0062",
+ "babengali": "\u09ac",
+ "backslash": "\u005c",
+ "backslashmonospace": "\uff3c",
+ "badeva": "\u092c",
+ "bagujarati": "\u0aac",
+ "bagurmukhi": "\u0a2c",
+ "bahiragana": "\u3070",
+ "bahtthai": "\u0e3f",
+ "bakatakana": "\u30d0",
+ "bar": "\u007c",
+ "barmonospace": "\uff5c",
+ "bbopomofo": "\u3105",
+ "bcircle": "\u24d1",
+ "bdotaccent": "\u1e03",
+ "bdotbelow": "\u1e05",
+ "beamedsixteenthnotes": "\u266c",
+ "because": "\u2235",
+ "becyrillic": "\u0431",
+ "beharabic": "\u0628",
+ "behfinalarabic": "\ufe90",
+ "behinitialarabic": "\ufe91",
+ "behiragana": "\u3079",
+ "behmedialarabic": "\ufe92",
+ "behmeeminitialarabic": "\ufc9f",
+ "behmeemisolatedarabic": "\ufc08",
+ "behnoonfinalarabic": "\ufc6d",
+ "bekatakana": "\u30d9",
+ "benarmenian": "\u0562",
+ "bet": "\u05d1",
+ "beta": "\u03b2",
+ "betasymbolgreek": "\u03d0",
+ "betdagesh": "\ufb31",
+ "betdageshhebrew": "\ufb31",
+ "bethebrew": "\u05d1",
+ "betrafehebrew": "\ufb4c",
+ "bhabengali": "\u09ad",
+ "bhadeva": "\u092d",
+ "bhagujarati": "\u0aad",
+ "bhagurmukhi": "\u0a2d",
+ "bhook": "\u0253",
+ "bihiragana": "\u3073",
+ "bikatakana": "\u30d3",
+ "bilabialclick": "\u0298",
+ "bindigurmukhi": "\u0a02",
+ "birusquare": "\u3331",
+ "blackcircle": "\u25cf",
+ "blackdiamond": "\u25c6",
+ "blackdownpointingtriangle": "\u25bc",
+ "blackleftpointingpointer": "\u25c4",
+ "blackleftpointingtriangle": "\u25c0",
+ "blacklenticularbracketleft": "\u3010",
+ "blacklenticularbracketleftvertical": "\ufe3b",
+ "blacklenticularbracketright": "\u3011",
+ "blacklenticularbracketrightvertical": "\ufe3c",
+ "blacklowerlefttriangle": "\u25e3",
+ "blacklowerrighttriangle": "\u25e2",
+ "blackrectangle": "\u25ac",
+ "blackrightpointingpointer": "\u25ba",
+ "blackrightpointingtriangle": "\u25b6",
+ "blacksmallsquare": "\u25aa",
+ "blacksmilingface": "\u263b",
+ "blacksquare": "\u25a0",
+ "blackstar": "\u2605",
+ "blackupperlefttriangle": "\u25e4",
+ "blackupperrighttriangle": "\u25e5",
+ "blackuppointingsmalltriangle": "\u25b4",
+ "blackuppointingtriangle": "\u25b2",
+ "blank": "\u2423",
+ "blinebelow": "\u1e07",
+ "block": "\u2588",
+ "bmonospace": "\uff42",
+ "bobaimaithai": "\u0e1a",
+ "bohiragana": "\u307c",
+ "bokatakana": "\u30dc",
+ "bparen": "\u249d",
+ "bqsquare": "\u33c3",
+ "braceex": "\uf8f4",
+ "braceleft": "\u007b",
+ "braceleftbt": "\uf8f3",
+ "braceleftmid": "\uf8f2",
+ "braceleftmonospace": "\uff5b",
+ "braceleftsmall": "\ufe5b",
+ "bracelefttp": "\uf8f1",
+ "braceleftvertical": "\ufe37",
+ "braceright": "\u007d",
+ "bracerightbt": "\uf8fe",
+ "bracerightmid": "\uf8fd",
+ "bracerightmonospace": "\uff5d",
+ "bracerightsmall": "\ufe5c",
+ "bracerighttp": "\uf8fc",
+ "bracerightvertical": "\ufe38",
+ "bracketleft": "\u005b",
+ "bracketleftbt": "\uf8f0",
+ "bracketleftex": "\uf8ef",
+ "bracketleftmonospace": "\uff3b",
+ "bracketlefttp": "\uf8ee",
+ "bracketright": "\u005d",
+ "bracketrightbt": "\uf8fb",
+ "bracketrightex": "\uf8fa",
+ "bracketrightmonospace": "\uff3d",
+ "bracketrighttp": "\uf8f9",
+ "breve": "\u02d8",
+ "brevebelowcmb": "\u032e",
+ "brevecmb": "\u0306",
+ "breveinvertedbelowcmb": "\u032f",
+ "breveinvertedcmb": "\u0311",
+ "breveinverteddoublecmb": "\u0361",
+ "bridgebelowcmb": "\u032a",
+ "bridgeinvertedbelowcmb": "\u033a",
+ "brokenbar": "\u00a6",
+ "bstroke": "\u0180",
+ "bsuperior": "\uf6ea",
+ "btopbar": "\u0183",
+ "buhiragana": "\u3076",
+ "bukatakana": "\u30d6",
+ "bullet": "\u2022",
+ "bulletinverse": "\u25d8",
+ "bulletoperator": "\u2219",
+ "bullseye": "\u25ce",
+ "c": "\u0063",
+ "caarmenian": "\u056e",
+ "cabengali": "\u099a",
+ "cacute": "\u0107",
+ "cadeva": "\u091a",
+ "cagujarati": "\u0a9a",
+ "cagurmukhi": "\u0a1a",
+ "calsquare": "\u3388",
+ "candrabindubengali": "\u0981",
+ "candrabinducmb": "\u0310",
+ "candrabindudeva": "\u0901",
+ "candrabindugujarati": "\u0a81",
+ "capslock": "\u21ea",
+ "careof": "\u2105",
+ "caron": "\u02c7",
+ "caronbelowcmb": "\u032c",
+ "caroncmb": "\u030c",
+ "carriagereturn": "\u21b5",
+ "cbopomofo": "\u3118",
+ "ccaron": "\u010d",
+ "ccedilla": "\u00e7",
+ "ccedillaacute": "\u1e09",
+ "ccircle": "\u24d2",
+ "ccircumflex": "\u0109",
+ "ccurl": "\u0255",
+ "cdot": "\u010b",
+ "cdotaccent": "\u010b",
+ "cdsquare": "\u33c5",
+ "cedilla": "\u00b8",
+ "cedillacmb": "\u0327",
+ "cent": "\u00a2",
+ "centigrade": "\u2103",
+ "centinferior": "\uf6df",
+ "centmonospace": "\uffe0",
+ "centoldstyle": "\uf7a2",
+ "centsuperior": "\uf6e0",
+ "chaarmenian": "\u0579",
+ "chabengali": "\u099b",
+ "chadeva": "\u091b",
+ "chagujarati": "\u0a9b",
+ "chagurmukhi": "\u0a1b",
+ "chbopomofo": "\u3114",
+ "cheabkhasiancyrillic": "\u04bd",
+ "checkmark": "\u2713",
+ "checyrillic": "\u0447",
+ "chedescenderabkhasiancyrillic": "\u04bf",
+ "chedescendercyrillic": "\u04b7",
+ "chedieresiscyrillic": "\u04f5",
+ "cheharmenian": "\u0573",
+ "chekhakassiancyrillic": "\u04cc",
+ "cheverticalstrokecyrillic": "\u04b9",
+ "chi": "\u03c7",
+ "chieuchacirclekorean": "\u3277",
+ "chieuchaparenkorean": "\u3217",
+ "chieuchcirclekorean": "\u3269",
+ "chieuchkorean": "\u314a",
+ "chieuchparenkorean": "\u3209",
+ "chochangthai": "\u0e0a",
+ "chochanthai": "\u0e08",
+ "chochingthai": "\u0e09",
+ "chochoethai": "\u0e0c",
+ "chook": "\u0188",
+ "cieucacirclekorean": "\u3276",
+ "cieucaparenkorean": "\u3216",
+ "cieuccirclekorean": "\u3268",
+ "cieuckorean": "\u3148",
+ "cieucparenkorean": "\u3208",
+ "cieucuparenkorean": "\u321c",
+ "circle": "\u25cb",
+ "circlemultiply": "\u2297",
+ "circleot": "\u2299",
+ "circleplus": "\u2295",
+ "circlepostalmark": "\u3036",
+ "circlewithlefthalfblack": "\u25d0",
+ "circlewithrighthalfblack": "\u25d1",
+ "circumflex": "\u02c6",
+ "circumflexbelowcmb": "\u032d",
+ "circumflexcmb": "\u0302",
+ "clear": "\u2327",
+ "clickalveolar": "\u01c2",
+ "clickdental": "\u01c0",
+ "clicklateral": "\u01c1",
+ "clickretroflex": "\u01c3",
+ "club": "\u2663",
+ "clubsuitblack": "\u2663",
+ "clubsuitwhite": "\u2667",
+ "cmcubedsquare": "\u33a4",
+ "cmonospace": "\uff43",
+ "cmsquaredsquare": "\u33a0",
+ "coarmenian": "\u0581",
+ "colon": "\u003a",
+ "colonmonetary": "\u20a1",
+ "colonmonospace": "\uff1a",
+ "colonsign": "\u20a1",
+ "colonsmall": "\ufe55",
+ "colontriangularhalfmod": "\u02d1",
+ "colontriangularmod": "\u02d0",
+ "comma": "\u002c",
+ "commaabovecmb": "\u0313",
+ "commaaboverightcmb": "\u0315",
+ "commaaccent": "\uf6c3",
+ "commaarabic": "\u060c",
+ "commaarmenian": "\u055d",
+ "commainferior": "\uf6e1",
+ "commamonospace": "\uff0c",
+ "commareversedabovecmb": "\u0314",
+ "commareversedmod": "\u02bd",
+ "commasmall": "\ufe50",
+ "commasuperior": "\uf6e2",
+ "commaturnedabovecmb": "\u0312",
+ "commaturnedmod": "\u02bb",
+ "compass": "\u263c",
+ "congruent": "\u2245",
+ "contourintegral": "\u222e",
+ "control": "\u2303",
+ "controlACK": "\u0006",
+ "controlBEL": "\u0007",
+ "controlBS": "\u0008",
+ "controlCAN": "\u0018",
+ "controlCR": "\u000d",
+ "controlDC1": "\u0011",
+ "controlDC2": "\u0012",
+ "controlDC3": "\u0013",
+ "controlDC4": "\u0014",
+ "controlDEL": "\u007f",
+ "controlDLE": "\u0010",
+ "controlEM": "\u0019",
+ "controlENQ": "\u0005",
+ "controlEOT": "\u0004",
+ "controlESC": "\u001b",
+ "controlETB": "\u0017",
+ "controlETX": "\u0003",
+ "controlFF": "\u000c",
+ "controlFS": "\u001c",
+ "controlGS": "\u001d",
+ "controlHT": "\u0009",
+ "controlLF": "\u000a",
+ "controlNAK": "\u0015",
+ "controlRS": "\u001e",
+ "controlSI": "\u000f",
+ "controlSO": "\u000e",
+ "controlSOT": "\u0002",
+ "controlSTX": "\u0001",
+ "controlSUB": "\u001a",
+ "controlSYN": "\u0016",
+ "controlUS": "\u001f",
+ "controlVT": "\u000b",
+ "copyright": "\u00a9",
+ "copyrightsans": "\uf8e9",
+ "copyrightserif": "\uf6d9",
+ "cornerbracketleft": "\u300c",
+ "cornerbracketlefthalfwidth": "\uff62",
+ "cornerbracketleftvertical": "\ufe41",
+ "cornerbracketright": "\u300d",
+ "cornerbracketrighthalfwidth": "\uff63",
+ "cornerbracketrightvertical": "\ufe42",
+ "corporationsquare": "\u337f",
+ "cosquare": "\u33c7",
+ "coverkgsquare": "\u33c6",
+ "cparen": "\u249e",
+ "cruzeiro": "\u20a2",
+ "cstretched": "\u0297",
+ "curlyand": "\u22cf",
+ "curlyor": "\u22ce",
+ "currency": "\u00a4",
+ "cyrBreve": "\uf6d1",
+ "cyrFlex": "\uf6d2",
+ "cyrbreve": "\uf6d4",
+ "cyrflex": "\uf6d5",
+ "d": "\u0064",
+ "daarmenian": "\u0564",
+ "dabengali": "\u09a6",
+ "dadarabic": "\u0636",
+ "dadeva": "\u0926",
+ "dadfinalarabic": "\ufebe",
+ "dadinitialarabic": "\ufebf",
+ "dadmedialarabic": "\ufec0",
+ "dagesh": "\u05bc",
+ "dageshhebrew": "\u05bc",
+ "dagger": "\u2020",
+ "daggerdbl": "\u2021",
+ "dagujarati": "\u0aa6",
+ "dagurmukhi": "\u0a26",
+ "dahiragana": "\u3060",
+ "dakatakana": "\u30c0",
+ "dalarabic": "\u062f",
+ "dalet": "\u05d3",
+ "daletdagesh": "\ufb33",
+ "daletdageshhebrew": "\ufb33",
+ "dalethatafpatah": "\u05d3\u05b2",
+ "dalethatafpatahhebrew": "\u05d3\u05b2",
+ "dalethatafsegol": "\u05d3\u05b1",
+ "dalethatafsegolhebrew": "\u05d3\u05b1",
+ "dalethebrew": "\u05d3",
+ "dalethiriq": "\u05d3\u05b4",
+ "dalethiriqhebrew": "\u05d3\u05b4",
+ "daletholam": "\u05d3\u05b9",
+ "daletholamhebrew": "\u05d3\u05b9",
+ "daletpatah": "\u05d3\u05b7",
+ "daletpatahhebrew": "\u05d3\u05b7",
+ "daletqamats": "\u05d3\u05b8",
+ "daletqamatshebrew": "\u05d3\u05b8",
+ "daletqubuts": "\u05d3\u05bb",
+ "daletqubutshebrew": "\u05d3\u05bb",
+ "daletsegol": "\u05d3\u05b6",
+ "daletsegolhebrew": "\u05d3\u05b6",
+ "daletsheva": "\u05d3\u05b0",
+ "daletshevahebrew": "\u05d3\u05b0",
+ "dalettsere": "\u05d3\u05b5",
+ "dalettserehebrew": "\u05d3\u05b5",
+ "dalfinalarabic": "\ufeaa",
+ "dammaarabic": "\u064f",
+ "dammalowarabic": "\u064f",
+ "dammatanaltonearabic": "\u064c",
+ "dammatanarabic": "\u064c",
+ "danda": "\u0964",
+ "dargahebrew": "\u05a7",
+ "dargalefthebrew": "\u05a7",
+ "dasiapneumatacyrilliccmb": "\u0485",
+ "dblGrave": "\uf6d3",
+ "dblanglebracketleft": "\u300a",
+ "dblanglebracketleftvertical": "\ufe3d",
+ "dblanglebracketright": "\u300b",
+ "dblanglebracketrightvertical": "\ufe3e",
+ "dblarchinvertedbelowcmb": "\u032b",
+ "dblarrowleft": "\u21d4",
+ "dblarrowright": "\u21d2",
+ "dbldanda": "\u0965",
+ "dblgrave": "\uf6d6",
+ "dblgravecmb": "\u030f",
+ "dblintegral": "\u222c",
+ "dbllowline": "\u2017",
+ "dbllowlinecmb": "\u0333",
+ "dbloverlinecmb": "\u033f",
+ "dblprimemod": "\u02ba",
+ "dblverticalbar": "\u2016",
+ "dblverticallineabovecmb": "\u030e",
+ "dbopomofo": "\u3109",
+ "dbsquare": "\u33c8",
+ "dcaron": "\u010f",
+ "dcedilla": "\u1e11",
+ "dcircle": "\u24d3",
+ "dcircumflexbelow": "\u1e13",
+ "dcroat": "\u0111",
+ "ddabengali": "\u09a1",
+ "ddadeva": "\u0921",
+ "ddagujarati": "\u0aa1",
+ "ddagurmukhi": "\u0a21",
+ "ddalarabic": "\u0688",
+ "ddalfinalarabic": "\ufb89",
+ "dddhadeva": "\u095c",
+ "ddhabengali": "\u09a2",
+ "ddhadeva": "\u0922",
+ "ddhagujarati": "\u0aa2",
+ "ddhagurmukhi": "\u0a22",
+ "ddotaccent": "\u1e0b",
+ "ddotbelow": "\u1e0d",
+ "decimalseparatorarabic": "\u066b",
+ "decimalseparatorpersian": "\u066b",
+ "decyrillic": "\u0434",
+ "degree": "\u00b0",
+ "dehihebrew": "\u05ad",
+ "dehiragana": "\u3067",
+ "deicoptic": "\u03ef",
+ "dekatakana": "\u30c7",
+ "deleteleft": "\u232b",
+ "deleteright": "\u2326",
+ "delta": "\u03b4",
+ "deltaturned": "\u018d",
+ "denominatorminusonenumeratorbengali": "\u09f8",
+ "dezh": "\u02a4",
+ "dhabengali": "\u09a7",
+ "dhadeva": "\u0927",
+ "dhagujarati": "\u0aa7",
+ "dhagurmukhi": "\u0a27",
+ "dhook": "\u0257",
+ "dialytikatonos": "\u0385",
+ "dialytikatonoscmb": "\u0344",
+ "diamond": "\u2666",
+ "diamondsuitwhite": "\u2662",
+ "dieresis": "\u00a8",
+ "dieresisacute": "\uf6d7",
+ "dieresisbelowcmb": "\u0324",
+ "dieresiscmb": "\u0308",
+ "dieresisgrave": "\uf6d8",
+ "dieresistonos": "\u0385",
+ "dihiragana": "\u3062",
+ "dikatakana": "\u30c2",
+ "dittomark": "\u3003",
+ "divide": "\u00f7",
+ "divides": "\u2223",
+ "divisionslash": "\u2215",
+ "djecyrillic": "\u0452",
+ "dkshade": "\u2593",
+ "dlinebelow": "\u1e0f",
+ "dlsquare": "\u3397",
+ "dmacron": "\u0111",
+ "dmonospace": "\uff44",
+ "dnblock": "\u2584",
+ "dochadathai": "\u0e0e",
+ "dodekthai": "\u0e14",
+ "dohiragana": "\u3069",
+ "dokatakana": "\u30c9",
+ "dollar": "\u0024",
+ "dollarinferior": "\uf6e3",
+ "dollarmonospace": "\uff04",
+ "dollaroldstyle": "\uf724",
+ "dollarsmall": "\ufe69",
+ "dollarsuperior": "\uf6e4",
+ "dong": "\u20ab",
+ "dorusquare": "\u3326",
+ "dotaccent": "\u02d9",
+ "dotaccentcmb": "\u0307",
+ "dotbelowcmb": "\u0323",
+ "dotbelowcomb": "\u0323",
+ "dotkatakana": "\u30fb",
+ "dotlessi": "\u0131",
+ "dotlessj": "\uf6be",
+ "dotlessjstrokehook": "\u0284",
+ "dotmath": "\u22c5",
+ "dottedcircle": "\u25cc",
+ "doubleyodpatah": "\ufb1f",
+ "doubleyodpatahhebrew": "\ufb1f",
+ "downtackbelowcmb": "\u031e",
+ "downtackmod": "\u02d5",
+ "dparen": "\u249f",
+ "dsuperior": "\uf6eb",
+ "dtail": "\u0256",
+ "dtopbar": "\u018c",
+ "duhiragana": "\u3065",
+ "dukatakana": "\u30c5",
+ "dz": "\u01f3",
+ "dzaltone": "\u02a3",
+ "dzcaron": "\u01c6",
+ "dzcurl": "\u02a5",
+ "dzeabkhasiancyrillic": "\u04e1",
+ "dzecyrillic": "\u0455",
+ "dzhecyrillic": "\u045f",
+ "e": "\u0065",
+ "eacute": "\u00e9",
+ "earth": "\u2641",
+ "ebengali": "\u098f",
+ "ebopomofo": "\u311c",
+ "ebreve": "\u0115",
+ "ecandradeva": "\u090d",
+ "ecandragujarati": "\u0a8d",
+ "ecandravowelsigndeva": "\u0945",
+ "ecandravowelsigngujarati": "\u0ac5",
+ "ecaron": "\u011b",
+ "ecedillabreve": "\u1e1d",
+ "echarmenian": "\u0565",
+ "echyiwnarmenian": "\u0587",
+ "ecircle": "\u24d4",
+ "ecircumflex": "\u00ea",
+ "ecircumflexacute": "\u1ebf",
+ "ecircumflexbelow": "\u1e19",
+ "ecircumflexdotbelow": "\u1ec7",
+ "ecircumflexgrave": "\u1ec1",
+ "ecircumflexhookabove": "\u1ec3",
+ "ecircumflextilde": "\u1ec5",
+ "ecyrillic": "\u0454",
+ "edblgrave": "\u0205",
+ "edeva": "\u090f",
+ "edieresis": "\u00eb",
+ "edot": "\u0117",
+ "edotaccent": "\u0117",
+ "edotbelow": "\u1eb9",
+ "eegurmukhi": "\u0a0f",
+ "eematragurmukhi": "\u0a47",
+ "efcyrillic": "\u0444",
+ "egrave": "\u00e8",
+ "egujarati": "\u0a8f",
+ "eharmenian": "\u0567",
+ "ehbopomofo": "\u311d",
+ "ehiragana": "\u3048",
+ "ehookabove": "\u1ebb",
+ "eibopomofo": "\u311f",
+ "eight": "\u0038",
+ "eightarabic": "\u0668",
+ "eightbengali": "\u09ee",
+ "eightcircle": "\u2467",
+ "eightcircleinversesansserif": "\u2791",
+ "eightdeva": "\u096e",
+ "eighteencircle": "\u2471",
+ "eighteenparen": "\u2485",
+ "eighteenperiod": "\u2499",
+ "eightgujarati": "\u0aee",
+ "eightgurmukhi": "\u0a6e",
+ "eighthackarabic": "\u0668",
+ "eighthangzhou": "\u3028",
+ "eighthnotebeamed": "\u266b",
+ "eightideographicparen": "\u3227",
+ "eightinferior": "\u2088",
+ "eightmonospace": "\uff18",
+ "eightoldstyle": "\uf738",
+ "eightparen": "\u247b",
+ "eightperiod": "\u248f",
+ "eightpersian": "\u06f8",
+ "eightroman": "\u2177",
+ "eightsuperior": "\u2078",
+ "eightthai": "\u0e58",
+ "einvertedbreve": "\u0207",
+ "eiotifiedcyrillic": "\u0465",
+ "ekatakana": "\u30a8",
+ "ekatakanahalfwidth": "\uff74",
+ "ekonkargurmukhi": "\u0a74",
+ "ekorean": "\u3154",
+ "elcyrillic": "\u043b",
+ "element": "\u2208",
+ "elevencircle": "\u246a",
+ "elevenparen": "\u247e",
+ "elevenperiod": "\u2492",
+ "elevenroman": "\u217a",
+ "ellipsis": "\u2026",
+ "ellipsisvertical": "\u22ee",
+ "emacron": "\u0113",
+ "emacronacute": "\u1e17",
+ "emacrongrave": "\u1e15",
+ "emcyrillic": "\u043c",
+ "emdash": "\u2014",
+ "emdashvertical": "\ufe31",
+ "emonospace": "\uff45",
+ "emphasismarkarmenian": "\u055b",
+ "emptyset": "\u2205",
+ "enbopomofo": "\u3123",
+ "encyrillic": "\u043d",
+ "endash": "\u2013",
+ "endashvertical": "\ufe32",
+ "endescendercyrillic": "\u04a3",
+ "eng": "\u014b",
+ "engbopomofo": "\u3125",
+ "enghecyrillic": "\u04a5",
+ "enhookcyrillic": "\u04c8",
+ "enspace": "\u2002",
+ "eogonek": "\u0119",
+ "eokorean": "\u3153",
+ "eopen": "\u025b",
+ "eopenclosed": "\u029a",
+ "eopenreversed": "\u025c",
+ "eopenreversedclosed": "\u025e",
+ "eopenreversedhook": "\u025d",
+ "eparen": "\u24a0",
+ "epsilon": "\u03b5",
+ "epsilontonos": "\u03ad",
+ "equal": "\u003d",
+ "equalmonospace": "\uff1d",
+ "equalsmall": "\ufe66",
+ "equalsuperior": "\u207c",
+ "equivalence": "\u2261",
+ "erbopomofo": "\u3126",
+ "ercyrillic": "\u0440",
+ "ereversed": "\u0258",
+ "ereversedcyrillic": "\u044d",
+ "escyrillic": "\u0441",
+ "esdescendercyrillic": "\u04ab",
+ "esh": "\u0283",
+ "eshcurl": "\u0286",
+ "eshortdeva": "\u090e",
+ "eshortvowelsigndeva": "\u0946",
+ "eshreversedloop": "\u01aa",
+ "eshsquatreversed": "\u0285",
+ "esmallhiragana": "\u3047",
+ "esmallkatakana": "\u30a7",
+ "esmallkatakanahalfwidth": "\uff6a",
+ "estimated": "\u212e",
+ "esuperior": "\uf6ec",
+ "eta": "\u03b7",
+ "etarmenian": "\u0568",
+ "etatonos": "\u03ae",
+ "eth": "\u00f0",
+ "etilde": "\u1ebd",
+ "etildebelow": "\u1e1b",
+ "etnahtafoukhhebrew": "\u0591",
+ "etnahtafoukhlefthebrew": "\u0591",
+ "etnahtahebrew": "\u0591",
+ "etnahtalefthebrew": "\u0591",
+ "eturned": "\u01dd",
+ "eukorean": "\u3161",
+ "euro": "\u20ac",
+ "evowelsignbengali": "\u09c7",
+ "evowelsigndeva": "\u0947",
+ "evowelsigngujarati": "\u0ac7",
+ "exclam": "\u0021",
+ "exclamarmenian": "\u055c",
+ "exclamdbl": "\u203c",
+ "exclamdown": "\u00a1",
+ "exclamdownsmall": "\uf7a1",
+ "exclammonospace": "\uff01",
+ "exclamsmall": "\uf721",
+ "existential": "\u2203",
+ "ezh": "\u0292",
+ "ezhcaron": "\u01ef",
+ "ezhcurl": "\u0293",
+ "ezhreversed": "\u01b9",
+ "ezhtail": "\u01ba",
+ "f": "\u0066",
+ "fadeva": "\u095e",
+ "fagurmukhi": "\u0a5e",
+ "fahrenheit": "\u2109",
+ "fathaarabic": "\u064e",
+ "fathalowarabic": "\u064e",
+ "fathatanarabic": "\u064b",
+ "fbopomofo": "\u3108",
+ "fcircle": "\u24d5",
+ "fdotaccent": "\u1e1f",
+ "feharabic": "\u0641",
+ "feharmenian": "\u0586",
+ "fehfinalarabic": "\ufed2",
+ "fehinitialarabic": "\ufed3",
+ "fehmedialarabic": "\ufed4",
+ "feicoptic": "\u03e5",
+ "female": "\u2640",
+ "ff": "\ufb00",
+ "ffi": "\ufb03",
+ "ffl": "\ufb04",
+ "fi": "\ufb01",
+ "fifteencircle": "\u246e",
+ "fifteenparen": "\u2482",
+ "fifteenperiod": "\u2496",
+ "figuredash": "\u2012",
+ "filledbox": "\u25a0",
+ "filledrect": "\u25ac",
+ "finalkaf": "\u05da",
+ "finalkafdagesh": "\ufb3a",
+ "finalkafdageshhebrew": "\ufb3a",
+ "finalkafhebrew": "\u05da",
+ "finalkafqamats": "\u05da\u05b8",
+ "finalkafqamatshebrew": "\u05da\u05b8",
+ "finalkafsheva": "\u05da\u05b0",
+ "finalkafshevahebrew": "\u05da\u05b0",
+ "finalmem": "\u05dd",
+ "finalmemhebrew": "\u05dd",
+ "finalnun": "\u05df",
+ "finalnunhebrew": "\u05df",
+ "finalpe": "\u05e3",
+ "finalpehebrew": "\u05e3",
+ "finaltsadi": "\u05e5",
+ "finaltsadihebrew": "\u05e5",
+ "firsttonechinese": "\u02c9",
+ "fisheye": "\u25c9",
+ "fitacyrillic": "\u0473",
+ "five": "\u0035",
+ "fivearabic": "\u0665",
+ "fivebengali": "\u09eb",
+ "fivecircle": "\u2464",
+ "fivecircleinversesansserif": "\u278e",
+ "fivedeva": "\u096b",
+ "fiveeighths": "\u215d",
+ "fivegujarati": "\u0aeb",
+ "fivegurmukhi": "\u0a6b",
+ "fivehackarabic": "\u0665",
+ "fivehangzhou": "\u3025",
+ "fiveideographicparen": "\u3224",
+ "fiveinferior": "\u2085",
+ "fivemonospace": "\uff15",
+ "fiveoldstyle": "\uf735",
+ "fiveparen": "\u2478",
+ "fiveperiod": "\u248c",
+ "fivepersian": "\u06f5",
+ "fiveroman": "\u2174",
+ "fivesuperior": "\u2075",
+ "fivethai": "\u0e55",
+ "fl": "\ufb02",
+ "florin": "\u0192",
+ "fmonospace": "\uff46",
+ "fmsquare": "\u3399",
+ "fofanthai": "\u0e1f",
+ "fofathai": "\u0e1d",
+ "fongmanthai": "\u0e4f",
+ "forall": "\u2200",
+ "four": "\u0034",
+ "fourarabic": "\u0664",
+ "fourbengali": "\u09ea",
+ "fourcircle": "\u2463",
+ "fourcircleinversesansserif": "\u278d",
+ "fourdeva": "\u096a",
+ "fourgujarati": "\u0aea",
+ "fourgurmukhi": "\u0a6a",
+ "fourhackarabic": "\u0664",
+ "fourhangzhou": "\u3024",
+ "fourideographicparen": "\u3223",
+ "fourinferior": "\u2084",
+ "fourmonospace": "\uff14",
+ "fournumeratorbengali": "\u09f7",
+ "fouroldstyle": "\uf734",
+ "fourparen": "\u2477",
+ "fourperiod": "\u248b",
+ "fourpersian": "\u06f4",
+ "fourroman": "\u2173",
+ "foursuperior": "\u2074",
+ "fourteencircle": "\u246d",
+ "fourteenparen": "\u2481",
+ "fourteenperiod": "\u2495",
+ "fourthai": "\u0e54",
+ "fourthtonechinese": "\u02cb",
+ "fparen": "\u24a1",
+ "fraction": "\u2044",
+ "franc": "\u20a3",
+ "g": "\u0067",
+ "gabengali": "\u0997",
+ "gacute": "\u01f5",
+ "gadeva": "\u0917",
+ "gafarabic": "\u06af",
+ "gaffinalarabic": "\ufb93",
+ "gafinitialarabic": "\ufb94",
+ "gafmedialarabic": "\ufb95",
+ "gagujarati": "\u0a97",
+ "gagurmukhi": "\u0a17",
+ "gahiragana": "\u304c",
+ "gakatakana": "\u30ac",
+ "gamma": "\u03b3",
+ "gammalatinsmall": "\u0263",
+ "gammasuperior": "\u02e0",
+ "gangiacoptic": "\u03eb",
+ "gbopomofo": "\u310d",
+ "gbreve": "\u011f",
+ "gcaron": "\u01e7",
+ "gcedilla": "\u0123",
+ "gcircle": "\u24d6",
+ "gcircumflex": "\u011d",
+ "gcommaaccent": "\u0123",
+ "gdot": "\u0121",
+ "gdotaccent": "\u0121",
+ "gecyrillic": "\u0433",
+ "gehiragana": "\u3052",
+ "gekatakana": "\u30b2",
+ "geometricallyequal": "\u2251",
+ "gereshaccenthebrew": "\u059c",
+ "gereshhebrew": "\u05f3",
+ "gereshmuqdamhebrew": "\u059d",
+ "germandbls": "\u00df",
+ "gershayimaccenthebrew": "\u059e",
+ "gershayimhebrew": "\u05f4",
+ "getamark": "\u3013",
+ "ghabengali": "\u0998",
+ "ghadarmenian": "\u0572",
+ "ghadeva": "\u0918",
+ "ghagujarati": "\u0a98",
+ "ghagurmukhi": "\u0a18",
+ "ghainarabic": "\u063a",
+ "ghainfinalarabic": "\ufece",
+ "ghaininitialarabic": "\ufecf",
+ "ghainmedialarabic": "\ufed0",
+ "ghemiddlehookcyrillic": "\u0495",
+ "ghestrokecyrillic": "\u0493",
+ "gheupturncyrillic": "\u0491",
+ "ghhadeva": "\u095a",
+ "ghhagurmukhi": "\u0a5a",
+ "ghook": "\u0260",
+ "ghzsquare": "\u3393",
+ "gihiragana": "\u304e",
+ "gikatakana": "\u30ae",
+ "gimarmenian": "\u0563",
+ "gimel": "\u05d2",
+ "gimeldagesh": "\ufb32",
+ "gimeldageshhebrew": "\ufb32",
+ "gimelhebrew": "\u05d2",
+ "gjecyrillic": "\u0453",
+ "glottalinvertedstroke": "\u01be",
+ "glottalstop": "\u0294",
+ "glottalstopinverted": "\u0296",
+ "glottalstopmod": "\u02c0",
+ "glottalstopreversed": "\u0295",
+ "glottalstopreversedmod": "\u02c1",
+ "glottalstopreversedsuperior": "\u02e4",
+ "glottalstopstroke": "\u02a1",
+ "glottalstopstrokereversed": "\u02a2",
+ "gmacron": "\u1e21",
+ "gmonospace": "\uff47",
+ "gohiragana": "\u3054",
+ "gokatakana": "\u30b4",
+ "gparen": "\u24a2",
+ "gpasquare": "\u33ac",
+ "gradient": "\u2207",
+ "grave": "\u0060",
+ "gravebelowcmb": "\u0316",
+ "gravecmb": "\u0300",
+ "gravecomb": "\u0300",
+ "gravedeva": "\u0953",
+ "gravelowmod": "\u02ce",
+ "gravemonospace": "\uff40",
+ "gravetonecmb": "\u0340",
+ "greater": "\u003e",
+ "greaterequal": "\u2265",
+ "greaterequalorless": "\u22db",
+ "greatermonospace": "\uff1e",
+ "greaterorequivalent": "\u2273",
+ "greaterorless": "\u2277",
+ "greateroverequal": "\u2267",
+ "greatersmall": "\ufe65",
+ "gscript": "\u0261",
+ "gstroke": "\u01e5",
+ "guhiragana": "\u3050",
+ "guillemotleft": "\u00ab",
+ "guillemotright": "\u00bb",
+ "guilsinglleft": "\u2039",
+ "guilsinglright": "\u203a",
+ "gukatakana": "\u30b0",
+ "guramusquare": "\u3318",
+ "gysquare": "\u33c9",
+ "h": "\u0068",
+ "haabkhasiancyrillic": "\u04a9",
+ "haaltonearabic": "\u06c1",
+ "habengali": "\u09b9",
+ "hadescendercyrillic": "\u04b3",
+ "hadeva": "\u0939",
+ "hagujarati": "\u0ab9",
+ "hagurmukhi": "\u0a39",
+ "haharabic": "\u062d",
+ "hahfinalarabic": "\ufea2",
+ "hahinitialarabic": "\ufea3",
+ "hahiragana": "\u306f",
+ "hahmedialarabic": "\ufea4",
+ "haitusquare": "\u332a",
+ "hakatakana": "\u30cf",
+ "hakatakanahalfwidth": "\uff8a",
+ "halantgurmukhi": "\u0a4d",
+ "hamzaarabic": "\u0621",
+ "hamzadammaarabic": "\u0621\u064f",
+ "hamzadammatanarabic": "\u0621\u064c",
+ "hamzafathaarabic": "\u0621\u064e",
+ "hamzafathatanarabic": "\u0621\u064b",
+ "hamzalowarabic": "\u0621",
+ "hamzalowkasraarabic": "\u0621\u0650",
+ "hamzalowkasratanarabic": "\u0621\u064d",
+ "hamzasukunarabic": "\u0621\u0652",
+ "hangulfiller": "\u3164",
+ "hardsigncyrillic": "\u044a",
+ "harpoonleftbarbup": "\u21bc",
+ "harpoonrightbarbup": "\u21c0",
+ "hasquare": "\u33ca",
+ "hatafpatah": "\u05b2",
+ "hatafpatah16": "\u05b2",
+ "hatafpatah23": "\u05b2",
+ "hatafpatah2f": "\u05b2",
+ "hatafpatahhebrew": "\u05b2",
+ "hatafpatahnarrowhebrew": "\u05b2",
+ "hatafpatahquarterhebrew": "\u05b2",
+ "hatafpatahwidehebrew": "\u05b2",
+ "hatafqamats": "\u05b3",
+ "hatafqamats1b": "\u05b3",
+ "hatafqamats28": "\u05b3",
+ "hatafqamats34": "\u05b3",
+ "hatafqamatshebrew": "\u05b3",
+ "hatafqamatsnarrowhebrew": "\u05b3",
+ "hatafqamatsquarterhebrew": "\u05b3",
+ "hatafqamatswidehebrew": "\u05b3",
+ "hatafsegol": "\u05b1",
+ "hatafsegol17": "\u05b1",
+ "hatafsegol24": "\u05b1",
+ "hatafsegol30": "\u05b1",
+ "hatafsegolhebrew": "\u05b1",
+ "hatafsegolnarrowhebrew": "\u05b1",
+ "hatafsegolquarterhebrew": "\u05b1",
+ "hatafsegolwidehebrew": "\u05b1",
+ "hbar": "\u0127",
+ "hbopomofo": "\u310f",
+ "hbrevebelow": "\u1e2b",
+ "hcedilla": "\u1e29",
+ "hcircle": "\u24d7",
+ "hcircumflex": "\u0125",
+ "hdieresis": "\u1e27",
+ "hdotaccent": "\u1e23",
+ "hdotbelow": "\u1e25",
+ "he": "\u05d4",
+ "heart": "\u2665",
+ "heartsuitblack": "\u2665",
+ "heartsuitwhite": "\u2661",
+ "hedagesh": "\ufb34",
+ "hedageshhebrew": "\ufb34",
+ "hehaltonearabic": "\u06c1",
+ "heharabic": "\u0647",
+ "hehebrew": "\u05d4",
+ "hehfinalaltonearabic": "\ufba7",
+ "hehfinalalttwoarabic": "\ufeea",
+ "hehfinalarabic": "\ufeea",
+ "hehhamzaabovefinalarabic": "\ufba5",
+ "hehhamzaaboveisolatedarabic": "\ufba4",
+ "hehinitialaltonearabic": "\ufba8",
+ "hehinitialarabic": "\ufeeb",
+ "hehiragana": "\u3078",
+ "hehmedialaltonearabic": "\ufba9",
+ "hehmedialarabic": "\ufeec",
+ "heiseierasquare": "\u337b",
+ "hekatakana": "\u30d8",
+ "hekatakanahalfwidth": "\uff8d",
+ "hekutaarusquare": "\u3336",
+ "henghook": "\u0267",
+ "herutusquare": "\u3339",
+ "het": "\u05d7",
+ "hethebrew": "\u05d7",
+ "hhook": "\u0266",
+ "hhooksuperior": "\u02b1",
+ "hieuhacirclekorean": "\u327b",
+ "hieuhaparenkorean": "\u321b",
+ "hieuhcirclekorean": "\u326d",
+ "hieuhkorean": "\u314e",
+ "hieuhparenkorean": "\u320d",
+ "hihiragana": "\u3072",
+ "hikatakana": "\u30d2",
+ "hikatakanahalfwidth": "\uff8b",
+ "hiriq": "\u05b4",
+ "hiriq14": "\u05b4",
+ "hiriq21": "\u05b4",
+ "hiriq2d": "\u05b4",
+ "hiriqhebrew": "\u05b4",
+ "hiriqnarrowhebrew": "\u05b4",
+ "hiriqquarterhebrew": "\u05b4",
+ "hiriqwidehebrew": "\u05b4",
+ "hlinebelow": "\u1e96",
+ "hmonospace": "\uff48",
+ "hoarmenian": "\u0570",
+ "hohipthai": "\u0e2b",
+ "hohiragana": "\u307b",
+ "hokatakana": "\u30db",
+ "hokatakanahalfwidth": "\uff8e",
+ "holam": "\u05b9",
+ "holam19": "\u05b9",
+ "holam26": "\u05b9",
+ "holam32": "\u05b9",
+ "holamhebrew": "\u05b9",
+ "holamnarrowhebrew": "\u05b9",
+ "holamquarterhebrew": "\u05b9",
+ "holamwidehebrew": "\u05b9",
+ "honokhukthai": "\u0e2e",
+ "hookabovecomb": "\u0309",
+ "hookcmb": "\u0309",
+ "hookpalatalizedbelowcmb": "\u0321",
+ "hookretroflexbelowcmb": "\u0322",
+ "hoonsquare": "\u3342",
+ "horicoptic": "\u03e9",
+ "horizontalbar": "\u2015",
+ "horncmb": "\u031b",
+ "hotsprings": "\u2668",
+ "house": "\u2302",
+ "hparen": "\u24a3",
+ "hsuperior": "\u02b0",
+ "hturned": "\u0265",
+ "huhiragana": "\u3075",
+ "huiitosquare": "\u3333",
+ "hukatakana": "\u30d5",
+ "hukatakanahalfwidth": "\uff8c",
+ "hungarumlaut": "\u02dd",
+ "hungarumlautcmb": "\u030b",
+ "hv": "\u0195",
+ "hyphen": "\u002d",
+ "hypheninferior": "\uf6e5",
+ "hyphenmonospace": "\uff0d",
+ "hyphensmall": "\ufe63",
+ "hyphensuperior": "\uf6e6",
+ "hyphentwo": "\u2010",
+ "i": "\u0069",
+ "iacute": "\u00ed",
+ "iacyrillic": "\u044f",
+ "ibengali": "\u0987",
+ "ibopomofo": "\u3127",
+ "ibreve": "\u012d",
+ "icaron": "\u01d0",
+ "icircle": "\u24d8",
+ "icircumflex": "\u00ee",
+ "icyrillic": "\u0456",
+ "idblgrave": "\u0209",
+ "ideographearthcircle": "\u328f",
+ "ideographfirecircle": "\u328b",
+ "ideographicallianceparen": "\u323f",
+ "ideographiccallparen": "\u323a",
+ "ideographiccentrecircle": "\u32a5",
+ "ideographicclose": "\u3006",
+ "ideographiccomma": "\u3001",
+ "ideographiccommaleft": "\uff64",
+ "ideographiccongratulationparen": "\u3237",
+ "ideographiccorrectcircle": "\u32a3",
+ "ideographicearthparen": "\u322f",
+ "ideographicenterpriseparen": "\u323d",
+ "ideographicexcellentcircle": "\u329d",
+ "ideographicfestivalparen": "\u3240",
+ "ideographicfinancialcircle": "\u3296",
+ "ideographicfinancialparen": "\u3236",
+ "ideographicfireparen": "\u322b",
+ "ideographichaveparen": "\u3232",
+ "ideographichighcircle": "\u32a4",
+ "ideographiciterationmark": "\u3005",
+ "ideographiclaborcircle": "\u3298",
+ "ideographiclaborparen": "\u3238",
+ "ideographicleftcircle": "\u32a7",
+ "ideographiclowcircle": "\u32a6",
+ "ideographicmedicinecircle": "\u32a9",
+ "ideographicmetalparen": "\u322e",
+ "ideographicmoonparen": "\u322a",
+ "ideographicnameparen": "\u3234",
+ "ideographicperiod": "\u3002",
+ "ideographicprintcircle": "\u329e",
+ "ideographicreachparen": "\u3243",
+ "ideographicrepresentparen": "\u3239",
+ "ideographicresourceparen": "\u323e",
+ "ideographicrightcircle": "\u32a8",
+ "ideographicsecretcircle": "\u3299",
+ "ideographicselfparen": "\u3242",
+ "ideographicsocietyparen": "\u3233",
+ "ideographicspace": "\u3000",
+ "ideographicspecialparen": "\u3235",
+ "ideographicstockparen": "\u3231",
+ "ideographicstudyparen": "\u323b",
+ "ideographicsunparen": "\u3230",
+ "ideographicsuperviseparen": "\u323c",
+ "ideographicwaterparen": "\u322c",
+ "ideographicwoodparen": "\u322d",
+ "ideographiczero": "\u3007",
+ "ideographmetalcircle": "\u328e",
+ "ideographmooncircle": "\u328a",
+ "ideographnamecircle": "\u3294",
+ "ideographsuncircle": "\u3290",
+ "ideographwatercircle": "\u328c",
+ "ideographwoodcircle": "\u328d",
+ "ideva": "\u0907",
+ "idieresis": "\u00ef",
+ "idieresisacute": "\u1e2f",
+ "idieresiscyrillic": "\u04e5",
+ "idotbelow": "\u1ecb",
+ "iebrevecyrillic": "\u04d7",
+ "iecyrillic": "\u0435",
+ "ieungacirclekorean": "\u3275",
+ "ieungaparenkorean": "\u3215",
+ "ieungcirclekorean": "\u3267",
+ "ieungkorean": "\u3147",
+ "ieungparenkorean": "\u3207",
+ "igrave": "\u00ec",
+ "igujarati": "\u0a87",
+ "igurmukhi": "\u0a07",
+ "ihiragana": "\u3044",
+ "ihookabove": "\u1ec9",
+ "iibengali": "\u0988",
+ "iicyrillic": "\u0438",
+ "iideva": "\u0908",
+ "iigujarati": "\u0a88",
+ "iigurmukhi": "\u0a08",
+ "iimatragurmukhi": "\u0a40",
+ "iinvertedbreve": "\u020b",
+ "iishortcyrillic": "\u0439",
+ "iivowelsignbengali": "\u09c0",
+ "iivowelsigndeva": "\u0940",
+ "iivowelsigngujarati": "\u0ac0",
+ "ij": "\u0133",
+ "ikatakana": "\u30a4",
+ "ikatakanahalfwidth": "\uff72",
+ "ikorean": "\u3163",
+ "ilde": "\u02dc",
+ "iluyhebrew": "\u05ac",
+ "imacron": "\u012b",
+ "imacroncyrillic": "\u04e3",
+ "imageorapproximatelyequal": "\u2253",
+ "imatragurmukhi": "\u0a3f",
+ "imonospace": "\uff49",
+ "increment": "\u2206",
+ "infinity": "\u221e",
+ "iniarmenian": "\u056b",
+ "integral": "\u222b",
+ "integralbottom": "\u2321",
+ "integralbt": "\u2321",
+ "integralex": "\uf8f5",
+ "integraltop": "\u2320",
+ "integraltp": "\u2320",
+ "intersection": "\u2229",
+ "intisquare": "\u3305",
+ "invbullet": "\u25d8",
+ "invcircle": "\u25d9",
+ "invsmileface": "\u263b",
+ "iocyrillic": "\u0451",
+ "iogonek": "\u012f",
+ "iota": "\u03b9",
+ "iotadieresis": "\u03ca",
+ "iotadieresistonos": "\u0390",
+ "iotalatin": "\u0269",
+ "iotatonos": "\u03af",
+ "iparen": "\u24a4",
+ "irigurmukhi": "\u0a72",
+ "ismallhiragana": "\u3043",
+ "ismallkatakana": "\u30a3",
+ "ismallkatakanahalfwidth": "\uff68",
+ "issharbengali": "\u09fa",
+ "istroke": "\u0268",
+ "isuperior": "\uf6ed",
+ "iterationhiragana": "\u309d",
+ "iterationkatakana": "\u30fd",
+ "itilde": "\u0129",
+ "itildebelow": "\u1e2d",
+ "iubopomofo": "\u3129",
+ "iucyrillic": "\u044e",
+ "ivowelsignbengali": "\u09bf",
+ "ivowelsigndeva": "\u093f",
+ "ivowelsigngujarati": "\u0abf",
+ "izhitsacyrillic": "\u0475",
+ "izhitsadblgravecyrillic": "\u0477",
+ "j": "\u006a",
+ "jaarmenian": "\u0571",
+ "jabengali": "\u099c",
+ "jadeva": "\u091c",
+ "jagujarati": "\u0a9c",
+ "jagurmukhi": "\u0a1c",
+ "jbopomofo": "\u3110",
+ "jcaron": "\u01f0",
+ "jcircle": "\u24d9",
+ "jcircumflex": "\u0135",
+ "jcrossedtail": "\u029d",
+ "jdotlessstroke": "\u025f",
+ "jecyrillic": "\u0458",
+ "jeemarabic": "\u062c",
+ "jeemfinalarabic": "\ufe9e",
+ "jeeminitialarabic": "\ufe9f",
+ "jeemmedialarabic": "\ufea0",
+ "jeharabic": "\u0698",
+ "jehfinalarabic": "\ufb8b",
+ "jhabengali": "\u099d",
+ "jhadeva": "\u091d",
+ "jhagujarati": "\u0a9d",
+ "jhagurmukhi": "\u0a1d",
+ "jheharmenian": "\u057b",
+ "jis": "\u3004",
+ "jmonospace": "\uff4a",
+ "jparen": "\u24a5",
+ "jsuperior": "\u02b2",
+ "k": "\u006b",
+ "kabashkircyrillic": "\u04a1",
+ "kabengali": "\u0995",
+ "kacute": "\u1e31",
+ "kacyrillic": "\u043a",
+ "kadescendercyrillic": "\u049b",
+ "kadeva": "\u0915",
+ "kaf": "\u05db",
+ "kafarabic": "\u0643",
+ "kafdagesh": "\ufb3b",
+ "kafdageshhebrew": "\ufb3b",
+ "kaffinalarabic": "\ufeda",
+ "kafhebrew": "\u05db",
+ "kafinitialarabic": "\ufedb",
+ "kafmedialarabic": "\ufedc",
+ "kafrafehebrew": "\ufb4d",
+ "kagujarati": "\u0a95",
+ "kagurmukhi": "\u0a15",
+ "kahiragana": "\u304b",
+ "kahookcyrillic": "\u04c4",
+ "kakatakana": "\u30ab",
+ "kakatakanahalfwidth": "\uff76",
+ "kappa": "\u03ba",
+ "kappasymbolgreek": "\u03f0",
+ "kapyeounmieumkorean": "\u3171",
+ "kapyeounphieuphkorean": "\u3184",
+ "kapyeounpieupkorean": "\u3178",
+ "kapyeounssangpieupkorean": "\u3179",
+ "karoriisquare": "\u330d",
+ "kashidaautoarabic": "\u0640",
+ "kashidaautonosidebearingarabic": "\u0640",
+ "kasmallkatakana": "\u30f5",
+ "kasquare": "\u3384",
+ "kasraarabic": "\u0650",
+ "kasratanarabic": "\u064d",
+ "kastrokecyrillic": "\u049f",
+ "katahiraprolongmarkhalfwidth": "\uff70",
+ "kaverticalstrokecyrillic": "\u049d",
+ "kbopomofo": "\u310e",
+ "kcalsquare": "\u3389",
+ "kcaron": "\u01e9",
+ "kcedilla": "\u0137",
+ "kcircle": "\u24da",
+ "kcommaaccent": "\u0137",
+ "kdotbelow": "\u1e33",
+ "keharmenian": "\u0584",
+ "kehiragana": "\u3051",
+ "kekatakana": "\u30b1",
+ "kekatakanahalfwidth": "\uff79",
+ "kenarmenian": "\u056f",
+ "kesmallkatakana": "\u30f6",
+ "kgreenlandic": "\u0138",
+ "khabengali": "\u0996",
+ "khacyrillic": "\u0445",
+ "khadeva": "\u0916",
+ "khagujarati": "\u0a96",
+ "khagurmukhi": "\u0a16",
+ "khaharabic": "\u062e",
+ "khahfinalarabic": "\ufea6",
+ "khahinitialarabic": "\ufea7",
+ "khahmedialarabic": "\ufea8",
+ "kheicoptic": "\u03e7",
+ "khhadeva": "\u0959",
+ "khhagurmukhi": "\u0a59",
+ "khieukhacirclekorean": "\u3278",
+ "khieukhaparenkorean": "\u3218",
+ "khieukhcirclekorean": "\u326a",
+ "khieukhkorean": "\u314b",
+ "khieukhparenkorean": "\u320a",
+ "khokhaithai": "\u0e02",
+ "khokhonthai": "\u0e05",
+ "khokhuatthai": "\u0e03",
+ "khokhwaithai": "\u0e04",
+ "khomutthai": "\u0e5b",
+ "khook": "\u0199",
+ "khorakhangthai": "\u0e06",
+ "khzsquare": "\u3391",
+ "kihiragana": "\u304d",
+ "kikatakana": "\u30ad",
+ "kikatakanahalfwidth": "\uff77",
+ "kiroguramusquare": "\u3315",
+ "kiromeetorusquare": "\u3316",
+ "kirosquare": "\u3314",
+ "kiyeokacirclekorean": "\u326e",
+ "kiyeokaparenkorean": "\u320e",
+ "kiyeokcirclekorean": "\u3260",
+ "kiyeokkorean": "\u3131",
+ "kiyeokparenkorean": "\u3200",
+ "kiyeoksioskorean": "\u3133",
+ "kjecyrillic": "\u045c",
+ "klinebelow": "\u1e35",
+ "klsquare": "\u3398",
+ "kmcubedsquare": "\u33a6",
+ "kmonospace": "\uff4b",
+ "kmsquaredsquare": "\u33a2",
+ "kohiragana": "\u3053",
+ "kohmsquare": "\u33c0",
+ "kokaithai": "\u0e01",
+ "kokatakana": "\u30b3",
+ "kokatakanahalfwidth": "\uff7a",
+ "kooposquare": "\u331e",
+ "koppacyrillic": "\u0481",
+ "koreanstandardsymbol": "\u327f",
+ "koroniscmb": "\u0343",
+ "kparen": "\u24a6",
+ "kpasquare": "\u33aa",
+ "ksicyrillic": "\u046f",
+ "ktsquare": "\u33cf",
+ "kturned": "\u029e",
+ "kuhiragana": "\u304f",
+ "kukatakana": "\u30af",
+ "kukatakanahalfwidth": "\uff78",
+ "kvsquare": "\u33b8",
+ "kwsquare": "\u33be",
+ "l": "\u006c",
+ "labengali": "\u09b2",
+ "lacute": "\u013a",
+ "ladeva": "\u0932",
+ "lagujarati": "\u0ab2",
+ "lagurmukhi": "\u0a32",
+ "lakkhangyaothai": "\u0e45",
+ "lamaleffinalarabic": "\ufefc",
+ "lamalefhamzaabovefinalarabic": "\ufef8",
+ "lamalefhamzaaboveisolatedarabic": "\ufef7",
+ "lamalefhamzabelowfinalarabic": "\ufefa",
+ "lamalefhamzabelowisolatedarabic": "\ufef9",
+ "lamalefisolatedarabic": "\ufefb",
+ "lamalefmaddaabovefinalarabic": "\ufef6",
+ "lamalefmaddaaboveisolatedarabic": "\ufef5",
+ "lamarabic": "\u0644",
+ "lambda": "\u03bb",
+ "lambdastroke": "\u019b",
+ "lamed": "\u05dc",
+ "lameddagesh": "\ufb3c",
+ "lameddageshhebrew": "\ufb3c",
+ "lamedhebrew": "\u05dc",
+ "lamedholam": "\u05dc\u05b9",
+ "lamedholamdagesh": "\u05dc\u05b9\u05bc",
+ "lamedholamdageshhebrew": "\u05dc\u05b9\u05bc",
+ "lamedholamhebrew": "\u05dc\u05b9",
+ "lamfinalarabic": "\ufede",
+ "lamhahinitialarabic": "\ufcca",
+ "laminitialarabic": "\ufedf",
+ "lamjeeminitialarabic": "\ufcc9",
+ "lamkhahinitialarabic": "\ufccb",
+ "lamlamhehisolatedarabic": "\ufdf2",
+ "lammedialarabic": "\ufee0",
+ "lammeemhahinitialarabic": "\ufd88",
+ "lammeeminitialarabic": "\ufccc",
+ "lammeemjeeminitialarabic": "\ufedf\ufee4\ufea0",
+ "lammeemkhahinitialarabic": "\ufedf\ufee4\ufea8",
+ "largecircle": "\u25ef",
+ "lbar": "\u019a",
+ "lbelt": "\u026c",
+ "lbopomofo": "\u310c",
+ "lcaron": "\u013e",
+ "lcedilla": "\u013c",
+ "lcircle": "\u24db",
+ "lcircumflexbelow": "\u1e3d",
+ "lcommaaccent": "\u013c",
+ "ldot": "\u0140",
+ "ldotaccent": "\u0140",
+ "ldotbelow": "\u1e37",
+ "ldotbelowmacron": "\u1e39",
+ "leftangleabovecmb": "\u031a",
+ "lefttackbelowcmb": "\u0318",
+ "less": "\u003c",
+ "lessequal": "\u2264",
+ "lessequalorgreater": "\u22da",
+ "lessmonospace": "\uff1c",
+ "lessorequivalent": "\u2272",
+ "lessorgreater": "\u2276",
+ "lessoverequal": "\u2266",
+ "lesssmall": "\ufe64",
+ "lezh": "\u026e",
+ "lfblock": "\u258c",
+ "lhookretroflex": "\u026d",
+ "lira": "\u20a4",
+ "liwnarmenian": "\u056c",
+ "lj": "\u01c9",
+ "ljecyrillic": "\u0459",
+ "ll": "\uf6c0",
+ "lladeva": "\u0933",
+ "llagujarati": "\u0ab3",
+ "llinebelow": "\u1e3b",
+ "llladeva": "\u0934",
+ "llvocalicbengali": "\u09e1",
+ "llvocalicdeva": "\u0961",
+ "llvocalicvowelsignbengali": "\u09e3",
+ "llvocalicvowelsigndeva": "\u0963",
+ "lmiddletilde": "\u026b",
+ "lmonospace": "\uff4c",
+ "lmsquare": "\u33d0",
+ "lochulathai": "\u0e2c",
+ "logicaland": "\u2227",
+ "logicalnot": "\u00ac",
+ "logicalnotreversed": "\u2310",
+ "logicalor": "\u2228",
+ "lolingthai": "\u0e25",
+ "longs": "\u017f",
+ "lowlinecenterline": "\ufe4e",
+ "lowlinecmb": "\u0332",
+ "lowlinedashed": "\ufe4d",
+ "lozenge": "\u25ca",
+ "lparen": "\u24a7",
+ "lslash": "\u0142",
+ "lsquare": "\u2113",
+ "lsuperior": "\uf6ee",
+ "ltshade": "\u2591",
+ "luthai": "\u0e26",
+ "lvocalicbengali": "\u098c",
+ "lvocalicdeva": "\u090c",
+ "lvocalicvowelsignbengali": "\u09e2",
+ "lvocalicvowelsigndeva": "\u0962",
+ "lxsquare": "\u33d3",
+ "m": "\u006d",
+ "mabengali": "\u09ae",
+ "macron": "\u00af",
+ "macronbelowcmb": "\u0331",
+ "macroncmb": "\u0304",
+ "macronlowmod": "\u02cd",
+ "macronmonospace": "\uffe3",
+ "macute": "\u1e3f",
+ "madeva": "\u092e",
+ "magujarati": "\u0aae",
+ "magurmukhi": "\u0a2e",
+ "mahapakhhebrew": "\u05a4",
+ "mahapakhlefthebrew": "\u05a4",
+ "mahiragana": "\u307e",
+ "maichattawalowleftthai": "\uf895",
+ "maichattawalowrightthai": "\uf894",
+ "maichattawathai": "\u0e4b",
+ "maichattawaupperleftthai": "\uf893",
+ "maieklowleftthai": "\uf88c",
+ "maieklowrightthai": "\uf88b",
+ "maiekthai": "\u0e48",
+ "maiekupperleftthai": "\uf88a",
+ "maihanakatleftthai": "\uf884",
+ "maihanakatthai": "\u0e31",
+ "maitaikhuleftthai": "\uf889",
+ "maitaikhuthai": "\u0e47",
+ "maitholowleftthai": "\uf88f",
+ "maitholowrightthai": "\uf88e",
+ "maithothai": "\u0e49",
+ "maithoupperleftthai": "\uf88d",
+ "maitrilowleftthai": "\uf892",
+ "maitrilowrightthai": "\uf891",
+ "maitrithai": "\u0e4a",
+ "maitriupperleftthai": "\uf890",
+ "maiyamokthai": "\u0e46",
+ "makatakana": "\u30de",
+ "makatakanahalfwidth": "\uff8f",
+ "male": "\u2642",
+ "mansyonsquare": "\u3347",
+ "maqafhebrew": "\u05be",
+ "mars": "\u2642",
+ "masoracirclehebrew": "\u05af",
+ "masquare": "\u3383",
+ "mbopomofo": "\u3107",
+ "mbsquare": "\u33d4",
+ "mcircle": "\u24dc",
+ "mcubedsquare": "\u33a5",
+ "mdotaccent": "\u1e41",
+ "mdotbelow": "\u1e43",
+ "meemarabic": "\u0645",
+ "meemfinalarabic": "\ufee2",
+ "meeminitialarabic": "\ufee3",
+ "meemmedialarabic": "\ufee4",
+ "meemmeeminitialarabic": "\ufcd1",
+ "meemmeemisolatedarabic": "\ufc48",
+ "meetorusquare": "\u334d",
+ "mehiragana": "\u3081",
+ "meizierasquare": "\u337e",
+ "mekatakana": "\u30e1",
+ "mekatakanahalfwidth": "\uff92",
+ "mem": "\u05de",
+ "memdagesh": "\ufb3e",
+ "memdageshhebrew": "\ufb3e",
+ "memhebrew": "\u05de",
+ "menarmenian": "\u0574",
+ "merkhahebrew": "\u05a5",
+ "merkhakefulahebrew": "\u05a6",
+ "merkhakefulalefthebrew": "\u05a6",
+ "merkhalefthebrew": "\u05a5",
+ "mhook": "\u0271",
+ "mhzsquare": "\u3392",
+ "middledotkatakanahalfwidth": "\uff65",
+ "middot": "\u00b7",
+ "mieumacirclekorean": "\u3272",
+ "mieumaparenkorean": "\u3212",
+ "mieumcirclekorean": "\u3264",
+ "mieumkorean": "\u3141",
+ "mieumpansioskorean": "\u3170",
+ "mieumparenkorean": "\u3204",
+ "mieumpieupkorean": "\u316e",
+ "mieumsioskorean": "\u316f",
+ "mihiragana": "\u307f",
+ "mikatakana": "\u30df",
+ "mikatakanahalfwidth": "\uff90",
+ "minus": "\u2212",
+ "minusbelowcmb": "\u0320",
+ "minuscircle": "\u2296",
+ "minusmod": "\u02d7",
+ "minusplus": "\u2213",
+ "minute": "\u2032",
+ "miribaarusquare": "\u334a",
+ "mirisquare": "\u3349",
+ "mlonglegturned": "\u0270",
+ "mlsquare": "\u3396",
+ "mmcubedsquare": "\u33a3",
+ "mmonospace": "\uff4d",
+ "mmsquaredsquare": "\u339f",
+ "mohiragana": "\u3082",
+ "mohmsquare": "\u33c1",
+ "mokatakana": "\u30e2",
+ "mokatakanahalfwidth": "\uff93",
+ "molsquare": "\u33d6",
+ "momathai": "\u0e21",
+ "moverssquare": "\u33a7",
+ "moverssquaredsquare": "\u33a8",
+ "mparen": "\u24a8",
+ "mpasquare": "\u33ab",
+ "mssquare": "\u33b3",
+ "msuperior": "\uf6ef",
+ "mturned": "\u026f",
+ "mu": "\u00b5",
+ "mu1": "\u00b5",
+ "muasquare": "\u3382",
+ "muchgreater": "\u226b",
+ "muchless": "\u226a",
+ "mufsquare": "\u338c",
+ "mugreek": "\u03bc",
+ "mugsquare": "\u338d",
+ "muhiragana": "\u3080",
+ "mukatakana": "\u30e0",
+ "mukatakanahalfwidth": "\uff91",
+ "mulsquare": "\u3395",
+ "multiply": "\u00d7",
+ "mumsquare": "\u339b",
+ "munahhebrew": "\u05a3",
+ "munahlefthebrew": "\u05a3",
+ "musicalnote": "\u266a",
+ "musicalnotedbl": "\u266b",
+ "musicflatsign": "\u266d",
+ "musicsharpsign": "\u266f",
+ "mussquare": "\u33b2",
+ "muvsquare": "\u33b6",
+ "muwsquare": "\u33bc",
+ "mvmegasquare": "\u33b9",
+ "mvsquare": "\u33b7",
+ "mwmegasquare": "\u33bf",
+ "mwsquare": "\u33bd",
+ "n": "\u006e",
+ "nabengali": "\u09a8",
+ "nabla": "\u2207",
+ "nacute": "\u0144",
+ "nadeva": "\u0928",
+ "nagujarati": "\u0aa8",
+ "nagurmukhi": "\u0a28",
+ "nahiragana": "\u306a",
+ "nakatakana": "\u30ca",
+ "nakatakanahalfwidth": "\uff85",
+ "napostrophe": "\u0149",
+ "nasquare": "\u3381",
+ "nbopomofo": "\u310b",
+ "nbspace": "\u00a0",
+ "ncaron": "\u0148",
+ "ncedilla": "\u0146",
+ "ncircle": "\u24dd",
+ "ncircumflexbelow": "\u1e4b",
+ "ncommaaccent": "\u0146",
+ "ndotaccent": "\u1e45",
+ "ndotbelow": "\u1e47",
+ "nehiragana": "\u306d",
+ "nekatakana": "\u30cd",
+ "nekatakanahalfwidth": "\uff88",
+ "newsheqelsign": "\u20aa",
+ "nfsquare": "\u338b",
+ "ngabengali": "\u0999",
+ "ngadeva": "\u0919",
+ "ngagujarati": "\u0a99",
+ "ngagurmukhi": "\u0a19",
+ "ngonguthai": "\u0e07",
+ "nhiragana": "\u3093",
+ "nhookleft": "\u0272",
+ "nhookretroflex": "\u0273",
+ "nieunacirclekorean": "\u326f",
+ "nieunaparenkorean": "\u320f",
+ "nieuncieuckorean": "\u3135",
+ "nieuncirclekorean": "\u3261",
+ "nieunhieuhkorean": "\u3136",
+ "nieunkorean": "\u3134",
+ "nieunpansioskorean": "\u3168",
+ "nieunparenkorean": "\u3201",
+ "nieunsioskorean": "\u3167",
+ "nieuntikeutkorean": "\u3166",
+ "nihiragana": "\u306b",
+ "nikatakana": "\u30cb",
+ "nikatakanahalfwidth": "\uff86",
+ "nikhahitleftthai": "\uf899",
+ "nikhahitthai": "\u0e4d",
+ "nine": "\u0039",
+ "ninearabic": "\u0669",
+ "ninebengali": "\u09ef",
+ "ninecircle": "\u2468",
+ "ninecircleinversesansserif": "\u2792",
+ "ninedeva": "\u096f",
+ "ninegujarati": "\u0aef",
+ "ninegurmukhi": "\u0a6f",
+ "ninehackarabic": "\u0669",
+ "ninehangzhou": "\u3029",
+ "nineideographicparen": "\u3228",
+ "nineinferior": "\u2089",
+ "ninemonospace": "\uff19",
+ "nineoldstyle": "\uf739",
+ "nineparen": "\u247c",
+ "nineperiod": "\u2490",
+ "ninepersian": "\u06f9",
+ "nineroman": "\u2178",
+ "ninesuperior": "\u2079",
+ "nineteencircle": "\u2472",
+ "nineteenparen": "\u2486",
+ "nineteenperiod": "\u249a",
+ "ninethai": "\u0e59",
+ "nj": "\u01cc",
+ "njecyrillic": "\u045a",
+ "nkatakana": "\u30f3",
+ "nkatakanahalfwidth": "\uff9d",
+ "nlegrightlong": "\u019e",
+ "nlinebelow": "\u1e49",
+ "nmonospace": "\uff4e",
+ "nmsquare": "\u339a",
+ "nnabengali": "\u09a3",
+ "nnadeva": "\u0923",
+ "nnagujarati": "\u0aa3",
+ "nnagurmukhi": "\u0a23",
+ "nnnadeva": "\u0929",
+ "nohiragana": "\u306e",
+ "nokatakana": "\u30ce",
+ "nokatakanahalfwidth": "\uff89",
+ "nonbreakingspace": "\u00a0",
+ "nonenthai": "\u0e13",
+ "nonuthai": "\u0e19",
+ "noonarabic": "\u0646",
+ "noonfinalarabic": "\ufee6",
+ "noonghunnaarabic": "\u06ba",
+ "noonghunnafinalarabic": "\ufb9f",
+ "noonhehinitialarabic": "\ufee7\ufeec",
+ "nooninitialarabic": "\ufee7",
+ "noonjeeminitialarabic": "\ufcd2",
+ "noonjeemisolatedarabic": "\ufc4b",
+ "noonmedialarabic": "\ufee8",
+ "noonmeeminitialarabic": "\ufcd5",
+ "noonmeemisolatedarabic": "\ufc4e",
+ "noonnoonfinalarabic": "\ufc8d",
+ "notcontains": "\u220c",
+ "notelement": "\u2209",
+ "notelementof": "\u2209",
+ "notequal": "\u2260",
+ "notgreater": "\u226f",
+ "notgreaternorequal": "\u2271",
+ "notgreaternorless": "\u2279",
+ "notidentical": "\u2262",
+ "notless": "\u226e",
+ "notlessnorequal": "\u2270",
+ "notparallel": "\u2226",
+ "notprecedes": "\u2280",
+ "notsubset": "\u2284",
+ "notsucceeds": "\u2281",
+ "notsuperset": "\u2285",
+ "nowarmenian": "\u0576",
+ "nparen": "\u24a9",
+ "nssquare": "\u33b1",
+ "nsuperior": "\u207f",
+ "ntilde": "\u00f1",
+ "nu": "\u03bd",
+ "nuhiragana": "\u306c",
+ "nukatakana": "\u30cc",
+ "nukatakanahalfwidth": "\uff87",
+ "nuktabengali": "\u09bc",
+ "nuktadeva": "\u093c",
+ "nuktagujarati": "\u0abc",
+ "nuktagurmukhi": "\u0a3c",
+ "numbersign": "\u0023",
+ "numbersignmonospace": "\uff03",
+ "numbersignsmall": "\ufe5f",
+ "numeralsigngreek": "\u0374",
+ "numeralsignlowergreek": "\u0375",
+ "numero": "\u2116",
+ "nun": "\u05e0",
+ "nundagesh": "\ufb40",
+ "nundageshhebrew": "\ufb40",
+ "nunhebrew": "\u05e0",
+ "nvsquare": "\u33b5",
+ "nwsquare": "\u33bb",
+ "nyabengali": "\u099e",
+ "nyadeva": "\u091e",
+ "nyagujarati": "\u0a9e",
+ "nyagurmukhi": "\u0a1e",
+ "o": "\u006f",
+ "oacute": "\u00f3",
+ "oangthai": "\u0e2d",
+ "obarred": "\u0275",
+ "obarredcyrillic": "\u04e9",
+ "obarreddieresiscyrillic": "\u04eb",
+ "obengali": "\u0993",
+ "obopomofo": "\u311b",
+ "obreve": "\u014f",
+ "ocandradeva": "\u0911",
+ "ocandragujarati": "\u0a91",
+ "ocandravowelsigndeva": "\u0949",
+ "ocandravowelsigngujarati": "\u0ac9",
+ "ocaron": "\u01d2",
+ "ocircle": "\u24de",
+ "ocircumflex": "\u00f4",
+ "ocircumflexacute": "\u1ed1",
+ "ocircumflexdotbelow": "\u1ed9",
+ "ocircumflexgrave": "\u1ed3",
+ "ocircumflexhookabove": "\u1ed5",
+ "ocircumflextilde": "\u1ed7",
+ "ocyrillic": "\u043e",
+ "odblacute": "\u0151",
+ "odblgrave": "\u020d",
+ "odeva": "\u0913",
+ "odieresis": "\u00f6",
+ "odieresiscyrillic": "\u04e7",
+ "odotbelow": "\u1ecd",
+ "oe": "\u0153",
+ "oekorean": "\u315a",
+ "ogonek": "\u02db",
+ "ogonekcmb": "\u0328",
+ "ograve": "\u00f2",
+ "ogujarati": "\u0a93",
+ "oharmenian": "\u0585",
+ "ohiragana": "\u304a",
+ "ohookabove": "\u1ecf",
+ "ohorn": "\u01a1",
+ "ohornacute": "\u1edb",
+ "ohorndotbelow": "\u1ee3",
+ "ohorngrave": "\u1edd",
+ "ohornhookabove": "\u1edf",
+ "ohorntilde": "\u1ee1",
+ "ohungarumlaut": "\u0151",
+ "oi": "\u01a3",
+ "oinvertedbreve": "\u020f",
+ "okatakana": "\u30aa",
+ "okatakanahalfwidth": "\uff75",
+ "okorean": "\u3157",
+ "olehebrew": "\u05ab",
+ "omacron": "\u014d",
+ "omacronacute": "\u1e53",
+ "omacrongrave": "\u1e51",
+ "omdeva": "\u0950",
+ "omega": "\u03c9",
+ "omega1": "\u03d6",
+ "omegacyrillic": "\u0461",
+ "omegalatinclosed": "\u0277",
+ "omegaroundcyrillic": "\u047b",
+ "omegatitlocyrillic": "\u047d",
+ "omegatonos": "\u03ce",
+ "omgujarati": "\u0ad0",
+ "omicron": "\u03bf",
+ "omicrontonos": "\u03cc",
+ "omonospace": "\uff4f",
+ "one": "\u0031",
+ "onearabic": "\u0661",
+ "onebengali": "\u09e7",
+ "onecircle": "\u2460",
+ "onecircleinversesansserif": "\u278a",
+ "onedeva": "\u0967",
+ "onedotenleader": "\u2024",
+ "oneeighth": "\u215b",
+ "onefitted": "\uf6dc",
+ "onegujarati": "\u0ae7",
+ "onegurmukhi": "\u0a67",
+ "onehackarabic": "\u0661",
+ "onehalf": "\u00bd",
+ "onehangzhou": "\u3021",
+ "oneideographicparen": "\u3220",
+ "oneinferior": "\u2081",
+ "onemonospace": "\uff11",
+ "onenumeratorbengali": "\u09f4",
+ "oneoldstyle": "\uf731",
+ "oneparen": "\u2474",
+ "oneperiod": "\u2488",
+ "onepersian": "\u06f1",
+ "onequarter": "\u00bc",
+ "oneroman": "\u2170",
+ "onesuperior": "\u00b9",
+ "onethai": "\u0e51",
+ "onethird": "\u2153",
+ "oogonek": "\u01eb",
+ "oogonekmacron": "\u01ed",
+ "oogurmukhi": "\u0a13",
+ "oomatragurmukhi": "\u0a4b",
+ "oopen": "\u0254",
+ "oparen": "\u24aa",
+ "openbullet": "\u25e6",
+ "option": "\u2325",
+ "ordfeminine": "\u00aa",
+ "ordmasculine": "\u00ba",
+ "orthogonal": "\u221f",
+ "oshortdeva": "\u0912",
+ "oshortvowelsigndeva": "\u094a",
+ "oslash": "\u00f8",
+ "oslashacute": "\u01ff",
+ "osmallhiragana": "\u3049",
+ "osmallkatakana": "\u30a9",
+ "osmallkatakanahalfwidth": "\uff6b",
+ "ostrokeacute": "\u01ff",
+ "osuperior": "\uf6f0",
+ "otcyrillic": "\u047f",
+ "otilde": "\u00f5",
+ "otildeacute": "\u1e4d",
+ "otildedieresis": "\u1e4f",
+ "oubopomofo": "\u3121",
+ "overline": "\u203e",
+ "overlinecenterline": "\ufe4a",
+ "overlinecmb": "\u0305",
+ "overlinedashed": "\ufe49",
+ "overlinedblwavy": "\ufe4c",
+ "overlinewavy": "\ufe4b",
+ "overscore": "\u00af",
+ "ovowelsignbengali": "\u09cb",
+ "ovowelsigndeva": "\u094b",
+ "ovowelsigngujarati": "\u0acb",
+ "p": "\u0070",
+ "paampssquare": "\u3380",
+ "paasentosquare": "\u332b",
+ "pabengali": "\u09aa",
+ "pacute": "\u1e55",
+ "padeva": "\u092a",
+ "pagedown": "\u21df",
+ "pageup": "\u21de",
+ "pagujarati": "\u0aaa",
+ "pagurmukhi": "\u0a2a",
+ "pahiragana": "\u3071",
+ "paiyannoithai": "\u0e2f",
+ "pakatakana": "\u30d1",
+ "palatalizationcyrilliccmb": "\u0484",
+ "palochkacyrillic": "\u04c0",
+ "pansioskorean": "\u317f",
+ "paragraph": "\u00b6",
+ "parallel": "\u2225",
+ "parenleft": "\u0028",
+ "parenleftaltonearabic": "\ufd3e",
+ "parenleftbt": "\uf8ed",
+ "parenleftex": "\uf8ec",
+ "parenleftinferior": "\u208d",
+ "parenleftmonospace": "\uff08",
+ "parenleftsmall": "\ufe59",
+ "parenleftsuperior": "\u207d",
+ "parenlefttp": "\uf8eb",
+ "parenleftvertical": "\ufe35",
+ "parenright": "\u0029",
+ "parenrightaltonearabic": "\ufd3f",
+ "parenrightbt": "\uf8f8",
+ "parenrightex": "\uf8f7",
+ "parenrightinferior": "\u208e",
+ "parenrightmonospace": "\uff09",
+ "parenrightsmall": "\ufe5a",
+ "parenrightsuperior": "\u207e",
+ "parenrighttp": "\uf8f6",
+ "parenrightvertical": "\ufe36",
+ "partialdiff": "\u2202",
+ "paseqhebrew": "\u05c0",
+ "pashtahebrew": "\u0599",
+ "pasquare": "\u33a9",
+ "patah": "\u05b7",
+ "patah11": "\u05b7",
+ "patah1d": "\u05b7",
+ "patah2a": "\u05b7",
+ "patahhebrew": "\u05b7",
+ "patahnarrowhebrew": "\u05b7",
+ "patahquarterhebrew": "\u05b7",
+ "patahwidehebrew": "\u05b7",
+ "pazerhebrew": "\u05a1",
+ "pbopomofo": "\u3106",
+ "pcircle": "\u24df",
+ "pdotaccent": "\u1e57",
+ "pe": "\u05e4",
+ "pecyrillic": "\u043f",
+ "pedagesh": "\ufb44",
+ "pedageshhebrew": "\ufb44",
+ "peezisquare": "\u333b",
+ "pefinaldageshhebrew": "\ufb43",
+ "peharabic": "\u067e",
+ "peharmenian": "\u057a",
+ "pehebrew": "\u05e4",
+ "pehfinalarabic": "\ufb57",
+ "pehinitialarabic": "\ufb58",
+ "pehiragana": "\u307a",
+ "pehmedialarabic": "\ufb59",
+ "pekatakana": "\u30da",
+ "pemiddlehookcyrillic": "\u04a7",
+ "perafehebrew": "\ufb4e",
+ "percent": "\u0025",
+ "percentarabic": "\u066a",
+ "percentmonospace": "\uff05",
+ "percentsmall": "\ufe6a",
+ "period": "\u002e",
+ "periodarmenian": "\u0589",
+ "periodcentered": "\u00b7",
+ "periodhalfwidth": "\uff61",
+ "periodinferior": "\uf6e7",
+ "periodmonospace": "\uff0e",
+ "periodsmall": "\ufe52",
+ "periodsuperior": "\uf6e8",
+ "perispomenigreekcmb": "\u0342",
+ "perpendicular": "\u22a5",
+ "perthousand": "\u2030",
+ "peseta": "\u20a7",
+ "pfsquare": "\u338a",
+ "phabengali": "\u09ab",
+ "phadeva": "\u092b",
+ "phagujarati": "\u0aab",
+ "phagurmukhi": "\u0a2b",
+ "phi": "\u03c6",
+ "phi1": "\u03d5",
+ "phieuphacirclekorean": "\u327a",
+ "phieuphaparenkorean": "\u321a",
+ "phieuphcirclekorean": "\u326c",
+ "phieuphkorean": "\u314d",
+ "phieuphparenkorean": "\u320c",
+ "philatin": "\u0278",
+ "phinthuthai": "\u0e3a",
+ "phisymbolgreek": "\u03d5",
+ "phook": "\u01a5",
+ "phophanthai": "\u0e1e",
+ "phophungthai": "\u0e1c",
+ "phosamphaothai": "\u0e20",
+ "pi": "\u03c0",
+ "pieupacirclekorean": "\u3273",
+ "pieupaparenkorean": "\u3213",
+ "pieupcieuckorean": "\u3176",
+ "pieupcirclekorean": "\u3265",
+ "pieupkiyeokkorean": "\u3172",
+ "pieupkorean": "\u3142",
+ "pieupparenkorean": "\u3205",
+ "pieupsioskiyeokkorean": "\u3174",
+ "pieupsioskorean": "\u3144",
+ "pieupsiostikeutkorean": "\u3175",
+ "pieupthieuthkorean": "\u3177",
+ "pieuptikeutkorean": "\u3173",
+ "pihiragana": "\u3074",
+ "pikatakana": "\u30d4",
+ "pisymbolgreek": "\u03d6",
+ "piwrarmenian": "\u0583",
+ "plus": "\u002b",
+ "plusbelowcmb": "\u031f",
+ "pluscircle": "\u2295",
+ "plusminus": "\u00b1",
+ "plusmod": "\u02d6",
+ "plusmonospace": "\uff0b",
+ "plussmall": "\ufe62",
+ "plussuperior": "\u207a",
+ "pmonospace": "\uff50",
+ "pmsquare": "\u33d8",
+ "pohiragana": "\u307d",
+ "pointingindexdownwhite": "\u261f",
+ "pointingindexleftwhite": "\u261c",
+ "pointingindexrightwhite": "\u261e",
+ "pointingindexupwhite": "\u261d",
+ "pokatakana": "\u30dd",
+ "poplathai": "\u0e1b",
+ "postalmark": "\u3012",
+ "postalmarkface": "\u3020",
+ "pparen": "\u24ab",
+ "precedes": "\u227a",
+ "prescription": "\u211e",
+ "primemod": "\u02b9",
+ "primereversed": "\u2035",
+ "product": "\u220f",
+ "projective": "\u2305",
+ "prolongedkana": "\u30fc",
+ "propellor": "\u2318",
+ "propersubset": "\u2282",
+ "propersuperset": "\u2283",
+ "proportion": "\u2237",
+ "proportional": "\u221d",
+ "psi": "\u03c8",
+ "psicyrillic": "\u0471",
+ "psilipneumatacyrilliccmb": "\u0486",
+ "pssquare": "\u33b0",
+ "puhiragana": "\u3077",
+ "pukatakana": "\u30d7",
+ "pvsquare": "\u33b4",
+ "pwsquare": "\u33ba",
+ "q": "\u0071",
+ "qadeva": "\u0958",
+ "qadmahebrew": "\u05a8",
+ "qafarabic": "\u0642",
+ "qaffinalarabic": "\ufed6",
+ "qafinitialarabic": "\ufed7",
+ "qafmedialarabic": "\ufed8",
+ "qamats": "\u05b8",
+ "qamats10": "\u05b8",
+ "qamats1a": "\u05b8",
+ "qamats1c": "\u05b8",
+ "qamats27": "\u05b8",
+ "qamats29": "\u05b8",
+ "qamats33": "\u05b8",
+ "qamatsde": "\u05b8",
+ "qamatshebrew": "\u05b8",
+ "qamatsnarrowhebrew": "\u05b8",
+ "qamatsqatanhebrew": "\u05b8",
+ "qamatsqatannarrowhebrew": "\u05b8",
+ "qamatsqatanquarterhebrew": "\u05b8",
+ "qamatsqatanwidehebrew": "\u05b8",
+ "qamatsquarterhebrew": "\u05b8",
+ "qamatswidehebrew": "\u05b8",
+ "qarneyparahebrew": "\u059f",
+ "qbopomofo": "\u3111",
+ "qcircle": "\u24e0",
+ "qhook": "\u02a0",
+ "qmonospace": "\uff51",
+ "qof": "\u05e7",
+ "qofdagesh": "\ufb47",
+ "qofdageshhebrew": "\ufb47",
+ "qofhatafpatah": "\u05e7\u05b2",
+ "qofhatafpatahhebrew": "\u05e7\u05b2",
+ "qofhatafsegol": "\u05e7\u05b1",
+ "qofhatafsegolhebrew": "\u05e7\u05b1",
+ "qofhebrew": "\u05e7",
+ "qofhiriq": "\u05e7\u05b4",
+ "qofhiriqhebrew": "\u05e7\u05b4",
+ "qofholam": "\u05e7\u05b9",
+ "qofholamhebrew": "\u05e7\u05b9",
+ "qofpatah": "\u05e7\u05b7",
+ "qofpatahhebrew": "\u05e7\u05b7",
+ "qofqamats": "\u05e7\u05b8",
+ "qofqamatshebrew": "\u05e7\u05b8",
+ "qofqubuts": "\u05e7\u05bb",
+ "qofqubutshebrew": "\u05e7\u05bb",
+ "qofsegol": "\u05e7\u05b6",
+ "qofsegolhebrew": "\u05e7\u05b6",
+ "qofsheva": "\u05e7\u05b0",
+ "qofshevahebrew": "\u05e7\u05b0",
+ "qoftsere": "\u05e7\u05b5",
+ "qoftserehebrew": "\u05e7\u05b5",
+ "qparen": "\u24ac",
+ "quarternote": "\u2669",
+ "qubuts": "\u05bb",
+ "qubuts18": "\u05bb",
+ "qubuts25": "\u05bb",
+ "qubuts31": "\u05bb",
+ "qubutshebrew": "\u05bb",
+ "qubutsnarrowhebrew": "\u05bb",
+ "qubutsquarterhebrew": "\u05bb",
+ "qubutswidehebrew": "\u05bb",
+ "question": "\u003f",
+ "questionarabic": "\u061f",
+ "questionarmenian": "\u055e",
+ "questiondown": "\u00bf",
+ "questiondownsmall": "\uf7bf",
+ "questiongreek": "\u037e",
+ "questionmonospace": "\uff1f",
+ "questionsmall": "\uf73f",
+ "quotedbl": "\u0022",
+ "quotedblbase": "\u201e",
+ "quotedblleft": "\u201c",
+ "quotedblmonospace": "\uff02",
+ "quotedblprime": "\u301e",
+ "quotedblprimereversed": "\u301d",
+ "quotedblright": "\u201d",
+ "quoteleft": "\u2018",
+ "quoteleftreversed": "\u201b",
+ "quotereversed": "\u201b",
+ "quoteright": "\u2019",
+ "quoterightn": "\u0149",
+ "quotesinglbase": "\u201a",
+ "quotesingle": "\u0027",
+ "quotesinglemonospace": "\uff07",
+ "r": "\u0072",
+ "raarmenian": "\u057c",
+ "rabengali": "\u09b0",
+ "racute": "\u0155",
+ "radeva": "\u0930",
+ "radical": "\u221a",
+ "radicalex": "\uf8e5",
+ "radoverssquare": "\u33ae",
+ "radoverssquaredsquare": "\u33af",
+ "radsquare": "\u33ad",
+ "rafe": "\u05bf",
+ "rafehebrew": "\u05bf",
+ "ragujarati": "\u0ab0",
+ "ragurmukhi": "\u0a30",
+ "rahiragana": "\u3089",
+ "rakatakana": "\u30e9",
+ "rakatakanahalfwidth": "\uff97",
+ "ralowerdiagonalbengali": "\u09f1",
+ "ramiddlediagonalbengali": "\u09f0",
+ "ramshorn": "\u0264",
+ "ratio": "\u2236",
+ "rbopomofo": "\u3116",
+ "rcaron": "\u0159",
+ "rcedilla": "\u0157",
+ "rcircle": "\u24e1",
+ "rcommaaccent": "\u0157",
+ "rdblgrave": "\u0211",
+ "rdotaccent": "\u1e59",
+ "rdotbelow": "\u1e5b",
+ "rdotbelowmacron": "\u1e5d",
+ "referencemark": "\u203b",
+ "reflexsubset": "\u2286",
+ "reflexsuperset": "\u2287",
+ "registered": "\u00ae",
+ "registersans": "\uf8e8",
+ "registerserif": "\uf6da",
+ "reharabic": "\u0631",
+ "reharmenian": "\u0580",
+ "rehfinalarabic": "\ufeae",
+ "rehiragana": "\u308c",
+ "rehyehaleflamarabic": "\u0631\ufef3\ufe8e\u0644",
+ "rekatakana": "\u30ec",
+ "rekatakanahalfwidth": "\uff9a",
+ "resh": "\u05e8",
+ "reshdageshhebrew": "\ufb48",
+ "reshhatafpatah": "\u05e8\u05b2",
+ "reshhatafpatahhebrew": "\u05e8\u05b2",
+ "reshhatafsegol": "\u05e8\u05b1",
+ "reshhatafsegolhebrew": "\u05e8\u05b1",
+ "reshhebrew": "\u05e8",
+ "reshhiriq": "\u05e8\u05b4",
+ "reshhiriqhebrew": "\u05e8\u05b4",
+ "reshholam": "\u05e8\u05b9",
+ "reshholamhebrew": "\u05e8\u05b9",
+ "reshpatah": "\u05e8\u05b7",
+ "reshpatahhebrew": "\u05e8\u05b7",
+ "reshqamats": "\u05e8\u05b8",
+ "reshqamatshebrew": "\u05e8\u05b8",
+ "reshqubuts": "\u05e8\u05bb",
+ "reshqubutshebrew": "\u05e8\u05bb",
+ "reshsegol": "\u05e8\u05b6",
+ "reshsegolhebrew": "\u05e8\u05b6",
+ "reshsheva": "\u05e8\u05b0",
+ "reshshevahebrew": "\u05e8\u05b0",
+ "reshtsere": "\u05e8\u05b5",
+ "reshtserehebrew": "\u05e8\u05b5",
+ "reversedtilde": "\u223d",
+ "reviahebrew": "\u0597",
+ "reviamugrashhebrew": "\u0597",
+ "revlogicalnot": "\u2310",
+ "rfishhook": "\u027e",
+ "rfishhookreversed": "\u027f",
+ "rhabengali": "\u09dd",
+ "rhadeva": "\u095d",
+ "rho": "\u03c1",
+ "rhook": "\u027d",
+ "rhookturned": "\u027b",
+ "rhookturnedsuperior": "\u02b5",
+ "rhosymbolgreek": "\u03f1",
+ "rhotichookmod": "\u02de",
+ "rieulacirclekorean": "\u3271",
+ "rieulaparenkorean": "\u3211",
+ "rieulcirclekorean": "\u3263",
+ "rieulhieuhkorean": "\u3140",
+ "rieulkiyeokkorean": "\u313a",
+ "rieulkiyeoksioskorean": "\u3169",
+ "rieulkorean": "\u3139",
+ "rieulmieumkorean": "\u313b",
+ "rieulpansioskorean": "\u316c",
+ "rieulparenkorean": "\u3203",
+ "rieulphieuphkorean": "\u313f",
+ "rieulpieupkorean": "\u313c",
+ "rieulpieupsioskorean": "\u316b",
+ "rieulsioskorean": "\u313d",
+ "rieulthieuthkorean": "\u313e",
+ "rieultikeutkorean": "\u316a",
+ "rieulyeorinhieuhkorean": "\u316d",
+ "rightangle": "\u221f",
+ "righttackbelowcmb": "\u0319",
+ "righttriangle": "\u22bf",
+ "rihiragana": "\u308a",
+ "rikatakana": "\u30ea",
+ "rikatakanahalfwidth": "\uff98",
+ "ring": "\u02da",
+ "ringbelowcmb": "\u0325",
+ "ringcmb": "\u030a",
+ "ringhalfleft": "\u02bf",
+ "ringhalfleftarmenian": "\u0559",
+ "ringhalfleftbelowcmb": "\u031c",
+ "ringhalfleftcentered": "\u02d3",
+ "ringhalfright": "\u02be",
+ "ringhalfrightbelowcmb": "\u0339",
+ "ringhalfrightcentered": "\u02d2",
+ "rinvertedbreve": "\u0213",
+ "rittorusquare": "\u3351",
+ "rlinebelow": "\u1e5f",
+ "rlongleg": "\u027c",
+ "rlonglegturned": "\u027a",
+ "rmonospace": "\uff52",
+ "rohiragana": "\u308d",
+ "rokatakana": "\u30ed",
+ "rokatakanahalfwidth": "\uff9b",
+ "roruathai": "\u0e23",
+ "rparen": "\u24ad",
+ "rrabengali": "\u09dc",
+ "rradeva": "\u0931",
+ "rragurmukhi": "\u0a5c",
+ "rreharabic": "\u0691",
+ "rrehfinalarabic": "\ufb8d",
+ "rrvocalicbengali": "\u09e0",
+ "rrvocalicdeva": "\u0960",
+ "rrvocalicgujarati": "\u0ae0",
+ "rrvocalicvowelsignbengali": "\u09c4",
+ "rrvocalicvowelsigndeva": "\u0944",
+ "rrvocalicvowelsigngujarati": "\u0ac4",
+ "rsuperior": "\uf6f1",
+ "rtblock": "\u2590",
+ "rturned": "\u0279",
+ "rturnedsuperior": "\u02b4",
+ "ruhiragana": "\u308b",
+ "rukatakana": "\u30eb",
+ "rukatakanahalfwidth": "\uff99",
+ "rupeemarkbengali": "\u09f2",
+ "rupeesignbengali": "\u09f3",
+ "rupiah": "\uf6dd",
+ "ruthai": "\u0e24",
+ "rvocalicbengali": "\u098b",
+ "rvocalicdeva": "\u090b",
+ "rvocalicgujarati": "\u0a8b",
+ "rvocalicvowelsignbengali": "\u09c3",
+ "rvocalicvowelsigndeva": "\u0943",
+ "rvocalicvowelsigngujarati": "\u0ac3",
+ "s": "\u0073",
+ "sabengali": "\u09b8",
+ "sacute": "\u015b",
+ "sacutedotaccent": "\u1e65",
+ "sadarabic": "\u0635",
+ "sadeva": "\u0938",
+ "sadfinalarabic": "\ufeba",
+ "sadinitialarabic": "\ufebb",
+ "sadmedialarabic": "\ufebc",
+ "sagujarati": "\u0ab8",
+ "sagurmukhi": "\u0a38",
+ "sahiragana": "\u3055",
+ "sakatakana": "\u30b5",
+ "sakatakanahalfwidth": "\uff7b",
+ "sallallahoualayhewasallamarabic": "\ufdfa",
+ "samekh": "\u05e1",
+ "samekhdagesh": "\ufb41",
+ "samekhdageshhebrew": "\ufb41",
+ "samekhhebrew": "\u05e1",
+ "saraaathai": "\u0e32",
+ "saraaethai": "\u0e41",
+ "saraaimaimalaithai": "\u0e44",
+ "saraaimaimuanthai": "\u0e43",
+ "saraamthai": "\u0e33",
+ "saraathai": "\u0e30",
+ "saraethai": "\u0e40",
+ "saraiileftthai": "\uf886",
+ "saraiithai": "\u0e35",
+ "saraileftthai": "\uf885",
+ "saraithai": "\u0e34",
+ "saraothai": "\u0e42",
+ "saraueeleftthai": "\uf888",
+ "saraueethai": "\u0e37",
+ "saraueleftthai": "\uf887",
+ "sarauethai": "\u0e36",
+ "sarauthai": "\u0e38",
+ "sarauuthai": "\u0e39",
+ "sbopomofo": "\u3119",
+ "scaron": "\u0161",
+ "scarondotaccent": "\u1e67",
+ "scedilla": "\u015f",
+ "schwa": "\u0259",
+ "schwacyrillic": "\u04d9",
+ "schwadieresiscyrillic": "\u04db",
+ "schwahook": "\u025a",
+ "scircle": "\u24e2",
+ "scircumflex": "\u015d",
+ "scommaaccent": "\u0219",
+ "sdotaccent": "\u1e61",
+ "sdotbelow": "\u1e63",
+ "sdotbelowdotaccent": "\u1e69",
+ "seagullbelowcmb": "\u033c",
+ "second": "\u2033",
+ "secondtonechinese": "\u02ca",
+ "section": "\u00a7",
+ "seenarabic": "\u0633",
+ "seenfinalarabic": "\ufeb2",
+ "seeninitialarabic": "\ufeb3",
+ "seenmedialarabic": "\ufeb4",
+ "segol": "\u05b6",
+ "segol13": "\u05b6",
+ "segol1f": "\u05b6",
+ "segol2c": "\u05b6",
+ "segolhebrew": "\u05b6",
+ "segolnarrowhebrew": "\u05b6",
+ "segolquarterhebrew": "\u05b6",
+ "segoltahebrew": "\u0592",
+ "segolwidehebrew": "\u05b6",
+ "seharmenian": "\u057d",
+ "sehiragana": "\u305b",
+ "sekatakana": "\u30bb",
+ "sekatakanahalfwidth": "\uff7e",
+ "semicolon": "\u003b",
+ "semicolonarabic": "\u061b",
+ "semicolonmonospace": "\uff1b",
+ "semicolonsmall": "\ufe54",
+ "semivoicedmarkkana": "\u309c",
+ "semivoicedmarkkanahalfwidth": "\uff9f",
+ "sentisquare": "\u3322",
+ "sentosquare": "\u3323",
+ "seven": "\u0037",
+ "sevenarabic": "\u0667",
+ "sevenbengali": "\u09ed",
+ "sevencircle": "\u2466",
+ "sevencircleinversesansserif": "\u2790",
+ "sevendeva": "\u096d",
+ "seveneighths": "\u215e",
+ "sevengujarati": "\u0aed",
+ "sevengurmukhi": "\u0a6d",
+ "sevenhackarabic": "\u0667",
+ "sevenhangzhou": "\u3027",
+ "sevenideographicparen": "\u3226",
+ "seveninferior": "\u2087",
+ "sevenmonospace": "\uff17",
+ "sevenoldstyle": "\uf737",
+ "sevenparen": "\u247a",
+ "sevenperiod": "\u248e",
+ "sevenpersian": "\u06f7",
+ "sevenroman": "\u2176",
+ "sevensuperior": "\u2077",
+ "seventeencircle": "\u2470",
+ "seventeenparen": "\u2484",
+ "seventeenperiod": "\u2498",
+ "seventhai": "\u0e57",
+ "sfthyphen": "\u00ad",
+ "shaarmenian": "\u0577",
+ "shabengali": "\u09b6",
+ "shacyrillic": "\u0448",
+ "shaddaarabic": "\u0651",
+ "shaddadammaarabic": "\ufc61",
+ "shaddadammatanarabic": "\ufc5e",
+ "shaddafathaarabic": "\ufc60",
+ "shaddafathatanarabic": "\u0651\u064b",
+ "shaddakasraarabic": "\ufc62",
+ "shaddakasratanarabic": "\ufc5f",
+ "shade": "\u2592",
+ "shadedark": "\u2593",
+ "shadelight": "\u2591",
+ "shademedium": "\u2592",
+ "shadeva": "\u0936",
+ "shagujarati": "\u0ab6",
+ "shagurmukhi": "\u0a36",
+ "shalshelethebrew": "\u0593",
+ "shbopomofo": "\u3115",
+ "shchacyrillic": "\u0449",
+ "sheenarabic": "\u0634",
+ "sheenfinalarabic": "\ufeb6",
+ "sheeninitialarabic": "\ufeb7",
+ "sheenmedialarabic": "\ufeb8",
+ "sheicoptic": "\u03e3",
+ "sheqel": "\u20aa",
+ "sheqelhebrew": "\u20aa",
+ "sheva": "\u05b0",
+ "sheva115": "\u05b0",
+ "sheva15": "\u05b0",
+ "sheva22": "\u05b0",
+ "sheva2e": "\u05b0",
+ "shevahebrew": "\u05b0",
+ "shevanarrowhebrew": "\u05b0",
+ "shevaquarterhebrew": "\u05b0",
+ "shevawidehebrew": "\u05b0",
+ "shhacyrillic": "\u04bb",
+ "shimacoptic": "\u03ed",
+ "shin": "\u05e9",
+ "shindagesh": "\ufb49",
+ "shindageshhebrew": "\ufb49",
+ "shindageshshindot": "\ufb2c",
+ "shindageshshindothebrew": "\ufb2c",
+ "shindageshsindot": "\ufb2d",
+ "shindageshsindothebrew": "\ufb2d",
+ "shindothebrew": "\u05c1",
+ "shinhebrew": "\u05e9",
+ "shinshindot": "\ufb2a",
+ "shinshindothebrew": "\ufb2a",
+ "shinsindot": "\ufb2b",
+ "shinsindothebrew": "\ufb2b",
+ "shook": "\u0282",
+ "sigma": "\u03c3",
+ "sigma1": "\u03c2",
+ "sigmafinal": "\u03c2",
+ "sigmalunatesymbolgreek": "\u03f2",
+ "sihiragana": "\u3057",
+ "sikatakana": "\u30b7",
+ "sikatakanahalfwidth": "\uff7c",
+ "siluqhebrew": "\u05bd",
+ "siluqlefthebrew": "\u05bd",
+ "similar": "\u223c",
+ "sindothebrew": "\u05c2",
+ "siosacirclekorean": "\u3274",
+ "siosaparenkorean": "\u3214",
+ "sioscieuckorean": "\u317e",
+ "sioscirclekorean": "\u3266",
+ "sioskiyeokkorean": "\u317a",
+ "sioskorean": "\u3145",
+ "siosnieunkorean": "\u317b",
+ "siosparenkorean": "\u3206",
+ "siospieupkorean": "\u317d",
+ "siostikeutkorean": "\u317c",
+ "six": "\u0036",
+ "sixarabic": "\u0666",
+ "sixbengali": "\u09ec",
+ "sixcircle": "\u2465",
+ "sixcircleinversesansserif": "\u278f",
+ "sixdeva": "\u096c",
+ "sixgujarati": "\u0aec",
+ "sixgurmukhi": "\u0a6c",
+ "sixhackarabic": "\u0666",
+ "sixhangzhou": "\u3026",
+ "sixideographicparen": "\u3225",
+ "sixinferior": "\u2086",
+ "sixmonospace": "\uff16",
+ "sixoldstyle": "\uf736",
+ "sixparen": "\u2479",
+ "sixperiod": "\u248d",
+ "sixpersian": "\u06f6",
+ "sixroman": "\u2175",
+ "sixsuperior": "\u2076",
+ "sixteencircle": "\u246f",
+ "sixteencurrencydenominatorbengali": "\u09f9",
+ "sixteenparen": "\u2483",
+ "sixteenperiod": "\u2497",
+ "sixthai": "\u0e56",
+ "slash": "\u002f",
+ "slashmonospace": "\uff0f",
+ "slong": "\u017f",
+ "slongdotaccent": "\u1e9b",
+ "smileface": "\u263a",
+ "smonospace": "\uff53",
+ "sofpasuqhebrew": "\u05c3",
+ "softhyphen": "\u00ad",
+ "softsigncyrillic": "\u044c",
+ "sohiragana": "\u305d",
+ "sokatakana": "\u30bd",
+ "sokatakanahalfwidth": "\uff7f",
+ "soliduslongoverlaycmb": "\u0338",
+ "solidusshortoverlaycmb": "\u0337",
+ "sorusithai": "\u0e29",
+ "sosalathai": "\u0e28",
+ "sosothai": "\u0e0b",
+ "sosuathai": "\u0e2a",
+ "space": "\u0020",
+ "spacehackarabic": "\u0020",
+ "spade": "\u2660",
+ "spadesuitblack": "\u2660",
+ "spadesuitwhite": "\u2664",
+ "sparen": "\u24ae",
+ "squarebelowcmb": "\u033b",
+ "squarecc": "\u33c4",
+ "squarecm": "\u339d",
+ "squarediagonalcrosshatchfill": "\u25a9",
+ "squarehorizontalfill": "\u25a4",
+ "squarekg": "\u338f",
+ "squarekm": "\u339e",
+ "squarekmcapital": "\u33ce",
+ "squareln": "\u33d1",
+ "squarelog": "\u33d2",
+ "squaremg": "\u338e",
+ "squaremil": "\u33d5",
+ "squaremm": "\u339c",
+ "squaremsquared": "\u33a1",
+ "squareorthogonalcrosshatchfill": "\u25a6",
+ "squareupperlefttolowerrightfill": "\u25a7",
+ "squareupperrighttolowerleftfill": "\u25a8",
+ "squareverticalfill": "\u25a5",
+ "squarewhitewithsmallblack": "\u25a3",
+ "srsquare": "\u33db",
+ "ssabengali": "\u09b7",
+ "ssadeva": "\u0937",
+ "ssagujarati": "\u0ab7",
+ "ssangcieuckorean": "\u3149",
+ "ssanghieuhkorean": "\u3185",
+ "ssangieungkorean": "\u3180",
+ "ssangkiyeokkorean": "\u3132",
+ "ssangnieunkorean": "\u3165",
+ "ssangpieupkorean": "\u3143",
+ "ssangsioskorean": "\u3146",
+ "ssangtikeutkorean": "\u3138",
+ "ssuperior": "\uf6f2",
+ "sterling": "\u00a3",
+ "sterlingmonospace": "\uffe1",
+ "strokelongoverlaycmb": "\u0336",
+ "strokeshortoverlaycmb": "\u0335",
+ "subset": "\u2282",
+ "subsetnotequal": "\u228a",
+ "subsetorequal": "\u2286",
+ "succeeds": "\u227b",
+ "suchthat": "\u220b",
+ "suhiragana": "\u3059",
+ "sukatakana": "\u30b9",
+ "sukatakanahalfwidth": "\uff7d",
+ "sukunarabic": "\u0652",
+ "summation": "\u2211",
+ "sun": "\u263c",
+ "superset": "\u2283",
+ "supersetnotequal": "\u228b",
+ "supersetorequal": "\u2287",
+ "svsquare": "\u33dc",
+ "syouwaerasquare": "\u337c",
+ "t": "\u0074",
+ "tabengali": "\u09a4",
+ "tackdown": "\u22a4",
+ "tackleft": "\u22a3",
+ "tadeva": "\u0924",
+ "tagujarati": "\u0aa4",
+ "tagurmukhi": "\u0a24",
+ "taharabic": "\u0637",
+ "tahfinalarabic": "\ufec2",
+ "tahinitialarabic": "\ufec3",
+ "tahiragana": "\u305f",
+ "tahmedialarabic": "\ufec4",
+ "taisyouerasquare": "\u337d",
+ "takatakana": "\u30bf",
+ "takatakanahalfwidth": "\uff80",
+ "tatweelarabic": "\u0640",
+ "tau": "\u03c4",
+ "tav": "\u05ea",
+ "tavdages": "\ufb4a",
+ "tavdagesh": "\ufb4a",
+ "tavdageshhebrew": "\ufb4a",
+ "tavhebrew": "\u05ea",
+ "tbar": "\u0167",
+ "tbopomofo": "\u310a",
+ "tcaron": "\u0165",
+ "tccurl": "\u02a8",
+ "tcedilla": "\u0163",
+ "tcheharabic": "\u0686",
+ "tchehfinalarabic": "\ufb7b",
+ "tchehinitialarabic": "\ufb7c",
+ "tchehmedialarabic": "\ufb7d",
+ "tchehmeeminitialarabic": "\ufb7c\ufee4",
+ "tcircle": "\u24e3",
+ "tcircumflexbelow": "\u1e71",
+ "tcommaaccent": "\u0163",
+ "tdieresis": "\u1e97",
+ "tdotaccent": "\u1e6b",
+ "tdotbelow": "\u1e6d",
+ "tecyrillic": "\u0442",
+ "tedescendercyrillic": "\u04ad",
+ "teharabic": "\u062a",
+ "tehfinalarabic": "\ufe96",
+ "tehhahinitialarabic": "\ufca2",
+ "tehhahisolatedarabic": "\ufc0c",
+ "tehinitialarabic": "\ufe97",
+ "tehiragana": "\u3066",
+ "tehjeeminitialarabic": "\ufca1",
+ "tehjeemisolatedarabic": "\ufc0b",
+ "tehmarbutaarabic": "\u0629",
+ "tehmarbutafinalarabic": "\ufe94",
+ "tehmedialarabic": "\ufe98",
+ "tehmeeminitialarabic": "\ufca4",
+ "tehmeemisolatedarabic": "\ufc0e",
+ "tehnoonfinalarabic": "\ufc73",
+ "tekatakana": "\u30c6",
+ "tekatakanahalfwidth": "\uff83",
+ "telephone": "\u2121",
+ "telephoneblack": "\u260e",
+ "telishagedolahebrew": "\u05a0",
+ "telishaqetanahebrew": "\u05a9",
+ "tencircle": "\u2469",
+ "tenideographicparen": "\u3229",
+ "tenparen": "\u247d",
+ "tenperiod": "\u2491",
+ "tenroman": "\u2179",
+ "tesh": "\u02a7",
+ "tet": "\u05d8",
+ "tetdagesh": "\ufb38",
+ "tetdageshhebrew": "\ufb38",
+ "tethebrew": "\u05d8",
+ "tetsecyrillic": "\u04b5",
+ "tevirhebrew": "\u059b",
+ "tevirlefthebrew": "\u059b",
+ "thabengali": "\u09a5",
+ "thadeva": "\u0925",
+ "thagujarati": "\u0aa5",
+ "thagurmukhi": "\u0a25",
+ "thalarabic": "\u0630",
+ "thalfinalarabic": "\ufeac",
+ "thanthakhatlowleftthai": "\uf898",
+ "thanthakhatlowrightthai": "\uf897",
+ "thanthakhatthai": "\u0e4c",
+ "thanthakhatupperleftthai": "\uf896",
+ "theharabic": "\u062b",
+ "thehfinalarabic": "\ufe9a",
+ "thehinitialarabic": "\ufe9b",
+ "thehmedialarabic": "\ufe9c",
+ "thereexists": "\u2203",
+ "therefore": "\u2234",
+ "theta": "\u03b8",
+ "theta1": "\u03d1",
+ "thetasymbolgreek": "\u03d1",
+ "thieuthacirclekorean": "\u3279",
+ "thieuthaparenkorean": "\u3219",
+ "thieuthcirclekorean": "\u326b",
+ "thieuthkorean": "\u314c",
+ "thieuthparenkorean": "\u320b",
+ "thirteencircle": "\u246c",
+ "thirteenparen": "\u2480",
+ "thirteenperiod": "\u2494",
+ "thonangmonthothai": "\u0e11",
+ "thook": "\u01ad",
+ "thophuthaothai": "\u0e12",
+ "thorn": "\u00fe",
+ "thothahanthai": "\u0e17",
+ "thothanthai": "\u0e10",
+ "thothongthai": "\u0e18",
+ "thothungthai": "\u0e16",
+ "thousandcyrillic": "\u0482",
+ "thousandsseparatorarabic": "\u066c",
+ "thousandsseparatorpersian": "\u066c",
+ "three": "\u0033",
+ "threearabic": "\u0663",
+ "threebengali": "\u09e9",
+ "threecircle": "\u2462",
+ "threecircleinversesansserif": "\u278c",
+ "threedeva": "\u0969",
+ "threeeighths": "\u215c",
+ "threegujarati": "\u0ae9",
+ "threegurmukhi": "\u0a69",
+ "threehackarabic": "\u0663",
+ "threehangzhou": "\u3023",
+ "threeideographicparen": "\u3222",
+ "threeinferior": "\u2083",
+ "threemonospace": "\uff13",
+ "threenumeratorbengali": "\u09f6",
+ "threeoldstyle": "\uf733",
+ "threeparen": "\u2476",
+ "threeperiod": "\u248a",
+ "threepersian": "\u06f3",
+ "threequarters": "\u00be",
+ "threequartersemdash": "\uf6de",
+ "threeroman": "\u2172",
+ "threesuperior": "\u00b3",
+ "threethai": "\u0e53",
+ "thzsquare": "\u3394",
+ "tihiragana": "\u3061",
+ "tikatakana": "\u30c1",
+ "tikatakanahalfwidth": "\uff81",
+ "tikeutacirclekorean": "\u3270",
+ "tikeutaparenkorean": "\u3210",
+ "tikeutcirclekorean": "\u3262",
+ "tikeutkorean": "\u3137",
+ "tikeutparenkorean": "\u3202",
+ "tilde": "\u02dc",
+ "tildebelowcmb": "\u0330",
+ "tildecmb": "\u0303",
+ "tildecomb": "\u0303",
+ "tildedoublecmb": "\u0360",
+ "tildeoperator": "\u223c",
+ "tildeoverlaycmb": "\u0334",
+ "tildeverticalcmb": "\u033e",
+ "timescircle": "\u2297",
+ "tipehahebrew": "\u0596",
+ "tipehalefthebrew": "\u0596",
+ "tippigurmukhi": "\u0a70",
+ "titlocyrilliccmb": "\u0483",
+ "tiwnarmenian": "\u057f",
+ "tlinebelow": "\u1e6f",
+ "tmonospace": "\uff54",
+ "toarmenian": "\u0569",
+ "tohiragana": "\u3068",
+ "tokatakana": "\u30c8",
+ "tokatakanahalfwidth": "\uff84",
+ "tonebarextrahighmod": "\u02e5",
+ "tonebarextralowmod": "\u02e9",
+ "tonebarhighmod": "\u02e6",
+ "tonebarlowmod": "\u02e8",
+ "tonebarmidmod": "\u02e7",
+ "tonefive": "\u01bd",
+ "tonesix": "\u0185",
+ "tonetwo": "\u01a8",
+ "tonos": "\u0384",
+ "tonsquare": "\u3327",
+ "topatakthai": "\u0e0f",
+ "tortoiseshellbracketleft": "\u3014",
+ "tortoiseshellbracketleftsmall": "\ufe5d",
+ "tortoiseshellbracketleftvertical": "\ufe39",
+ "tortoiseshellbracketright": "\u3015",
+ "tortoiseshellbracketrightsmall": "\ufe5e",
+ "tortoiseshellbracketrightvertical": "\ufe3a",
+ "totaothai": "\u0e15",
+ "tpalatalhook": "\u01ab",
+ "tparen": "\u24af",
+ "trademark": "\u2122",
+ "trademarksans": "\uf8ea",
+ "trademarkserif": "\uf6db",
+ "tretroflexhook": "\u0288",
+ "triagdn": "\u25bc",
+ "triaglf": "\u25c4",
+ "triagrt": "\u25ba",
+ "triagup": "\u25b2",
+ "ts": "\u02a6",
+ "tsadi": "\u05e6",
+ "tsadidagesh": "\ufb46",
+ "tsadidageshhebrew": "\ufb46",
+ "tsadihebrew": "\u05e6",
+ "tsecyrillic": "\u0446",
+ "tsere": "\u05b5",
+ "tsere12": "\u05b5",
+ "tsere1e": "\u05b5",
+ "tsere2b": "\u05b5",
+ "tserehebrew": "\u05b5",
+ "tserenarrowhebrew": "\u05b5",
+ "tserequarterhebrew": "\u05b5",
+ "tserewidehebrew": "\u05b5",
+ "tshecyrillic": "\u045b",
+ "tsuperior": "\uf6f3",
+ "ttabengali": "\u099f",
+ "ttadeva": "\u091f",
+ "ttagujarati": "\u0a9f",
+ "ttagurmukhi": "\u0a1f",
+ "tteharabic": "\u0679",
+ "ttehfinalarabic": "\ufb67",
+ "ttehinitialarabic": "\ufb68",
+ "ttehmedialarabic": "\ufb69",
+ "tthabengali": "\u09a0",
+ "tthadeva": "\u0920",
+ "tthagujarati": "\u0aa0",
+ "tthagurmukhi": "\u0a20",
+ "tturned": "\u0287",
+ "tuhiragana": "\u3064",
+ "tukatakana": "\u30c4",
+ "tukatakanahalfwidth": "\uff82",
+ "tusmallhiragana": "\u3063",
+ "tusmallkatakana": "\u30c3",
+ "tusmallkatakanahalfwidth": "\uff6f",
+ "twelvecircle": "\u246b",
+ "twelveparen": "\u247f",
+ "twelveperiod": "\u2493",
+ "twelveroman": "\u217b",
+ "twentycircle": "\u2473",
+ "twentyhangzhou": "\u5344",
+ "twentyparen": "\u2487",
+ "twentyperiod": "\u249b",
+ "two": "\u0032",
+ "twoarabic": "\u0662",
+ "twobengali": "\u09e8",
+ "twocircle": "\u2461",
+ "twocircleinversesansserif": "\u278b",
+ "twodeva": "\u0968",
+ "twodotenleader": "\u2025",
+ "twodotleader": "\u2025",
+ "twodotleadervertical": "\ufe30",
+ "twogujarati": "\u0ae8",
+ "twogurmukhi": "\u0a68",
+ "twohackarabic": "\u0662",
+ "twohangzhou": "\u3022",
+ "twoideographicparen": "\u3221",
+ "twoinferior": "\u2082",
+ "twomonospace": "\uff12",
+ "twonumeratorbengali": "\u09f5",
+ "twooldstyle": "\uf732",
+ "twoparen": "\u2475",
+ "twoperiod": "\u2489",
+ "twopersian": "\u06f2",
+ "tworoman": "\u2171",
+ "twostroke": "\u01bb",
+ "twosuperior": "\u00b2",
+ "twothai": "\u0e52",
+ "twothirds": "\u2154",
+ "u": "\u0075",
+ "uacute": "\u00fa",
+ "ubar": "\u0289",
+ "ubengali": "\u0989",
+ "ubopomofo": "\u3128",
+ "ubreve": "\u016d",
+ "ucaron": "\u01d4",
+ "ucircle": "\u24e4",
+ "ucircumflex": "\u00fb",
+ "ucircumflexbelow": "\u1e77",
+ "ucyrillic": "\u0443",
+ "udattadeva": "\u0951",
+ "udblacute": "\u0171",
+ "udblgrave": "\u0215",
+ "udeva": "\u0909",
+ "udieresis": "\u00fc",
+ "udieresisacute": "\u01d8",
+ "udieresisbelow": "\u1e73",
+ "udieresiscaron": "\u01da",
+ "udieresiscyrillic": "\u04f1",
+ "udieresisgrave": "\u01dc",
+ "udieresismacron": "\u01d6",
+ "udotbelow": "\u1ee5",
+ "ugrave": "\u00f9",
+ "ugujarati": "\u0a89",
+ "ugurmukhi": "\u0a09",
+ "uhiragana": "\u3046",
+ "uhookabove": "\u1ee7",
+ "uhorn": "\u01b0",
+ "uhornacute": "\u1ee9",
+ "uhorndotbelow": "\u1ef1",
+ "uhorngrave": "\u1eeb",
+ "uhornhookabove": "\u1eed",
+ "uhorntilde": "\u1eef",
+ "uhungarumlaut": "\u0171",
+ "uhungarumlautcyrillic": "\u04f3",
+ "uinvertedbreve": "\u0217",
+ "ukatakana": "\u30a6",
+ "ukatakanahalfwidth": "\uff73",
+ "ukcyrillic": "\u0479",
+ "ukorean": "\u315c",
+ "umacron": "\u016b",
+ "umacroncyrillic": "\u04ef",
+ "umacrondieresis": "\u1e7b",
+ "umatragurmukhi": "\u0a41",
+ "umonospace": "\uff55",
+ "underscore": "\u005f",
+ "underscoredbl": "\u2017",
+ "underscoremonospace": "\uff3f",
+ "underscorevertical": "\ufe33",
+ "underscorewavy": "\ufe4f",
+ "union": "\u222a",
+ "universal": "\u2200",
+ "uogonek": "\u0173",
+ "uparen": "\u24b0",
+ "upblock": "\u2580",
+ "upperdothebrew": "\u05c4",
+ "upsilon": "\u03c5",
+ "upsilondieresis": "\u03cb",
+ "upsilondieresistonos": "\u03b0",
+ "upsilonlatin": "\u028a",
+ "upsilontonos": "\u03cd",
+ "uptackbelowcmb": "\u031d",
+ "uptackmod": "\u02d4",
+ "uragurmukhi": "\u0a73",
+ "uring": "\u016f",
+ "ushortcyrillic": "\u045e",
+ "usmallhiragana": "\u3045",
+ "usmallkatakana": "\u30a5",
+ "usmallkatakanahalfwidth": "\uff69",
+ "ustraightcyrillic": "\u04af",
+ "ustraightstrokecyrillic": "\u04b1",
+ "utilde": "\u0169",
+ "utildeacute": "\u1e79",
+ "utildebelow": "\u1e75",
+ "uubengali": "\u098a",
+ "uudeva": "\u090a",
+ "uugujarati": "\u0a8a",
+ "uugurmukhi": "\u0a0a",
+ "uumatragurmukhi": "\u0a42",
+ "uuvowelsignbengali": "\u09c2",
+ "uuvowelsigndeva": "\u0942",
+ "uuvowelsigngujarati": "\u0ac2",
+ "uvowelsignbengali": "\u09c1",
+ "uvowelsigndeva": "\u0941",
+ "uvowelsigngujarati": "\u0ac1",
+ "v": "\u0076",
+ "vadeva": "\u0935",
+ "vagujarati": "\u0ab5",
+ "vagurmukhi": "\u0a35",
+ "vakatakana": "\u30f7",
+ "vav": "\u05d5",
+ "vavdagesh": "\ufb35",
+ "vavdagesh65": "\ufb35",
+ "vavdageshhebrew": "\ufb35",
+ "vavhebrew": "\u05d5",
+ "vavholam": "\ufb4b",
+ "vavholamhebrew": "\ufb4b",
+ "vavvavhebrew": "\u05f0",
+ "vavyodhebrew": "\u05f1",
+ "vcircle": "\u24e5",
+ "vdotbelow": "\u1e7f",
+ "vecyrillic": "\u0432",
+ "veharabic": "\u06a4",
+ "vehfinalarabic": "\ufb6b",
+ "vehinitialarabic": "\ufb6c",
+ "vehmedialarabic": "\ufb6d",
+ "vekatakana": "\u30f9",
+ "venus": "\u2640",
+ "verticalbar": "\u007c",
+ "verticallineabovecmb": "\u030d",
+ "verticallinebelowcmb": "\u0329",
+ "verticallinelowmod": "\u02cc",
+ "verticallinemod": "\u02c8",
+ "vewarmenian": "\u057e",
+ "vhook": "\u028b",
+ "vikatakana": "\u30f8",
+ "viramabengali": "\u09cd",
+ "viramadeva": "\u094d",
+ "viramagujarati": "\u0acd",
+ "visargabengali": "\u0983",
+ "visargadeva": "\u0903",
+ "visargagujarati": "\u0a83",
+ "vmonospace": "\uff56",
+ "voarmenian": "\u0578",
+ "voicediterationhiragana": "\u309e",
+ "voicediterationkatakana": "\u30fe",
+ "voicedmarkkana": "\u309b",
+ "voicedmarkkanahalfwidth": "\uff9e",
+ "vokatakana": "\u30fa",
+ "vparen": "\u24b1",
+ "vtilde": "\u1e7d",
+ "vturned": "\u028c",
+ "vuhiragana": "\u3094",
+ "vukatakana": "\u30f4",
+ "w": "\u0077",
+ "wacute": "\u1e83",
+ "waekorean": "\u3159",
+ "wahiragana": "\u308f",
+ "wakatakana": "\u30ef",
+ "wakatakanahalfwidth": "\uff9c",
+ "wakorean": "\u3158",
+ "wasmallhiragana": "\u308e",
+ "wasmallkatakana": "\u30ee",
+ "wattosquare": "\u3357",
+ "wavedash": "\u301c",
+ "wavyunderscorevertical": "\ufe34",
+ "wawarabic": "\u0648",
+ "wawfinalarabic": "\ufeee",
+ "wawhamzaabovearabic": "\u0624",
+ "wawhamzaabovefinalarabic": "\ufe86",
+ "wbsquare": "\u33dd",
+ "wcircle": "\u24e6",
+ "wcircumflex": "\u0175",
+ "wdieresis": "\u1e85",
+ "wdotaccent": "\u1e87",
+ "wdotbelow": "\u1e89",
+ "wehiragana": "\u3091",
+ "weierstrass": "\u2118",
+ "wekatakana": "\u30f1",
+ "wekorean": "\u315e",
+ "weokorean": "\u315d",
+ "wgrave": "\u1e81",
+ "whitebullet": "\u25e6",
+ "whitecircle": "\u25cb",
+ "whitecircleinverse": "\u25d9",
+ "whitecornerbracketleft": "\u300e",
+ "whitecornerbracketleftvertical": "\ufe43",
+ "whitecornerbracketright": "\u300f",
+ "whitecornerbracketrightvertical": "\ufe44",
+ "whitediamond": "\u25c7",
+ "whitediamondcontainingblacksmalldiamond": "\u25c8",
+ "whitedownpointingsmalltriangle": "\u25bf",
+ "whitedownpointingtriangle": "\u25bd",
+ "whiteleftpointingsmalltriangle": "\u25c3",
+ "whiteleftpointingtriangle": "\u25c1",
+ "whitelenticularbracketleft": "\u3016",
+ "whitelenticularbracketright": "\u3017",
+ "whiterightpointingsmalltriangle": "\u25b9",
+ "whiterightpointingtriangle": "\u25b7",
+ "whitesmallsquare": "\u25ab",
+ "whitesmilingface": "\u263a",
+ "whitesquare": "\u25a1",
+ "whitestar": "\u2606",
+ "whitetelephone": "\u260f",
+ "whitetortoiseshellbracketleft": "\u3018",
+ "whitetortoiseshellbracketright": "\u3019",
+ "whiteuppointingsmalltriangle": "\u25b5",
+ "whiteuppointingtriangle": "\u25b3",
+ "wihiragana": "\u3090",
+ "wikatakana": "\u30f0",
+ "wikorean": "\u315f",
+ "wmonospace": "\uff57",
+ "wohiragana": "\u3092",
+ "wokatakana": "\u30f2",
+ "wokatakanahalfwidth": "\uff66",
+ "won": "\u20a9",
+ "wonmonospace": "\uffe6",
+ "wowaenthai": "\u0e27",
+ "wparen": "\u24b2",
+ "wring": "\u1e98",
+ "wsuperior": "\u02b7",
+ "wturned": "\u028d",
+ "wynn": "\u01bf",
+ "x": "\u0078",
+ "xabovecmb": "\u033d",
+ "xbopomofo": "\u3112",
+ "xcircle": "\u24e7",
+ "xdieresis": "\u1e8d",
+ "xdotaccent": "\u1e8b",
+ "xeharmenian": "\u056d",
+ "xi": "\u03be",
+ "xmonospace": "\uff58",
+ "xparen": "\u24b3",
+ "xsuperior": "\u02e3",
+ "y": "\u0079",
+ "yaadosquare": "\u334e",
+ "yabengali": "\u09af",
+ "yacute": "\u00fd",
+ "yadeva": "\u092f",
+ "yaekorean": "\u3152",
+ "yagujarati": "\u0aaf",
+ "yagurmukhi": "\u0a2f",
+ "yahiragana": "\u3084",
+ "yakatakana": "\u30e4",
+ "yakatakanahalfwidth": "\uff94",
+ "yakorean": "\u3151",
+ "yamakkanthai": "\u0e4e",
+ "yasmallhiragana": "\u3083",
+ "yasmallkatakana": "\u30e3",
+ "yasmallkatakanahalfwidth": "\uff6c",
+ "yatcyrillic": "\u0463",
+ "ycircle": "\u24e8",
+ "ycircumflex": "\u0177",
+ "ydieresis": "\u00ff",
+ "ydotaccent": "\u1e8f",
+ "ydotbelow": "\u1ef5",
+ "yeharabic": "\u064a",
+ "yehbarreearabic": "\u06d2",
+ "yehbarreefinalarabic": "\ufbaf",
+ "yehfinalarabic": "\ufef2",
+ "yehhamzaabovearabic": "\u0626",
+ "yehhamzaabovefinalarabic": "\ufe8a",
+ "yehhamzaaboveinitialarabic": "\ufe8b",
+ "yehhamzaabovemedialarabic": "\ufe8c",
+ "yehinitialarabic": "\ufef3",
+ "yehmedialarabic": "\ufef4",
+ "yehmeeminitialarabic": "\ufcdd",
+ "yehmeemisolatedarabic": "\ufc58",
+ "yehnoonfinalarabic": "\ufc94",
+ "yehthreedotsbelowarabic": "\u06d1",
+ "yekorean": "\u3156",
+ "yen": "\u00a5",
+ "yenmonospace": "\uffe5",
+ "yeokorean": "\u3155",
+ "yeorinhieuhkorean": "\u3186",
+ "yerahbenyomohebrew": "\u05aa",
+ "yerahbenyomolefthebrew": "\u05aa",
+ "yericyrillic": "\u044b",
+ "yerudieresiscyrillic": "\u04f9",
+ "yesieungkorean": "\u3181",
+ "yesieungpansioskorean": "\u3183",
+ "yesieungsioskorean": "\u3182",
+ "yetivhebrew": "\u059a",
+ "ygrave": "\u1ef3",
+ "yhook": "\u01b4",
+ "yhookabove": "\u1ef7",
+ "yiarmenian": "\u0575",
+ "yicyrillic": "\u0457",
+ "yikorean": "\u3162",
+ "yinyang": "\u262f",
+ "yiwnarmenian": "\u0582",
+ "ymonospace": "\uff59",
+ "yod": "\u05d9",
+ "yoddagesh": "\ufb39",
+ "yoddageshhebrew": "\ufb39",
+ "yodhebrew": "\u05d9",
+ "yodyodhebrew": "\u05f2",
+ "yodyodpatahhebrew": "\ufb1f",
+ "yohiragana": "\u3088",
+ "yoikorean": "\u3189",
+ "yokatakana": "\u30e8",
+ "yokatakanahalfwidth": "\uff96",
+ "yokorean": "\u315b",
+ "yosmallhiragana": "\u3087",
+ "yosmallkatakana": "\u30e7",
+ "yosmallkatakanahalfwidth": "\uff6e",
+ "yotgreek": "\u03f3",
+ "yoyaekorean": "\u3188",
+ "yoyakorean": "\u3187",
+ "yoyakthai": "\u0e22",
+ "yoyingthai": "\u0e0d",
+ "yparen": "\u24b4",
+ "ypogegrammeni": "\u037a",
+ "ypogegrammenigreekcmb": "\u0345",
+ "yr": "\u01a6",
+ "yring": "\u1e99",
+ "ysuperior": "\u02b8",
+ "ytilde": "\u1ef9",
+ "yturned": "\u028e",
+ "yuhiragana": "\u3086",
+ "yuikorean": "\u318c",
+ "yukatakana": "\u30e6",
+ "yukatakanahalfwidth": "\uff95",
+ "yukorean": "\u3160",
+ "yusbigcyrillic": "\u046b",
+ "yusbigiotifiedcyrillic": "\u046d",
+ "yuslittlecyrillic": "\u0467",
+ "yuslittleiotifiedcyrillic": "\u0469",
+ "yusmallhiragana": "\u3085",
+ "yusmallkatakana": "\u30e5",
+ "yusmallkatakanahalfwidth": "\uff6d",
+ "yuyekorean": "\u318b",
+ "yuyeokorean": "\u318a",
+ "yyabengali": "\u09df",
+ "yyadeva": "\u095f",
+ "z": "\u007a",
+ "zaarmenian": "\u0566",
+ "zacute": "\u017a",
+ "zadeva": "\u095b",
+ "zagurmukhi": "\u0a5b",
+ "zaharabic": "\u0638",
+ "zahfinalarabic": "\ufec6",
+ "zahinitialarabic": "\ufec7",
+ "zahiragana": "\u3056",
+ "zahmedialarabic": "\ufec8",
+ "zainarabic": "\u0632",
+ "zainfinalarabic": "\ufeb0",
+ "zakatakana": "\u30b6",
+ "zaqefgadolhebrew": "\u0595",
+ "zaqefqatanhebrew": "\u0594",
+ "zarqahebrew": "\u0598",
+ "zayin": "\u05d6",
+ "zayindagesh": "\ufb36",
+ "zayindageshhebrew": "\ufb36",
+ "zayinhebrew": "\u05d6",
+ "zbopomofo": "\u3117",
+ "zcaron": "\u017e",
+ "zcircle": "\u24e9",
+ "zcircumflex": "\u1e91",
+ "zcurl": "\u0291",
+ "zdot": "\u017c",
+ "zdotaccent": "\u017c",
+ "zdotbelow": "\u1e93",
+ "zecyrillic": "\u0437",
+ "zedescendercyrillic": "\u0499",
+ "zedieresiscyrillic": "\u04df",
+ "zehiragana": "\u305c",
+ "zekatakana": "\u30bc",
+ "zero": "\u0030",
+ "zeroarabic": "\u0660",
+ "zerobengali": "\u09e6",
+ "zerodeva": "\u0966",
+ "zerogujarati": "\u0ae6",
+ "zerogurmukhi": "\u0a66",
+ "zerohackarabic": "\u0660",
+ "zeroinferior": "\u2080",
+ "zeromonospace": "\uff10",
+ "zerooldstyle": "\uf730",
+ "zeropersian": "\u06f0",
+ "zerosuperior": "\u2070",
+ "zerothai": "\u0e50",
+ "zerowidthjoiner": "\ufeff",
+ "zerowidthnonjoiner": "\u200c",
+ "zerowidthspace": "\u200b",
+ "zeta": "\u03b6",
+ "zhbopomofo": "\u3113",
+ "zhearmenian": "\u056a",
+ "zhebrevecyrillic": "\u04c2",
+ "zhecyrillic": "\u0436",
+ "zhedescendercyrillic": "\u0497",
+ "zhedieresiscyrillic": "\u04dd",
+ "zihiragana": "\u3058",
+ "zikatakana": "\u30b8",
+ "zinorhebrew": "\u05ae",
+ "zlinebelow": "\u1e95",
+ "zmonospace": "\uff5a",
+ "zohiragana": "\u305e",
+ "zokatakana": "\u30be",
+ "zparen": "\u24b5",
+ "zretroflexhook": "\u0290",
+ "zstroke": "\u01b6",
+ "zuhiragana": "\u305a",
+ "zukatakana": "\u30ba",
+}
+# --end
diff --git a/babeldoc/pdfminer/high_level.py b/babeldoc/pdfminer/high_level.py
new file mode 100644
index 0000000000000000000000000000000000000000..3930caa7cd11ddcd4915ef07f274df5b5ba42c01
--- /dev/null
+++ b/babeldoc/pdfminer/high_level.py
@@ -0,0 +1,233 @@
+"""Functions that can be used for the most common use-cases for pdfminer.six"""
+
+import logging
+import sys
+from collections.abc import Container
+from collections.abc import Iterator
+from io import StringIO
+from typing import Any
+from typing import BinaryIO
+from typing import cast
+
+from babeldoc.pdfminer.converter import HOCRConverter
+from babeldoc.pdfminer.converter import HTMLConverter
+from babeldoc.pdfminer.converter import PDFPageAggregator
+from babeldoc.pdfminer.converter import TextConverter
+from babeldoc.pdfminer.converter import XMLConverter
+from babeldoc.pdfminer.image import ImageWriter
+from babeldoc.pdfminer.layout import LAParams
+from babeldoc.pdfminer.layout import LTPage
+from babeldoc.pdfminer.pdfdevice import PDFDevice
+from babeldoc.pdfminer.pdfdevice import TagExtractor
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+from babeldoc.pdfminer.pdfinterp import PDFPageInterpreter
+from babeldoc.pdfminer.pdfinterp import PDFResourceManager
+from babeldoc.pdfminer.pdfpage import PDFPage
+from babeldoc.pdfminer.utils import AnyIO
+from babeldoc.pdfminer.utils import FileOrName
+from babeldoc.pdfminer.utils import open_filename
+
+
+def extract_text_to_fp(
+ inf: BinaryIO,
+ outfp: AnyIO,
+ output_type: str = "text",
+ codec: str = "utf-8",
+ laparams: LAParams | None = None,
+ maxpages: int = 0,
+ page_numbers: Container[int] | None = None,
+ password: str = "",
+ scale: float = 1.0,
+ rotation: int = 0,
+ layoutmode: str = "normal",
+ output_dir: str | None = None,
+ strip_control: bool = False,
+ debug: bool = False,
+ disable_caching: bool = False,
+ **kwargs: Any,
+) -> None:
+ """Parses text from inf-file and writes to outfp file-like object.
+
+ Takes loads of optional arguments but the defaults are somewhat sane.
+ Beware laparams: Including an empty LAParams is not the same as passing
+ None!
+
+ :param inf: a file-like object to read PDF structure from, such as a
+ file handler (using the builtin `open()` function) or a `BytesIO`.
+ :param outfp: a file-like object to write the text to.
+ :param output_type: May be 'text', 'xml', 'html', 'hocr', 'tag'.
+ Only 'text' works properly.
+ :param codec: Text decoding codec
+ :param laparams: An LAParams object from babeldoc.pdfminer.layout. Default is None
+ but may not layout correctly.
+ :param maxpages: How many pages to stop parsing after
+ :param page_numbers: zero-indexed page numbers to operate on.
+ :param password: For encrypted PDFs, the password to decrypt.
+ :param scale: Scale factor
+ :param rotation: Rotation factor
+ :param layoutmode: Default is 'normal', see
+ pdfminer.converter.HTMLConverter
+ :param output_dir: If given, creates an ImageWriter for extracted images.
+ :param strip_control: Does what it says on the tin
+ :param debug: Output more logging data
+ :param disable_caching: Does what it says on the tin
+ :param other:
+ :return: nothing, acting as it does on two streams. Use StringIO to get
+ strings.
+ """
+ if debug:
+ logging.getLogger().setLevel(logging.DEBUG)
+
+ imagewriter = None
+ if output_dir:
+ imagewriter = ImageWriter(output_dir)
+
+ rsrcmgr = PDFResourceManager(caching=not disable_caching)
+ device: PDFDevice | None = None
+
+ if output_type != "text" and outfp == sys.stdout:
+ outfp = sys.stdout.buffer
+
+ if output_type == "text":
+ device = TextConverter(
+ rsrcmgr,
+ outfp,
+ codec=codec,
+ laparams=laparams,
+ imagewriter=imagewriter,
+ )
+
+ elif output_type == "xml":
+ device = XMLConverter(
+ rsrcmgr,
+ outfp,
+ codec=codec,
+ laparams=laparams,
+ imagewriter=imagewriter,
+ stripcontrol=strip_control,
+ )
+
+ elif output_type == "html":
+ device = HTMLConverter(
+ rsrcmgr,
+ outfp,
+ codec=codec,
+ scale=scale,
+ layoutmode=layoutmode,
+ laparams=laparams,
+ imagewriter=imagewriter,
+ )
+
+ elif output_type == "hocr":
+ device = HOCRConverter(
+ rsrcmgr,
+ outfp,
+ codec=codec,
+ laparams=laparams,
+ stripcontrol=strip_control,
+ )
+
+ elif output_type == "tag":
+ # Binary I/O is required, but we have no good way to test it here.
+ device = TagExtractor(rsrcmgr, cast(BinaryIO, outfp), codec=codec)
+
+ else:
+ msg = f"Output type can be text, html, xml or tag but is {output_type}"
+ raise PDFValueError(msg)
+
+ assert device is not None
+ interpreter = PDFPageInterpreter(rsrcmgr, device)
+ for page in PDFPage.get_pages(
+ inf,
+ page_numbers,
+ maxpages=maxpages,
+ password=password,
+ caching=not disable_caching,
+ ):
+ page.rotate = (page.rotate + rotation) % 360
+ interpreter.process_page(page)
+
+ device.close()
+
+
+def extract_text(
+ pdf_file: FileOrName,
+ password: str = "",
+ page_numbers: Container[int] | None = None,
+ maxpages: int = 0,
+ caching: bool = True,
+ codec: str = "utf-8",
+ laparams: LAParams | None = None,
+) -> str:
+ """Parse and return the text contained in a PDF file.
+
+ :param pdf_file: Either a file path or a file-like object for the PDF file
+ to be worked on.
+ :param password: For encrypted PDFs, the password to decrypt.
+ :param page_numbers: List of zero-indexed page numbers to extract.
+ :param maxpages: The maximum number of pages to parse
+ :param caching: If resources should be cached
+ :param codec: Text decoding codec
+ :param laparams: An LAParams object from babeldoc.pdfminer.layout. If None, uses
+ some default settings that often work well.
+ :return: a string containing all of the text extracted.
+ """
+ if laparams is None:
+ laparams = LAParams()
+
+ with open_filename(pdf_file, "rb") as fp, StringIO() as output_string:
+ fp = cast(BinaryIO, fp) # we opened in binary mode
+ rsrcmgr = PDFResourceManager(caching=caching)
+ device = TextConverter(rsrcmgr, output_string, codec=codec, laparams=laparams)
+ interpreter = PDFPageInterpreter(rsrcmgr, device)
+
+ for page in PDFPage.get_pages(
+ fp,
+ page_numbers,
+ maxpages=maxpages,
+ password=password,
+ caching=caching,
+ ):
+ interpreter.process_page(page)
+
+ return output_string.getvalue()
+
+
+def extract_pages(
+ pdf_file: FileOrName,
+ password: str = "",
+ page_numbers: Container[int] | None = None,
+ maxpages: int = 0,
+ caching: bool = True,
+ laparams: LAParams | None = None,
+) -> Iterator[LTPage]:
+ """Extract and yield LTPage objects
+
+ :param pdf_file: Either a file path or a file-like object for the PDF file
+ to be worked on.
+ :param password: For encrypted PDFs, the password to decrypt.
+ :param page_numbers: List of zero-indexed page numbers to extract.
+ :param maxpages: The maximum number of pages to parse
+ :param caching: If resources should be cached
+ :param laparams: An LAParams object from babeldoc.pdfminer.layout. If None, uses
+ some default settings that often work well.
+ :return: LTPage objects
+ """
+ if laparams is None:
+ laparams = LAParams()
+
+ with open_filename(pdf_file, "rb") as fp:
+ fp = cast(BinaryIO, fp) # we opened in binary mode
+ resource_manager = PDFResourceManager(caching=caching)
+ device = PDFPageAggregator(resource_manager, laparams=laparams)
+ interpreter = PDFPageInterpreter(resource_manager, device)
+ for page in PDFPage.get_pages(
+ fp,
+ page_numbers,
+ maxpages=maxpages,
+ password=password,
+ caching=caching,
+ ):
+ interpreter.process_page(page)
+ layout = device.get_result()
+ yield layout
diff --git a/babeldoc/pdfminer/image.py b/babeldoc/pdfminer/image.py
new file mode 100644
index 0000000000000000000000000000000000000000..5bfd8f8477ecb75e7bdd843727b63a65511695bf
--- /dev/null
+++ b/babeldoc/pdfminer/image.py
@@ -0,0 +1,288 @@
+import os
+import os.path
+import struct
+from io import BytesIO
+from typing import BinaryIO
+from typing import Literal
+
+from babeldoc.pdfminer.jbig2 import JBIG2StreamReader
+from babeldoc.pdfminer.jbig2 import JBIG2StreamWriter
+from babeldoc.pdfminer.layout import LTImage
+from babeldoc.pdfminer.pdfcolor import LITERAL_DEVICE_CMYK
+from babeldoc.pdfminer.pdfcolor import LITERAL_DEVICE_GRAY
+from babeldoc.pdfminer.pdfcolor import LITERAL_DEVICE_RGB
+from babeldoc.pdfminer.pdfcolor import LITERAL_INLINE_DEVICE_GRAY
+from babeldoc.pdfminer.pdfcolor import LITERAL_INLINE_DEVICE_RGB
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+from babeldoc.pdfminer.pdftypes import LITERALS_DCT_DECODE
+from babeldoc.pdfminer.pdftypes import LITERALS_FLATE_DECODE
+from babeldoc.pdfminer.pdftypes import LITERALS_JBIG2_DECODE
+from babeldoc.pdfminer.pdftypes import LITERALS_JPX_DECODE
+
+PIL_ERROR_MESSAGE = (
+ "Could not import Pillow. This dependency of pdfminer.six is not "
+ "installed by default. You need it to to save jpg images to a file. Install it "
+ "with `pip install 'pdfminer.six[image]'`"
+)
+
+
+def align32(x: int) -> int:
+ return ((x + 3) // 4) * 4
+
+
+class BMPWriter:
+ def __init__(self, fp: BinaryIO, bits: int, width: int, height: int) -> None:
+ self.fp = fp
+ self.bits = bits
+ self.width = width
+ self.height = height
+ if bits == 1:
+ ncols = 2
+ elif bits == 8:
+ ncols = 256
+ elif bits == 24:
+ ncols = 0
+ else:
+ raise PDFValueError(bits)
+ self.linesize = align32((self.width * self.bits + 7) // 8)
+ self.datasize = self.linesize * self.height
+ headersize = 14 + 40 + ncols * 4
+ info = struct.pack(
+ " None:
+ self.fp.seek(self.pos1 - (y + 1) * self.linesize)
+ self.fp.write(data)
+
+
+class ImageWriter:
+ """Write image to a file
+
+ Supports various image types: JPEG, JBIG2 and bitmaps
+ """
+
+ def __init__(self, outdir: str) -> None:
+ self.outdir = outdir
+ if not os.path.exists(self.outdir):
+ os.makedirs(self.outdir)
+
+ def export_image(self, image: LTImage) -> str:
+ """Save an LTImage to disk"""
+ (width, height) = image.srcsize
+
+ filters = image.stream.get_filters()
+
+ if filters[-1][0] in LITERALS_DCT_DECODE:
+ name = self._save_jpeg(image)
+
+ elif filters[-1][0] in LITERALS_JPX_DECODE:
+ name = self._save_jpeg2000(image)
+
+ elif self._is_jbig2_iamge(image):
+ name = self._save_jbig2(image)
+
+ elif image.bits == 1:
+ name = self._save_bmp(image, width, height, (width + 7) // 8, image.bits)
+
+ elif image.bits == 8 and (
+ LITERAL_DEVICE_RGB in image.colorspace
+ or LITERAL_INLINE_DEVICE_RGB in image.colorspace
+ ):
+ name = self._save_bmp(image, width, height, width * 3, image.bits * 3)
+
+ elif image.bits == 8 and (
+ LITERAL_DEVICE_GRAY in image.colorspace
+ or LITERAL_INLINE_DEVICE_GRAY in image.colorspace
+ ):
+ name = self._save_bmp(image, width, height, width, image.bits)
+
+ elif len(filters) == 1 and filters[0][0] in LITERALS_FLATE_DECODE:
+ name = self._save_bytes(image)
+
+ else:
+ name = self._save_raw(image)
+
+ return name
+
+ def _save_jpeg(self, image: LTImage) -> str:
+ """Save a JPEG encoded image"""
+ data = image.stream.get_data()
+
+ name, path = self._create_unique_image_name(image, ".jpg")
+ with open(path, "wb") as fp:
+ if LITERAL_DEVICE_CMYK in image.colorspace:
+ try:
+ from PIL import Image # type: ignore[import]
+ from PIL import ImageChops # type: ignore[import]
+ except ImportError:
+ raise ImportError(PIL_ERROR_MESSAGE)
+
+ ifp = BytesIO(data)
+ i = Image.open(ifp)
+ i = ImageChops.invert(i)
+ i = i.convert("RGB")
+ i.save(fp, "JPEG")
+ else:
+ fp.write(data)
+
+ return name
+
+ def _save_jpeg2000(self, image: LTImage) -> str:
+ """Save a JPEG 2000 encoded image"""
+ data = image.stream.get_data()
+
+ name, path = self._create_unique_image_name(image, ".jp2")
+ with open(path, "wb") as fp:
+ try:
+ from PIL import Image # type: ignore[import]
+ except ImportError:
+ raise ImportError(PIL_ERROR_MESSAGE)
+
+ # if we just write the raw data, most image programs
+ # that I have tried cannot open the file. However,
+ # open and saving with PIL produces a file that
+ # seems to be easily opened by other programs
+ ifp = BytesIO(data)
+ i = Image.open(ifp)
+ i.save(fp, "JPEG2000")
+ return name
+
+ def _save_jbig2(self, image: LTImage) -> str:
+ """Save a JBIG2 encoded image"""
+ name, path = self._create_unique_image_name(image, ".jb2")
+ with open(path, "wb") as fp:
+ input_stream = BytesIO()
+
+ global_streams = []
+ filters = image.stream.get_filters()
+ for filter_name, params in filters:
+ if filter_name in LITERALS_JBIG2_DECODE:
+ global_streams.append(params["JBIG2Globals"].resolve())
+
+ if len(global_streams) > 1:
+ msg = (
+ "There should never be more than one JBIG2Globals "
+ "associated with a JBIG2 embedded image"
+ )
+ raise PDFValueError(msg)
+ if len(global_streams) == 1:
+ input_stream.write(global_streams[0].get_data().rstrip(b"\n"))
+ input_stream.write(image.stream.get_data())
+ input_stream.seek(0)
+ reader = JBIG2StreamReader(input_stream)
+ segments = reader.get_segments()
+
+ writer = JBIG2StreamWriter(fp)
+ writer.write_file(segments)
+ return name
+
+ def _save_bmp(
+ self,
+ image: LTImage,
+ width: int,
+ height: int,
+ bytes_per_line: int,
+ bits: int,
+ ) -> str:
+ """Save a BMP encoded image"""
+ name, path = self._create_unique_image_name(image, ".bmp")
+ with open(path, "wb") as fp:
+ bmp = BMPWriter(fp, bits, width, height)
+ data = image.stream.get_data()
+ i = 0
+ for y in range(height):
+ bmp.write_line(y, data[i : i + bytes_per_line])
+ i += bytes_per_line
+ return name
+
+ def _save_bytes(self, image: LTImage) -> str:
+ """Save an image without encoding, just bytes"""
+ name, path = self._create_unique_image_name(image, ".jpg")
+ width, height = image.srcsize
+ channels = len(image.stream.get_data()) / width / height / (image.bits / 8)
+ with open(path, "wb") as fp:
+ try:
+ from PIL import Image # type: ignore[import]
+ from PIL import ImageOps
+ except ImportError:
+ raise ImportError(PIL_ERROR_MESSAGE)
+
+ mode: Literal["1", "L", "RGB", "CMYK"]
+ if image.bits == 1:
+ mode = "1"
+ elif image.bits == 8 and channels == 1:
+ mode = "L"
+ elif image.bits == 8 and channels == 3:
+ mode = "RGB"
+ elif image.bits == 8 and channels == 4:
+ mode = "CMYK"
+
+ img = Image.frombytes(mode, image.srcsize, image.stream.get_data(), "raw")
+ if mode == "L":
+ img = ImageOps.invert(img)
+
+ img.save(fp)
+
+ return name
+
+ def _save_raw(self, image: LTImage) -> str:
+ """Save an image with unknown encoding"""
+ ext = ".%d.%dx%d.img" % (image.bits, image.srcsize[0], image.srcsize[1])
+ name, path = self._create_unique_image_name(image, ext)
+
+ with open(path, "wb") as fp:
+ fp.write(image.stream.get_data())
+ return name
+
+ @staticmethod
+ def _is_jbig2_iamge(image: LTImage) -> bool:
+ filters = image.stream.get_filters()
+ for filter_name, params in filters:
+ if filter_name in LITERALS_JBIG2_DECODE:
+ return True
+ return False
+
+ def _create_unique_image_name(self, image: LTImage, ext: str) -> tuple[str, str]:
+ name = image.name + ext
+ path = os.path.join(self.outdir, name)
+ img_index = 0
+ while os.path.exists(path):
+ name = "%s.%d%s" % (image.name, img_index, ext)
+ path = os.path.join(self.outdir, name)
+ img_index += 1
+ return name, path
diff --git a/babeldoc/pdfminer/jbig2.py b/babeldoc/pdfminer/jbig2.py
new file mode 100644
index 0000000000000000000000000000000000000000..fd3f6e605d77fda0f91c8d85b79750e9e4e2ccb7
--- /dev/null
+++ b/babeldoc/pdfminer/jbig2.py
@@ -0,0 +1,377 @@
+import math
+import os
+from collections.abc import Iterable
+from struct import calcsize
+from struct import pack
+from struct import unpack
+from typing import BinaryIO
+from typing import cast
+
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+
+# segment structure base
+SEG_STRUCT = [
+ (">L", "number"),
+ (">B", "flags"),
+ (">B", "retention_flags"),
+ (">B", "page_assoc"),
+ (">L", "data_length"),
+]
+
+# segment header literals
+HEADER_FLAG_DEFERRED = 0b10000000
+HEADER_FLAG_PAGE_ASSOC_LONG = 0b01000000
+
+SEG_TYPE_MASK = 0b00111111
+
+REF_COUNT_SHORT_MASK = 0b11100000
+REF_COUNT_LONG_MASK = 0x1FFFFFFF
+REF_COUNT_LONG = 7
+
+DATA_LEN_UNKNOWN = 0xFFFFFFFF
+
+# segment types
+SEG_TYPE_IMMEDIATE_GEN_REGION = 38
+SEG_TYPE_END_OF_PAGE = 49
+SEG_TYPE_END_OF_FILE = 51
+
+# file literals
+FILE_HEADER_ID = b"\x97\x4a\x42\x32\x0d\x0a\x1a\x0a"
+FILE_HEAD_FLAG_SEQUENTIAL = 0b00000001
+
+
+def bit_set(bit_pos: int, value: int) -> bool:
+ return bool((value >> bit_pos) & 1)
+
+
+def check_flag(flag: int, value: int) -> bool:
+ return bool(flag & value)
+
+
+def masked_value(mask: int, value: int) -> int:
+ for bit_pos in range(31):
+ if bit_set(bit_pos, mask):
+ return (value & mask) >> bit_pos
+
+ raise PDFValueError("Invalid mask or value")
+
+
+def mask_value(mask: int, value: int) -> int:
+ for bit_pos in range(31):
+ if bit_set(bit_pos, mask):
+ return (value & (mask >> bit_pos)) << bit_pos
+
+ raise PDFValueError("Invalid mask or value")
+
+
+def unpack_int(format: str, buffer: bytes) -> int:
+ assert format in {">B", ">I", ">L"}
+ [result] = cast(tuple[int], unpack(format, buffer))
+ return result
+
+
+JBIG2SegmentFlags = dict[str, int | bool]
+JBIG2RetentionFlags = dict[str, int | list[int] | list[bool]]
+JBIG2Segment = dict[
+ str,
+ bool | int | bytes | JBIG2SegmentFlags | JBIG2RetentionFlags,
+]
+
+
+class JBIG2StreamReader:
+ """Read segments from a JBIG2 byte stream"""
+
+ def __init__(self, stream: BinaryIO) -> None:
+ self.stream = stream
+
+ def get_segments(self) -> list[JBIG2Segment]:
+ segments: list[JBIG2Segment] = []
+ while not self.is_eof():
+ segment: JBIG2Segment = {}
+ for field_format, name in SEG_STRUCT:
+ field_len = calcsize(field_format)
+ field = self.stream.read(field_len)
+ if len(field) < field_len:
+ segment["_error"] = True
+ break
+ value = unpack_int(field_format, field)
+ parser = getattr(self, "parse_%s" % name, None)
+ if callable(parser):
+ value = parser(segment, value, field)
+ segment[name] = value
+
+ if not segment.get("_error"):
+ segments.append(segment)
+ return segments
+
+ def is_eof(self) -> bool:
+ if self.stream.read(1) == b"":
+ return True
+ else:
+ self.stream.seek(-1, os.SEEK_CUR)
+ return False
+
+ def parse_flags(
+ self,
+ segment: JBIG2Segment,
+ flags: int,
+ field: bytes,
+ ) -> JBIG2SegmentFlags:
+ return {
+ "deferred": check_flag(HEADER_FLAG_DEFERRED, flags),
+ "page_assoc_long": check_flag(HEADER_FLAG_PAGE_ASSOC_LONG, flags),
+ "type": masked_value(SEG_TYPE_MASK, flags),
+ }
+
+ def parse_retention_flags(
+ self,
+ segment: JBIG2Segment,
+ flags: int,
+ field: bytes,
+ ) -> JBIG2RetentionFlags:
+ ref_count = masked_value(REF_COUNT_SHORT_MASK, flags)
+ retain_segments = []
+ ref_segments = []
+
+ if ref_count < REF_COUNT_LONG:
+ for bit_pos in range(5):
+ retain_segments.append(bit_set(bit_pos, flags))
+ else:
+ field += self.stream.read(3)
+ ref_count = unpack_int(">L", field)
+ ref_count = masked_value(REF_COUNT_LONG_MASK, ref_count)
+ ret_bytes_count = int(math.ceil((ref_count + 1) / 8))
+ for ret_byte_index in range(ret_bytes_count):
+ ret_byte = unpack_int(">B", self.stream.read(1))
+ for bit_pos in range(7):
+ retain_segments.append(bit_set(bit_pos, ret_byte))
+
+ seg_num = segment["number"]
+ assert isinstance(seg_num, int)
+ if seg_num <= 256:
+ ref_format = ">B"
+ elif seg_num <= 65536:
+ ref_format = ">I"
+ else:
+ ref_format = ">L"
+
+ ref_size = calcsize(ref_format)
+
+ for ref_index in range(ref_count):
+ ref_data = self.stream.read(ref_size)
+ ref = unpack_int(ref_format, ref_data)
+ ref_segments.append(ref)
+
+ return {
+ "ref_count": ref_count,
+ "retain_segments": retain_segments,
+ "ref_segments": ref_segments,
+ }
+
+ def parse_page_assoc(self, segment: JBIG2Segment, page: int, field: bytes) -> int:
+ if cast(JBIG2SegmentFlags, segment["flags"])["page_assoc_long"]:
+ field += self.stream.read(3)
+ page = unpack_int(">L", field)
+ return page
+
+ def parse_data_length(
+ self,
+ segment: JBIG2Segment,
+ length: int,
+ field: bytes,
+ ) -> int:
+ if length:
+ if (
+ cast(JBIG2SegmentFlags, segment["flags"])["type"]
+ == SEG_TYPE_IMMEDIATE_GEN_REGION
+ ) and (length == DATA_LEN_UNKNOWN):
+ raise NotImplementedError(
+ "Working with unknown segment length is not implemented yet",
+ )
+ else:
+ segment["raw_data"] = self.stream.read(length)
+
+ return length
+
+
+class JBIG2StreamWriter:
+ """Write JBIG2 segments to a file in JBIG2 format"""
+
+ EMPTY_RETENTION_FLAGS: JBIG2RetentionFlags = {
+ "ref_count": 0,
+ "ref_segments": cast(list[int], []),
+ "retain_segments": cast(list[bool], []),
+ }
+
+ def __init__(self, stream: BinaryIO) -> None:
+ self.stream = stream
+
+ def write_segments(
+ self,
+ segments: Iterable[JBIG2Segment],
+ fix_last_page: bool = True,
+ ) -> int:
+ data_len = 0
+ current_page: int | None = None
+ seg_num: int | None = None
+
+ for segment in segments:
+ data = self.encode_segment(segment)
+ self.stream.write(data)
+ data_len += len(data)
+
+ seg_num = cast(int | None, segment["number"])
+
+ if fix_last_page:
+ seg_page = cast(int, segment.get("page_assoc"))
+
+ if (
+ cast(JBIG2SegmentFlags, segment["flags"])["type"]
+ == SEG_TYPE_END_OF_PAGE
+ ):
+ current_page = None
+ elif seg_page:
+ current_page = seg_page
+
+ if fix_last_page and current_page and (seg_num is not None):
+ segment = self.get_eop_segment(seg_num + 1, current_page)
+ data = self.encode_segment(segment)
+ self.stream.write(data)
+ data_len += len(data)
+
+ return data_len
+
+ def write_file(
+ self,
+ segments: Iterable[JBIG2Segment],
+ fix_last_page: bool = True,
+ ) -> int:
+ header = FILE_HEADER_ID
+ header_flags = FILE_HEAD_FLAG_SEQUENTIAL
+ header += pack(">B", header_flags)
+ # The embedded JBIG2 files in a PDF always
+ # only have one page
+ number_of_pages = pack(">L", 1)
+ header += number_of_pages
+ self.stream.write(header)
+ data_len = len(header)
+
+ data_len += self.write_segments(segments, fix_last_page)
+
+ seg_num = 0
+ for segment in segments:
+ seg_num = cast(int, segment["number"])
+
+ if fix_last_page:
+ seg_num_offset = 2
+ else:
+ seg_num_offset = 1
+ eof_segment = self.get_eof_segment(seg_num + seg_num_offset)
+ data = self.encode_segment(eof_segment)
+
+ self.stream.write(data)
+ data_len += len(data)
+
+ return data_len
+
+ def encode_segment(self, segment: JBIG2Segment) -> bytes:
+ data = b""
+ for field_format, name in SEG_STRUCT:
+ value = segment.get(name)
+ encoder = getattr(self, "encode_%s" % name, None)
+ if callable(encoder):
+ field = encoder(value, segment)
+ else:
+ field = pack(field_format, value)
+ data += field
+ return data
+
+ def encode_flags(self, value: JBIG2SegmentFlags, segment: JBIG2Segment) -> bytes:
+ flags = 0
+ if value.get("deferred"):
+ flags |= HEADER_FLAG_DEFERRED
+
+ if "page_assoc_long" in value:
+ flags |= HEADER_FLAG_PAGE_ASSOC_LONG if value["page_assoc_long"] else flags
+ else:
+ flags |= (
+ HEADER_FLAG_PAGE_ASSOC_LONG
+ if cast(int, segment.get("page", 0)) > 255
+ else flags
+ )
+
+ flags |= mask_value(SEG_TYPE_MASK, value["type"])
+
+ return pack(">B", flags)
+
+ def encode_retention_flags(
+ self,
+ value: JBIG2RetentionFlags,
+ segment: JBIG2Segment,
+ ) -> bytes:
+ flags = []
+ flags_format = ">B"
+ ref_count = value["ref_count"]
+ assert isinstance(ref_count, int)
+ retain_segments = cast(list[bool], value.get("retain_segments", []))
+
+ if ref_count <= 4:
+ flags_byte = mask_value(REF_COUNT_SHORT_MASK, ref_count)
+ for ref_index, ref_retain in enumerate(retain_segments):
+ if ref_retain:
+ flags_byte |= 1 << ref_index
+ flags.append(flags_byte)
+ else:
+ bytes_count = math.ceil((ref_count + 1) / 8)
+ flags_format = ">L" + ("B" * bytes_count)
+ flags_dword = mask_value(REF_COUNT_SHORT_MASK, REF_COUNT_LONG) << 24
+ flags.append(flags_dword)
+
+ for byte_index in range(bytes_count):
+ ret_byte = 0
+ ret_part = retain_segments[byte_index * 8 : byte_index * 8 + 8]
+ for bit_pos, ret_seg in enumerate(ret_part):
+ ret_byte |= 1 << bit_pos if ret_seg else ret_byte
+
+ flags.append(ret_byte)
+
+ ref_segments = cast(list[int], value.get("ref_segments", []))
+
+ seg_num = cast(int, segment["number"])
+ if seg_num <= 256:
+ ref_format = "B"
+ elif seg_num <= 65536:
+ ref_format = "I"
+ else:
+ ref_format = "L"
+
+ for ref in ref_segments:
+ flags_format += ref_format
+ flags.append(ref)
+
+ return pack(flags_format, *flags)
+
+ def encode_data_length(self, value: int, segment: JBIG2Segment) -> bytes:
+ data = pack(">L", value)
+ data += cast(bytes, segment["raw_data"])
+ return data
+
+ def get_eop_segment(self, seg_number: int, page_number: int) -> JBIG2Segment:
+ return {
+ "data_length": 0,
+ "flags": {"deferred": False, "type": SEG_TYPE_END_OF_PAGE},
+ "number": seg_number,
+ "page_assoc": page_number,
+ "raw_data": b"",
+ "retention_flags": JBIG2StreamWriter.EMPTY_RETENTION_FLAGS,
+ }
+
+ def get_eof_segment(self, seg_number: int) -> JBIG2Segment:
+ return {
+ "data_length": 0,
+ "flags": {"deferred": False, "type": SEG_TYPE_END_OF_FILE},
+ "number": seg_number,
+ "page_assoc": 0,
+ "raw_data": b"",
+ "retention_flags": JBIG2StreamWriter.EMPTY_RETENTION_FLAGS,
+ }
diff --git a/babeldoc/pdfminer/latin_enc.py b/babeldoc/pdfminer/latin_enc.py
new file mode 100644
index 0000000000000000000000000000000000000000..e83c09cfdde9b13ca7fb65adffdac7ef6d4963c8
--- /dev/null
+++ b/babeldoc/pdfminer/latin_enc.py
@@ -0,0 +1,244 @@
+"""Standard encoding tables used in PDF.
+
+This table is extracted from PDF Reference Manual 1.6, pp.925
+ "D.1 Latin Character Set and Encodings"
+
+"""
+
+EncodingRow = tuple[str, int | None, int | None, int | None, int | None]
+
+ENCODING: list[EncodingRow] = [
+ # (name, std, mac, win, pdf)
+ ("A", 65, 65, 65, 65),
+ ("AE", 225, 174, 198, 198),
+ ("Aacute", None, 231, 193, 193),
+ ("Acircumflex", None, 229, 194, 194),
+ ("Adieresis", None, 128, 196, 196),
+ ("Agrave", None, 203, 192, 192),
+ ("Aring", None, 129, 197, 197),
+ ("Atilde", None, 204, 195, 195),
+ ("B", 66, 66, 66, 66),
+ ("C", 67, 67, 67, 67),
+ ("Ccedilla", None, 130, 199, 199),
+ ("D", 68, 68, 68, 68),
+ ("E", 69, 69, 69, 69),
+ ("Eacute", None, 131, 201, 201),
+ ("Ecircumflex", None, 230, 202, 202),
+ ("Edieresis", None, 232, 203, 203),
+ ("Egrave", None, 233, 200, 200),
+ ("Eth", None, None, 208, 208),
+ ("Euro", None, None, 128, 160),
+ ("F", 70, 70, 70, 70),
+ ("G", 71, 71, 71, 71),
+ ("H", 72, 72, 72, 72),
+ ("I", 73, 73, 73, 73),
+ ("Iacute", None, 234, 205, 205),
+ ("Icircumflex", None, 235, 206, 206),
+ ("Idieresis", None, 236, 207, 207),
+ ("Igrave", None, 237, 204, 204),
+ ("J", 74, 74, 74, 74),
+ ("K", 75, 75, 75, 75),
+ ("L", 76, 76, 76, 76),
+ ("Lslash", 232, None, None, 149),
+ ("M", 77, 77, 77, 77),
+ ("N", 78, 78, 78, 78),
+ ("Ntilde", None, 132, 209, 209),
+ ("O", 79, 79, 79, 79),
+ ("OE", 234, 206, 140, 150),
+ ("Oacute", None, 238, 211, 211),
+ ("Ocircumflex", None, 239, 212, 212),
+ ("Odieresis", None, 133, 214, 214),
+ ("Ograve", None, 241, 210, 210),
+ ("Oslash", 233, 175, 216, 216),
+ ("Otilde", None, 205, 213, 213),
+ ("P", 80, 80, 80, 80),
+ ("Q", 81, 81, 81, 81),
+ ("R", 82, 82, 82, 82),
+ ("S", 83, 83, 83, 83),
+ ("Scaron", None, None, 138, 151),
+ ("T", 84, 84, 84, 84),
+ ("Thorn", None, None, 222, 222),
+ ("U", 85, 85, 85, 85),
+ ("Uacute", None, 242, 218, 218),
+ ("Ucircumflex", None, 243, 219, 219),
+ ("Udieresis", None, 134, 220, 220),
+ ("Ugrave", None, 244, 217, 217),
+ ("V", 86, 86, 86, 86),
+ ("W", 87, 87, 87, 87),
+ ("X", 88, 88, 88, 88),
+ ("Y", 89, 89, 89, 89),
+ ("Yacute", None, None, 221, 221),
+ ("Ydieresis", None, 217, 159, 152),
+ ("Z", 90, 90, 90, 90),
+ ("Zcaron", None, None, 142, 153),
+ ("a", 97, 97, 97, 97),
+ ("aacute", None, 135, 225, 225),
+ ("acircumflex", None, 137, 226, 226),
+ ("acute", 194, 171, 180, 180),
+ ("adieresis", None, 138, 228, 228),
+ ("ae", 241, 190, 230, 230),
+ ("agrave", None, 136, 224, 224),
+ ("ampersand", 38, 38, 38, 38),
+ ("aring", None, 140, 229, 229),
+ ("asciicircum", 94, 94, 94, 94),
+ ("asciitilde", 126, 126, 126, 126),
+ ("asterisk", 42, 42, 42, 42),
+ ("at", 64, 64, 64, 64),
+ ("atilde", None, 139, 227, 227),
+ ("b", 98, 98, 98, 98),
+ ("backslash", 92, 92, 92, 92),
+ ("bar", 124, 124, 124, 124),
+ ("braceleft", 123, 123, 123, 123),
+ ("braceright", 125, 125, 125, 125),
+ ("bracketleft", 91, 91, 91, 91),
+ ("bracketright", 93, 93, 93, 93),
+ ("breve", 198, 249, None, 24),
+ ("brokenbar", None, None, 166, 166),
+ ("bullet", 183, 165, 149, 128),
+ ("c", 99, 99, 99, 99),
+ ("caron", 207, 255, None, 25),
+ ("ccedilla", None, 141, 231, 231),
+ ("cedilla", 203, 252, 184, 184),
+ ("cent", 162, 162, 162, 162),
+ ("circumflex", 195, 246, 136, 26),
+ ("colon", 58, 58, 58, 58),
+ ("comma", 44, 44, 44, 44),
+ ("copyright", None, 169, 169, 169),
+ ("currency", 168, 219, 164, 164),
+ ("d", 100, 100, 100, 100),
+ ("dagger", 178, 160, 134, 129),
+ ("daggerdbl", 179, 224, 135, 130),
+ ("degree", None, 161, 176, 176),
+ ("dieresis", 200, 172, 168, 168),
+ ("divide", None, 214, 247, 247),
+ ("dollar", 36, 36, 36, 36),
+ ("dotaccent", 199, 250, None, 27),
+ ("dotlessi", 245, 245, None, 154),
+ ("e", 101, 101, 101, 101),
+ ("eacute", None, 142, 233, 233),
+ ("ecircumflex", None, 144, 234, 234),
+ ("edieresis", None, 145, 235, 235),
+ ("egrave", None, 143, 232, 232),
+ ("eight", 56, 56, 56, 56),
+ ("ellipsis", 188, 201, 133, 131),
+ ("emdash", 208, 209, 151, 132),
+ ("endash", 177, 208, 150, 133),
+ ("equal", 61, 61, 61, 61),
+ ("eth", None, None, 240, 240),
+ ("exclam", 33, 33, 33, 33),
+ ("exclamdown", 161, 193, 161, 161),
+ ("f", 102, 102, 102, 102),
+ ("fi", 174, 222, None, 147),
+ ("five", 53, 53, 53, 53),
+ ("fl", 175, 223, None, 148),
+ ("florin", 166, 196, 131, 134),
+ ("four", 52, 52, 52, 52),
+ ("fraction", 164, 218, None, 135),
+ ("g", 103, 103, 103, 103),
+ ("germandbls", 251, 167, 223, 223),
+ ("grave", 193, 96, 96, 96),
+ ("greater", 62, 62, 62, 62),
+ ("guillemotleft", 171, 199, 171, 171),
+ ("guillemotright", 187, 200, 187, 187),
+ ("guilsinglleft", 172, 220, 139, 136),
+ ("guilsinglright", 173, 221, 155, 137),
+ ("h", 104, 104, 104, 104),
+ ("hungarumlaut", 205, 253, None, 28),
+ ("hyphen", 45, 45, 45, 45),
+ ("i", 105, 105, 105, 105),
+ ("iacute", None, 146, 237, 237),
+ ("icircumflex", None, 148, 238, 238),
+ ("idieresis", None, 149, 239, 239),
+ ("igrave", None, 147, 236, 236),
+ ("j", 106, 106, 106, 106),
+ ("k", 107, 107, 107, 107),
+ ("l", 108, 108, 108, 108),
+ ("less", 60, 60, 60, 60),
+ ("logicalnot", None, 194, 172, 172),
+ ("lslash", 248, None, None, 155),
+ ("m", 109, 109, 109, 109),
+ ("macron", 197, 248, 175, 175),
+ ("minus", None, None, None, 138),
+ ("mu", None, 181, 181, 181),
+ ("multiply", None, None, 215, 215),
+ ("n", 110, 110, 110, 110),
+ ("nbspace", None, 202, 160, None),
+ ("nine", 57, 57, 57, 57),
+ ("ntilde", None, 150, 241, 241),
+ ("numbersign", 35, 35, 35, 35),
+ ("o", 111, 111, 111, 111),
+ ("oacute", None, 151, 243, 243),
+ ("ocircumflex", None, 153, 244, 244),
+ ("odieresis", None, 154, 246, 246),
+ ("oe", 250, 207, 156, 156),
+ ("ogonek", 206, 254, None, 29),
+ ("ograve", None, 152, 242, 242),
+ ("one", 49, 49, 49, 49),
+ ("onehalf", None, None, 189, 189),
+ ("onequarter", None, None, 188, 188),
+ ("onesuperior", None, None, 185, 185),
+ ("ordfeminine", 227, 187, 170, 170),
+ ("ordmasculine", 235, 188, 186, 186),
+ ("oslash", 249, 191, 248, 248),
+ ("otilde", None, 155, 245, 245),
+ ("p", 112, 112, 112, 112),
+ ("paragraph", 182, 166, 182, 182),
+ ("parenleft", 40, 40, 40, 40),
+ ("parenright", 41, 41, 41, 41),
+ ("percent", 37, 37, 37, 37),
+ ("period", 46, 46, 46, 46),
+ ("periodcentered", 180, 225, 183, 183),
+ ("perthousand", 189, 228, 137, 139),
+ ("plus", 43, 43, 43, 43),
+ ("plusminus", None, 177, 177, 177),
+ ("q", 113, 113, 113, 113),
+ ("question", 63, 63, 63, 63),
+ ("questiondown", 191, 192, 191, 191),
+ ("quotedbl", 34, 34, 34, 34),
+ ("quotedblbase", 185, 227, 132, 140),
+ ("quotedblleft", 170, 210, 147, 141),
+ ("quotedblright", 186, 211, 148, 142),
+ ("quoteleft", 96, 212, 145, 143),
+ ("quoteright", 39, 213, 146, 144),
+ ("quotesinglbase", 184, 226, 130, 145),
+ ("quotesingle", 169, 39, 39, 39),
+ ("r", 114, 114, 114, 114),
+ ("registered", None, 168, 174, 174),
+ ("ring", 202, 251, None, 30),
+ ("s", 115, 115, 115, 115),
+ ("scaron", None, None, 154, 157),
+ ("section", 167, 164, 167, 167),
+ ("semicolon", 59, 59, 59, 59),
+ ("seven", 55, 55, 55, 55),
+ ("six", 54, 54, 54, 54),
+ ("slash", 47, 47, 47, 47),
+ ("space", 32, 32, 32, 32),
+ ("space", None, 202, 160, None),
+ ("space", None, 202, 173, None),
+ ("sterling", 163, 163, 163, 163),
+ ("t", 116, 116, 116, 116),
+ ("thorn", None, None, 254, 254),
+ ("three", 51, 51, 51, 51),
+ ("threequarters", None, None, 190, 190),
+ ("threesuperior", None, None, 179, 179),
+ ("tilde", 196, 247, 152, 31),
+ ("trademark", None, 170, 153, 146),
+ ("two", 50, 50, 50, 50),
+ ("twosuperior", None, None, 178, 178),
+ ("u", 117, 117, 117, 117),
+ ("uacute", None, 156, 250, 250),
+ ("ucircumflex", None, 158, 251, 251),
+ ("udieresis", None, 159, 252, 252),
+ ("ugrave", None, 157, 249, 249),
+ ("underscore", 95, 95, 95, 95),
+ ("v", 118, 118, 118, 118),
+ ("w", 119, 119, 119, 119),
+ ("x", 120, 120, 120, 120),
+ ("y", 121, 121, 121, 121),
+ ("yacute", None, None, 253, 253),
+ ("ydieresis", None, 216, 255, 255),
+ ("yen", 165, 180, 165, 165),
+ ("z", 122, 122, 122, 122),
+ ("zcaron", None, None, 158, 158),
+ ("zero", 48, 48, 48, 48),
+]
diff --git a/babeldoc/pdfminer/layout.py b/babeldoc/pdfminer/layout.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c1bbbf024295b1e7eef931375a91220670d51ab
--- /dev/null
+++ b/babeldoc/pdfminer/layout.py
@@ -0,0 +1,979 @@
+import heapq
+import logging
+from collections.abc import Iterable
+from collections.abc import Iterator
+from collections.abc import Sequence
+from typing import Generic
+from typing import TypeVar
+from typing import Union
+from typing import cast
+
+from babeldoc.format.pdf.babelpdf.utils import guarded_bbox
+from babeldoc.pdfminer.pdfcolor import PDFColorSpace
+from babeldoc.pdfminer.pdfexceptions import PDFTypeError
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+from babeldoc.pdfminer.pdffont import PDFFont
+from babeldoc.pdfminer.pdfinterp import Color
+from babeldoc.pdfminer.pdfinterp import PDFGraphicState
+from babeldoc.pdfminer.pdftypes import PDFStream
+from babeldoc.pdfminer.utils import INF
+from babeldoc.pdfminer.utils import LTComponentT
+from babeldoc.pdfminer.utils import Matrix
+from babeldoc.pdfminer.utils import PathSegment
+from babeldoc.pdfminer.utils import Plane
+from babeldoc.pdfminer.utils import Point
+from babeldoc.pdfminer.utils import Rect
+from babeldoc.pdfminer.utils import apply_matrix_pt
+from babeldoc.pdfminer.utils import bbox2str
+from babeldoc.pdfminer.utils import fsplit
+from babeldoc.pdfminer.utils import get_bound
+from babeldoc.pdfminer.utils import matrix2str
+from babeldoc.pdfminer.utils import uniq
+
+logger = logging.getLogger(__name__)
+
+
+class IndexAssigner:
+ def __init__(self, index: int = 0) -> None:
+ self.index = index
+
+ def run(self, obj: "LTItem") -> None:
+ if isinstance(obj, LTTextBox):
+ obj.index = self.index
+ self.index += 1
+ elif isinstance(obj, LTTextGroup):
+ for x in obj:
+ self.run(x)
+
+
+class LAParams:
+ """Parameters for layout analysis
+
+ :param line_overlap: If two characters have more overlap than this they
+ are considered to be on the same line. The overlap is specified
+ relative to the minimum height of both characters.
+ :param char_margin: If two characters are closer together than this
+ margin they are considered part of the same line. The margin is
+ specified relative to the width of the character.
+ :param word_margin: If two characters on the same line are further apart
+ than this margin then they are considered to be two separate words, and
+ an intermediate space will be added for readability. The margin is
+ specified relative to the width of the character.
+ :param line_margin: If two lines are are close together they are
+ considered to be part of the same paragraph. The margin is
+ specified relative to the height of a line.
+ :param boxes_flow: Specifies how much a horizontal and vertical position
+ of a text matters when determining the order of text boxes. The value
+ should be within the range of -1.0 (only horizontal position
+ matters) to +1.0 (only vertical position matters). You can also pass
+ `None` to disable advanced layout analysis, and instead return text
+ based on the position of the bottom left corner of the text box.
+ :param detect_vertical: If vertical text should be considered during
+ layout analysis
+ :param all_texts: If layout analysis should be performed on text in
+ figures.
+ """
+
+ def __init__(
+ self,
+ line_overlap: float = 0.5,
+ char_margin: float = 2.0,
+ line_margin: float = 0.5,
+ word_margin: float = 0.1,
+ boxes_flow: float | None = 0.5,
+ detect_vertical: bool = False,
+ all_texts: bool = False,
+ ) -> None:
+ self.line_overlap = line_overlap
+ self.char_margin = char_margin
+ self.line_margin = line_margin
+ self.word_margin = word_margin
+ self.boxes_flow = boxes_flow
+ self.detect_vertical = detect_vertical
+ self.all_texts = all_texts
+
+ self._validate()
+
+ def _validate(self) -> None:
+ if self.boxes_flow is not None:
+ boxes_flow_err_msg = (
+ "LAParam boxes_flow should be None, or a number between -1 and +1"
+ )
+ if not (
+ isinstance(self.boxes_flow, int) or isinstance(self.boxes_flow, float)
+ ):
+ raise PDFTypeError(boxes_flow_err_msg)
+ if not -1 <= self.boxes_flow <= 1:
+ raise PDFValueError(boxes_flow_err_msg)
+
+ def __repr__(self) -> str:
+ return (
+ ""
+ % (self.char_margin, self.line_margin, self.word_margin, self.all_texts)
+ )
+
+
+class LTItem:
+ """Interface for things that can be analyzed"""
+
+ def analyze(self, laparams: LAParams) -> None:
+ """Perform the layout analysis."""
+
+
+class LTText:
+ """Interface for things that have text"""
+
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__} {self.get_text()!r}>"
+
+ def get_text(self) -> str:
+ """Text contained in this object"""
+ raise NotImplementedError
+
+
+class LTComponent(LTItem):
+ """Object with a bounding box"""
+
+ def __init__(self, bbox: Rect) -> None:
+ LTItem.__init__(self)
+ self.set_bbox(bbox)
+
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__} {bbox2str(self.bbox)}>"
+
+ # Disable comparison.
+ def __lt__(self, _: object) -> bool:
+ raise PDFValueError
+
+ def __le__(self, _: object) -> bool:
+ raise PDFValueError
+
+ def __gt__(self, _: object) -> bool:
+ raise PDFValueError
+
+ def __ge__(self, _: object) -> bool:
+ raise PDFValueError
+
+ def set_bbox(self, bbox: Rect) -> None:
+ (x0, y0, x1, y1) = bbox
+ self.x0 = x0
+ self.y0 = y0
+ self.x1 = x1
+ self.y1 = y1
+ self.width = x1 - x0
+ self.height = y1 - y0
+ self.bbox = bbox
+
+ def is_empty(self) -> bool:
+ return self.width <= 0 or self.height <= 0
+
+ def is_hoverlap(self, obj: "LTComponent") -> bool:
+ assert isinstance(obj, LTComponent), str(type(obj))
+ return obj.x0 <= self.x1 and self.x0 <= obj.x1
+
+ def hdistance(self, obj: "LTComponent") -> float:
+ assert isinstance(obj, LTComponent), str(type(obj))
+ if self.is_hoverlap(obj):
+ return 0
+ else:
+ return min(abs(self.x0 - obj.x1), abs(self.x1 - obj.x0))
+
+ def hoverlap(self, obj: "LTComponent") -> float:
+ assert isinstance(obj, LTComponent), str(type(obj))
+ if self.is_hoverlap(obj):
+ return min(abs(self.x0 - obj.x1), abs(self.x1 - obj.x0))
+ else:
+ return 0
+
+ def is_voverlap(self, obj: "LTComponent") -> bool:
+ assert isinstance(obj, LTComponent), str(type(obj))
+ return obj.y0 <= self.y1 and self.y0 <= obj.y1
+
+ def vdistance(self, obj: "LTComponent") -> float:
+ assert isinstance(obj, LTComponent), str(type(obj))
+ if self.is_voverlap(obj):
+ return 0
+ else:
+ return min(abs(self.y0 - obj.y1), abs(self.y1 - obj.y0))
+
+ def voverlap(self, obj: "LTComponent") -> float:
+ assert isinstance(obj, LTComponent), str(type(obj))
+ if self.is_voverlap(obj):
+ return min(abs(self.y0 - obj.y1), abs(self.y1 - obj.y0))
+ else:
+ return 0
+
+
+class LTCurve(LTComponent):
+ """A generic Bezier curve
+
+ The parameter `original_path` contains the original
+ pathing information from the pdf (e.g. for reconstructing Bezier Curves).
+
+ `dashing_style` contains the Dashing information if any.
+ """
+
+ def __init__(
+ self,
+ linewidth: float,
+ pts: list[Point],
+ stroke: bool = False,
+ fill: bool = False,
+ evenodd: bool = False,
+ stroking_color: Color | None = None,
+ non_stroking_color: Color | None = None,
+ original_path: list[PathSegment] | None = None,
+ dashing_style: tuple[object, object] | None = None,
+ ) -> None:
+ LTComponent.__init__(self, get_bound(pts))
+ self.pts = pts
+ self.linewidth = linewidth
+ self.stroke = stroke
+ self.fill = fill
+ self.evenodd = evenodd
+ self.stroking_color = stroking_color
+ self.non_stroking_color = non_stroking_color
+ self.original_path = original_path
+ self.dashing_style = dashing_style
+
+ def get_pts(self) -> str:
+ return ",".join("%.3f,%.3f" % p for p in self.pts)
+
+
+class LTLine(LTCurve):
+ """A single straight line.
+
+ Could be used for separating text or figures.
+ """
+
+ def __init__(
+ self,
+ linewidth: float,
+ p0: Point,
+ p1: Point,
+ stroke: bool = False,
+ fill: bool = False,
+ evenodd: bool = False,
+ stroking_color: Color | None = None,
+ non_stroking_color: Color | None = None,
+ original_path: list[PathSegment] | None = None,
+ dashing_style: tuple[object, object] | None = None,
+ ) -> None:
+ LTCurve.__init__(
+ self,
+ linewidth,
+ [p0, p1],
+ stroke,
+ fill,
+ evenodd,
+ stroking_color,
+ non_stroking_color,
+ original_path,
+ dashing_style,
+ )
+
+
+class LTRect(LTCurve):
+ """A rectangle.
+
+ Could be used for framing another pictures or figures.
+ """
+
+ def __init__(
+ self,
+ linewidth: float,
+ bbox: Rect,
+ stroke: bool = False,
+ fill: bool = False,
+ evenodd: bool = False,
+ stroking_color: Color | None = None,
+ non_stroking_color: Color | None = None,
+ original_path: list[PathSegment] | None = None,
+ dashing_style: tuple[object, object] | None = None,
+ ) -> None:
+ (x0, y0, x1, y1) = bbox
+ LTCurve.__init__(
+ self,
+ linewidth,
+ [(x0, y0), (x1, y0), (x1, y1), (x0, y1)],
+ stroke,
+ fill,
+ evenodd,
+ stroking_color,
+ non_stroking_color,
+ original_path,
+ dashing_style,
+ )
+
+
+class LTImage(LTComponent):
+ """An image object.
+
+ Embedded images can be in JPEG, Bitmap or JBIG2.
+ """
+
+ def __init__(self, name: str, stream: PDFStream, bbox: Rect) -> None:
+ LTComponent.__init__(self, bbox)
+ self.name = name
+ self.stream = stream
+ self.srcsize = (stream.get_any(("W", "Width")), stream.get_any(("H", "Height")))
+ self.imagemask = stream.get_any(("IM", "ImageMask"))
+ self.bits = stream.get_any(("BPC", "BitsPerComponent"), 1)
+ self.colorspace = stream.get_any(("CS", "ColorSpace"))
+ if not isinstance(self.colorspace, list):
+ self.colorspace = [self.colorspace]
+
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__}({self.name}) {bbox2str(self.bbox)} {self.srcsize!r}>"
+
+
+class LTAnno(LTItem, LTText):
+ """Actual letter in the text as a Unicode string.
+
+ Note that, while a LTChar object has actual boundaries, LTAnno objects does
+ not, as these are "virtual" characters, inserted by a layout analyzer
+ according to the relationship between two characters (e.g. a space).
+ """
+
+ def __init__(self, text: str) -> None:
+ self._text = text
+
+ def get_text(self) -> str:
+ return self._text
+
+
+class LTChar(LTComponent, LTText):
+ """Actual letter in the text as a Unicode string."""
+
+ def __init__(
+ self,
+ matrix: Matrix,
+ font: PDFFont,
+ fontsize: float,
+ scaling: float,
+ rise: float,
+ text: str,
+ textwidth: float,
+ textdisp: float | tuple[float | None, float],
+ ncs: PDFColorSpace,
+ graphicstate: PDFGraphicState,
+ ) -> None:
+ LTText.__init__(self)
+ self._text = text
+ self.matrix = matrix
+ self.fontname = font.fontname
+ self.ncs = ncs
+ self.graphicstate = graphicstate
+ self.adv = textwidth * fontsize * scaling
+ # compute the boundary rectangle.
+ if font.is_vertical():
+ # vertical
+ assert isinstance(textdisp, tuple)
+ (vx, vy) = textdisp
+ if vx is None:
+ vx = fontsize * 0.5
+ else:
+ vx = vx * fontsize * 0.001
+ vy = (1000 - vy) * fontsize * 0.001
+ bbox_lower_left = (-vx, vy + rise + self.adv)
+ bbox_upper_right = (-vx + fontsize, vy + rise)
+ else:
+ # horizontal
+ descent = font.get_descent() * fontsize
+ bbox_lower_left = (0, descent + rise)
+ bbox_upper_right = (self.adv, descent + rise + fontsize)
+ (a, b, c, d, e, f) = self.matrix
+ self.upright = a * d * scaling > 0 and b * c <= 0
+ (x0, y0) = apply_matrix_pt(self.matrix, bbox_lower_left)
+ (x1, y1) = apply_matrix_pt(self.matrix, bbox_upper_right)
+ if x1 < x0:
+ (x0, x1) = (x1, x0)
+ if y1 < y0:
+ (y0, y1) = (y1, y0)
+ LTComponent.__init__(self, (x0, y0, x1, y1))
+ if font.is_vertical():
+ self.size = self.width
+ else:
+ self.size = self.height
+
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__} {bbox2str(self.bbox)} matrix={matrix2str(self.matrix)} font={self.fontname!r} adv={self.adv} text={self.get_text()!r}>"
+
+ def get_text(self) -> str:
+ return self._text
+
+
+LTItemT = TypeVar("LTItemT", bound=LTItem)
+
+
+class LTContainer(LTComponent, Generic[LTItemT]):
+ """Object that can be extended and analyzed"""
+
+ def __init__(self, bbox: Rect) -> None:
+ LTComponent.__init__(self, bbox)
+ self._objs: list[LTItemT] = []
+
+ def __iter__(self) -> Iterator[LTItemT]:
+ return iter(self._objs)
+
+ def __len__(self) -> int:
+ return len(self._objs)
+
+ def add(self, obj: LTItemT) -> None:
+ self._objs.append(obj)
+
+ def extend(self, objs: Iterable[LTItemT]) -> None:
+ for obj in objs:
+ self.add(obj)
+
+ def analyze(self, laparams: LAParams) -> None:
+ for obj in self._objs:
+ obj.analyze(laparams)
+
+
+class LTExpandableContainer(LTContainer[LTItemT]):
+ def __init__(self) -> None:
+ LTContainer.__init__(self, (+INF, +INF, -INF, -INF))
+
+ # Incompatible override: we take an LTComponent (with bounding box), but
+ # super() LTContainer only considers LTItem (no bounding box).
+ def add(self, obj: LTComponent) -> None: # type: ignore[override]
+ LTContainer.add(self, cast(LTItemT, obj))
+ self.set_bbox(
+ (
+ min(self.x0, obj.x0),
+ min(self.y0, obj.y0),
+ max(self.x1, obj.x1),
+ max(self.y1, obj.y1),
+ ),
+ )
+
+
+class LTTextContainer(LTExpandableContainer[LTItemT], LTText):
+ def __init__(self) -> None:
+ LTText.__init__(self)
+ LTExpandableContainer.__init__(self)
+
+ def get_text(self) -> str:
+ return "".join(
+ cast(LTText, obj).get_text() for obj in self if isinstance(obj, LTText)
+ )
+
+
+TextLineElement = Union[LTChar, LTAnno]
+
+
+class LTTextLine(LTTextContainer[TextLineElement]):
+ """Contains a list of LTChar objects that represent a single text line.
+
+ The characters are aligned either horizontally or vertically, depending on
+ the text's writing mode.
+ """
+
+ def __init__(self, word_margin: float) -> None:
+ super().__init__()
+ self.word_margin = word_margin
+
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__} {bbox2str(self.bbox)} {self.get_text()!r}>"
+
+ def analyze(self, laparams: LAParams) -> None:
+ for obj in self._objs:
+ obj.analyze(laparams)
+ LTContainer.add(self, LTAnno("\n"))
+
+ def find_neighbors(
+ self,
+ plane: Plane[LTComponentT],
+ ratio: float,
+ ) -> list["LTTextLine"]:
+ raise NotImplementedError
+
+ def is_empty(self) -> bool:
+ return super().is_empty() or self.get_text().isspace()
+
+
+class LTTextLineHorizontal(LTTextLine):
+ def __init__(self, word_margin: float) -> None:
+ LTTextLine.__init__(self, word_margin)
+ self._x1: float = +INF
+
+ # Incompatible override: we take an LTComponent (with bounding box), but
+ # LTContainer only considers LTItem (no bounding box).
+ def add(self, obj: LTComponent) -> None: # type: ignore[override]
+ if isinstance(obj, LTChar) and self.word_margin:
+ margin = self.word_margin * max(obj.width, obj.height)
+ if self._x1 < obj.x0 - margin:
+ LTContainer.add(self, LTAnno(" "))
+ self._x1 = obj.x1
+ super().add(obj)
+
+ def find_neighbors(
+ self,
+ plane: Plane[LTComponentT],
+ ratio: float,
+ ) -> list[LTTextLine]:
+ """Finds neighboring LTTextLineHorizontals in the plane.
+
+ Returns a list of other LTTestLineHorizontals in the plane which are
+ close to self. "Close" can be controlled by ratio. The returned objects
+ will be the same height as self, and also either left-, right-, or
+ centrally-aligned.
+ """
+ d = ratio * self.height
+ objs = plane.find((self.x0, self.y0 - d, self.x1, self.y1 + d))
+ return [
+ obj
+ for obj in objs
+ if (
+ isinstance(obj, LTTextLineHorizontal)
+ and self._is_same_height_as(obj, tolerance=d)
+ and (
+ self._is_left_aligned_with(obj, tolerance=d)
+ or self._is_right_aligned_with(obj, tolerance=d)
+ or self._is_centrally_aligned_with(obj, tolerance=d)
+ )
+ )
+ ]
+
+ def _is_left_aligned_with(self, other: LTComponent, tolerance: float = 0) -> bool:
+ """Whether the left-hand edge of `other` is within `tolerance`."""
+ return abs(other.x0 - self.x0) <= tolerance
+
+ def _is_right_aligned_with(self, other: LTComponent, tolerance: float = 0) -> bool:
+ """Whether the right-hand edge of `other` is within `tolerance`."""
+ return abs(other.x1 - self.x1) <= tolerance
+
+ def _is_centrally_aligned_with(
+ self,
+ other: LTComponent,
+ tolerance: float = 0,
+ ) -> bool:
+ """Whether the horizontal center of `other` is within `tolerance`."""
+ return abs((other.x0 + other.x1) / 2 - (self.x0 + self.x1) / 2) <= tolerance
+
+ def _is_same_height_as(self, other: LTComponent, tolerance: float = 0) -> bool:
+ return abs(other.height - self.height) <= tolerance
+
+
+class LTTextLineVertical(LTTextLine):
+ def __init__(self, word_margin: float) -> None:
+ LTTextLine.__init__(self, word_margin)
+ self._y0: float = -INF
+
+ # Incompatible override: we take an LTComponent (with bounding box), but
+ # LTContainer only considers LTItem (no bounding box).
+ def add(self, obj: LTComponent) -> None: # type: ignore[override]
+ if isinstance(obj, LTChar) and self.word_margin:
+ margin = self.word_margin * max(obj.width, obj.height)
+ if obj.y1 + margin < self._y0:
+ LTContainer.add(self, LTAnno(" "))
+ self._y0 = obj.y0
+ super().add(obj)
+
+ def find_neighbors(
+ self,
+ plane: Plane[LTComponentT],
+ ratio: float,
+ ) -> list[LTTextLine]:
+ """Finds neighboring LTTextLineVerticals in the plane.
+
+ Returns a list of other LTTextLineVerticals in the plane which are
+ close to self. "Close" can be controlled by ratio. The returned objects
+ will be the same width as self, and also either upper-, lower-, or
+ centrally-aligned.
+ """
+ d = ratio * self.width
+ objs = plane.find((self.x0 - d, self.y0, self.x1 + d, self.y1))
+ return [
+ obj
+ for obj in objs
+ if (
+ isinstance(obj, LTTextLineVertical)
+ and self._is_same_width_as(obj, tolerance=d)
+ and (
+ self._is_lower_aligned_with(obj, tolerance=d)
+ or self._is_upper_aligned_with(obj, tolerance=d)
+ or self._is_centrally_aligned_with(obj, tolerance=d)
+ )
+ )
+ ]
+
+ def _is_lower_aligned_with(self, other: LTComponent, tolerance: float = 0) -> bool:
+ """Whether the lower edge of `other` is within `tolerance`."""
+ return abs(other.y0 - self.y0) <= tolerance
+
+ def _is_upper_aligned_with(self, other: LTComponent, tolerance: float = 0) -> bool:
+ """Whether the upper edge of `other` is within `tolerance`."""
+ return abs(other.y1 - self.y1) <= tolerance
+
+ def _is_centrally_aligned_with(
+ self,
+ other: LTComponent,
+ tolerance: float = 0,
+ ) -> bool:
+ """Whether the vertical center of `other` is within `tolerance`."""
+ return abs((other.y0 + other.y1) / 2 - (self.y0 + self.y1) / 2) <= tolerance
+
+ def _is_same_width_as(self, other: LTComponent, tolerance: float) -> bool:
+ return abs(other.width - self.width) <= tolerance
+
+
+class LTTextBox(LTTextContainer[LTTextLine]):
+ """Represents a group of text chunks in a rectangular area.
+
+ Note that this box is created by geometric analysis and does not
+ necessarily represents a logical boundary of the text. It contains a list
+ of LTTextLine objects.
+ """
+
+ def __init__(self) -> None:
+ LTTextContainer.__init__(self)
+ self.index: int = -1
+
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__}({self.index}) {bbox2str(self.bbox)} {self.get_text()!r}>"
+
+ def get_writing_mode(self) -> str:
+ raise NotImplementedError
+
+
+class LTTextBoxHorizontal(LTTextBox):
+ def analyze(self, laparams: LAParams) -> None:
+ super().analyze(laparams)
+ self._objs.sort(key=lambda obj: -obj.y1)
+
+ def get_writing_mode(self) -> str:
+ return "lr-tb"
+
+
+class LTTextBoxVertical(LTTextBox):
+ def analyze(self, laparams: LAParams) -> None:
+ super().analyze(laparams)
+ self._objs.sort(key=lambda obj: -obj.x1)
+
+ def get_writing_mode(self) -> str:
+ return "tb-rl"
+
+
+TextGroupElement = Union[LTTextBox, "LTTextGroup"]
+
+
+class LTTextGroup(LTTextContainer[TextGroupElement]):
+ def __init__(self, objs: Iterable[TextGroupElement]) -> None:
+ super().__init__()
+ self.extend(objs)
+
+
+class LTTextGroupLRTB(LTTextGroup):
+ def analyze(self, laparams: LAParams) -> None:
+ super().analyze(laparams)
+ assert laparams.boxes_flow is not None
+ boxes_flow = laparams.boxes_flow
+ # reorder the objects from top-left to bottom-right.
+ self._objs.sort(
+ key=lambda obj: (1 - boxes_flow) * obj.x0
+ - (1 + boxes_flow) * (obj.y0 + obj.y1),
+ )
+
+
+class LTTextGroupTBRL(LTTextGroup):
+ def analyze(self, laparams: LAParams) -> None:
+ super().analyze(laparams)
+ assert laparams.boxes_flow is not None
+ boxes_flow = laparams.boxes_flow
+ # reorder the objects from top-right to bottom-left.
+ self._objs.sort(
+ key=lambda obj: -(1 + boxes_flow) * (obj.x0 + obj.x1)
+ - (1 - boxes_flow) * obj.y1,
+ )
+
+
+class LTLayoutContainer(LTContainer[LTComponent]):
+ def __init__(self, bbox: Rect) -> None:
+ LTContainer.__init__(self, bbox)
+ self.groups: list[LTTextGroup] | None = None
+
+ # group_objects: group text object to textlines.
+ def group_objects(
+ self,
+ laparams: LAParams,
+ objs: Iterable[LTComponent],
+ ) -> Iterator[LTTextLine]:
+ obj0 = None
+ line = None
+ for obj1 in objs:
+ if obj0 is not None:
+ # halign: obj0 and obj1 is horizontally aligned.
+ #
+ # +------+ - - -
+ # | obj0 | - - +------+ -
+ # | | | obj1 | | (line_overlap)
+ # +------+ - - | | -
+ # - - - +------+
+ #
+ # |<--->|
+ # (char_margin)
+ halign = (
+ obj0.is_voverlap(obj1)
+ and min(obj0.height, obj1.height) * laparams.line_overlap
+ < obj0.voverlap(obj1)
+ and obj0.hdistance(obj1)
+ < max(obj0.width, obj1.width) * laparams.char_margin
+ )
+
+ # valign: obj0 and obj1 is vertically aligned.
+ #
+ # +------+
+ # | obj0 |
+ # | |
+ # +------+ - - -
+ # | | | (char_margin)
+ # +------+ - -
+ # | obj1 |
+ # | |
+ # +------+
+ #
+ # |<-->|
+ # (line_overlap)
+ valign = (
+ laparams.detect_vertical
+ and obj0.is_hoverlap(obj1)
+ and min(obj0.width, obj1.width) * laparams.line_overlap
+ < obj0.hoverlap(obj1)
+ and obj0.vdistance(obj1)
+ < max(obj0.height, obj1.height) * laparams.char_margin
+ )
+
+ if (halign and isinstance(line, LTTextLineHorizontal)) or (
+ valign and isinstance(line, LTTextLineVertical)
+ ):
+ line.add(obj1)
+ elif line is not None:
+ yield line
+ line = None
+ elif valign and not halign:
+ line = LTTextLineVertical(laparams.word_margin)
+ line.add(obj0)
+ line.add(obj1)
+ elif halign and not valign:
+ line = LTTextLineHorizontal(laparams.word_margin)
+ line.add(obj0)
+ line.add(obj1)
+ else:
+ line = LTTextLineHorizontal(laparams.word_margin)
+ line.add(obj0)
+ yield line
+ line = None
+ obj0 = obj1
+ if line is None:
+ line = LTTextLineHorizontal(laparams.word_margin)
+ assert obj0 is not None
+ line.add(obj0)
+ yield line
+
+ def group_textlines(
+ self,
+ laparams: LAParams,
+ lines: Iterable[LTTextLine],
+ ) -> Iterator[LTTextBox]:
+ """Group neighboring lines to textboxes"""
+ plane: Plane[LTTextLine] = Plane(self.bbox)
+ plane.extend(lines)
+ boxes: dict[LTTextLine, LTTextBox] = {}
+ for line in lines:
+ neighbors = line.find_neighbors(plane, laparams.line_margin)
+ members = [line]
+ for obj1 in neighbors:
+ members.append(obj1)
+ if obj1 in boxes:
+ members.extend(boxes.pop(obj1))
+ if isinstance(line, LTTextLineHorizontal):
+ box: LTTextBox = LTTextBoxHorizontal()
+ else:
+ box = LTTextBoxVertical()
+ for obj in uniq(members):
+ box.add(obj)
+ boxes[obj] = box
+ done = set()
+ for line in lines:
+ if line not in boxes:
+ continue
+ box = boxes[line]
+ if box in done:
+ continue
+ done.add(box)
+ if not box.is_empty():
+ yield box
+
+ def group_textboxes(
+ self,
+ laparams: LAParams,
+ boxes: Sequence[LTTextBox],
+ ) -> list[LTTextGroup]:
+ """Group textboxes hierarchically.
+
+ Get pair-wise distances, via dist func defined below, and then merge
+ from the closest textbox pair. Once obj1 and obj2 are merged /
+ grouped, the resulting group is considered as a new object, and its
+ distances to other objects & groups are added to the process queue.
+
+ For performance reason, pair-wise distances and object pair info are
+ maintained in a heap of (idx, dist, id(obj1), id(obj2), obj1, obj2)
+ tuples. It ensures quick access to the smallest element. Note that
+ since comparison operators, e.g., __lt__, are disabled for
+ LTComponent, id(obj) has to appear before obj in element tuples.
+
+ :param laparams: LAParams object.
+ :param boxes: All textbox objects to be grouped.
+ :return: a list that has only one element, the final top level group.
+ """
+ ElementT = Union[LTTextBox, LTTextGroup]
+ plane: Plane[ElementT] = Plane(self.bbox)
+
+ def dist(obj1: LTComponent, obj2: LTComponent) -> float:
+ """A distance function between two TextBoxes.
+
+ Consider the bounding rectangle for obj1 and obj2.
+ Return its area less the areas of obj1 and obj2,
+ shown as 'www' below. This value may be negative.
+ +------+..........+ (x1, y1)
+ | obj1 |wwwwwwwwww:
+ +------+www+------+
+ :wwwwwwwwww| obj2 |
+ (x0, y0) +..........+------+
+ """
+ x0 = min(obj1.x0, obj2.x0)
+ y0 = min(obj1.y0, obj2.y0)
+ x1 = max(obj1.x1, obj2.x1)
+ y1 = max(obj1.y1, obj2.y1)
+ return (
+ (x1 - x0) * (y1 - y0)
+ - obj1.width * obj1.height
+ - obj2.width * obj2.height
+ )
+
+ def isany(obj1: ElementT, obj2: ElementT) -> set[ElementT]:
+ """Check if there's any other object between obj1 and obj2."""
+ x0 = min(obj1.x0, obj2.x0)
+ y0 = min(obj1.y0, obj2.y0)
+ x1 = max(obj1.x1, obj2.x1)
+ y1 = max(obj1.y1, obj2.y1)
+ objs = set(plane.find((x0, y0, x1, y1)))
+ return objs.difference((obj1, obj2))
+
+ dists: list[tuple[bool, float, int, int, ElementT, ElementT]] = []
+ for i in range(len(boxes)):
+ box1 = boxes[i]
+ for j in range(i + 1, len(boxes)):
+ box2 = boxes[j]
+ dists.append((False, dist(box1, box2), id(box1), id(box2), box1, box2))
+ heapq.heapify(dists)
+
+ plane.extend(boxes)
+ done = set()
+ while len(dists) > 0:
+ (skip_isany, d, id1, id2, obj1, obj2) = heapq.heappop(dists)
+ # Skip objects that are already merged
+ if (id1 not in done) and (id2 not in done):
+ if not skip_isany and isany(obj1, obj2):
+ heapq.heappush(dists, (True, d, id1, id2, obj1, obj2))
+ continue
+ if isinstance(obj1, (LTTextBoxVertical, LTTextGroupTBRL)) or isinstance(
+ obj2,
+ (LTTextBoxVertical, LTTextGroupTBRL),
+ ):
+ group: LTTextGroup = LTTextGroupTBRL([obj1, obj2])
+ else:
+ group = LTTextGroupLRTB([obj1, obj2])
+ plane.remove(obj1)
+ plane.remove(obj2)
+ done.update([id1, id2])
+
+ for other in plane:
+ heapq.heappush(
+ dists,
+ (False, dist(group, other), id(group), id(other), group, other),
+ )
+ plane.add(group)
+ # By now only groups are in the plane
+ return list(cast(LTTextGroup, g) for g in plane)
+
+ def analyze(self, laparams: LAParams) -> None:
+ # textobjs is a list of LTChar objects, i.e.
+ # it has all the individual characters in the page.
+ (textobjs, otherobjs) = fsplit(lambda obj: isinstance(obj, LTChar), self)
+ for obj in otherobjs:
+ obj.analyze(laparams)
+ if not textobjs:
+ return
+ textlines = list(self.group_objects(laparams, textobjs))
+ (empties, textlines) = fsplit(lambda obj: obj.is_empty(), textlines)
+ for obj in empties:
+ obj.analyze(laparams)
+ textboxes = list(self.group_textlines(laparams, textlines))
+ if laparams.boxes_flow is None:
+ for textbox in textboxes:
+ textbox.analyze(laparams)
+
+ def getkey(box: LTTextBox) -> tuple[int, float, float]:
+ if isinstance(box, LTTextBoxVertical):
+ return (0, -box.x1, -box.y0)
+ else:
+ return (1, -box.y0, box.x0)
+
+ textboxes.sort(key=getkey)
+ else:
+ self.groups = self.group_textboxes(laparams, textboxes)
+ assigner = IndexAssigner()
+ for group in self.groups:
+ group.analyze(laparams)
+ assigner.run(group)
+ textboxes.sort(key=lambda box: box.index)
+ self._objs = (
+ cast(list[LTComponent], textboxes)
+ + otherobjs
+ + cast(list[LTComponent], empties)
+ )
+
+
+class LTFigure(LTLayoutContainer):
+ """Represents an area used by PDF Form objects.
+
+ PDF Forms can be used to present figures or pictures by embedding yet
+ another PDF document within a page. Note that LTFigure objects can appear
+ recursively.
+ """
+
+ def __init__(self, name: str, bbox: Rect, matrix: Matrix) -> None:
+ self.name = name
+ self.matrix = matrix
+ (x, y, w, h) = guarded_bbox(bbox)
+ bounds = ((x, y), (x + w, y), (x, y + h), (x + w, y + h))
+ bbox = get_bound(apply_matrix_pt(matrix, (p, q)) for (p, q) in bounds)
+ LTLayoutContainer.__init__(self, bbox)
+
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__}({self.name}) {bbox2str(self.bbox)} matrix={matrix2str(self.matrix)}>"
+
+ def analyze(self, laparams: LAParams) -> None:
+ if not laparams.all_texts:
+ return
+ LTLayoutContainer.analyze(self, laparams)
+
+
+class LTPage(LTLayoutContainer):
+ """Represents an entire page.
+
+ Like any other LTLayoutContainer, an LTPage can be iterated to obtain child
+ objects like LTTextBox, LTFigure, LTImage, LTRect, LTCurve and LTLine.
+ """
+
+ def __init__(self, pageid: int, bbox: Rect, rotate: float = 0) -> None:
+ LTLayoutContainer.__init__(self, bbox)
+ self.pageid = pageid
+ self.rotate = rotate
+
+ def __repr__(self) -> str:
+ return f"<{self.__class__.__name__}({self.pageid!r}) {bbox2str(self.bbox)} rotate={self.rotate!r}>"
diff --git a/babeldoc/pdfminer/lzw.py b/babeldoc/pdfminer/lzw.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e4ce36e72e348337f49a330b8eb483865b19a61
--- /dev/null
+++ b/babeldoc/pdfminer/lzw.py
@@ -0,0 +1,108 @@
+import logging
+from collections.abc import Iterator
+from io import BytesIO
+from typing import BinaryIO
+from typing import cast
+
+from babeldoc.pdfminer.pdfexceptions import PDFEOFError
+from babeldoc.pdfminer.pdfexceptions import PDFException
+
+logger = logging.getLogger(__name__)
+
+
+class CorruptDataError(PDFException):
+ pass
+
+
+class LZWDecoder:
+ def __init__(self, fp: BinaryIO) -> None:
+ self.fp = fp
+ self.buff = 0
+ self.bpos = 8
+ self.nbits = 9
+ # NB: self.table stores None only in indices 256 and 257
+ self.table: list[bytes | None] = []
+ self.prevbuf: bytes | None = None
+
+ def readbits(self, bits: int) -> int:
+ v = 0
+ while 1:
+ # the number of remaining bits we can get from the current buffer.
+ r = 8 - self.bpos
+ if bits <= r:
+ # |-----8-bits-----|
+ # |-bpos-|-bits-| |
+ # | |----r----|
+ v = (v << bits) | ((self.buff >> (r - bits)) & ((1 << bits) - 1))
+ self.bpos += bits
+ break
+ else:
+ # |-----8-bits-----|
+ # |-bpos-|---bits----...
+ # | |----r----|
+ v = (v << r) | (self.buff & ((1 << r) - 1))
+ bits -= r
+ x = self.fp.read(1)
+ if not x:
+ raise PDFEOFError
+ self.buff = ord(x)
+ self.bpos = 0
+ return v
+
+ def feed(self, code: int) -> bytes:
+ x = b""
+ if code == 256:
+ self.table = [bytes((c,)) for c in range(256)] # 0-255
+ self.table.append(None) # 256
+ self.table.append(None) # 257
+ self.prevbuf = b""
+ self.nbits = 9
+ elif code == 257:
+ pass
+ elif not self.prevbuf:
+ x = self.prevbuf = cast(bytes, self.table[code]) # assume not None
+ else:
+ if code < len(self.table):
+ x = cast(bytes, self.table[code]) # assume not None
+ self.table.append(self.prevbuf + x[:1])
+ elif code == len(self.table):
+ self.table.append(self.prevbuf + self.prevbuf[:1])
+ x = cast(bytes, self.table[code])
+ else:
+ raise CorruptDataError
+ table_length = len(self.table)
+ if table_length == 511:
+ self.nbits = 10
+ elif table_length == 1023:
+ self.nbits = 11
+ elif table_length == 2047:
+ self.nbits = 12
+ self.prevbuf = x
+ return x
+
+ def run(self) -> Iterator[bytes]:
+ while 1:
+ try:
+ code = self.readbits(self.nbits)
+ except EOFError:
+ break
+ try:
+ x = self.feed(code)
+ except CorruptDataError:
+ # just ignore corrupt data and stop yielding there
+ break
+ yield x
+
+ logger.debug(
+ "nbits=%d, code=%d, output=%r, table=%r",
+ self.nbits,
+ code,
+ x,
+ self.table[258:],
+ )
+
+
+def lzwdecode(data: bytes) -> bytes:
+ fp = BytesIO(data)
+ s = LZWDecoder(fp).run()
+ return b"".join(s)
diff --git a/babeldoc/pdfminer/pdfcolor.py b/babeldoc/pdfminer/pdfcolor.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a264c2743589f0ab2116462fa0b4bcbdf33f614
--- /dev/null
+++ b/babeldoc/pdfminer/pdfcolor.py
@@ -0,0 +1,36 @@
+import collections
+
+from babeldoc.pdfminer.psparser import LIT
+
+LITERAL_DEVICE_GRAY = LIT("DeviceGray")
+LITERAL_DEVICE_RGB = LIT("DeviceRGB")
+LITERAL_DEVICE_CMYK = LIT("DeviceCMYK")
+# Abbreviations for inline images
+LITERAL_INLINE_DEVICE_GRAY = LIT("G")
+LITERAL_INLINE_DEVICE_RGB = LIT("RGB")
+LITERAL_INLINE_DEVICE_CMYK = LIT("CMYK")
+
+
+class PDFColorSpace:
+ def __init__(self, name: str, ncomponents: int) -> None:
+ self.name = name
+ self.ncomponents = ncomponents
+
+ def __repr__(self) -> str:
+ return "" % (self.name, self.ncomponents)
+
+
+PREDEFINED_COLORSPACE: dict[str, PDFColorSpace] = collections.OrderedDict()
+
+for name, n in [
+ ("DeviceGray", 1), # default value first
+ ("CalRGB", 3),
+ ("CalGray", 1),
+ ("Lab", 3),
+ ("DeviceRGB", 3),
+ ("DeviceCMYK", 4),
+ ("Separation", 1),
+ ("Indexed", 1),
+ ("Pattern", 1),
+]:
+ PREDEFINED_COLORSPACE[name] = PDFColorSpace(name, n)
diff --git a/babeldoc/pdfminer/pdfdevice.py b/babeldoc/pdfminer/pdfdevice.py
new file mode 100644
index 0000000000000000000000000000000000000000..d9a8d65307d6e09b4445398fe32817f06ba5a1f4
--- /dev/null
+++ b/babeldoc/pdfminer/pdfdevice.py
@@ -0,0 +1,326 @@
+import logging
+from collections.abc import Iterable
+from collections.abc import Sequence
+from typing import TYPE_CHECKING
+from typing import BinaryIO
+from typing import Optional
+from typing import cast
+
+from babeldoc.pdfminer.pdfcolor import PDFColorSpace
+from babeldoc.pdfminer.pdffont import PDFFont
+from babeldoc.pdfminer.pdffont import PDFUnicodeNotDefined
+from babeldoc.pdfminer.pdfpage import PDFPage
+from babeldoc.pdfminer.pdftypes import PDFStream
+from babeldoc.pdfminer.psparser import PSLiteral
+from babeldoc.pdfminer.utils import Matrix
+from babeldoc.pdfminer.utils import PathSegment
+from babeldoc.pdfminer.utils import Point
+from babeldoc.pdfminer.utils import Rect
+from babeldoc.pdfminer import utils
+
+if TYPE_CHECKING:
+ from babeldoc.pdfminer.pdfinterp import PDFGraphicState
+ from babeldoc.pdfminer.pdfinterp import PDFResourceManager
+ from babeldoc.pdfminer.pdfinterp import PDFStackT
+ from babeldoc.pdfminer.pdfinterp import PDFTextState
+
+
+PDFTextSeq = Iterable[int | float | bytes]
+
+logger = logging.getLogger(__name__)
+
+
+class PDFDevice:
+ """Translate the output of PDFPageInterpreter to the output that is needed"""
+
+ def __init__(self, rsrcmgr: "PDFResourceManager") -> None:
+ self.rsrcmgr = rsrcmgr
+ self.ctm: Matrix | None = None
+
+ def __repr__(self) -> str:
+ return ""
+
+ def __enter__(self) -> "PDFDevice":
+ return self
+
+ def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
+ self.close()
+
+ def close(self) -> None:
+ pass
+
+ def set_ctm(self, ctm: Matrix) -> None:
+ self.ctm = ctm
+
+ def begin_tag(self, tag: PSLiteral, props: Optional["PDFStackT"] = None) -> None:
+ pass
+
+ def end_tag(self) -> None:
+ pass
+
+ def do_tag(self, tag: PSLiteral, props: Optional["PDFStackT"] = None) -> None:
+ pass
+
+ def begin_page(self, page: PDFPage, ctm: Matrix) -> None:
+ pass
+
+ def end_page(self, page: PDFPage) -> None:
+ pass
+
+ def begin_figure(self, name: str, bbox: Rect, matrix: Matrix) -> None:
+ pass
+
+ def end_figure(self, name: str) -> None:
+ pass
+
+ def paint_path(
+ self,
+ graphicstate: "PDFGraphicState",
+ stroke: bool,
+ fill: bool,
+ evenodd: bool,
+ path: Sequence[PathSegment],
+ ) -> None:
+ pass
+
+ def render_image(self, name: str, stream: PDFStream) -> None:
+ pass
+
+ def render_string(
+ self,
+ textstate: "PDFTextState",
+ seq: PDFTextSeq,
+ ncs: PDFColorSpace,
+ graphicstate: "PDFGraphicState",
+ ) -> None:
+ pass
+
+
+class PDFTextDevice(PDFDevice):
+ def render_string(
+ self,
+ textstate: "PDFTextState",
+ seq: PDFTextSeq,
+ ncs: PDFColorSpace,
+ graphicstate: "PDFGraphicState",
+ ) -> None:
+ assert self.ctm is not None
+ matrix = utils.mult_matrix(textstate.matrix, self.ctm)
+ font = textstate.font
+ font.font_id_temp = getattr(textstate, "font_id", None)
+ fontsize = textstate.fontsize
+ scaling = textstate.scaling * 0.01
+ charspace = textstate.charspace * scaling
+ wordspace = textstate.wordspace * scaling
+ rise = textstate.rise
+ assert font is not None
+ if font.is_multibyte():
+ wordspace = 0
+ dxscale = 0.001 * fontsize * scaling
+ if font.is_vertical():
+ textstate.linematrix = self.render_string_vertical(
+ seq,
+ matrix,
+ textstate.linematrix,
+ font,
+ fontsize,
+ scaling,
+ charspace,
+ wordspace,
+ rise,
+ dxscale,
+ ncs,
+ graphicstate,
+ )
+ else:
+ textstate.linematrix = self.render_string_horizontal(
+ seq,
+ matrix,
+ textstate.linematrix,
+ font,
+ fontsize,
+ scaling,
+ charspace,
+ wordspace,
+ rise,
+ dxscale,
+ ncs,
+ graphicstate,
+ )
+
+ def render_string_horizontal(
+ self,
+ seq: PDFTextSeq,
+ matrix: Matrix,
+ pos: Point,
+ font: PDFFont,
+ fontsize: float,
+ scaling: float,
+ charspace: float,
+ wordspace: float,
+ rise: float,
+ dxscale: float,
+ ncs: PDFColorSpace,
+ graphicstate: "PDFGraphicState",
+ ) -> Point:
+ (x, y) = pos
+ needcharspace = False
+ for obj in seq:
+ if isinstance(obj, (int, float)):
+ x -= obj * dxscale
+ needcharspace = True
+ elif isinstance(obj, bytes):
+ for cid in font.decode(obj):
+ if needcharspace:
+ x += charspace
+ x += self.render_char(
+ utils.translate_matrix(matrix, (x, y)),
+ font,
+ fontsize,
+ scaling,
+ rise,
+ cid,
+ ncs,
+ graphicstate,
+ )
+ if cid == 32 and wordspace:
+ x += wordspace
+ needcharspace = True
+ else:
+ logger.warning(
+ f"Cannot render horizontal string because {obj!r} is not a valid int, float or bytes."
+ )
+ return (x, y)
+
+ def render_string_vertical(
+ self,
+ seq: PDFTextSeq,
+ matrix: Matrix,
+ pos: Point,
+ font: PDFFont,
+ fontsize: float,
+ scaling: float,
+ charspace: float,
+ wordspace: float,
+ rise: float,
+ dxscale: float,
+ ncs: PDFColorSpace,
+ graphicstate: "PDFGraphicState",
+ ) -> Point:
+ (x, y) = pos
+ needcharspace = False
+ for obj in seq:
+ if isinstance(obj, (int, float)):
+ y -= obj * dxscale
+ needcharspace = True
+ elif isinstance(obj, bytes):
+ for cid in font.decode(obj):
+ if needcharspace:
+ y += charspace
+ y += self.render_char(
+ utils.translate_matrix(matrix, (x, y)),
+ font,
+ fontsize,
+ scaling,
+ rise,
+ cid,
+ ncs,
+ graphicstate,
+ )
+ if cid == 32 and wordspace:
+ y += wordspace
+ needcharspace = True
+ else:
+ logger.warning(
+ f"Cannot render vertical string because {obj!r} is not a valid int, float or bytes."
+ )
+ return (x, y)
+
+ def render_char(
+ self,
+ matrix: Matrix,
+ font: PDFFont,
+ fontsize: float,
+ scaling: float,
+ rise: float,
+ cid: int,
+ ncs: PDFColorSpace,
+ graphicstate: "PDFGraphicState",
+ ) -> float:
+ return 0
+
+
+class TagExtractor(PDFDevice):
+ def __init__(
+ self,
+ rsrcmgr: "PDFResourceManager",
+ outfp: BinaryIO,
+ codec: str = "utf-8",
+ ) -> None:
+ PDFDevice.__init__(self, rsrcmgr)
+ self.outfp = outfp
+ self.codec = codec
+ self.pageno = 0
+ self._stack: list[PSLiteral] = []
+
+ def render_string(
+ self,
+ textstate: "PDFTextState",
+ seq: PDFTextSeq,
+ ncs: PDFColorSpace,
+ graphicstate: "PDFGraphicState",
+ ) -> None:
+ font = textstate.font
+ assert font is not None
+ text = ""
+ for obj in seq:
+ if isinstance(obj, str):
+ obj = utils.make_compat_bytes(obj)
+ if not isinstance(obj, bytes):
+ continue
+ chars = font.decode(obj)
+ for cid in chars:
+ try:
+ char = font.to_unichr(cid)
+ text += char
+ except PDFUnicodeNotDefined:
+ pass
+ self._write(utils.enc(text))
+
+ def begin_page(self, page: PDFPage, ctm: Matrix) -> None:
+ output = '' % (
+ self.pageno,
+ utils.bbox2str(page.mediabox),
+ page.rotate,
+ )
+ self._write(output)
+
+ def end_page(self, page: PDFPage) -> None:
+ self._write("\n")
+ self.pageno += 1
+
+ def begin_tag(self, tag: PSLiteral, props: Optional["PDFStackT"] = None) -> None:
+ s = ""
+ if isinstance(props, dict):
+ s = "".join(
+ [
+ f' {utils.enc(k)}="{utils.make_compat_str(v)}"'
+ for (k, v) in sorted(props.items())
+ ],
+ )
+ out_s = f"<{utils.enc(cast(str, tag.name))}{s}>"
+ self._write(out_s)
+ self._stack.append(tag)
+
+ def end_tag(self) -> None:
+ assert self._stack, str(self.pageno)
+ tag = self._stack.pop(-1)
+ out_s = "" % utils.enc(cast(str, tag.name))
+ self._write(out_s)
+
+ def do_tag(self, tag: PSLiteral, props: Optional["PDFStackT"] = None) -> None:
+ self.begin_tag(tag, props)
+ self._stack.pop(-1)
+
+ def _write(self, s: str) -> None:
+ self.outfp.write(s.encode(self.codec))
diff --git a/babeldoc/pdfminer/pdfdocument.py b/babeldoc/pdfminer/pdfdocument.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c13cd7226d836ae3f06c463f84dcd463ff21964
--- /dev/null
+++ b/babeldoc/pdfminer/pdfdocument.py
@@ -0,0 +1,1072 @@
+import itertools
+import logging
+import re
+import struct
+from collections.abc import Callable
+from collections.abc import Iterable
+from collections.abc import Iterator
+from collections.abc import KeysView
+from collections.abc import Sequence
+from hashlib import md5
+from hashlib import sha256
+from hashlib import sha384
+from hashlib import sha512
+from typing import Any
+from typing import cast
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.ciphers import Cipher
+from cryptography.hazmat.primitives.ciphers import algorithms
+from cryptography.hazmat.primitives.ciphers import modes
+
+from babeldoc.pdfminer.arcfour import Arcfour
+from babeldoc.pdfminer.casting import safe_int
+from babeldoc.pdfminer.data_structures import NumberTree
+from babeldoc.pdfminer.pdfexceptions import PDFException
+from babeldoc.pdfminer.pdfexceptions import PDFKeyError
+from babeldoc.pdfminer.pdfexceptions import PDFObjectNotFound
+from babeldoc.pdfminer.pdfexceptions import PDFTypeError
+from babeldoc.pdfminer.pdfparser import PDFParser
+from babeldoc.pdfminer.pdfparser import PDFStreamParser
+from babeldoc.pdfminer.pdfparser import PDFSyntaxError
+from babeldoc.pdfminer.pdftypes import DecipherCallable
+from babeldoc.pdfminer.pdftypes import PDFStream
+from babeldoc.pdfminer.pdftypes import decipher_all
+from babeldoc.pdfminer.pdftypes import dict_value
+from babeldoc.pdfminer.pdftypes import int_value
+from babeldoc.pdfminer.pdftypes import list_value
+from babeldoc.pdfminer.pdftypes import str_value
+from babeldoc.pdfminer.pdftypes import stream_value
+from babeldoc.pdfminer.pdftypes import uint_value
+from babeldoc.pdfminer.psexceptions import PSEOF
+from babeldoc.pdfminer.psparser import KWD
+from babeldoc.pdfminer.psparser import LIT
+from babeldoc.pdfminer.psparser import literal_name
+from babeldoc.pdfminer.utils import choplist
+from babeldoc.pdfminer.utils import decode_text
+from babeldoc.pdfminer.utils import format_int_alpha
+from babeldoc.pdfminer.utils import format_int_roman
+from babeldoc.pdfminer.utils import nunpack
+from babeldoc.pdfminer import settings
+
+log = logging.getLogger(__name__)
+
+
+class PDFNoValidXRef(PDFSyntaxError):
+ pass
+
+
+class PDFNoValidXRefWarning(SyntaxWarning):
+ """Legacy warning for missing xref.
+
+ Not used anymore because warnings.warn is replaced by logger.Logger.warn.
+ """
+
+
+class PDFNoOutlines(PDFException):
+ pass
+
+
+class PDFNoPageLabels(PDFException):
+ pass
+
+
+class PDFDestinationNotFound(PDFException):
+ pass
+
+
+class PDFEncryptionError(PDFException):
+ pass
+
+
+class PDFPasswordIncorrect(PDFEncryptionError):
+ pass
+
+
+class PDFEncryptionWarning(UserWarning):
+ """Legacy warning for failed decryption.
+
+ Not used anymore because warnings.warn is replaced by logger.Logger.warn.
+ """
+
+
+class PDFTextExtractionNotAllowedWarning(UserWarning):
+ """Legacy warning for PDF that does not allow extraction.
+
+ Not used anymore because warnings.warn is replaced by logger.Logger.warn.
+ """
+
+
+class PDFTextExtractionNotAllowed(PDFEncryptionError):
+ pass
+
+
+# some predefined literals and keywords.
+LITERAL_OBJSTM = LIT("ObjStm")
+LITERAL_XREF = LIT("XRef")
+LITERAL_CATALOG = LIT("Catalog")
+
+
+class PDFBaseXRef:
+ def get_trailer(self) -> dict[str, Any]:
+ raise NotImplementedError
+
+ def get_objids(self) -> Iterable[int]:
+ return []
+
+ # Must return
+ # (strmid, index, genno)
+ # or (None, pos, genno)
+ def get_pos(self, objid: int) -> tuple[int | None, int, int]:
+ raise PDFKeyError(objid)
+
+ def load(self, parser: PDFParser) -> None:
+ raise NotImplementedError
+
+
+class PDFXRef(PDFBaseXRef):
+ def __init__(self) -> None:
+ self.offsets: dict[int, tuple[int | None, int, int]] = {}
+ self.trailer: dict[str, Any] = {}
+
+ def __repr__(self) -> str:
+ return "" % (self.offsets.keys())
+
+ def load(self, parser: PDFParser) -> None:
+ while True:
+ try:
+ (pos, line) = parser.nextline()
+ line = line.strip()
+ if not line:
+ continue
+ except PSEOF:
+ raise PDFNoValidXRef("Unexpected EOF - file corrupted?")
+ if line.startswith(b"trailer"):
+ parser.seek(pos)
+ break
+ f = line.split(b" ")
+ if len(f) != 2:
+ error_msg = f"Trailer not found: {parser!r}: line={line!r}"
+ raise PDFNoValidXRef(error_msg)
+ try:
+ (start, nobjs) = map(int, f)
+ except ValueError:
+ error_msg = f"Invalid line: {parser!r}: line={line!r}"
+ raise PDFNoValidXRef(error_msg)
+ for objid in range(start, start + nobjs):
+ try:
+ (_, line) = parser.nextline()
+ line = line.strip()
+ except PSEOF:
+ raise PDFNoValidXRef("Unexpected EOF - file corrupted?")
+ f = line.split(b" ")
+ if len(f) != 3:
+ error_msg = f"Invalid XRef format: {parser!r}, line={line!r}"
+ raise PDFNoValidXRef(error_msg)
+ (pos_b, genno_b, use_b) = f
+ if use_b != b"n":
+ continue
+
+ pos_i = safe_int(pos_b)
+ genno_i = safe_int(genno_b)
+ if pos_i is not None and genno_i is not None:
+ self.offsets[objid] = (None, pos_i, genno_i)
+ else:
+ log.warning(
+ f"Not adding object {objid} to xref because position {pos_b!r} "
+ f"or generation number {genno_b!r} cannot be parsed as an int"
+ )
+
+ log.debug("xref objects: %r", self.offsets)
+ self.load_trailer(parser)
+
+ def load_trailer(self, parser: PDFParser) -> None:
+ try:
+ (_, kwd) = parser.nexttoken()
+ assert kwd is KWD(b"trailer"), str(kwd)
+ (_, dic) = parser.nextobject()
+ except PSEOF:
+ x = parser.pop(1)
+ if not x:
+ raise PDFNoValidXRef("Unexpected EOF - file corrupted")
+ (_, dic) = x[0]
+ self.trailer.update(dict_value(dic))
+ log.debug("trailer=%r", self.trailer)
+
+ def get_trailer(self) -> dict[str, Any]:
+ return self.trailer
+
+ def get_objids(self) -> KeysView[int]:
+ return self.offsets.keys()
+
+ def get_pos(self, objid: int) -> tuple[int | None, int, int]:
+ return self.offsets[objid]
+
+
+class PDFXRefFallback(PDFXRef):
+ def __repr__(self) -> str:
+ return "" % (self.offsets.keys())
+
+ PDFOBJ_CUE = re.compile(r"^(\d+)\s+(\d+)\s+obj\b")
+
+ def load(self, parser: PDFParser) -> None:
+ parser.seek(0)
+ while 1:
+ try:
+ (pos, line_bytes) = parser.nextline()
+ except PSEOF:
+ break
+ if line_bytes.startswith(b"trailer"):
+ parser.seek(pos)
+ self.load_trailer(parser)
+ log.debug("trailer: %r", self.trailer)
+ break
+ line = line_bytes.decode("latin-1") # default pdf encoding
+ m = self.PDFOBJ_CUE.match(line)
+ if not m:
+ continue
+ (objid_s, genno_s) = m.groups()
+ objid = int(objid_s)
+ genno = int(genno_s)
+ self.offsets[objid] = (None, pos, genno)
+ # expand ObjStm.
+ parser.seek(pos)
+ (_, obj) = parser.nextobject()
+ if isinstance(obj, PDFStream) and obj.get("Type") is LITERAL_OBJSTM:
+ stream = stream_value(obj)
+ try:
+ n = stream["N"]
+ except KeyError:
+ if settings.STRICT:
+ raise PDFSyntaxError("N is not defined: %r" % stream)
+ n = 0
+ parser1 = PDFStreamParser(stream.get_data())
+ objs: list[int] = []
+ try:
+ while 1:
+ (_, obj) = parser1.nextobject()
+ objs.append(cast(int, obj))
+ except PSEOF:
+ pass
+ n = min(n, len(objs) // 2)
+ for index in range(n):
+ objid1 = objs[index * 2]
+ self.offsets[objid1] = (objid, index, 0)
+
+
+class PDFXRefStream(PDFBaseXRef):
+ def __init__(self) -> None:
+ self.data: bytes | None = None
+ self.entlen: int | None = None
+ self.fl1: int | None = None
+ self.fl2: int | None = None
+ self.fl3: int | None = None
+ self.ranges: list[tuple[int, int]] = []
+
+ def __repr__(self) -> str:
+ return "" % (self.ranges)
+
+ def load(self, parser: PDFParser) -> None:
+ (_, objid) = parser.nexttoken() # ignored
+ (_, genno) = parser.nexttoken() # ignored
+ (_, kwd) = parser.nexttoken()
+ (_, stream) = parser.nextobject()
+ if not isinstance(stream, PDFStream) or stream.get("Type") is not LITERAL_XREF:
+ raise PDFNoValidXRef("Invalid PDF stream spec.")
+ size = stream["Size"]
+ index_array = stream.get("Index", (0, size))
+ if len(index_array) % 2 != 0:
+ raise PDFSyntaxError("Invalid index number")
+ self.ranges.extend(cast(Iterator[tuple[int, int]], choplist(2, index_array)))
+ (self.fl1, self.fl2, self.fl3) = stream["W"]
+ assert self.fl1 is not None and self.fl2 is not None and self.fl3 is not None
+ self.data = stream.get_data()
+ self.entlen = self.fl1 + self.fl2 + self.fl3
+ self.trailer = stream.attrs
+ log.debug(
+ "xref stream: objid=%s, fields=%d,%d,%d",
+ ", ".join(map(repr, self.ranges)),
+ self.fl1,
+ self.fl2,
+ self.fl3,
+ )
+
+ def get_trailer(self) -> dict[str, Any]:
+ return self.trailer
+
+ def get_objids(self) -> Iterator[int]:
+ for start, nobjs in self.ranges:
+ for i in range(nobjs):
+ assert self.entlen is not None
+ assert self.data is not None
+ offset = self.entlen * i
+ ent = self.data[offset : offset + self.entlen]
+ f1 = nunpack(ent[: self.fl1], 1)
+ if f1 == 1 or f1 == 2:
+ yield start + i
+
+ def get_pos(self, objid: int) -> tuple[int | None, int, int]:
+ index = 0
+ for start, nobjs in self.ranges:
+ if start <= objid and objid < start + nobjs:
+ index += objid - start
+ break
+ else:
+ index += nobjs
+ else:
+ raise PDFKeyError(objid)
+ assert self.entlen is not None
+ assert self.data is not None
+ assert self.fl1 is not None and self.fl2 is not None and self.fl3 is not None
+ offset = self.entlen * index
+ ent = self.data[offset : offset + self.entlen]
+ f1 = nunpack(ent[: self.fl1], 1)
+ f2 = nunpack(ent[self.fl1 : self.fl1 + self.fl2])
+ f3 = nunpack(ent[self.fl1 + self.fl2 :])
+ if f1 == 1:
+ return (None, f2, f3)
+ elif f1 == 2:
+ return (f2, f3, 0)
+ else:
+ # this is a free object
+ raise PDFKeyError(objid)
+
+
+class PDFStandardSecurityHandler:
+ PASSWORD_PADDING = (
+ b"(\xbfN^Nu\x8aAd\x00NV\xff\xfa\x01\x08..\x00\xb6\xd0h>\x80/\x0c\xa9\xfedSiz"
+ )
+ supported_revisions: tuple[int, ...] = (2, 3)
+
+ def __init__(
+ self,
+ docid: Sequence[bytes],
+ param: dict[str, Any],
+ password: str = "",
+ ) -> None:
+ self.docid = docid
+ self.param = param
+ self.password = password
+ self.init()
+
+ def init(self) -> None:
+ self.init_params()
+ if self.r not in self.supported_revisions:
+ error_msg = "Unsupported revision: param=%r" % self.param
+ raise PDFEncryptionError(error_msg)
+ self.init_key()
+
+ def init_params(self) -> None:
+ self.v = int_value(self.param.get("V", 0))
+ self.r = int_value(self.param["R"])
+ self.p = uint_value(self.param["P"], 32)
+ self.o = str_value(self.param["O"])
+ self.u = str_value(self.param["U"])
+ self.length = int_value(self.param.get("Length", 40))
+
+ def init_key(self) -> None:
+ self.key = self.authenticate(self.password)
+ if self.key is None:
+ raise PDFPasswordIncorrect
+
+ def is_printable(self) -> bool:
+ return bool(self.p & 4)
+
+ def is_modifiable(self) -> bool:
+ return bool(self.p & 8)
+
+ def is_extractable(self) -> bool:
+ return bool(self.p & 16)
+
+ def compute_u(self, key: bytes) -> bytes:
+ if self.r == 2:
+ # Algorithm 3.4
+ return Arcfour(key).encrypt(self.PASSWORD_PADDING) # 2
+ else:
+ # Algorithm 3.5
+ hash = md5(self.PASSWORD_PADDING) # 2
+ hash.update(self.docid[0]) # 3
+ result = Arcfour(key).encrypt(hash.digest()) # 4
+ for i in range(1, 20): # 5
+ k = b"".join(bytes((c ^ i,)) for c in iter(key))
+ result = Arcfour(k).encrypt(result)
+ result += result # 6
+ return result
+
+ def compute_encryption_key(self, password: bytes) -> bytes:
+ # Algorithm 3.2
+ password = (password + self.PASSWORD_PADDING)[:32] # 1
+ hash = md5(password) # 2
+ hash.update(self.o) # 3
+ # See https://github.com/pdfminer/pdfminer.six/issues/186
+ hash.update(struct.pack("= 4:
+ if not cast(PDFStandardSecurityHandlerV4, self).encrypt_metadata:
+ hash.update(b"\xff\xff\xff\xff")
+ result = hash.digest()
+ n = 5
+ if self.r >= 3:
+ n = self.length // 8
+ for _ in range(50):
+ result = md5(result[:n]).digest()
+ return result[:n]
+
+ def authenticate(self, password: str) -> bytes | None:
+ password_bytes = password.encode("latin1")
+ key = self.authenticate_user_password(password_bytes)
+ if key is None:
+ key = self.authenticate_owner_password(password_bytes)
+ return key
+
+ def authenticate_user_password(self, password: bytes) -> bytes | None:
+ key = self.compute_encryption_key(password)
+ if self.verify_encryption_key(key):
+ return key
+ else:
+ return None
+
+ def verify_encryption_key(self, key: bytes) -> bool:
+ # Algorithm 3.6
+ u = self.compute_u(key)
+ if self.r == 2:
+ return u == self.u
+ return u[:16] == self.u[:16]
+
+ def authenticate_owner_password(self, password: bytes) -> bytes | None:
+ # Algorithm 3.7
+ password = (password + self.PASSWORD_PADDING)[:32]
+ hash = md5(password)
+ if self.r >= 3:
+ for _ in range(50):
+ hash = md5(hash.digest())
+ n = 5
+ if self.r >= 3:
+ n = self.length // 8
+ key = hash.digest()[:n]
+ if self.r == 2:
+ user_password = Arcfour(key).decrypt(self.o)
+ else:
+ user_password = self.o
+ for i in range(19, -1, -1):
+ k = b"".join(bytes((c ^ i,)) for c in iter(key))
+ user_password = Arcfour(k).decrypt(user_password)
+ return self.authenticate_user_password(user_password)
+
+ def decrypt(
+ self,
+ objid: int,
+ genno: int,
+ data: bytes,
+ attrs: dict[str, Any] | None = None,
+ ) -> bytes:
+ return self.decrypt_rc4(objid, genno, data)
+
+ def decrypt_rc4(self, objid: int, genno: int, data: bytes) -> bytes:
+ assert self.key is not None
+ key = self.key + struct.pack(" None:
+ super().init_params()
+ self.length = 128
+ self.cf = dict_value(self.param.get("CF"))
+ self.stmf = literal_name(self.param["StmF"])
+ self.strf = literal_name(self.param["StrF"])
+ self.encrypt_metadata = bool(self.param.get("EncryptMetadata", True))
+ if self.stmf != self.strf:
+ error_msg = "Unsupported crypt filter: param=%r" % self.param
+ raise PDFEncryptionError(error_msg)
+ self.cfm = {}
+ for k, v in self.cf.items():
+ f = self.get_cfm(literal_name(v["CFM"]))
+ if f is None:
+ error_msg = "Unknown crypt filter method: param=%r" % self.param
+ raise PDFEncryptionError(error_msg)
+ self.cfm[k] = f
+ self.cfm["Identity"] = self.decrypt_identity
+ if self.strf not in self.cfm:
+ error_msg = "Undefined crypt filter: param=%r" % self.param
+ raise PDFEncryptionError(error_msg)
+
+ def get_cfm(self, name: str) -> Callable[[int, int, bytes], bytes] | None:
+ if name == "V2":
+ return self.decrypt_rc4
+ elif name == "AESV2":
+ return self.decrypt_aes128
+ else:
+ return None
+
+ def decrypt(
+ self,
+ objid: int,
+ genno: int,
+ data: bytes,
+ attrs: dict[str, Any] | None = None,
+ name: str | None = None,
+ ) -> bytes:
+ if not self.encrypt_metadata and attrs is not None:
+ t = attrs.get("Type")
+ if t is not None and literal_name(t) == "Metadata":
+ return data
+ if name is None:
+ name = self.strf
+ return self.cfm[name](objid, genno, data)
+
+ def decrypt_identity(self, objid: int, genno: int, data: bytes) -> bytes:
+ return data
+
+ def decrypt_aes128(self, objid: int, genno: int, data: bytes) -> bytes:
+ assert self.key is not None
+ key = (
+ self.key
+ + struct.pack(" None:
+ super().init_params()
+ self.length = 256
+ self.oe = str_value(self.param["OE"])
+ self.ue = str_value(self.param["UE"])
+ self.o_hash = self.o[:32]
+ self.o_validation_salt = self.o[32:40]
+ self.o_key_salt = self.o[40:]
+ self.u_hash = self.u[:32]
+ self.u_validation_salt = self.u[32:40]
+ self.u_key_salt = self.u[40:]
+
+ def get_cfm(self, name: str) -> Callable[[int, int, bytes], bytes] | None:
+ if name == "AESV3":
+ return self.decrypt_aes256
+ else:
+ return None
+
+ def authenticate(self, password: str) -> bytes | None:
+ password_b = self._normalize_password(password)
+ hash = self._password_hash(password_b, self.o_validation_salt, self.u)
+ if hash == self.o_hash:
+ hash = self._password_hash(password_b, self.o_key_salt, self.u)
+ cipher = Cipher(
+ algorithms.AES(hash),
+ modes.CBC(b"\0" * 16),
+ backend=default_backend(),
+ ) # type: ignore
+ return cipher.decryptor().update(self.oe) # type: ignore
+ hash = self._password_hash(password_b, self.u_validation_salt)
+ if hash == self.u_hash:
+ hash = self._password_hash(password_b, self.u_key_salt)
+ cipher = Cipher(
+ algorithms.AES(hash),
+ modes.CBC(b"\0" * 16),
+ backend=default_backend(),
+ ) # type: ignore
+ return cipher.decryptor().update(self.ue) # type: ignore
+ return None
+
+ def _normalize_password(self, password: str) -> bytes:
+ if self.r == 6:
+ # saslprep expects non-empty strings, apparently
+ if not password:
+ return b""
+ from babeldoc.pdfminer._saslprep import saslprep
+
+ password = saslprep(password)
+ return password.encode("utf-8")[:127]
+
+ def _password_hash(
+ self,
+ password: bytes,
+ salt: bytes,
+ vector: bytes | None = None,
+ ) -> bytes:
+ """Compute password hash depending on revision number"""
+ if self.r == 5:
+ return self._r5_password(password, salt, vector)
+ return self._r6_password(password, salt[0:8], vector)
+
+ def _r5_password(
+ self,
+ password: bytes,
+ salt: bytes,
+ vector: bytes | None = None,
+ ) -> bytes:
+ """Compute the password for revision 5"""
+ hash = sha256(password)
+ hash.update(salt)
+ if vector is not None:
+ hash.update(vector)
+ return hash.digest()
+
+ def _r6_password(
+ self,
+ password: bytes,
+ salt: bytes,
+ vector: bytes | None = None,
+ ) -> bytes:
+ """Compute the password for revision 6"""
+ initial_hash = sha256(password)
+ initial_hash.update(salt)
+ if vector is not None:
+ initial_hash.update(vector)
+ k = initial_hash.digest()
+ hashes = (sha256, sha384, sha512)
+ round_no = last_byte_val = 0
+ while round_no < 64 or last_byte_val > round_no - 32:
+ k1 = (password + k + (vector or b"")) * 64
+ e = self._aes_cbc_encrypt(key=k[:16], iv=k[16:32], data=k1)
+ # compute the first 16 bytes of e,
+ # interpreted as an unsigned integer mod 3
+ next_hash = hashes[self._bytes_mod_3(e[:16])]
+ k = next_hash(e).digest()
+ last_byte_val = e[len(e) - 1]
+ round_no += 1
+ return k[:32]
+
+ @staticmethod
+ def _bytes_mod_3(input_bytes: bytes) -> int:
+ # 256 is 1 mod 3, so we can just sum 'em
+ return sum(b % 3 for b in input_bytes) % 3
+
+ def _aes_cbc_encrypt(self, key: bytes, iv: bytes, data: bytes) -> bytes:
+ cipher = Cipher(algorithms.AES(key), modes.CBC(iv))
+ encryptor = cipher.encryptor() # type: ignore
+ return encryptor.update(data) + encryptor.finalize() # type: ignore
+
+ def decrypt_aes256(self, objid: int, genno: int, data: bytes) -> bytes:
+ initialization_vector = data[:16]
+ ciphertext = data[16:]
+ assert self.key is not None
+ cipher = Cipher(
+ algorithms.AES(self.key),
+ modes.CBC(initialization_vector),
+ backend=default_backend(),
+ ) # type: ignore
+ return cipher.decryptor().update(ciphertext) # type: ignore
+
+
+class PDFDocument:
+ """PDFDocument object represents a PDF document.
+
+ Since a PDF file can be very big, normally it is not loaded at
+ once. So PDF document has to cooperate with a PDF parser in order to
+ dynamically import the data as processing goes.
+
+ Typical usage:
+ doc = PDFDocument(parser, password)
+ obj = doc.getobj(objid)
+
+ """
+
+ security_handler_registry: dict[int, type[PDFStandardSecurityHandler]] = {
+ 1: PDFStandardSecurityHandler,
+ 2: PDFStandardSecurityHandler,
+ 4: PDFStandardSecurityHandlerV4,
+ 5: PDFStandardSecurityHandlerV5,
+ }
+
+ def __init__(
+ self,
+ parser: PDFParser,
+ password: str = "",
+ caching: bool = True,
+ fallback: bool = True,
+ ) -> None:
+ """Set the document to use a given PDFParser object."""
+ self.caching = caching
+ self.xrefs: list[PDFBaseXRef] = []
+ self.info = []
+ self.catalog: dict[str, Any] = {}
+ self.encryption: tuple[Any, Any] | None = None
+ self.decipher: DecipherCallable | None = None
+ self._parser = None
+ self._cached_objs: dict[int, tuple[object, int]] = {}
+ self._parsed_objs: dict[int, tuple[list[object], int]] = {}
+ self._parser = parser
+ self._parser.set_document(self)
+ self.is_printable = self.is_modifiable = self.is_extractable = True
+ # Retrieve the information of each header that was appended
+ # (maybe multiple times) at the end of the document.
+ try:
+ pos = self.find_xref(parser)
+ self.read_xref_from(parser, pos, self.xrefs)
+ except PDFNoValidXRef:
+ if fallback:
+ parser.fallback = True
+ newxref = PDFXRefFallback()
+ newxref.load(parser)
+ self.xrefs.append(newxref)
+
+ for xref in self.xrefs:
+ trailer = xref.get_trailer()
+ if not trailer:
+ continue
+ # If there's an encryption info, remember it.
+ if "Encrypt" in trailer:
+ if "ID" in trailer:
+ id_value = list_value(trailer["ID"])
+ else:
+ # Some documents may not have a /ID, use two empty
+ # byte strings instead. Solves
+ # https://github.com/pdfminer/pdfminer.six/issues/594
+ id_value = (b"", b"")
+ self.encryption = (id_value, dict_value(trailer["Encrypt"]))
+ self._initialize_password(password)
+ if "Info" in trailer:
+ self.info.append(dict_value(trailer["Info"]))
+ if "Root" in trailer:
+ # Every PDF file must have exactly one /Root dictionary.
+ self.catalog = dict_value(trailer["Root"])
+ break
+ else:
+ raise PDFSyntaxError("No /Root object! - Is this really a PDF?")
+ if self.catalog.get("Type") is not LITERAL_CATALOG:
+ if settings.STRICT:
+ raise PDFSyntaxError("Catalog not found!")
+
+ KEYWORD_OBJ = KWD(b"obj")
+
+ # _initialize_password(password=b'')
+ # Perform the initialization with a given password.
+ def _initialize_password(self, password: str = "") -> None:
+ assert self.encryption is not None
+ (docid, param) = self.encryption
+ if literal_name(param.get("Filter")) != "Standard":
+ raise PDFEncryptionError("Unknown filter: param=%r" % param)
+ v = int_value(param.get("V", 0))
+ factory = self.security_handler_registry.get(v)
+ if factory is None:
+ raise PDFEncryptionError("Unknown algorithm: param=%r" % param)
+ handler = factory(docid, param, password)
+ self.decipher = handler.decrypt
+ self.is_printable = handler.is_printable()
+ self.is_modifiable = handler.is_modifiable()
+ self.is_extractable = handler.is_extractable()
+ assert self._parser is not None
+ self._parser.fallback = False # need to read streams with exact length
+
+ def _getobj_objstm(self, stream: PDFStream, index: int, objid: int) -> object:
+ if stream.objid in self._parsed_objs:
+ (objs, n) = self._parsed_objs[stream.objid]
+ else:
+ (objs, n) = self._get_objects(stream)
+ if self.caching:
+ assert stream.objid is not None
+ self._parsed_objs[stream.objid] = (objs, n)
+ i = n * 2 + index
+ try:
+ obj = objs[i]
+ except IndexError:
+ raise PDFSyntaxError("index too big: %r" % index)
+ return obj
+
+ def _get_objects(self, stream: PDFStream) -> tuple[list[object], int]:
+ if stream.get("Type") is not LITERAL_OBJSTM:
+ if settings.STRICT:
+ raise PDFSyntaxError("Not a stream object: %r" % stream)
+ try:
+ n = cast(int, stream["N"])
+ except KeyError:
+ if settings.STRICT:
+ raise PDFSyntaxError("N is not defined: %r" % stream)
+ n = 0
+ parser = PDFStreamParser(stream.get_data())
+ parser.set_document(self)
+ objs: list[object] = []
+ try:
+ while 1:
+ (_, obj) = parser.nextobject()
+ objs.append(obj)
+ except PSEOF:
+ pass
+ return (objs, n)
+
+ def _getobj_parse(self, pos: int, objid: int) -> object:
+ assert self._parser is not None
+ self._parser.seek(pos)
+ (_, objid1) = self._parser.nexttoken() # objid
+ (_, genno) = self._parser.nexttoken() # genno
+ (_, kwd) = self._parser.nexttoken()
+ # hack around malformed pdf files
+ # copied from https://github.com/jaepil/pdfminer3k/blob/master/
+ # pdfminer/pdfparser.py#L399
+ # to solve https://github.com/pdfminer/pdfminer.six/issues/56
+ # assert objid1 == objid, str((objid1, objid))
+ if objid1 != objid:
+ x = []
+ while kwd is not self.KEYWORD_OBJ:
+ (_, kwd) = self._parser.nexttoken()
+ x.append(kwd)
+ if len(x) >= 2:
+ objid1 = x[-2]
+ # #### end hack around malformed pdf files
+ if objid1 != objid:
+ raise PDFSyntaxError(f"objid mismatch: {objid1!r}={objid!r}")
+
+ if kwd != KWD(b"obj"):
+ raise PDFSyntaxError("Invalid object spec: offset=%r" % pos)
+ (_, obj) = self._parser.nextobject()
+ return obj
+
+ # can raise PDFObjectNotFound
+ def getobj(self, objid: int) -> object:
+ """Get object from PDF
+
+ :raises PDFException if PDFDocument is not initialized
+ :raises PDFObjectNotFound if objid does not exist in PDF
+ """
+ if not self.xrefs:
+ raise PDFException("PDFDocument is not initialized")
+ log.debug("getobj: objid=%r", objid)
+ if objid in self._cached_objs:
+ (obj, genno) = self._cached_objs[objid]
+ else:
+ for xref in self.xrefs:
+ try:
+ (strmid, index, genno) = xref.get_pos(objid)
+ except KeyError:
+ continue
+ try:
+ if strmid is not None:
+ stream = stream_value(self.getobj(strmid))
+ obj = self._getobj_objstm(stream, index, objid)
+ else:
+ obj = self._getobj_parse(index, objid)
+ if self.decipher:
+ obj = decipher_all(self.decipher, objid, genno, obj)
+
+ if isinstance(obj, PDFStream):
+ obj.set_objid(objid, genno)
+ break
+ except (PSEOF, PDFSyntaxError):
+ continue
+ else:
+ raise PDFObjectNotFound(objid)
+ log.debug("register: objid=%r: %r", objid, obj)
+ if self.caching:
+ self._cached_objs[objid] = (obj, genno)
+ return obj
+
+ OutlineType = tuple[Any, Any, Any, Any, Any]
+
+ def get_outlines(self) -> Iterator[OutlineType]:
+ if "Outlines" not in self.catalog:
+ raise PDFNoOutlines
+
+ def search(entry: object, level: int) -> Iterator[PDFDocument.OutlineType]:
+ entry = dict_value(entry)
+ if "Title" in entry:
+ if "A" in entry or "Dest" in entry:
+ title = decode_text(str_value(entry["Title"]))
+ dest = entry.get("Dest")
+ action = entry.get("A")
+ se = entry.get("SE")
+ yield (level, title, dest, action, se)
+ if "First" in entry and "Last" in entry:
+ yield from search(entry["First"], level + 1)
+ if "Next" in entry:
+ yield from search(entry["Next"], level)
+
+ return search(self.catalog["Outlines"], 0)
+
+ def get_page_labels(self) -> Iterator[str]:
+ """Generate page label strings for the PDF document.
+
+ If the document includes page labels, generates strings, one per page.
+ If not, raises PDFNoPageLabels.
+
+ The resulting iteration is unbounded.
+ """
+ assert self.catalog is not None
+
+ try:
+ page_labels = PageLabels(self.catalog["PageLabels"])
+ except (PDFTypeError, KeyError):
+ raise PDFNoPageLabels
+
+ return page_labels.labels
+
+ def lookup_name(self, cat: str, key: str | bytes) -> Any:
+ try:
+ names = dict_value(self.catalog["Names"])
+ except (PDFTypeError, KeyError):
+ raise PDFKeyError((cat, key))
+ # may raise KeyError
+ d0 = dict_value(names[cat])
+
+ def lookup(d: dict[str, Any]) -> Any:
+ if "Limits" in d:
+ (k1, k2) = list_value(d["Limits"])
+ if key < k1 or k2 < key:
+ return None
+ if "Names" in d:
+ objs = list_value(d["Names"])
+ names = dict(
+ cast(Iterator[tuple[str | bytes, Any]], choplist(2, objs)),
+ )
+ return names[key]
+ if "Kids" in d:
+ for c in list_value(d["Kids"]):
+ v = lookup(dict_value(c))
+ if v:
+ return v
+ raise PDFKeyError((cat, key))
+
+ return lookup(d0)
+
+ def get_dest(self, name: str | bytes) -> Any:
+ try:
+ # PDF-1.2 or later
+ obj = self.lookup_name("Dests", name)
+ except KeyError:
+ # PDF-1.1 or prior
+ if "Dests" not in self.catalog:
+ raise PDFDestinationNotFound(name)
+ d0 = dict_value(self.catalog["Dests"])
+ if name not in d0:
+ raise PDFDestinationNotFound(name)
+ obj = d0[name]
+ return obj
+
+ # find_xref
+ def find_xref(self, parser: PDFParser) -> int:
+ """Internal function used to locate the first XRef."""
+ # search the last xref table by scanning the file backwards.
+ prev = b""
+ for line in parser.revreadlines():
+ line = line.strip()
+ log.debug("find_xref: %r", line)
+
+ if line == b"startxref":
+ log.debug("xref found: pos=%r", prev)
+
+ if not prev.isdigit():
+ raise PDFNoValidXRef(f"Invalid xref position: {prev!r}")
+
+ start = int(prev)
+
+ if not start >= 0:
+ raise PDFNoValidXRef(f"Invalid negative xref position: {start}")
+
+ return start
+
+ if line:
+ prev = line
+
+ raise PDFNoValidXRef("Unexpected EOF")
+
+ # read xref table
+ def read_xref_from(
+ self,
+ parser: PDFParser,
+ start: int,
+ xrefs: list[PDFBaseXRef],
+ ) -> None:
+ """Reads XRefs from the given location."""
+ parser.seek(start)
+ parser.reset()
+ try:
+ (pos, token) = parser.nexttoken()
+ except PSEOF:
+ raise PDFNoValidXRef("Unexpected EOF")
+ log.debug("read_xref_from: start=%d, token=%r", start, token)
+ if isinstance(token, int):
+ # XRefStream: PDF-1.5
+ parser.seek(pos)
+ parser.reset()
+ xref: PDFBaseXRef = PDFXRefStream()
+ xref.load(parser)
+ else:
+ if token is parser.KEYWORD_XREF:
+ parser.nextline()
+ xref = PDFXRef()
+ xref.load(parser)
+ xrefs.append(xref)
+ trailer = xref.get_trailer()
+ log.debug("trailer: %r", trailer)
+ if "XRefStm" in trailer:
+ pos = int_value(trailer["XRefStm"])
+ self.read_xref_from(parser, pos, xrefs)
+ if "Prev" in trailer:
+ # find previous xref
+ pos = int_value(trailer["Prev"])
+ self.read_xref_from(parser, pos, xrefs)
+
+
+class PageLabels(NumberTree):
+ """PageLabels from the document catalog.
+
+ See Section 8.3.1 in the PDF Reference.
+ """
+
+ @property
+ def labels(self) -> Iterator[str]:
+ ranges = self.values
+
+ # The tree must begin with page index 0
+ if len(ranges) == 0 or ranges[0][0] != 0:
+ if settings.STRICT:
+ raise PDFSyntaxError("PageLabels is missing page index 0")
+ else:
+ # Try to cope, by assuming empty labels for the initial pages
+ ranges.insert(0, (0, {}))
+
+ for next, (start, label_dict_unchecked) in enumerate(ranges, 1):
+ label_dict = dict_value(label_dict_unchecked)
+ style = label_dict.get("S")
+ prefix = decode_text(str_value(label_dict.get("P", b"")))
+ first_value = int_value(label_dict.get("St", 1))
+
+ if next == len(ranges):
+ # This is the last specified range. It continues until the end
+ # of the document.
+ values: Iterable[int] = itertools.count(first_value)
+ else:
+ end, _ = ranges[next]
+ range_length = end - start
+ values = range(first_value, first_value + range_length)
+
+ for value in values:
+ label = self._format_page_label(value, style)
+ yield prefix + label
+
+ @staticmethod
+ def _format_page_label(value: int, style: Any) -> str:
+ """Format page label value in a specific style"""
+ if style is None:
+ label = ""
+ elif style is LIT("D"): # Decimal arabic numerals
+ label = str(value)
+ elif style is LIT("R"): # Uppercase roman numerals
+ label = format_int_roman(value).upper()
+ elif style is LIT("r"): # Lowercase roman numerals
+ label = format_int_roman(value)
+ elif style is LIT("A"): # Uppercase letters A-Z, AA-ZZ...
+ label = format_int_alpha(value).upper()
+ elif style is LIT("a"): # Lowercase letters a-z, aa-zz...
+ label = format_int_alpha(value)
+ else:
+ log.warning("Unknown page label style: %r", style)
+ label = ""
+ return label
diff --git a/babeldoc/pdfminer/pdfexceptions.py b/babeldoc/pdfminer/pdfexceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2e86eea09071edfc039890c350cb9bca67895a2
--- /dev/null
+++ b/babeldoc/pdfminer/pdfexceptions.py
@@ -0,0 +1,33 @@
+from babeldoc.pdfminer.psexceptions import PSException
+
+
+class PDFException(PSException):
+ pass
+
+
+class PDFTypeError(PDFException, TypeError):
+ pass
+
+
+class PDFValueError(PDFException, ValueError):
+ pass
+
+
+class PDFObjectNotFound(PDFException):
+ pass
+
+
+class PDFNotImplementedError(PDFException, NotImplementedError):
+ pass
+
+
+class PDFKeyError(PDFException, KeyError):
+ pass
+
+
+class PDFEOFError(PDFException, EOFError):
+ pass
+
+
+class PDFIOError(PDFException, IOError):
+ pass
diff --git a/babeldoc/pdfminer/pdffont.py b/babeldoc/pdfminer/pdffont.py
new file mode 100644
index 0000000000000000000000000000000000000000..59fc1cec7d712c269dbed87edc5245d038ea7aa0
--- /dev/null
+++ b/babeldoc/pdfminer/pdffont.py
@@ -0,0 +1,1137 @@
+import logging
+import struct
+from collections.abc import Iterable
+from collections.abc import Iterator
+from collections.abc import Mapping
+from io import BytesIO
+from typing import TYPE_CHECKING
+from typing import Any
+from typing import BinaryIO
+from typing import cast
+import freetype
+
+from babeldoc.pdfminer.casting import safe_float
+from babeldoc.pdfminer.casting import safe_rect_list
+from babeldoc.pdfminer.cmapdb import CMap
+from babeldoc.pdfminer.cmapdb import CMapBase
+from babeldoc.pdfminer.cmapdb import CMapDB
+from babeldoc.pdfminer.cmapdb import CMapParser
+from babeldoc.pdfminer.cmapdb import FileUnicodeMap
+from babeldoc.pdfminer.cmapdb import IdentityUnicodeMap
+from babeldoc.pdfminer.cmapdb import UnicodeMap
+from babeldoc.pdfminer.encodingdb import EncodingDB
+from babeldoc.pdfminer.encodingdb import name2unicode
+from babeldoc.pdfminer.fontmetrics import FONT_METRICS
+from babeldoc.pdfminer.pdfexceptions import PDFException
+from babeldoc.pdfminer.pdfexceptions import PDFKeyError
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+from babeldoc.pdfminer.pdftypes import PDFStream
+from babeldoc.pdfminer.pdftypes import dict_value
+from babeldoc.pdfminer.pdftypes import int_value
+from babeldoc.pdfminer.pdftypes import list_value
+from babeldoc.pdfminer.pdftypes import num_value
+from babeldoc.pdfminer.pdftypes import resolve1
+from babeldoc.pdfminer.pdftypes import resolve_all
+from babeldoc.pdfminer.pdftypes import stream_value
+from babeldoc.pdfminer.psexceptions import PSEOF
+from babeldoc.pdfminer.psparser import KWD
+from babeldoc.pdfminer.psparser import LIT
+from babeldoc.pdfminer.psparser import PSKeyword
+from babeldoc.pdfminer.psparser import PSLiteral
+from babeldoc.pdfminer.psparser import PSStackParser
+from babeldoc.pdfminer.psparser import literal_name
+from babeldoc.pdfminer.utils import Matrix
+from babeldoc.pdfminer.utils import Point
+from babeldoc.pdfminer.utils import Rect
+from babeldoc.pdfminer.utils import apply_matrix_norm
+from babeldoc.pdfminer.utils import choplist
+from babeldoc.pdfminer.utils import nunpack
+from babeldoc.pdfminer import settings
+
+if TYPE_CHECKING:
+ from babeldoc.pdfminer.pdfinterp import PDFResourceManager
+
+log = logging.getLogger(__name__)
+
+
+def get_widths(seq: Iterable[object]) -> dict[str | int, float]:
+ """Build a mapping of character widths for horizontal writing."""
+ widths: dict[int, float] = {}
+ r: list[float] = []
+ for v in seq:
+ v = resolve1(v)
+ if isinstance(v, list):
+ if r:
+ char1 = r[-1]
+ for i, w in enumerate(v):
+ widths[cast(int, char1) + i] = w
+ r = []
+ elif isinstance(v, (int, float)): # == utils.isnumber(v)
+ r.append(v)
+ if len(r) == 3:
+ (char1, char2, w) = r
+ if isinstance(char1, int) and isinstance(char2, int):
+ for i in range(cast(int, char1), cast(int, char2) + 1):
+ widths[i] = w
+ else:
+ log.warning(
+ f"Skipping invalid font width specification for {char1} to {char2} because either of them is not an int"
+ )
+ r = []
+ else:
+ log.warning(
+ f"Skipping invalid font width specification for {v} because it is not a number or a list"
+ )
+ return cast(dict[str | int, float], widths)
+
+
+def get_widths2(seq: Iterable[object]) -> dict[int, tuple[float, Point]]:
+ """Build a mapping of character widths for vertical writing."""
+ widths: dict[int, tuple[float, Point]] = {}
+ r: list[float] = []
+ for v in seq:
+ if isinstance(v, list):
+ if r:
+ char1 = r[-1]
+ for i, (w, vx, vy) in enumerate(choplist(3, v)):
+ widths[cast(int, char1) + i] = (w, (vx, vy))
+ r = []
+ elif isinstance(v, (int, float)): # == utils.isnumber(v)
+ r.append(v)
+ if len(r) == 5:
+ (char1, char2, w, vx, vy) = r
+ for i in range(cast(int, char1), cast(int, char2) + 1):
+ widths[i] = (w, (vx, vy))
+ r = []
+ return widths
+
+
+class FontMetricsDB:
+ @classmethod
+ def get_metrics(cls, fontname: str) -> tuple[dict[str, object], dict[str, int]]:
+ return FONT_METRICS[fontname]
+
+
+# int here means that we're not extending PSStackParser with additional types.
+class Type1FontHeaderParser(PSStackParser[int]):
+ KEYWORD_BEGIN = KWD(b"begin")
+ KEYWORD_END = KWD(b"end")
+ KEYWORD_DEF = KWD(b"def")
+ KEYWORD_PUT = KWD(b"put")
+ KEYWORD_DICT = KWD(b"dict")
+ KEYWORD_ARRAY = KWD(b"array")
+ KEYWORD_READONLY = KWD(b"readonly")
+ KEYWORD_FOR = KWD(b"for")
+
+ def __init__(self, data: BinaryIO) -> None:
+ PSStackParser.__init__(self, data)
+ self._cid2unicode: dict[int, str] = {}
+
+ def get_encoding(self) -> dict[int, str]:
+ """Parse the font encoding.
+
+ The Type1 font encoding maps character codes to character names. These
+ character names could either be standard Adobe glyph names, or
+ character names associated with custom CharStrings for this font. A
+ CharString is a sequence of operations that describe how the character
+ should be drawn. Currently, this function returns '' (empty string)
+ for character names that are associated with a CharStrings.
+
+ Reference: Adobe Systems Incorporated, Adobe Type 1 Font Format
+
+ :returns mapping of character identifiers (cid's) to unicode characters
+ """
+ while 1:
+ try:
+ (cid, name) = self.nextobject()
+ except PSEOF:
+ break
+ try:
+ self._cid2unicode[cid] = name2unicode(cast(str, name))
+ except KeyError as e:
+ log.debug(str(e))
+ return self._cid2unicode
+
+ def do_keyword(self, pos: int, token: PSKeyword) -> None:
+ if token is self.KEYWORD_PUT:
+ ((_, key), (_, value)) = self.pop(2)
+ if isinstance(key, int) and isinstance(value, PSLiteral):
+ self.add_results((key, literal_name(value)))
+
+
+NIBBLES = ("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ".", "e", "e-", None, "-")
+
+# Mapping of cmap names. Original cmap name is kept if not in the mapping.
+# (missing reference for why DLIdent is mapped to Identity)
+IDENTITY_ENCODER = {
+ "DLIdent-H": "Identity-H",
+ "DLIdent-V": "Identity-V",
+}
+
+
+def getdict(data: bytes) -> dict[int, list[float | int]]:
+ d: dict[int, list[float | int]] = {}
+ fp = BytesIO(data)
+ stack: list[float | int] = []
+ while 1:
+ c = fp.read(1)
+ if not c:
+ break
+ b0 = ord(c)
+ if b0 <= 21:
+ d[b0] = stack
+ stack = []
+ continue
+ if b0 == 30:
+ s = ""
+ loop = True
+ while loop:
+ b = ord(fp.read(1))
+ for n in (b >> 4, b & 15):
+ if n == 15:
+ loop = False
+ else:
+ nibble = NIBBLES[n]
+ assert nibble is not None
+ s += nibble
+ value = float(s)
+ elif b0 >= 32 and b0 <= 246:
+ value = b0 - 139
+ else:
+ b1 = ord(fp.read(1))
+ if b0 >= 247 and b0 <= 250:
+ value = ((b0 - 247) << 8) + b1 + 108
+ elif b0 >= 251 and b0 <= 254:
+ value = -((b0 - 251) << 8) - b1 - 108
+ else:
+ b2 = ord(fp.read(1))
+ if b1 >= 128:
+ b1 -= 256
+ if b0 == 28:
+ value = b1 << 8 | b2
+ else:
+ value = b1 << 24 | b2 << 16 | struct.unpack(">H", fp.read(2))[0]
+ stack.append(value)
+ return d
+
+
+class CFFFont:
+ STANDARD_STRINGS = (
+ ".notdef",
+ "space",
+ "exclam",
+ "quotedbl",
+ "numbersign",
+ "dollar",
+ "percent",
+ "ampersand",
+ "quoteright",
+ "parenleft",
+ "parenright",
+ "asterisk",
+ "plus",
+ "comma",
+ "hyphen",
+ "period",
+ "slash",
+ "zero",
+ "one",
+ "two",
+ "three",
+ "four",
+ "five",
+ "six",
+ "seven",
+ "eight",
+ "nine",
+ "colon",
+ "semicolon",
+ "less",
+ "equal",
+ "greater",
+ "question",
+ "at",
+ "A",
+ "B",
+ "C",
+ "D",
+ "E",
+ "F",
+ "G",
+ "H",
+ "I",
+ "J",
+ "K",
+ "L",
+ "M",
+ "N",
+ "O",
+ "P",
+ "Q",
+ "R",
+ "S",
+ "T",
+ "U",
+ "V",
+ "W",
+ "X",
+ "Y",
+ "Z",
+ "bracketleft",
+ "backslash",
+ "bracketright",
+ "asciicircum",
+ "underscore",
+ "quoteleft",
+ "a",
+ "b",
+ "c",
+ "d",
+ "e",
+ "f",
+ "g",
+ "h",
+ "i",
+ "j",
+ "k",
+ "l",
+ "m",
+ "n",
+ "o",
+ "p",
+ "q",
+ "r",
+ "s",
+ "t",
+ "u",
+ "v",
+ "w",
+ "x",
+ "y",
+ "z",
+ "braceleft",
+ "bar",
+ "braceright",
+ "asciitilde",
+ "exclamdown",
+ "cent",
+ "sterling",
+ "fraction",
+ "yen",
+ "florin",
+ "section",
+ "currency",
+ "quotesingle",
+ "quotedblleft",
+ "guillemotleft",
+ "guilsinglleft",
+ "guilsinglright",
+ "fi",
+ "fl",
+ "endash",
+ "dagger",
+ "daggerdbl",
+ "periodcentered",
+ "paragraph",
+ "bullet",
+ "quotesinglbase",
+ "quotedblbase",
+ "quotedblright",
+ "guillemotright",
+ "ellipsis",
+ "perthousand",
+ "questiondown",
+ "grave",
+ "acute",
+ "circumflex",
+ "tilde",
+ "macron",
+ "breve",
+ "dotaccent",
+ "dieresis",
+ "ring",
+ "cedilla",
+ "hungarumlaut",
+ "ogonek",
+ "caron",
+ "emdash",
+ "AE",
+ "ordfeminine",
+ "Lslash",
+ "Oslash",
+ "OE",
+ "ordmasculine",
+ "ae",
+ "dotlessi",
+ "lslash",
+ "oslash",
+ "oe",
+ "germandbls",
+ "onesuperior",
+ "logicalnot",
+ "mu",
+ "trademark",
+ "Eth",
+ "onehalf",
+ "plusminus",
+ "Thorn",
+ "onequarter",
+ "divide",
+ "brokenbar",
+ "degree",
+ "thorn",
+ "threequarters",
+ "twosuperior",
+ "registered",
+ "minus",
+ "eth",
+ "multiply",
+ "threesuperior",
+ "copyright",
+ "Aacute",
+ "Acircumflex",
+ "Adieresis",
+ "Agrave",
+ "Aring",
+ "Atilde",
+ "Ccedilla",
+ "Eacute",
+ "Ecircumflex",
+ "Edieresis",
+ "Egrave",
+ "Iacute",
+ "Icircumflex",
+ "Idieresis",
+ "Igrave",
+ "Ntilde",
+ "Oacute",
+ "Ocircumflex",
+ "Odieresis",
+ "Ograve",
+ "Otilde",
+ "Scaron",
+ "Uacute",
+ "Ucircumflex",
+ "Udieresis",
+ "Ugrave",
+ "Yacute",
+ "Ydieresis",
+ "Zcaron",
+ "aacute",
+ "acircumflex",
+ "adieresis",
+ "agrave",
+ "aring",
+ "atilde",
+ "ccedilla",
+ "eacute",
+ "ecircumflex",
+ "edieresis",
+ "egrave",
+ "iacute",
+ "icircumflex",
+ "idieresis",
+ "igrave",
+ "ntilde",
+ "oacute",
+ "ocircumflex",
+ "odieresis",
+ "ograve",
+ "otilde",
+ "scaron",
+ "uacute",
+ "ucircumflex",
+ "udieresis",
+ "ugrave",
+ "yacute",
+ "ydieresis",
+ "zcaron",
+ "exclamsmall",
+ "Hungarumlautsmall",
+ "dollaroldstyle",
+ "dollarsuperior",
+ "ampersandsmall",
+ "Acutesmall",
+ "parenleftsuperior",
+ "parenrightsuperior",
+ "twodotenleader",
+ "onedotenleader",
+ "zerooldstyle",
+ "oneoldstyle",
+ "twooldstyle",
+ "threeoldstyle",
+ "fouroldstyle",
+ "fiveoldstyle",
+ "sixoldstyle",
+ "sevenoldstyle",
+ "eightoldstyle",
+ "nineoldstyle",
+ "commasuperior",
+ "threequartersemdash",
+ "periodsuperior",
+ "questionsmall",
+ "asuperior",
+ "bsuperior",
+ "centsuperior",
+ "dsuperior",
+ "esuperior",
+ "isuperior",
+ "lsuperior",
+ "msuperior",
+ "nsuperior",
+ "osuperior",
+ "rsuperior",
+ "ssuperior",
+ "tsuperior",
+ "ff",
+ "ffi",
+ "ffl",
+ "parenleftinferior",
+ "parenrightinferior",
+ "Circumflexsmall",
+ "hyphensuperior",
+ "Gravesmall",
+ "Asmall",
+ "Bsmall",
+ "Csmall",
+ "Dsmall",
+ "Esmall",
+ "Fsmall",
+ "Gsmall",
+ "Hsmall",
+ "Ismall",
+ "Jsmall",
+ "Ksmall",
+ "Lsmall",
+ "Msmall",
+ "Nsmall",
+ "Osmall",
+ "Psmall",
+ "Qsmall",
+ "Rsmall",
+ "Ssmall",
+ "Tsmall",
+ "Usmall",
+ "Vsmall",
+ "Wsmall",
+ "Xsmall",
+ "Ysmall",
+ "Zsmall",
+ "colonmonetary",
+ "onefitted",
+ "rupiah",
+ "Tildesmall",
+ "exclamdownsmall",
+ "centoldstyle",
+ "Lslashsmall",
+ "Scaronsmall",
+ "Zcaronsmall",
+ "Dieresissmall",
+ "Brevesmall",
+ "Caronsmall",
+ "Dotaccentsmall",
+ "Macronsmall",
+ "figuredash",
+ "hypheninferior",
+ "Ogoneksmall",
+ "Ringsmall",
+ "Cedillasmall",
+ "questiondownsmall",
+ "oneeighth",
+ "threeeighths",
+ "fiveeighths",
+ "seveneighths",
+ "onethird",
+ "twothirds",
+ "zerosuperior",
+ "foursuperior",
+ "fivesuperior",
+ "sixsuperior",
+ "sevensuperior",
+ "eightsuperior",
+ "ninesuperior",
+ "zeroinferior",
+ "oneinferior",
+ "twoinferior",
+ "threeinferior",
+ "fourinferior",
+ "fiveinferior",
+ "sixinferior",
+ "seveninferior",
+ "eightinferior",
+ "nineinferior",
+ "centinferior",
+ "dollarinferior",
+ "periodinferior",
+ "commainferior",
+ "Agravesmall",
+ "Aacutesmall",
+ "Acircumflexsmall",
+ "Atildesmall",
+ "Adieresissmall",
+ "Aringsmall",
+ "AEsmall",
+ "Ccedillasmall",
+ "Egravesmall",
+ "Eacutesmall",
+ "Ecircumflexsmall",
+ "Edieresissmall",
+ "Igravesmall",
+ "Iacutesmall",
+ "Icircumflexsmall",
+ "Idieresissmall",
+ "Ethsmall",
+ "Ntildesmall",
+ "Ogravesmall",
+ "Oacutesmall",
+ "Ocircumflexsmall",
+ "Otildesmall",
+ "Odieresissmall",
+ "OEsmall",
+ "Oslashsmall",
+ "Ugravesmall",
+ "Uacutesmall",
+ "Ucircumflexsmall",
+ "Udieresissmall",
+ "Yacutesmall",
+ "Thornsmall",
+ "Ydieresissmall",
+ "001.000",
+ "001.001",
+ "001.002",
+ "001.003",
+ "Black",
+ "Bold",
+ "Book",
+ "Light",
+ "Medium",
+ "Regular",
+ "Roman",
+ "Semibold",
+ )
+
+ class INDEX:
+ def __init__(self, fp: BinaryIO) -> None:
+ self.fp = fp
+ self.offsets: list[int] = []
+ (count, offsize) = struct.unpack(">HB", self.fp.read(3))
+ for i in range(count + 1):
+ self.offsets.append(nunpack(self.fp.read(offsize)))
+ self.base = self.fp.tell() - 1
+ self.fp.seek(self.base + self.offsets[-1])
+
+ def __repr__(self) -> str:
+ return "" % len(self)
+
+ def __len__(self) -> int:
+ return len(self.offsets) - 1
+
+ def __getitem__(self, i: int) -> bytes:
+ self.fp.seek(self.base + self.offsets[i])
+ return self.fp.read(self.offsets[i + 1] - self.offsets[i])
+
+ def __iter__(self) -> Iterator[bytes]:
+ return iter(self[i] for i in range(len(self)))
+
+ def __init__(self, name: str, fp: BinaryIO) -> None:
+ self.name = name
+ self.fp = fp
+ # Header
+ (_major, _minor, hdrsize, offsize) = struct.unpack("BBBB", self.fp.read(4))
+ self.fp.read(hdrsize - 4)
+ # Name INDEX
+ self.name_index = self.INDEX(self.fp)
+ # Top DICT INDEX
+ self.dict_index = self.INDEX(self.fp)
+ # String INDEX
+ self.string_index = self.INDEX(self.fp)
+ # Global Subr INDEX
+ self.subr_index = self.INDEX(self.fp)
+ # Top DICT DATA
+ self.top_dict = getdict(self.dict_index[0])
+ (charset_pos,) = self.top_dict.get(15, [0])
+ (encoding_pos,) = self.top_dict.get(16, [0])
+ (charstring_pos,) = self.top_dict.get(17, [0])
+ # CharStrings
+ self.fp.seek(cast(int, charstring_pos))
+ self.charstring = self.INDEX(self.fp)
+ self.nglyphs = len(self.charstring)
+ # Encodings
+ self.code2gid = {}
+ self.gid2code = {}
+ self.fp.seek(cast(int, encoding_pos))
+ format = self.fp.read(1)
+ if format == b"\x00":
+ # Format 0
+ (n,) = struct.unpack("B", self.fp.read(1))
+ for code, gid in enumerate(struct.unpack("B" * n, self.fp.read(n))):
+ self.code2gid[code] = gid
+ self.gid2code[gid] = code
+ elif format == b"\x01":
+ # Format 1
+ (n,) = struct.unpack("B", self.fp.read(1))
+ code = 0
+ for i in range(n):
+ (first, nleft) = struct.unpack("BB", self.fp.read(2))
+ for gid in range(first, first + nleft + 1):
+ self.code2gid[code] = gid
+ self.gid2code[gid] = code
+ code += 1
+ else:
+ raise PDFValueError("unsupported encoding format: %r" % format)
+ # Charsets
+ self.name2gid = {}
+ self.gid2name = {}
+ self.fp.seek(cast(int, charset_pos))
+ format = self.fp.read(1)
+ if format == b"\x00":
+ # Format 0
+ n = self.nglyphs - 1
+ for gid, sid in enumerate(
+ cast(
+ tuple[int, ...], struct.unpack(">" + "H" * n, self.fp.read(2 * n))
+ ),
+ ):
+ gid += 1
+ sidname = self.getstr(sid)
+ self.name2gid[sidname] = gid
+ self.gid2name[gid] = sidname
+ elif format == b"\x01":
+ # Format 1
+ (n,) = struct.unpack("B", self.fp.read(1))
+ sid = 0
+ for i in range(n):
+ (first, nleft) = struct.unpack("BB", self.fp.read(2))
+ for gid in range(first, first + nleft + 1):
+ sidname = self.getstr(sid)
+ self.name2gid[sidname] = gid
+ self.gid2name[gid] = sidname
+ sid += 1
+ elif format == b"\x02":
+ # Format 2
+ assert False, str(("Unhandled", format))
+ else:
+ raise PDFValueError("unsupported charset format: %r" % format)
+
+ def getstr(self, sid: int) -> str | bytes:
+ # This returns str for one of the STANDARD_STRINGS but bytes otherwise,
+ # and appears to be a needless source of type complexity.
+ if sid < len(self.STANDARD_STRINGS):
+ return self.STANDARD_STRINGS[sid]
+ return self.string_index[sid - len(self.STANDARD_STRINGS)]
+
+
+class TrueTypeFont:
+ class CMapNotFound(PDFException):
+ pass
+
+ def __init__(self, name: str, fp: BinaryIO) -> None:
+ self.name = name
+ self.fp = fp
+ self.tables: dict[bytes, tuple[int, int]] = {}
+ self.fonttype = fp.read(4)
+ try:
+ (ntables, _1, _2, _3) = cast(
+ tuple[int, int, int, int],
+ struct.unpack(">HHHH", fp.read(8)),
+ )
+ for _ in range(ntables):
+ (name_bytes, tsum, offset, length) = cast(
+ tuple[bytes, int, int, int],
+ struct.unpack(">4sLLL", fp.read(16)),
+ )
+ self.tables[name_bytes] = (offset, length)
+ except struct.error:
+ # Do not fail if there are not enough bytes to read. Even for
+ # corrupted PDFs we would like to get as much information as
+ # possible, so continue.
+ pass
+
+ def create_unicode_map(self) -> FileUnicodeMap:
+ if b"cmap" not in self.tables:
+ raise TrueTypeFont.CMapNotFound
+ fp = self.fp
+ char2gid = []
+ try:
+ face = freetype.Face(fp)
+ char2gid = list(face.get_chars())
+ except Exception:
+ raise TrueTypeFont.CMapNotFound
+ # create unicode map
+ unicode_map = FileUnicodeMap()
+ for char, gid in char2gid:
+ unicode_map.add_cid2unichr(gid, char)
+ return unicode_map
+
+
+class PDFFontError(PDFException):
+ pass
+
+
+class PDFUnicodeNotDefined(PDFFontError):
+ pass
+
+
+LITERAL_STANDARD_ENCODING = LIT("StandardEncoding")
+LITERAL_TYPE1C = LIT("Type1C")
+
+# Font widths are maintained in a dict type that maps from *either* unicode
+# chars or integer character IDs.
+FontWidthDict = dict[int | str, float]
+
+
+class PDFFont:
+ def __init__(
+ self,
+ descriptor: Mapping[str, Any],
+ widths: FontWidthDict,
+ default_width: float | None = None,
+ ) -> None:
+ self.descriptor = descriptor
+ self.widths: FontWidthDict = resolve_all(widths)
+ self.fontname = resolve1(descriptor.get("FontName", "unknown"))
+ if isinstance(self.fontname, PSLiteral):
+ self.fontname = literal_name(self.fontname)
+ self.flags = int_value(descriptor.get("Flags", 0))
+ self.ascent = num_value(descriptor.get("Ascent", 0))
+ self.descent = num_value(descriptor.get("Descent", 0))
+ self.italic_angle = num_value(descriptor.get("ItalicAngle", 0))
+ if default_width is None:
+ self.default_width = num_value(descriptor.get("MissingWidth", 0))
+ else:
+ self.default_width = default_width
+ self.default_width = resolve1(self.default_width)
+ self.leading = num_value(descriptor.get("Leading", 0))
+ self.bbox = self._parse_bbox(descriptor)
+ self.hscale = self.vscale = 0.001
+
+ # PDF RM 9.8.1 specifies /Descent should always be a negative number.
+ # PScript5.dll seems to produce Descent with a positive number, but
+ # text analysis will be wrong if this is taken as correct. So force
+ # descent to negative.
+ if self.descent > 0:
+ self.descent = -self.descent
+
+ def __repr__(self) -> str:
+ return ""
+
+ def is_vertical(self) -> bool:
+ return False
+
+ def is_multibyte(self) -> bool:
+ return False
+
+ def decode(self, bytes: bytes) -> Iterable[int]:
+ return bytearray(bytes) # map(ord, bytes)
+
+ def get_ascent(self) -> float:
+ """Ascent above the baseline, in text space units"""
+ return self.ascent * self.vscale
+
+ def get_descent(self) -> float:
+ """Descent below the baseline, in text space units; always negative"""
+ return self.descent * self.vscale
+
+ def get_width(self) -> float:
+ w = self.bbox[2] - self.bbox[0]
+ if w == 0:
+ w = -self.default_width
+ return w * self.hscale
+
+ def get_height(self) -> float:
+ h = self.bbox[3] - self.bbox[1]
+ if h == 0:
+ h = self.ascent - self.descent
+ return h * self.vscale
+
+ def char_width(self, cid: int) -> float:
+ # Because character widths may be mapping either IDs or strings,
+ # we try to lookup the character ID first, then its str equivalent.
+ cid_width = safe_float(self.widths.get(cid))
+ if cid_width is not None:
+ return cid_width * self.hscale
+
+ try:
+ str_cid = self.to_unichr(cid)
+ cid_width = safe_float(self.widths.get(str_cid))
+ if cid_width is not None:
+ return cid_width * self.hscale
+
+ except PDFUnicodeNotDefined:
+ pass
+
+ return self.default_width * self.hscale
+
+ def char_disp(self, cid: int) -> float | tuple[float | None, float]:
+ """Returns an integer for horizontal fonts, a tuple for vertical fonts."""
+ return 0
+
+ def string_width(self, s: bytes) -> float:
+ return sum(self.char_width(cid) for cid in self.decode(s))
+
+ def to_unichr(self, cid: int) -> str:
+ raise NotImplementedError
+
+ @staticmethod
+ def _parse_bbox(descriptor: Mapping[str, Any]) -> Rect:
+ """Parse FontBBox from the fonts descriptor"""
+ font_bbox = resolve_all(descriptor.get("FontBBox"))
+ bbox = safe_rect_list(font_bbox)
+ if bbox is None:
+ log.warning(
+ f"Could get FontBBox from font descriptor because {font_bbox!r} cannot be parsed as 4 floats"
+ )
+ return 0.0, 0.0, 0.0, 0.0
+ return bbox
+
+
+class PDFSimpleFont(PDFFont):
+ def __init__(
+ self,
+ descriptor: Mapping[str, Any],
+ widths: FontWidthDict,
+ spec: Mapping[str, Any],
+ ) -> None:
+ # Font encoding is specified either by a name of
+ # built-in encoding or a dictionary that describes
+ # the differences.
+ if "Encoding" in spec:
+ encoding = resolve1(spec["Encoding"])
+ else:
+ encoding = LITERAL_STANDARD_ENCODING
+ if isinstance(encoding, dict):
+ name = literal_name(encoding.get("BaseEncoding", LITERAL_STANDARD_ENCODING))
+ diff = list_value(encoding.get("Differences", []))
+ self.cid2unicode = EncodingDB.get_encoding(name, diff)
+ else:
+ self.cid2unicode = EncodingDB.get_encoding(literal_name(encoding))
+ self.unicode_map: UnicodeMap | None = None
+ if "ToUnicode" in spec:
+ strm = stream_value(spec["ToUnicode"])
+ self.unicode_map = FileUnicodeMap()
+ CMapParser(self.unicode_map, BytesIO(strm.get_data())).run()
+ PDFFont.__init__(self, descriptor, widths)
+
+ def to_unichr(self, cid: int) -> str:
+ if self.unicode_map:
+ try:
+ return self.unicode_map.get_unichr(cid)
+ except KeyError:
+ pass
+ try:
+ return self.cid2unicode[cid]
+ except KeyError:
+ raise PDFUnicodeNotDefined(None, cid)
+
+
+class PDFType1Font(PDFSimpleFont):
+ def __init__(self, rsrcmgr: "PDFResourceManager", spec: Mapping[str, Any]) -> None:
+ try:
+ self.basefont = literal_name(spec["BaseFont"])
+ except KeyError:
+ if settings.STRICT:
+ raise PDFFontError("BaseFont is missing")
+ self.basefont = "unknown"
+
+ widths: FontWidthDict
+ try:
+ (descriptor, int_widths) = FontMetricsDB.get_metrics(self.basefont)
+ widths = cast(dict[str | int, float], int_widths) # implicit int->float
+ except KeyError:
+ descriptor = dict_value(spec.get("FontDescriptor", {}))
+ firstchar = int_value(spec.get("FirstChar", 0))
+ # lastchar = int_value(spec.get('LastChar', 255))
+ width_list = list_value(spec.get("Widths", [0] * 256))
+ widths = {i + firstchar: resolve1(w) for (i, w) in enumerate(width_list)}
+ PDFSimpleFont.__init__(self, descriptor, widths, spec)
+ if "Encoding" not in spec and "FontFile" in descriptor:
+ # try to recover the missing encoding info from the font file.
+ self.fontfile = stream_value(descriptor.get("FontFile"))
+ length1 = int_value(self.fontfile["Length1"])
+ data = self.fontfile.get_data()[:length1]
+ # awcm: quickfix for type 1 font which contains bad string literals
+ offset = 0
+ if enc_offset := data.index(b"/Encoding"):
+ offset = enc_offset
+ parser = Type1FontHeaderParser(BytesIO(data[offset:]))
+ self.cid2unicode = parser.get_encoding()
+
+ def __repr__(self) -> str:
+ return "" % self.basefont
+
+
+class PDFTrueTypeFont(PDFType1Font):
+ def __repr__(self) -> str:
+ return "" % self.basefont
+
+
+class PDFType3Font(PDFSimpleFont):
+ def __init__(self, rsrcmgr: "PDFResourceManager", spec: Mapping[str, Any]) -> None:
+ firstchar = int_value(spec.get("FirstChar", 0))
+ # lastchar = int_value(spec.get('LastChar', 0))
+ width_list = list_value(spec.get("Widths", [0] * 256))
+ widths: dict[str | int, float] = {
+ i + firstchar: w for (i, w) in enumerate(width_list)
+ }
+ if "FontDescriptor" in spec:
+ descriptor = dict_value(spec["FontDescriptor"])
+ else:
+ descriptor = {"Ascent": 0, "Descent": 0, "FontBBox": spec["FontBBox"]}
+ PDFSimpleFont.__init__(self, descriptor, widths, spec)
+ self.matrix = cast(Matrix, tuple(list_value(spec.get("FontMatrix"))))
+ (_, self.descent, _, self.ascent) = self.bbox
+ (self.hscale, self.vscale) = apply_matrix_norm(self.matrix, (1, 1))
+
+ def __repr__(self) -> str:
+ return ""
+
+
+class PDFCIDFont(PDFFont):
+ default_disp: float | tuple[float | None, float]
+
+ def __init__(
+ self,
+ rsrcmgr: "PDFResourceManager",
+ spec: Mapping[str, Any],
+ strict: bool = settings.STRICT,
+ ) -> None:
+ try:
+ self.basefont = literal_name(spec["BaseFont"])
+ except KeyError:
+ if strict:
+ raise PDFFontError("BaseFont is missing")
+ self.basefont = "unknown"
+ self.cidsysteminfo = dict_value(spec.get("CIDSystemInfo", {}))
+ cid_registry = resolve1(self.cidsysteminfo.get("Registry", b"unknown")).decode(
+ "latin1",
+ )
+ cid_ordering = resolve1(self.cidsysteminfo.get("Ordering", b"unknown")).decode(
+ "latin1",
+ )
+ self.cidcoding = f"{cid_registry.strip()}-{cid_ordering.strip()}"
+ self.cmap: CMapBase = self.get_cmap_from_spec(spec, strict)
+
+ try:
+ descriptor = dict_value(spec["FontDescriptor"])
+ except KeyError:
+ if strict:
+ raise PDFFontError("FontDescriptor is missing")
+ descriptor = {}
+ ttf = None
+ if "FontFile2" in descriptor:
+ self.fontfile = stream_value(descriptor.get("FontFile2"))
+ ttf = TrueTypeFont(self.basefont, BytesIO(self.fontfile.get_data()))
+ self.unicode_map: UnicodeMap | None = None
+ if "ToUnicode" in spec:
+ if isinstance(spec["ToUnicode"], PDFStream):
+ strm = stream_value(spec["ToUnicode"])
+ self.unicode_map = FileUnicodeMap()
+ CMapParser(self.unicode_map, BytesIO(strm.get_data())).run()
+ else:
+ cmap_name = literal_name(spec["ToUnicode"])
+ encoding = literal_name(spec["Encoding"])
+ if (
+ "Identity" in cid_ordering
+ or "Identity" in cmap_name
+ or "Identity" in encoding
+ ):
+ self.unicode_map = IdentityUnicodeMap()
+ elif self.cidcoding in ("Adobe-Identity", "Adobe-UCS"):
+ if ttf:
+ try:
+ self.unicode_map = ttf.create_unicode_map()
+ except TrueTypeFont.CMapNotFound:
+ pass
+ else:
+ try:
+ self.unicode_map = CMapDB.get_unicode_map(
+ self.cidcoding,
+ self.cmap.is_vertical(),
+ )
+ except CMapDB.CMapNotFound:
+ pass
+
+ self.vertical = self.cmap.is_vertical()
+ if self.vertical:
+ # writing mode: vertical
+ widths2 = get_widths2(list_value(spec.get("W2", [])))
+ self.disps = {cid: (vx, vy) for (cid, (_, (vx, vy))) in widths2.items()}
+ (vy, w) = resolve1(spec.get("DW2", [880, -1000]))
+ self.default_disp = (None, vy)
+ widths: dict[str | int, float] = {
+ cid: w for (cid, (w, _)) in widths2.items()
+ }
+ default_width = w
+ else:
+ # writing mode: horizontal
+ self.disps = {}
+ self.default_disp = 0
+ widths = get_widths(list_value(spec.get("W", [])))
+ default_width = spec.get("DW", 1000)
+ PDFFont.__init__(self, descriptor, widths, default_width=default_width)
+
+ def get_cmap_from_spec(self, spec: Mapping[str, Any], strict: bool) -> CMapBase:
+ """Get cmap from font specification
+
+ For certain PDFs, Encoding Type isn't mentioned as an attribute of
+ Encoding but as an attribute of CMapName, where CMapName is an
+ attribute of spec['Encoding'].
+ The horizontal/vertical modes are mentioned with different name
+ such as 'DLIdent-H/V','OneByteIdentityH/V','Identity-H/V'.
+ """
+ cmap_name = self._get_cmap_name(spec, strict)
+
+ try:
+ return CMapDB.get_cmap(cmap_name)
+ except CMapDB.CMapNotFound as e:
+ if strict:
+ raise PDFFontError(e)
+ return CMap()
+
+ @staticmethod
+ def _get_cmap_name(spec: Mapping[str, Any], strict: bool) -> str:
+ """Get cmap name from font specification"""
+ cmap_name = "unknown" # default value
+
+ try:
+ spec_encoding = spec["Encoding"]
+ if hasattr(spec_encoding, "name"):
+ cmap_name = literal_name(spec["Encoding"])
+ else:
+ cmap_name = literal_name(spec_encoding["CMapName"])
+ except KeyError:
+ if strict:
+ raise PDFFontError("Encoding is unspecified")
+
+ if type(cmap_name) is PDFStream: # type: ignore[comparison-overlap]
+ cmap_name_stream: PDFStream = cast(PDFStream, cmap_name)
+ if "CMapName" in cmap_name_stream:
+ cmap_name = cmap_name_stream.get("CMapName").name
+ elif strict:
+ raise PDFFontError("CMapName unspecified for encoding")
+
+ return IDENTITY_ENCODER.get(cmap_name, cmap_name)
+
+ def __repr__(self) -> str:
+ return f""
+
+ def is_vertical(self) -> bool:
+ return self.vertical
+
+ def is_multibyte(self) -> bool:
+ return True
+
+ def decode(self, bytes: bytes) -> Iterable[int]:
+ return self.cmap.decode(bytes)
+
+ def char_disp(self, cid: int) -> float | tuple[float | None, float]:
+ """Returns an integer for horizontal fonts, a tuple for vertical fonts."""
+ return self.disps.get(cid, self.default_disp)
+
+ def to_unichr(self, cid: int) -> str:
+ try:
+ if not self.unicode_map:
+ raise PDFKeyError(cid)
+ return self.unicode_map.get_unichr(cid)
+ except KeyError:
+ raise PDFUnicodeNotDefined(self.cidcoding, cid)
diff --git a/babeldoc/pdfminer/pdfinterp.py b/babeldoc/pdfminer/pdfinterp.py
new file mode 100644
index 0000000000000000000000000000000000000000..35249b41f94c0170b1b38a16ebecf0356197ce17
--- /dev/null
+++ b/babeldoc/pdfminer/pdfinterp.py
@@ -0,0 +1,1279 @@
+import logging
+import re
+from collections.abc import Mapping
+from collections.abc import Sequence
+from io import BytesIO
+from typing import Union
+from typing import cast
+
+from babeldoc.pdfminer.casting import safe_cmyk
+from babeldoc.pdfminer.casting import safe_float
+from babeldoc.pdfminer.casting import safe_int
+from babeldoc.pdfminer.casting import safe_matrix
+from babeldoc.pdfminer.casting import safe_rgb
+from babeldoc.pdfminer.cmapdb import CMap
+from babeldoc.pdfminer.cmapdb import CMapBase
+from babeldoc.pdfminer.cmapdb import CMapDB
+from babeldoc.pdfminer.pdfcolor import PREDEFINED_COLORSPACE
+from babeldoc.pdfminer.pdfcolor import PDFColorSpace
+from babeldoc.pdfminer.pdfdevice import PDFDevice
+from babeldoc.pdfminer.pdfdevice import PDFTextSeq
+from babeldoc.pdfminer.pdfexceptions import PDFException
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+from babeldoc.pdfminer.pdffont import PDFCIDFont
+from babeldoc.pdfminer.pdffont import PDFFont
+from babeldoc.pdfminer.pdffont import PDFFontError
+from babeldoc.pdfminer.pdffont import PDFTrueTypeFont
+from babeldoc.pdfminer.pdffont import PDFType1Font
+from babeldoc.pdfminer.pdffont import PDFType3Font
+from babeldoc.pdfminer.pdfpage import PDFPage
+from babeldoc.pdfminer.pdftypes import LITERALS_ASCII85_DECODE
+from babeldoc.pdfminer.pdftypes import PDFObjRef
+from babeldoc.pdfminer.pdftypes import PDFStream
+from babeldoc.pdfminer.pdftypes import dict_value
+from babeldoc.pdfminer.pdftypes import list_value
+from babeldoc.pdfminer.pdftypes import resolve1
+from babeldoc.pdfminer.pdftypes import stream_value
+from babeldoc.pdfminer.psexceptions import PSEOF
+from babeldoc.pdfminer.psexceptions import PSTypeError
+from babeldoc.pdfminer.psparser import KWD
+from babeldoc.pdfminer.psparser import LIT
+from babeldoc.pdfminer.psparser import PSKeyword
+from babeldoc.pdfminer.psparser import PSLiteral
+from babeldoc.pdfminer.psparser import PSStackParser
+from babeldoc.pdfminer.psparser import PSStackType
+from babeldoc.pdfminer.psparser import keyword_name
+from babeldoc.pdfminer.psparser import literal_name
+from babeldoc.pdfminer.utils import MATRIX_IDENTITY, apply_matrix_pt
+from babeldoc.pdfminer.utils import Matrix
+from babeldoc.pdfminer.utils import PathSegment
+from babeldoc.pdfminer.utils import Point
+from babeldoc.pdfminer.utils import Rect
+from babeldoc.pdfminer.utils import choplist
+from babeldoc.pdfminer.utils import mult_matrix
+from babeldoc.pdfminer import settings
+
+log = logging.getLogger(__name__)
+
+
+class PDFResourceError(PDFException):
+ pass
+
+
+class PDFInterpreterError(PDFException):
+ pass
+
+
+LITERAL_PDF = LIT("PDF")
+LITERAL_TEXT = LIT("Text")
+LITERAL_FONT = LIT("Font")
+LITERAL_FORM = LIT("Form")
+LITERAL_IMAGE = LIT("Image")
+
+
+class PDFTextState:
+ matrix: Matrix
+ linematrix: Point
+
+ def __init__(self) -> None:
+ self.font: PDFFont | None = None
+ self.fontsize: float = 0
+ self.charspace: float = 0
+ self.wordspace: float = 0
+ self.scaling: float = 100
+ self.leading: float = 0
+ self.render: int = 0
+ self.rise: float = 0
+ self.reset()
+ # self.matrix is set
+ # self.linematrix is set
+
+ def __repr__(self) -> str:
+ return (
+ ""
+ % (
+ self.font,
+ self.fontsize,
+ self.charspace,
+ self.wordspace,
+ self.scaling,
+ self.leading,
+ self.render,
+ self.rise,
+ self.matrix,
+ self.linematrix,
+ )
+ )
+
+ def copy(self) -> "PDFTextState":
+ obj = PDFTextState()
+ obj.font = self.font
+ obj.fontsize = self.fontsize
+ obj.charspace = self.charspace
+ obj.wordspace = self.wordspace
+ obj.scaling = self.scaling
+ obj.leading = self.leading
+ obj.render = self.render
+ obj.rise = self.rise
+ obj.matrix = self.matrix
+ obj.linematrix = self.linematrix
+ obj.font_id = getattr(self, "font_id", None)
+ return obj
+
+ def reset(self) -> None:
+ self.matrix = MATRIX_IDENTITY
+ self.linematrix = (0, 0)
+
+
+Color = Union[
+ float, # Greyscale
+ tuple[float, float, float], # R, G, B
+ tuple[float, float, float, float], # C, M, Y, K
+]
+
+
+class PDFGraphicState:
+ def __init__(self) -> None:
+ self.linewidth: float = 0
+ self.linecap: object | None = None
+ self.linejoin: object | None = None
+ self.miterlimit: object | None = None
+ self.dash: tuple[object, object] | None = None
+ self.intent: object | None = None
+ self.flatness: object | None = None
+
+ # stroking color
+ self.scolor: Color | None = None
+
+ # non stroking color
+ self.ncolor: Color | None = None
+
+ def copy(self) -> "PDFGraphicState":
+ obj = PDFGraphicState()
+ obj.linewidth = self.linewidth
+ obj.linecap = self.linecap
+ obj.linejoin = self.linejoin
+ obj.miterlimit = self.miterlimit
+ obj.dash = self.dash
+ obj.intent = self.intent
+ obj.flatness = self.flatness
+ obj.scolor = self.scolor
+ obj.ncolor = self.ncolor
+ return obj
+
+ def __repr__(self) -> str:
+ return (
+ ""
+ % (
+ self.linewidth,
+ self.linecap,
+ self.linejoin,
+ self.miterlimit,
+ self.dash,
+ self.intent,
+ self.flatness,
+ self.scolor,
+ self.ncolor,
+ )
+ )
+
+
+class PDFResourceManager:
+ """Repository of shared resources.
+
+ ResourceManager facilitates reuse of shared resources
+ such as fonts and images so that large objects are not
+ allocated multiple times.
+ """
+
+ def __init__(self, caching: bool = True) -> None:
+ self.caching = caching
+ self._cached_fonts: dict[object, PDFFont] = {}
+
+ def get_procset(self, procs: Sequence[object]) -> None:
+ for proc in procs:
+ if proc is LITERAL_PDF or proc is LITERAL_TEXT:
+ pass
+ else:
+ pass
+
+ def get_cmap(self, cmapname: str, strict: bool = False) -> CMapBase:
+ try:
+ return CMapDB.get_cmap(cmapname)
+ except CMapDB.CMapNotFound:
+ if strict:
+ raise
+ return CMap()
+
+ def get_font(self, objid: object, spec: Mapping[str, object]) -> PDFFont:
+ if objid and objid in self._cached_fonts:
+ font = self._cached_fonts[objid]
+ else:
+ log.debug("get_font: create: objid=%r, spec=%r", objid, spec)
+ if settings.STRICT:
+ if spec["Type"] is not LITERAL_FONT:
+ raise PDFFontError("Type is not /Font")
+ # Create a Font object.
+ if "Subtype" in spec:
+ subtype = literal_name(spec["Subtype"])
+ else:
+ if settings.STRICT:
+ raise PDFFontError("Font Subtype is not specified.")
+ subtype = "Type1"
+ if subtype in ("Type1", "MMType1"):
+ # Type1 Font
+ font = PDFType1Font(self, spec)
+ elif subtype == "TrueType":
+ # TrueType Font
+ font = PDFTrueTypeFont(self, spec)
+ elif subtype == "Type3":
+ # Type3 Font
+ font = PDFType3Font(self, spec)
+ elif subtype in ("CIDFontType0", "CIDFontType2"):
+ # CID Font
+ font = PDFCIDFont(self, spec)
+ elif subtype == "Type0":
+ # Type0 Font
+ dfonts = list_value(spec["DescendantFonts"])
+ assert dfonts
+ subspec = dict_value(dfonts[0]).copy()
+ for k in ("Encoding", "ToUnicode"):
+ if k in spec:
+ subspec[k] = resolve1(spec[k])
+ font = self.get_font(None, subspec)
+ else:
+ if settings.STRICT:
+ raise PDFFontError("Invalid Font spec: %r" % spec)
+ font = PDFType1Font(self, spec) # this is so wrong!
+ if objid and self.caching:
+ self._cached_fonts[objid] = font
+ return font
+
+
+class PDFContentParser(PSStackParser[Union[PSKeyword, PDFStream]]):
+ def __init__(self, streams: Sequence[object]) -> None:
+ self.streams = streams
+ self.istream = 0
+ # PSStackParser.__init__(fp=None) is safe only because we've overloaded
+ # all the methods that would attempt to access self.fp without first
+ # calling self.fillfp().
+ PSStackParser.__init__(self, None) # type: ignore[arg-type]
+
+ def fillfp(self) -> None:
+ if not self.fp:
+ if self.istream < len(self.streams):
+ strm = stream_value(self.streams[self.istream])
+ self.istream += 1
+ else:
+ raise PSEOF("Unexpected EOF, file truncated?")
+ self.fp = BytesIO(strm.get_data())
+
+ def seek(self, pos: int) -> None:
+ self.fillfp()
+ PSStackParser.seek(self, pos)
+
+ def fillbuf(self) -> None:
+ if self.charpos < len(self.buf):
+ return
+ while 1:
+ self.fillfp()
+ self.bufpos = self.fp.tell()
+ self.buf = self.fp.read(self.BUFSIZ)
+ if self.buf:
+ break
+ self.fp = None # type: ignore[assignment]
+ self.charpos = 0
+
+ def get_inline_data(self, pos: int, target: bytes = b"EI") -> tuple[int, bytes]:
+ self.seek(pos)
+ i = 0
+ data = b""
+ while i <= len(target):
+ self.fillbuf()
+ if i:
+ ci = self.buf[self.charpos]
+ c = bytes((ci,))
+ data += c
+ self.charpos += 1
+ if (
+ len(target) <= i
+ and c.isspace()
+ or i < len(target)
+ and c == (bytes((target[i],)))
+ ):
+ i += 1
+ else:
+ i = 0
+ else:
+ try:
+ j = self.buf.index(target[0], self.charpos)
+ data += self.buf[self.charpos : j + 1]
+ self.charpos = j + 1
+ i = 1
+ except ValueError:
+ data += self.buf[self.charpos :]
+ self.charpos = len(self.buf)
+ data = data[: -(len(target) + 1)] # strip the last part
+ data = re.sub(rb"(\x0d\x0a|[\x0d\x0a])$", b"", data)
+ return (pos, data)
+
+ def flush(self) -> None:
+ self.add_results(*self.popall())
+
+ KEYWORD_BI = KWD(b"BI")
+ KEYWORD_ID = KWD(b"ID")
+ KEYWORD_EI = KWD(b"EI")
+
+ def do_keyword(self, pos: int, token: PSKeyword) -> None:
+ if token is self.KEYWORD_BI:
+ # inline image within a content stream
+ self.start_type(pos, "inline")
+ elif token is self.KEYWORD_ID:
+ try:
+ (_, objs) = self.end_type("inline")
+ if len(objs) % 2 != 0:
+ error_msg = f"Invalid dictionary construct: {objs!r}"
+ raise PSTypeError(error_msg)
+ d = {literal_name(k): resolve1(v) for (k, v) in choplist(2, objs)}
+ eos = b"EI"
+ filter = d.get("F", None)
+ if filter is not None:
+ if isinstance(filter, PSLiteral):
+ filter = [filter]
+ if filter[0] in LITERALS_ASCII85_DECODE:
+ eos = b"~>"
+ (pos, data) = self.get_inline_data(pos + len(b"ID "), target=eos)
+ if eos != b"EI": # it may be necessary for decoding
+ data += eos
+ obj = PDFStream(d, data)
+ self.push((pos, obj))
+ if eos == b"EI": # otherwise it is still in the stream
+ self.push((pos, self.KEYWORD_EI))
+ except PSTypeError:
+ if settings.STRICT:
+ raise
+ else:
+ self.push((pos, token))
+
+
+PDFStackT = PSStackType[PDFStream]
+"""Types that may appear on the PDF argument stack."""
+
+
+class PDFPageInterpreter:
+ """Processor for the content of a PDF page
+
+ Reference: PDF Reference, Appendix A, Operator Summary
+ """
+
+ def __init__(self, rsrcmgr: PDFResourceManager, device: PDFDevice) -> None:
+ self.rsrcmgr = rsrcmgr
+ self.device = device
+
+ def dup(self) -> "PDFPageInterpreter":
+ return self.__class__(self.rsrcmgr, self.device)
+
+ def init_resources(self, resources: dict[object, object]) -> None:
+ """Prepare the fonts and XObjects listed in the Resource attribute."""
+ self.resources = resources
+ self.fontmap: dict[object, PDFFont] = {}
+ self.xobjmap = {}
+ self.csmap: dict[str, PDFColorSpace] = PREDEFINED_COLORSPACE.copy()
+ if not resources:
+ return
+
+ def get_colorspace(spec: object) -> PDFColorSpace | None:
+ if isinstance(spec, list):
+ name = literal_name(spec[0])
+ else:
+ name = literal_name(spec)
+ if name == "ICCBased" and isinstance(spec, list) and len(spec) >= 2:
+ return PDFColorSpace(name, stream_value(spec[1])["N"])
+ elif name == "DeviceN" and isinstance(spec, list) and len(spec) >= 2:
+ return PDFColorSpace(name, len(list_value(spec[1])))
+ else:
+ return PREDEFINED_COLORSPACE.get(name)
+
+ for k, v in dict_value(resources).items():
+ log.debug("Resource: %r: %r", k, v)
+ if k == "Font":
+ for fontid, spec in dict_value(v).items():
+ objid = None
+ if isinstance(spec, PDFObjRef):
+ objid = spec.objid
+ spec = dict_value(spec)
+ self.fontmap[fontid] = self.rsrcmgr.get_font(objid, spec)
+ elif k == "ColorSpace":
+ for csid, spec in dict_value(v).items():
+ colorspace = get_colorspace(resolve1(spec))
+ if colorspace is not None:
+ self.csmap[csid] = colorspace
+ elif k == "ProcSet":
+ self.rsrcmgr.get_procset(list_value(v))
+ elif k == "XObject":
+ for xobjid, xobjstrm in dict_value(v).items():
+ self.xobjmap[xobjid] = xobjstrm
+
+ def init_state(self, ctm: Matrix) -> None:
+ """Initialize the text and graphic states for rendering a page."""
+ # gstack: stack for graphical states.
+ self.gstack: list[tuple[Matrix, PDFTextState, PDFGraphicState]] = []
+ self.ctm = ctm
+ self.device.set_ctm(self.ctm)
+ self.textstate = PDFTextState()
+ self.graphicstate = PDFGraphicState()
+ self.curpath: list[PathSegment] = []
+ # argstack: stack for command arguments.
+ self.argstack: list[PDFStackT] = []
+ # set some global states.
+ self.scs: PDFColorSpace | None = None
+ self.ncs: PDFColorSpace | None = None
+ if self.csmap:
+ self.scs = self.ncs = next(iter(self.csmap.values()))
+
+ def push(self, obj: PDFStackT) -> None:
+ self.argstack.append(obj)
+
+ def pop(self, n: int) -> list[PDFStackT]:
+ if n == 0:
+ return []
+ x = self.argstack[-n:]
+ self.argstack = self.argstack[:-n]
+ return x
+
+ def get_current_state(self) -> tuple[Matrix, PDFTextState, PDFGraphicState]:
+ return (self.ctm, self.textstate.copy(), self.graphicstate.copy())
+
+ def set_current_state(
+ self,
+ state: tuple[Matrix, PDFTextState, PDFGraphicState],
+ ) -> None:
+ (self.ctm, self.textstate, self.graphicstate) = state
+ self.device.set_ctm(self.ctm)
+
+ def do_q(self) -> None:
+ """Save graphics state"""
+ self.gstack.append(self.get_current_state())
+
+ def do_Q(self) -> None:
+ """Restore graphics state"""
+ if self.gstack:
+ self.set_current_state(self.gstack.pop())
+
+ def do_cm(
+ self,
+ a1: PDFStackT,
+ b1: PDFStackT,
+ c1: PDFStackT,
+ d1: PDFStackT,
+ e1: PDFStackT,
+ f1: PDFStackT,
+ ) -> None:
+ """Concatenate matrix to current transformation matrix"""
+ matrix = safe_matrix(a1, b1, c1, d1, e1, f1)
+
+ if matrix is None:
+ log.warning(
+ f"Cannot concatenate matrix to current transformation matrix because not all values in {(a1, b1, c1, d1, e1, f1)!r} can be parsed as floats"
+ )
+ else:
+ self.ctm = mult_matrix(matrix, self.ctm)
+ self.device.set_ctm(self.ctm)
+
+ def do_w(self, linewidth: PDFStackT) -> None:
+ """Set line width"""
+ linewidth_f = safe_float(linewidth)
+ if linewidth_f is None:
+ log.warning(
+ f"Cannot set line width because {linewidth!r} is an invalid float value"
+ )
+ else:
+ self.graphicstate.linewidth = linewidth_f
+
+ def do_J(self, linecap: PDFStackT) -> None:
+ """Set line cap style"""
+ self.graphicstate.linecap = linecap
+
+ def do_j(self, linejoin: PDFStackT) -> None:
+ """Set line join style"""
+ self.graphicstate.linejoin = linejoin
+
+ def do_M(self, miterlimit: PDFStackT) -> None:
+ """Set miter limit"""
+ self.graphicstate.miterlimit = miterlimit
+
+ def do_d(self, dash: PDFStackT, phase: PDFStackT) -> None:
+ """Set line dash pattern"""
+ self.graphicstate.dash = (dash, phase)
+
+ def do_ri(self, intent: PDFStackT) -> None:
+ """Set color rendering intent"""
+ self.graphicstate.intent = intent
+
+ def do_i(self, flatness: PDFStackT) -> None:
+ """Set flatness tolerance"""
+ self.graphicstate.flatness = flatness
+
+ def do_gs(self, name: PDFStackT) -> None:
+ """Set parameters from graphics state parameter dictionary"""
+ # to do
+
+ def do_m(self, x: PDFStackT, y: PDFStackT) -> None:
+ """Begin new subpath"""
+ x_f = safe_float(x)
+ y_f = safe_float(y)
+
+ if x_f is None or y_f is None:
+ point = ("m", x, y)
+ log.warning(
+ f"Cannot start new subpath because not all values in {point!r} can be parsed as floats"
+ )
+ else:
+ point = ("m", x_f, y_f)
+ self.curpath.append(point)
+
+ def do_l(self, x: PDFStackT, y: PDFStackT) -> None:
+ """Append straight line segment to path"""
+ x_f = safe_float(x)
+ y_f = safe_float(y)
+ if x_f is None or y_f is None:
+ point = ("l", x, y)
+ log.warning(
+ f"Cannot append straight line segment to path because not all values in {point!r} can be parsed as floats"
+ )
+ else:
+ point = ("l", x_f, y_f)
+ self.curpath.append(point)
+
+ def do_c(
+ self,
+ x1: PDFStackT,
+ y1: PDFStackT,
+ x2: PDFStackT,
+ y2: PDFStackT,
+ x3: PDFStackT,
+ y3: PDFStackT,
+ ) -> None:
+ """Append curved segment to path (three control points)"""
+ x1_f = safe_float(x1)
+ y1_f = safe_float(y1)
+ x2_f = safe_float(x2)
+ y2_f = safe_float(y2)
+ x3_f = safe_float(x3)
+ y3_f = safe_float(y3)
+ if (
+ x1_f is None
+ or y1_f is None
+ or x2_f is None
+ or y2_f is None
+ or x3_f is None
+ or y3_f is None
+ ):
+ point = ("c", x1, y1, x2, y2, x3, y3)
+ log.warning(
+ f"Cannot append curved segment to path because not all values in {point!r} can be parsed as floats"
+ )
+ else:
+ point = ("c", x1_f, y1_f, x2_f, y2_f, x3_f, y3_f)
+ self.curpath.append(point)
+
+ def do_v(self, x2: PDFStackT, y2: PDFStackT, x3: PDFStackT, y3: PDFStackT) -> None:
+ """Append curved segment to path (initial point replicated)"""
+ x2_f = safe_float(x2)
+ y2_f = safe_float(y2)
+ x3_f = safe_float(x3)
+ y3_f = safe_float(y3)
+ if x2_f is None or y2_f is None or x3_f is None or y3_f is None:
+ point = ("v", x2, y2, x3, y3)
+ log.warning(
+ f"Cannot append curved segment to path because not all values in {point!r} can be parsed as floats"
+ )
+ else:
+ point = ("v", x2_f, y2_f, x3_f, y3_f)
+ self.curpath.append(point)
+
+ def do_y(self, x1: PDFStackT, y1: PDFStackT, x3: PDFStackT, y3: PDFStackT) -> None:
+ """Append curved segment to path (final point replicated)"""
+ x1_f = safe_float(x1)
+ y1_f = safe_float(y1)
+ x3_f = safe_float(x3)
+ y3_f = safe_float(y3)
+ if x1_f is None or y1_f is None or x3_f is None or y3_f is None:
+ point = ("y", x1, y1, x3, y3)
+ log.warning(
+ f"Cannot append curved segment to path because not all values in {point!r} can be parsed as floats"
+ )
+ else:
+ point = ("y", x1_f, y1_f, x3_f, y3_f)
+ self.curpath.append(point)
+
+ def do_h(self) -> None:
+ """Close subpath"""
+ self.curpath.append(("h",))
+
+ def do_re(self, x: PDFStackT, y: PDFStackT, w: PDFStackT, h: PDFStackT) -> None:
+ """Append rectangle to path"""
+ x_f = safe_float(x)
+ y_f = safe_float(y)
+ w_f = safe_float(w)
+ h_f = safe_float(h)
+
+ if x_f is None or y_f is None or w_f is None or h_f is None:
+ values = (x, y, w, h)
+ log.warning(
+ f"Cannot append rectangle to path because not all values in {values!r} can be parsed as floats"
+ )
+ else:
+ self.curpath.append(("m", x_f, y_f))
+ self.curpath.append(("l", x_f + w_f, y_f))
+ self.curpath.append(("l", x_f + w_f, y_f + h_f))
+ self.curpath.append(("l", x_f, y_f + h_f))
+ self.curpath.append(("h",))
+
+ def do_S(self) -> None:
+ """Stroke path"""
+ self.device.paint_path(self.graphicstate, True, False, False, self.curpath)
+ self.curpath = []
+
+ def do_s(self) -> None:
+ """Close and stroke path"""
+ self.do_h()
+ self.do_S()
+
+ def do_f(self) -> None:
+ """Fill path using nonzero winding number rule"""
+ self.device.paint_path(self.graphicstate, False, True, False, self.curpath)
+ self.curpath = []
+
+ def do_F(self) -> None:
+ """Fill path using nonzero winding number rule (obsolete)"""
+
+ def do_f_a(self) -> None:
+ """Fill path using even-odd rule"""
+ self.device.paint_path(self.graphicstate, False, True, True, self.curpath)
+ self.curpath = []
+
+ def do_B(self) -> None:
+ """Fill and stroke path using nonzero winding number rule"""
+ self.device.paint_path(self.graphicstate, True, True, False, self.curpath)
+ self.curpath = []
+
+ def do_B_a(self) -> None:
+ """Fill and stroke path using even-odd rule"""
+ self.device.paint_path(self.graphicstate, True, True, True, self.curpath)
+ self.curpath = []
+
+ def do_b(self) -> None:
+ """Close, fill, and stroke path using nonzero winding number rule"""
+ self.do_h()
+ self.do_B()
+
+ def do_b_a(self) -> None:
+ """Close, fill, and stroke path using even-odd rule"""
+ self.do_h()
+ self.do_B_a()
+
+ def do_n(self) -> None:
+ """End path without filling or stroking"""
+ self.curpath = []
+
+ def do_W(self) -> None:
+ """Set clipping path using nonzero winding number rule"""
+ pass
+
+ def do_W_a(self) -> None:
+ """Set clipping path using even-odd rule"""
+ pass
+
+ def do_CS(self, name: PDFStackT) -> None:
+ """Set color space for stroking operations
+
+ Introduced in PDF 1.1
+ """
+ try:
+ self.scs = self.csmap[literal_name(name)]
+ except KeyError:
+ if settings.STRICT:
+ raise PDFInterpreterError("Undefined ColorSpace: %r" % name)
+
+ def do_cs(self, name: PDFStackT) -> None:
+ """Set color space for nonstroking operations"""
+ try:
+ self.ncs = self.csmap[literal_name(name)]
+ except KeyError:
+ if settings.STRICT:
+ raise PDFInterpreterError("Undefined ColorSpace: %r" % name)
+
+ def do_G(self, gray: PDFStackT) -> None:
+ """Set gray level for stroking operations"""
+ gray_f = safe_float(gray)
+
+ if gray_f is None:
+ log.warning(
+ f"Cannot set gray level because {gray!r} is an invalid float value"
+ )
+ else:
+ self.graphicstate.scolor = gray_f
+ self.scs = self.csmap["DeviceGray"]
+
+ def do_g(self, gray: PDFStackT) -> None:
+ """Set gray level for nonstroking operations"""
+ gray_f = safe_float(gray)
+
+ if gray_f is None:
+ log.warning(
+ f"Cannot set gray level because {gray!r} is an invalid float value"
+ )
+ else:
+ self.graphicstate.ncolor = gray_f
+ self.ncs = self.csmap["DeviceGray"]
+
+ def do_RG(self, r: PDFStackT, g: PDFStackT, b: PDFStackT) -> None:
+ """Set RGB color for stroking operations"""
+ rgb = safe_rgb(r, g, b)
+
+ if rgb is None:
+ log.warning(
+ f"Cannot set RGB stroke color because not all values in {(r, g, b)!r} can be parsed as floats"
+ )
+ else:
+ self.graphicstate.scolor = rgb
+ self.scs = self.csmap["DeviceRGB"]
+
+ def do_rg(self, r: PDFStackT, g: PDFStackT, b: PDFStackT) -> None:
+ """Set RGB color for nonstroking operations"""
+ rgb = safe_rgb(r, g, b)
+
+ if rgb is None:
+ log.warning(
+ f"Cannot set RGB non-stroke color because not all values in {(r, g, b)!r} can be parsed as floats"
+ )
+ else:
+ self.graphicstate.ncolor = rgb
+ self.ncs = self.csmap["DeviceRGB"]
+
+ def do_K(self, c: PDFStackT, m: PDFStackT, y: PDFStackT, k: PDFStackT) -> None:
+ """Set CMYK color for stroking operations"""
+ cmyk = safe_cmyk(c, m, y, k)
+
+ if cmyk is None:
+ log.warning(
+ f"Cannot set CMYK stroke color because not all values in {(c, m, y, k)!r} can be parsed as floats"
+ )
+ else:
+ self.graphicstate.scolor = cmyk
+ self.scs = self.csmap["DeviceCMYK"]
+
+ def do_k(self, c: PDFStackT, m: PDFStackT, y: PDFStackT, k: PDFStackT) -> None:
+ """Set CMYK color for nonstroking operations"""
+ cmyk = safe_cmyk(c, m, y, k)
+
+ if cmyk is None:
+ log.warning(
+ f"Cannot set CMYK non-stroke color because not all values in {(c, m, y, k)!r} can be parsed as floats"
+ )
+ else:
+ self.graphicstate.ncolor = cmyk
+ self.ncs = self.csmap["DeviceCMYK"]
+
+ def do_SCN(self) -> None:
+ """Set color for stroking operations."""
+ if self.scs:
+ n = self.scs.ncomponents
+ else:
+ if settings.STRICT:
+ raise PDFInterpreterError("No colorspace specified!")
+ n = 1
+
+ if n == 1:
+ gray = self.pop(1)[0]
+ gray_f = safe_float(gray)
+ if gray_f is None:
+ log.warning(
+ f"Cannot set gray stroke color because {gray!r} is an invalid float value"
+ )
+ else:
+ self.graphicstate.scolor = gray_f
+
+ elif n == 3:
+ values = self.pop(3)
+ rgb = safe_rgb(*values)
+ if rgb is None:
+ log.warning(
+ f"Cannot set RGB stroke color because not all values in {values!r} can be parsed as floats"
+ )
+ else:
+ self.graphicstate.scolor = rgb
+
+ elif n == 4:
+ values = self.pop(4)
+ cmyk = safe_cmyk(*values)
+
+ if cmyk is None:
+ log.warning(
+ f"Cannot set CMYK stroke color because not all values in {values!r} can be parsed as floats"
+ )
+ else:
+ self.graphicstate.scolor = cmyk
+
+ else:
+ log.warning(
+ f"Cannot set stroke color because {n} components are specified but only 1 (grayscale), 3 (rgb) and 4 (cmyk) are supported"
+ )
+
+ def do_scn(self) -> None:
+ """Set color for nonstroking operations"""
+ if self.ncs:
+ n = self.ncs.ncomponents
+ else:
+ if settings.STRICT:
+ raise PDFInterpreterError("No colorspace specified!")
+ n = 1
+
+ if n == 1:
+ gray = self.pop(1)[0]
+ gray_f = safe_float(gray)
+ if gray_f is None:
+ log.warning(
+ f"Cannot set gray non-stroke color because {gray!r} is an invalid float value"
+ )
+ else:
+ self.graphicstate.ncolor = gray_f
+
+ elif n == 3:
+ values = self.pop(3)
+ rgb = safe_rgb(*values)
+
+ if rgb is None:
+ log.warning(
+ f"Cannot set RGB non-stroke color because not all values in {values!r} can be parsed as floats"
+ )
+ else:
+ self.graphicstate.ncolor = rgb
+
+ elif n == 4:
+ values = self.pop(4)
+ cmyk = safe_cmyk(*values)
+
+ if cmyk is None:
+ log.warning(
+ f"Cannot set CMYK non-stroke color because not all values in {values!r} can be parsed as floats"
+ )
+ else:
+ self.graphicstate.ncolor = cmyk
+
+ else:
+ log.warning(
+ f"Cannot set non-stroke color because {n} components are specified but only 1 (grayscale), 3 (rgb) and 4 (cmyk) are supported"
+ )
+
+ def do_SC(self) -> None:
+ """Set color for stroking operations"""
+ self.do_SCN()
+
+ def do_sc(self) -> None:
+ """Set color for nonstroking operations"""
+ self.do_scn()
+
+ def do_sh(self, name: object) -> None:
+ """Paint area defined by shading pattern"""
+
+ def do_BT(self) -> None:
+ """Begin text object
+
+ Initializing the text matrix, Tm, and the text line matrix, Tlm, to
+ the identity matrix. Text objects cannot be nested; a second BT cannot
+ appear before an ET.
+ """
+ self.textstate.reset()
+
+ def do_ET(self) -> None:
+ """End a text object"""
+
+ def do_BX(self) -> None:
+ """Begin compatibility section"""
+
+ def do_EX(self) -> None:
+ """End compatibility section"""
+
+ def do_MP(self, tag: PDFStackT) -> None:
+ """Define marked-content point"""
+ if isinstance(tag, PSLiteral):
+ self.device.do_tag(tag)
+ else:
+ log.warning(
+ f"Cannot define marked-content point because {tag!r} is not a PSLiteral"
+ )
+
+ def do_DP(self, tag: PDFStackT, props: PDFStackT) -> None:
+ """Define marked-content point with property list"""
+ if isinstance(tag, PSLiteral):
+ self.device.do_tag(tag, props)
+ else:
+ log.warning(
+ f"Cannot define marked-content point with property list because {tag!r} is not a PSLiteral"
+ )
+
+ def do_BMC(self, tag: PDFStackT) -> None:
+ """Begin marked-content sequence"""
+ if isinstance(tag, PSLiteral):
+ self.device.begin_tag(tag)
+ else:
+ log.warning(
+ f"Cannot begin marked-content sequence because {tag!r} is not a PSLiteral"
+ )
+
+ def do_BDC(self, tag: PDFStackT, props: PDFStackT) -> None:
+ """Begin marked-content sequence with property list"""
+ if isinstance(tag, PSLiteral):
+ self.device.begin_tag(tag, props)
+ else:
+ log.warning(
+ f"Cannot begin marked-content sequence with property list because {tag!r} is not a PSLiteral"
+ )
+
+ def do_EMC(self) -> None:
+ """End marked-content sequence"""
+ self.device.end_tag()
+
+ def do_Tc(self, space: PDFStackT) -> None:
+ """Set character spacing.
+
+ Character spacing is used by the Tj, TJ, and ' operators.
+
+ :param space: a number expressed in unscaled text space units.
+ """
+ charspace = safe_float(space)
+ if charspace is None:
+ log.warning(
+ f"Could not set character spacing because {space!r} is an invalid float value"
+ )
+ else:
+ self.textstate.charspace = charspace
+
+ def do_Tw(self, space: PDFStackT) -> None:
+ """Set the word spacing.
+
+ Word spacing is used by the Tj, TJ, and ' operators.
+
+ :param space: a number expressed in unscaled text space units
+ """
+ wordspace = safe_float(space)
+ if wordspace is None:
+ log.warning(
+ f"Could not set word spacing becuase {space!r} is an invalid float value"
+ )
+ else:
+ self.textstate.wordspace = wordspace
+
+ def do_Tz(self, scale: PDFStackT) -> None:
+ """Set the horizontal scaling.
+
+ :param scale: is a number specifying the percentage of the normal width
+ """
+ scale_f = safe_float(scale)
+
+ if scale_f is None:
+ log.warning(
+ f"Could not set horizontal scaling because {scale!r} is an invalid float value"
+ )
+ else:
+ self.textstate.scaling = scale_f
+
+ def do_TL(self, leading: PDFStackT) -> None:
+ """Set the text leading.
+
+ Text leading is used only by the T*, ', and " operators.
+
+ :param leading: a number expressed in unscaled text space units
+ """
+ leading_f = safe_float(leading)
+ if leading_f is None:
+ log.warning(
+ f"Could not set text leading because {leading!r} is an invalid float value"
+ )
+ else:
+ self.textstate.leading = -leading_f
+
+ def do_Tf(self, fontid: PDFStackT, fontsize: PDFStackT) -> None:
+ """Set the text font
+
+ :param fontid: the name of a font resource in the Font subdictionary
+ of the current resource dictionary
+ :param fontsize: size is a number representing a scale factor.
+ """
+ try:
+ self.textstate.font = self.fontmap[literal_name(fontid)]
+ self.textstate.font_id = literal_name(fontid)
+ except KeyError:
+ if settings.STRICT:
+ raise PDFInterpreterError("Undefined Font id: %r" % fontid)
+ self.textstate.font = self.rsrcmgr.get_font(None, {})
+
+ fontsize_f = safe_float(fontsize)
+ if fontsize_f is None:
+ log.warning(
+ f"Could not set text font because {fontsize!r} is an invalid float value"
+ )
+ else:
+ self.textstate.fontsize = fontsize_f
+
+ def do_Tr(self, render: PDFStackT) -> None:
+ """Set the text rendering mode"""
+ render_i = safe_int(render)
+
+ if render_i is None:
+ log.warning(
+ f"Could not set text rendering mode because {render!r} is an invalid int value"
+ )
+ else:
+ self.textstate.render = render_i
+
+ def do_Ts(self, rise: PDFStackT) -> None:
+ """Set the text rise
+
+ :param rise: a number expressed in unscaled text space units
+ """
+ rise_f = safe_float(rise)
+
+ if rise_f is None:
+ log.warning(
+ f"Could not set text rise because {rise!r} is an invalid float value"
+ )
+ else:
+ self.textstate.rise = rise_f
+
+ def do_Td(self, tx: PDFStackT, ty: PDFStackT) -> None:
+ """Move to the start of the next line
+
+ Offset from the start of the current line by (tx , ty).
+ """
+ tx_ = safe_float(tx)
+ ty_ = safe_float(ty)
+ if tx_ is not None and ty_ is not None:
+ (a, b, c, d, e, f) = self.textstate.matrix
+ e_new = tx_ * a + ty_ * c + e
+ f_new = tx_ * b + ty_ * d + f
+ self.textstate.matrix = (a, b, c, d, e_new, f_new)
+
+ elif settings.STRICT:
+ raise PDFValueError(f"Invalid offset ({tx!r}, {ty!r}) for Td")
+
+ self.textstate.linematrix = (0, 0)
+
+ def do_TD(self, tx: PDFStackT, ty: PDFStackT) -> None:
+ """Move to the start of the next line.
+
+ offset from the start of the current line by (tx , ty). As a side effect, this
+ operator sets the leading parameter in the text state.
+ """
+ tx_ = safe_float(tx)
+ ty_ = safe_float(ty)
+
+ if tx_ is not None and ty_ is not None:
+ (a, b, c, d, e, f) = self.textstate.matrix
+ e_new = tx_ * a + ty_ * c + e
+ f_new = tx_ * b + ty_ * d + f
+ self.textstate.matrix = (a, b, c, d, e_new, f_new)
+
+ elif settings.STRICT:
+ raise PDFValueError("Invalid offset ({tx}, {ty}) for TD")
+
+ if ty_ is not None:
+ self.textstate.leading = ty_
+
+ self.textstate.linematrix = (0, 0)
+
+ def do_Tm(
+ self,
+ a: PDFStackT,
+ b: PDFStackT,
+ c: PDFStackT,
+ d: PDFStackT,
+ e: PDFStackT,
+ f: PDFStackT,
+ ) -> None:
+ """Set text matrix and text line matrix"""
+ values = (a, b, c, d, e, f)
+ matrix = safe_matrix(*values)
+
+ if matrix is None:
+ log.warning(
+ f"Could not set text matrix because not all values in {values!r} can be parsed as floats"
+ )
+ else:
+ self.textstate.matrix = matrix
+ self.textstate.linematrix = (0, 0)
+
+ def do_T_a(self) -> None:
+ """Move to start of next text line"""
+ (a, b, c, d, e, f) = self.textstate.matrix
+ self.textstate.matrix = (
+ a,
+ b,
+ c,
+ d,
+ self.textstate.leading * c + e,
+ self.textstate.leading * d + f,
+ )
+ self.textstate.linematrix = (0, 0)
+
+ def do_TJ(self, seq: PDFStackT) -> None:
+ """Show text, allowing individual glyph positioning"""
+ if self.textstate.font is None:
+ if settings.STRICT:
+ raise PDFInterpreterError("No font specified!")
+ return
+ assert self.ncs is not None
+ self.device.render_string(
+ self.textstate,
+ cast(PDFTextSeq, seq),
+ self.ncs,
+ self.graphicstate.copy(),
+ )
+
+ def do_Tj(self, s: PDFStackT) -> None:
+ """Show text"""
+ self.do_TJ([s])
+
+ def do__q(self, s: PDFStackT) -> None:
+ """Move to next line and show text
+
+ The ' (single quote) operator.
+ """
+ self.do_T_a()
+ self.do_TJ([s])
+
+ def do__w(self, aw: PDFStackT, ac: PDFStackT, s: PDFStackT) -> None:
+ """Set word and character spacing, move to next line, and show text
+
+ The " (double quote) operator.
+ """
+ self.do_Tw(aw)
+ self.do_Tc(ac)
+ self.do_TJ([s])
+
+ def do_BI(self) -> None:
+ """Begin inline image object"""
+
+ def do_ID(self) -> None:
+ """Begin inline image data"""
+
+ def do_EI(self, obj: PDFStackT) -> None:
+ """End inline image object"""
+ if isinstance(obj, PDFStream) and "W" in obj and "H" in obj:
+ iobjid = str(id(obj))
+ self.device.begin_figure(iobjid, (0, 0, 1, 1), MATRIX_IDENTITY)
+ self.device.render_image(iobjid, obj)
+ self.device.end_figure(iobjid)
+
+ def do_Do(self, xobjid_arg: PDFStackT) -> None:
+ """Invoke named XObject"""
+ xobjid = literal_name(xobjid_arg)
+ try:
+ xobj = stream_value(self.xobjmap[xobjid])
+ except KeyError:
+ if settings.STRICT:
+ raise PDFInterpreterError("Undefined xobject id: %r" % xobjid)
+ return
+ log.debug("Processing xobj: %r", xobj)
+ subtype = xobj.get("Subtype")
+ if subtype is LITERAL_FORM and "BBox" in xobj:
+ interpreter = self.dup()
+ bbox = cast(Rect, list_value(xobj["BBox"]))
+ matrix = cast(Matrix, list_value(xobj.get("Matrix", MATRIX_IDENTITY)))
+ # According to PDF reference 1.7 section 4.9.1, XObjects in
+ # earlier PDFs (prior to v1.2) use the page's Resources entry
+ # instead of having their own Resources entry.
+ xobjres = xobj.get("Resources")
+ if xobjres:
+ resources = dict_value(xobjres)
+ else:
+ resources = self.resources.copy()
+ self.device.begin_figure(xobjid, bbox, matrix)
+ interpreter.render_contents(
+ resources,
+ [xobj],
+ ctm=mult_matrix(matrix, self.ctm),
+ )
+ self.device.end_figure(xobjid)
+ elif subtype is LITERAL_IMAGE and "Width" in xobj and "Height" in xobj:
+ self.device.begin_figure(xobjid, (0, 0, 1, 1), MATRIX_IDENTITY)
+ self.device.render_image(xobjid, xobj)
+ self.device.end_figure(xobjid)
+ else:
+ # unsupported xobject type.
+ pass
+
+ def process_page(self, page: PDFPage) -> None:
+ log.debug("Processing page: %r", page)
+ (x0, y0, x1, y1) = page.mediabox
+ if page.rotate == 90:
+ ctm = (0, -1, 1, 0, -y0, x1)
+ elif page.rotate == 180:
+ ctm = (-1, 0, 0, -1, x1, y1)
+ elif page.rotate == 270:
+ ctm = (0, 1, -1, 0, y1, -x0)
+ else:
+ ctm = (1, 0, 0, 1, -x0, -y0)
+ self.device.begin_page(page, ctm)
+ self.render_contents(page.resources, page.contents, ctm=ctm)
+ self.device.end_page(page)
+
+ def render_contents(
+ self,
+ resources: dict[object, object],
+ streams: Sequence[object],
+ ctm: Matrix = MATRIX_IDENTITY,
+ ) -> None:
+ """Render the content streams.
+
+ This method may be called recursively.
+ """
+ log.debug(
+ "render_contents: resources=%r, streams=%r, ctm=%r",
+ resources,
+ streams,
+ ctm,
+ )
+ self.init_resources(resources)
+ self.init_state(ctm)
+ self.execute(list_value(streams))
+
+ def execute(self, streams: Sequence[object]) -> None:
+ try:
+ parser = PDFContentParser(streams)
+ except PSEOF:
+ # empty page
+ return
+ while True:
+ try:
+ (_, obj) = parser.nextobject()
+ except PSEOF:
+ break
+ if isinstance(obj, PSKeyword):
+ name = keyword_name(obj)
+ method = "do_%s" % name.replace("*", "_a").replace('"', "_w").replace(
+ "'",
+ "_q",
+ )
+ if hasattr(self, method):
+ func = getattr(self, method)
+ nargs = func.__code__.co_argcount - 1
+ if nargs:
+ args = self.pop(nargs)
+ log.debug("exec: %s %r", name, args)
+ if len(args) == nargs:
+ func(*args)
+ else:
+ log.debug("exec: %s", name)
+ func()
+ elif settings.STRICT:
+ error_msg = "Unknown operator: %r" % name
+ raise PDFInterpreterError(error_msg)
+ else:
+ self.push(obj)
diff --git a/babeldoc/pdfminer/pdfpage.py b/babeldoc/pdfminer/pdfpage.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce38dfcae71e809821ac460fb151495e86f78c5b
--- /dev/null
+++ b/babeldoc/pdfminer/pdfpage.py
@@ -0,0 +1,232 @@
+import itertools
+import logging
+from collections.abc import Container
+from collections.abc import Iterator
+from typing import Any
+from typing import BinaryIO
+
+from babeldoc.pdfminer.pdfdocument import PDFDocument
+from babeldoc.pdfminer.pdfdocument import PDFNoPageLabels
+from babeldoc.pdfminer.pdfdocument import PDFTextExtractionNotAllowed
+from babeldoc.pdfminer.pdfexceptions import PDFObjectNotFound
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+from babeldoc.pdfminer.pdfparser import PDFParser
+from babeldoc.pdfminer.pdftypes import dict_value, PDFObjRef
+from babeldoc.pdfminer.pdftypes import int_value
+from babeldoc.pdfminer.pdftypes import list_value
+from babeldoc.pdfminer.pdftypes import resolve1
+from babeldoc.pdfminer.psparser import LIT
+from babeldoc.pdfminer.utils import Rect
+from babeldoc.pdfminer.utils import parse_rect
+from babeldoc.pdfminer import settings
+
+log = logging.getLogger(__name__)
+
+# some predefined literals and keywords.
+LITERAL_PAGE = LIT("Page")
+LITERAL_PAGES = LIT("Pages")
+
+
+class PDFPage:
+ """An object that holds the information about a page.
+
+ A PDFPage object is merely a convenience class that has a set
+ of keys and values, which describe the properties of a page
+ and point to its contents.
+
+ Attributes
+ ----------
+ doc: a PDFDocument object.
+ pageid: any Python object that can uniquely identify the page.
+ attrs: a dictionary of page attributes.
+ contents: a list of PDFStream objects that represents the page content.
+ lastmod: the last modified time of the page.
+ resources: a dictionary of resources used by the page.
+ mediabox: the physical size of the page.
+ cropbox: the crop rectangle of the page.
+ rotate: the page rotation (in degree).
+ annots: the page annotations.
+ beads: a chain that represents natural reading order.
+ label: the page's label (typically, the logical page number).
+
+ """
+
+ def __init__(
+ self,
+ doc: PDFDocument,
+ pageid: object,
+ attrs: object,
+ label: str | None,
+ ) -> None:
+ """Initialize a page object.
+
+ doc: a PDFDocument object.
+ pageid: any Python object that can uniquely identify the page.
+ attrs: a dictionary of page attributes.
+ label: page label string.
+ """
+ self.doc = doc
+ self.pageid = pageid
+ self.attrs = dict_value(attrs)
+ self.label = label
+ self.lastmod = resolve1(self.attrs.get("LastModified"))
+ self.resources: dict[object, object] = resolve1(
+ self.attrs.get("Resources", dict()),
+ )
+ try:
+ while isinstance(attrs["MediaBox"], PDFObjRef):
+ attrs["MediaBox"] = resolve1(attrs["MediaBox"])
+ except Exception:
+ log.exception(f"try to fix mediabox failed: {attrs}")
+
+ self.mediabox = self._parse_mediabox(self.attrs.get("MediaBox"))
+ try:
+ self.cropbox = self._parse_cropbox(self.attrs.get("CropBox"), self.mediabox)
+ except Exception:
+ self.cropbox = self.mediabox
+ self.contents = self._parse_contents(self.attrs.get("Contents"))
+
+ self.rotate = (int_value(self.attrs.get("Rotate", 0)) + 360) % 360
+ self.annots = self.attrs.get("Annots")
+ self.beads = self.attrs.get("B")
+
+ def __repr__(self) -> str:
+ return f""
+
+ INHERITABLE_ATTRS = {"Resources", "MediaBox", "CropBox", "Rotate"}
+
+ @classmethod
+ def create_pages(cls, document: PDFDocument) -> Iterator["PDFPage"]:
+ def depth_first_search(
+ obj: Any,
+ parent: dict[str, Any],
+ visited: set[Any] | None = None,
+ ) -> Iterator[tuple[int, dict[Any, dict[Any, Any]]]]:
+ if isinstance(obj, int):
+ object_id = obj
+ object_properties = dict_value(document.getobj(object_id)).copy()
+ else:
+ # This looks broken. obj.objid means obj could be either
+ # PDFObjRef or PDFStream, but neither is valid for dict_value.
+ object_id = obj.objid # type: ignore[attr-defined]
+ object_properties = dict_value(obj).copy()
+
+ # Avoid recursion errors by keeping track of visited nodes
+ if visited is None:
+ visited = set()
+ if object_id in visited:
+ return
+ visited.add(object_id)
+
+ for k, v in parent.items():
+ if k in cls.INHERITABLE_ATTRS and k not in object_properties:
+ object_properties[k] = v
+
+ object_type = object_properties.get("Type")
+ if object_type is None and not settings.STRICT: # See #64
+ object_type = object_properties.get("type")
+
+ if object_type is LITERAL_PAGES and "Kids" in object_properties:
+ log.debug("Pages: Kids=%r", object_properties["Kids"])
+ for child in list_value(object_properties["Kids"]):
+ yield from depth_first_search(child, object_properties, visited)
+
+ elif object_type is LITERAL_PAGE:
+ log.debug("Page: %r", object_properties)
+ yield (object_id, object_properties)
+
+ try:
+ page_labels: Iterator[str | None] = document.get_page_labels()
+ except PDFNoPageLabels:
+ page_labels = itertools.repeat(None)
+
+ pages = False
+ if "Pages" in document.catalog:
+ objects = depth_first_search(document.catalog["Pages"], document.catalog)
+ for objid, tree in objects:
+ yield cls(document, objid, tree, next(page_labels))
+ pages = True
+ if not pages:
+ # fallback when /Pages is missing.
+ for xref in document.xrefs:
+ for objid in xref.get_objids():
+ try:
+ obj = document.getobj(objid)
+ if isinstance(obj, dict) and obj.get("Type") is LITERAL_PAGE:
+ yield cls(document, objid, obj, next(page_labels))
+ except PDFObjectNotFound:
+ pass
+
+ @classmethod
+ def get_pages(
+ cls,
+ fp: BinaryIO,
+ pagenos: Container[int] | None = None,
+ maxpages: int = 0,
+ password: str = "",
+ caching: bool = True,
+ check_extractable: bool = False,
+ ) -> Iterator["PDFPage"]:
+ # Create a PDF parser object associated with the file object.
+ parser = PDFParser(fp)
+ # Create a PDF document object that stores the document structure.
+ doc = PDFDocument(parser, password=password, caching=caching)
+ # Check if the document allows text extraction.
+ # If not, warn the user and proceed.
+ if not doc.is_extractable:
+ if check_extractable:
+ error_msg = "Text extraction is not allowed: %r" % fp
+ raise PDFTextExtractionNotAllowed(error_msg)
+ else:
+ warning_msg = (
+ "The PDF %r contains a metadata field "
+ "indicating that it should not allow "
+ "text extraction. Ignoring this field "
+ "and proceeding. Use the check_extractable "
+ "if you want to raise an error in this case" % fp
+ )
+ log.warning(warning_msg)
+ # Process each page contained in the document.
+ for pageno, page in enumerate(cls.create_pages(doc)):
+ if pagenos and (pageno not in pagenos):
+ continue
+ yield page
+ if maxpages and maxpages <= pageno + 1:
+ break
+
+ def _parse_mediabox(self, value: Any) -> Rect:
+ us_letter = (0.0, 0.0, 612.0, 792.0)
+
+ if value is None:
+ log.warning(
+ "MediaBox missing from /Page (and not inherited), "
+ "defaulting to US Letter"
+ )
+ return us_letter
+
+ try:
+ return parse_rect(resolve1(val) for val in resolve1(value))
+
+ except PDFValueError:
+ log.warning("Invalid MediaBox in /Page, defaulting to US Letter")
+ return us_letter
+
+ def _parse_cropbox(self, value: Any, mediabox: Rect) -> Rect:
+ if value is None:
+ # CropBox is optional, and MediaBox is used if not specified.
+ return mediabox
+
+ try:
+ return parse_rect(resolve1(val) for val in resolve1(value))
+
+ except PDFValueError:
+ log.warning("Invalid CropBox in /Page, defaulting to MediaBox")
+ return mediabox
+
+ def _parse_contents(self, value: Any) -> list[Any]:
+ contents: list[Any] = []
+ if value is not None:
+ contents = resolve1(value)
+ if not isinstance(contents, list):
+ contents = [contents]
+ return contents
diff --git a/babeldoc/pdfminer/pdfparser.py b/babeldoc/pdfminer/pdfparser.py
new file mode 100644
index 0000000000000000000000000000000000000000..78a85a5f42e27d5864b7e98895a145bbfc53baf7
--- /dev/null
+++ b/babeldoc/pdfminer/pdfparser.py
@@ -0,0 +1,173 @@
+import logging
+from io import BytesIO
+from typing import TYPE_CHECKING
+from typing import BinaryIO
+from typing import Union
+
+from babeldoc.pdfminer.casting import safe_int
+from babeldoc.pdfminer.pdfexceptions import PDFException
+from babeldoc.pdfminer.pdftypes import PDFObjRef
+from babeldoc.pdfminer.pdftypes import PDFStream
+from babeldoc.pdfminer.pdftypes import dict_value
+from babeldoc.pdfminer.pdftypes import int_value
+from babeldoc.pdfminer.psexceptions import PSEOF
+from babeldoc.pdfminer.psparser import KWD
+from babeldoc.pdfminer.psparser import PSKeyword
+from babeldoc.pdfminer.psparser import PSStackParser
+from babeldoc.pdfminer import settings
+
+if TYPE_CHECKING:
+ from babeldoc.pdfminer.pdfdocument import PDFDocument
+
+log = logging.getLogger(__name__)
+
+
+class PDFSyntaxError(PDFException):
+ pass
+
+
+# PDFParser stack holds all the base types plus PDFStream, PDFObjRef, and None
+class PDFParser(PSStackParser[Union[PSKeyword, PDFStream, PDFObjRef, None]]):
+ """PDFParser fetch PDF objects from a file stream.
+ It can handle indirect references by referring to
+ a PDF document set by set_document method.
+ It also reads XRefs at the end of every PDF file.
+
+ Typical usage:
+ parser = PDFParser(fp)
+ parser.read_xref()
+ parser.read_xref(fallback=True) # optional
+ parser.set_document(doc)
+ parser.seek(offset)
+ parser.nextobject()
+
+ """
+
+ def __init__(self, fp: BinaryIO) -> None:
+ PSStackParser.__init__(self, fp)
+ self.doc: PDFDocument | None = None
+ self.fallback = False
+
+ def set_document(self, doc: "PDFDocument") -> None:
+ """Associates the parser with a PDFDocument object."""
+ self.doc = doc
+
+ KEYWORD_R = KWD(b"R")
+ KEYWORD_NULL = KWD(b"null")
+ KEYWORD_ENDOBJ = KWD(b"endobj")
+ KEYWORD_STREAM = KWD(b"stream")
+ KEYWORD_XREF = KWD(b"xref")
+ KEYWORD_STARTXREF = KWD(b"startxref")
+
+ def do_keyword(self, pos: int, token: PSKeyword) -> None:
+ """Handles PDF-related keywords."""
+ if token in (self.KEYWORD_XREF, self.KEYWORD_STARTXREF):
+ self.add_results(*self.pop(1))
+
+ elif token is self.KEYWORD_ENDOBJ:
+ self.add_results(*self.pop(4))
+
+ elif token is self.KEYWORD_NULL:
+ # null object
+ self.push((pos, None))
+
+ elif token is self.KEYWORD_R:
+ # reference to indirect object
+ if len(self.curstack) >= 2:
+ (_, _object_id), _ = self.pop(2)
+ object_id = safe_int(_object_id)
+ if object_id is not None:
+ obj = PDFObjRef(self.doc, object_id)
+ self.push((pos, obj))
+
+ elif token is self.KEYWORD_STREAM:
+ # stream object
+ ((_, dic),) = self.pop(1)
+ dic = dict_value(dic)
+ objlen = 0
+ if not self.fallback:
+ try:
+ objlen = int_value(dic["Length"])
+ except KeyError:
+ if settings.STRICT:
+ raise PDFSyntaxError("/Length is undefined: %r" % dic)
+ self.seek(pos)
+ try:
+ (_, line) = self.nextline() # 'stream'
+ except PSEOF:
+ if settings.STRICT:
+ raise PDFSyntaxError("Unexpected EOF")
+ return
+ pos += len(line)
+ self.fp.seek(pos)
+ data = bytearray(self.fp.read(objlen))
+ self.seek(pos + objlen)
+ while 1:
+ try:
+ (linepos, line) = self.nextline()
+ except PSEOF:
+ if settings.STRICT:
+ raise PDFSyntaxError("Unexpected EOF")
+ break
+ if b"endstream" in line:
+ i = line.index(b"endstream")
+ objlen += i
+ if self.fallback:
+ data += line[:i]
+ break
+ objlen += len(line)
+ if self.fallback:
+ data += line
+ self.seek(pos + objlen)
+ # XXX limit objlen not to exceed object boundary
+ log.debug(
+ "Stream: pos=%d, objlen=%d, dic=%r, data=%r...",
+ pos,
+ objlen,
+ dic,
+ data[:10],
+ )
+ assert self.doc is not None
+ stream = PDFStream(dic, bytes(data), self.doc.decipher)
+ self.push((pos, stream))
+
+ else:
+ # others
+ self.push((pos, token))
+
+
+class PDFStreamParser(PDFParser):
+ """PDFStreamParser is used to parse PDF content streams
+ that is contained in each page and has instructions
+ for rendering the page. A reference to a PDF document is
+ needed because a PDF content stream can also have
+ indirect references to other objects in the same document.
+ """
+
+ def __init__(self, data: bytes) -> None:
+ PDFParser.__init__(self, BytesIO(data))
+
+ def flush(self) -> None:
+ self.add_results(*self.popall())
+
+ KEYWORD_OBJ = KWD(b"obj")
+
+ def do_keyword(self, pos: int, token: PSKeyword) -> None:
+ if token is self.KEYWORD_R:
+ # reference to indirect object
+ (_, _object_id), _ = self.pop(2)
+ object_id = safe_int(_object_id)
+ if object_id is not None:
+ obj = PDFObjRef(self.doc, object_id)
+ self.push((pos, obj))
+ return
+
+ elif token in (self.KEYWORD_OBJ, self.KEYWORD_ENDOBJ):
+ if settings.STRICT:
+ # See PDF Spec 3.4.6: Only the object values are stored in the
+ # stream; the obj and endobj keywords are not used.
+ raise PDFSyntaxError("Keyword endobj found in stream")
+ return
+
+ # others
+ self.push((pos, token))
diff --git a/babeldoc/pdfminer/pdftypes.py b/babeldoc/pdfminer/pdftypes.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea72c9c70e0a195dabdff807e3d1b2d1d0dd32d8
--- /dev/null
+++ b/babeldoc/pdfminer/pdftypes.py
@@ -0,0 +1,394 @@
+import io
+import logging
+import zlib
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
+from typing import Any
+from typing import Optional
+from typing import Protocol
+from typing import cast
+from warnings import warn
+
+from babeldoc.pdfminer.ascii85 import ascii85decode
+from babeldoc.pdfminer.ascii85 import asciihexdecode
+from babeldoc.pdfminer.ccitt import ccittfaxdecode
+from babeldoc.pdfminer.lzw import lzwdecode
+from babeldoc.pdfminer.psparser import LIT
+from babeldoc.pdfminer.psparser import PSObject
+from babeldoc.pdfminer.runlength import rldecode
+from babeldoc.pdfminer.utils import apply_png_predictor
+from babeldoc.pdfminer import pdfexceptions
+from babeldoc.pdfminer import settings
+
+if TYPE_CHECKING:
+ from babeldoc.pdfminer.pdfdocument import PDFDocument
+
+logger = logging.getLogger(__name__)
+
+LITERAL_CRYPT = LIT("Crypt")
+
+# Abbreviation of Filter names in PDF 4.8.6. "Inline Images"
+LITERALS_FLATE_DECODE = (LIT("FlateDecode"), LIT("Fl"))
+LITERALS_LZW_DECODE = (LIT("LZWDecode"), LIT("LZW"))
+LITERALS_ASCII85_DECODE = (LIT("ASCII85Decode"), LIT("A85"))
+LITERALS_ASCIIHEX_DECODE = (LIT("ASCIIHexDecode"), LIT("AHx"))
+LITERALS_RUNLENGTH_DECODE = (LIT("RunLengthDecode"), LIT("RL"))
+LITERALS_CCITTFAX_DECODE = (LIT("CCITTFaxDecode"), LIT("CCF"))
+LITERALS_DCT_DECODE = (LIT("DCTDecode"), LIT("DCT"))
+LITERALS_JBIG2_DECODE = (LIT("JBIG2Decode"),)
+LITERALS_JPX_DECODE = (LIT("JPXDecode"),)
+
+
+class DecipherCallable(Protocol):
+ """Fully typed a decipher callback, with optional parameter."""
+
+ def __call__(
+ self,
+ objid: int,
+ genno: int,
+ data: bytes,
+ attrs: dict[str, Any] | None = None,
+ ) -> bytes:
+ raise NotImplementedError
+
+
+class PDFObject(PSObject):
+ pass
+
+
+# Adding aliases for these exceptions for backwards compatibility
+PDFException = pdfexceptions.PDFException
+PDFTypeError = pdfexceptions.PDFTypeError
+PDFValueError = pdfexceptions.PDFValueError
+PDFObjectNotFound = pdfexceptions.PDFObjectNotFound
+PDFNotImplementedError = pdfexceptions.PDFNotImplementedError
+
+_DEFAULT = object()
+
+
+class PDFObjRef(PDFObject):
+ def __init__(
+ self,
+ doc: Optional["PDFDocument"],
+ objid: int,
+ _: Any = _DEFAULT,
+ ) -> None:
+ """Reference to a PDF object.
+
+ :param doc: The PDF document.
+ :param objid: The object number.
+ :param _: Unused argument for backwards compatibility.
+ """
+ if _ is not _DEFAULT:
+ warn(
+ "The third argument of PDFObjRef is unused and will be removed after "
+ "2024",
+ DeprecationWarning,
+ )
+
+ if objid == 0:
+ if settings.STRICT:
+ raise PDFValueError("PDF object id cannot be 0.")
+
+ self.doc = doc
+ self.objid = objid
+
+ def __repr__(self) -> str:
+ return "" % (self.objid)
+
+ def resolve(self, default: object = None) -> Any:
+ assert self.doc is not None
+ try:
+ return self.doc.getobj(self.objid)
+ except PDFObjectNotFound:
+ return default
+
+
+def resolve1(x: object, default: object = None) -> Any:
+ """Resolves an object.
+
+ If this is an array or dictionary, it may still contains
+ some indirect objects inside.
+ """
+ while isinstance(x, PDFObjRef):
+ x = x.resolve(default=default)
+ return x
+
+
+def resolve_all(x: object, default: object = None) -> Any:
+ """Recursively resolves the given object and all the internals.
+
+ Make sure there is no indirect reference within the nested object.
+ This procedure might be slow.
+ """
+ while isinstance(x, PDFObjRef):
+ x = x.resolve(default=default)
+ if isinstance(x, list):
+ x = [resolve_all(v, default=default) for v in x]
+ elif isinstance(x, dict):
+ for k, v in x.items():
+ x[k] = resolve_all(v, default=default)
+ return x
+
+
+def decipher_all(decipher: DecipherCallable, objid: int, genno: int, x: object) -> Any:
+ """Recursively deciphers the given object."""
+ if isinstance(x, bytes):
+ if len(x) == 0:
+ return x
+ return decipher(objid, genno, x)
+ if isinstance(x, list):
+ x = [decipher_all(decipher, objid, genno, v) for v in x]
+ elif isinstance(x, dict):
+ for k, v in x.items():
+ x[k] = decipher_all(decipher, objid, genno, v)
+ return x
+
+
+def int_value(x: object) -> int:
+ x = resolve1(x)
+ if not isinstance(x, int):
+ if settings.STRICT:
+ raise PDFTypeError("Integer required: %r" % x)
+ return 0
+ return x
+
+
+def float_value(x: object) -> float:
+ x = resolve1(x)
+ if not isinstance(x, float):
+ if settings.STRICT:
+ raise PDFTypeError("Float required: %r" % x)
+ return 0.0
+ return x
+
+
+def num_value(x: object) -> float:
+ x = resolve1(x)
+ if not isinstance(x, (int, float)): # == utils.isnumber(x)
+ if settings.STRICT:
+ raise PDFTypeError("Int or Float required: %r" % x)
+ return 0
+ return x
+
+
+def uint_value(x: object, n_bits: int) -> int:
+ """Resolve number and interpret it as a two's-complement unsigned number"""
+ xi = int_value(x)
+ if xi > 0:
+ return xi
+ else:
+ return xi + cast(int, 2**n_bits)
+
+
+def str_value(x: object) -> bytes:
+ x = resolve1(x)
+ if not isinstance(x, bytes):
+ if settings.STRICT:
+ raise PDFTypeError("String required: %r" % x)
+ return b""
+ return x
+
+
+def list_value(x: object) -> list[Any] | tuple[Any, ...]:
+ x = resolve1(x)
+ if not isinstance(x, (list, tuple)):
+ if settings.STRICT:
+ raise PDFTypeError("List required: %r" % x)
+ return []
+ return x
+
+
+def dict_value(x: object) -> dict[Any, Any]:
+ x = resolve1(x)
+ if not isinstance(x, dict):
+ if settings.STRICT:
+ logger.error("PDFTypeError : Dict required: %r", x)
+ raise PDFTypeError("Dict required: %r" % x)
+ return {}
+ return x
+
+
+def stream_value(x: object) -> "PDFStream":
+ x = resolve1(x)
+ if not isinstance(x, PDFStream):
+ if settings.STRICT:
+ raise PDFTypeError("PDFStream required: %r" % x)
+ return PDFStream({}, b"")
+ return x
+
+
+def decompress_corrupted(data: bytes) -> bytes:
+ """Called on some data that can't be properly decoded because of CRC checksum
+ error. Attempt to decode it skipping the CRC.
+ """
+ d = zlib.decompressobj()
+ f = io.BytesIO(data)
+ result_str = b""
+ buffer = f.read(1)
+ i = 0
+ try:
+ while buffer:
+ result_str += d.decompress(buffer)
+ buffer = f.read(1)
+ i += 1
+ except zlib.error:
+ # Let the error propagates if we're not yet in the CRC checksum
+ if i < len(data) - 3:
+ logger.warning("Data-loss while decompressing corrupted data")
+ return result_str
+
+
+class PDFStream(PDFObject):
+ def __init__(
+ self,
+ attrs: dict[str, Any],
+ rawdata: bytes,
+ decipher: DecipherCallable | None = None,
+ ) -> None:
+ assert isinstance(attrs, dict), str(type(attrs))
+ self.attrs = attrs
+ self.rawdata: bytes | None = rawdata
+ self.decipher = decipher
+ self.data: bytes | None = None
+ self.objid: int | None = None
+ self.genno: int | None = None
+
+ def set_objid(self, objid: int, genno: int) -> None:
+ self.objid = objid
+ self.genno = genno
+
+ def __repr__(self) -> str:
+ if self.data is None:
+ assert self.rawdata is not None
+ return "" % (
+ self.objid,
+ len(self.rawdata),
+ self.attrs,
+ )
+ else:
+ assert self.data is not None
+ return "" % (
+ self.objid,
+ len(self.data),
+ self.attrs,
+ )
+
+ def __contains__(self, name: object) -> bool:
+ return name in self.attrs
+
+ def __getitem__(self, name: str) -> Any:
+ return self.attrs[name]
+
+ def get(self, name: str, default: object = None) -> Any:
+ return self.attrs.get(name, default)
+
+ def get_any(self, names: Iterable[str], default: object = None) -> Any:
+ for name in names:
+ if name in self.attrs:
+ return self.attrs[name]
+ return default
+
+ def get_filters(self) -> list[tuple[Any, Any]]:
+ filters = resolve1(self.get_any(("F", "Filter"), []))
+ params = resolve1(self.get_any(("DP", "DecodeParms", "FDecodeParms"), {}))
+ if not filters:
+ return []
+ if not isinstance(filters, list):
+ filters = [filters]
+ if not isinstance(params, list):
+ # Make sure the parameters list is the same as filters.
+ params = [params] * len(filters)
+ if settings.STRICT and len(params) != len(filters):
+ raise PDFException("Parameters len filter mismatch")
+
+ resolved_filters = [resolve1(f) for f in filters]
+ resolved_params = [resolve1(param) for param in params]
+ return list(zip(resolved_filters, resolved_params, strict=False))
+
+ def decode(self) -> None:
+ assert self.data is None and self.rawdata is not None, str(
+ (self.data, self.rawdata),
+ )
+ data = self.rawdata
+ if self.decipher:
+ # Handle encryption
+ assert self.objid is not None
+ assert self.genno is not None
+ data = self.decipher(self.objid, self.genno, data, self.attrs)
+ filters = self.get_filters()
+ if not filters:
+ self.data = data
+ self.rawdata = None
+ return
+ for f, params in filters:
+ if f in LITERALS_FLATE_DECODE:
+ # will get errors if the document is encrypted.
+ try:
+ data = zlib.decompress(data)
+
+ except zlib.error as e:
+ if settings.STRICT:
+ error_msg = f"Invalid zlib bytes: {e!r}, {data!r}"
+ raise PDFException(error_msg)
+
+ try:
+ data = decompress_corrupted(data)
+ except zlib.error:
+ data = b""
+
+ elif f in LITERALS_LZW_DECODE:
+ data = lzwdecode(data)
+ elif f in LITERALS_ASCII85_DECODE:
+ data = ascii85decode(data)
+ elif f in LITERALS_ASCIIHEX_DECODE:
+ data = asciihexdecode(data)
+ elif f in LITERALS_RUNLENGTH_DECODE:
+ data = rldecode(data)
+ elif f in LITERALS_CCITTFAX_DECODE:
+ data = ccittfaxdecode(data, params)
+ elif f in LITERALS_DCT_DECODE:
+ # This is probably a JPG stream
+ # it does not need to be decoded twice.
+ # Just return the stream to the user.
+ pass
+ elif f in LITERALS_JBIG2_DECODE or f in LITERALS_JPX_DECODE:
+ pass
+ elif f == LITERAL_CRYPT:
+ # not yet..
+ raise PDFNotImplementedError("/Crypt filter is unsupported")
+ else:
+ raise PDFNotImplementedError("Unsupported filter: %r" % f)
+ # apply predictors
+ if params and "Predictor" in params:
+ pred = int_value(params["Predictor"])
+ if pred == 1:
+ # no predictor
+ pass
+ elif pred >= 10:
+ # PNG predictor
+ colors = int_value(params.get("Colors", 1))
+ columns = int_value(params.get("Columns", 1))
+ raw_bits_per_component = params.get("BitsPerComponent", 8)
+ bitspercomponent = int_value(raw_bits_per_component)
+ data = apply_png_predictor(
+ pred,
+ colors,
+ columns,
+ bitspercomponent,
+ data,
+ )
+ else:
+ error_msg = "Unsupported predictor: %r" % pred
+ raise PDFNotImplementedError(error_msg)
+ self.data = data
+ self.rawdata = None
+
+ def get_data(self) -> bytes:
+ if self.data is None:
+ self.decode()
+ assert self.data is not None
+ return self.data
+
+ def get_rawdata(self) -> bytes | None:
+ return self.rawdata
diff --git a/babeldoc/pdfminer/psexceptions.py b/babeldoc/pdfminer/psexceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..b8291dc0915cd48c0d63f7a29f877476f1c39220
--- /dev/null
+++ b/babeldoc/pdfminer/psexceptions.py
@@ -0,0 +1,18 @@
+class PSException(Exception):
+ pass
+
+
+class PSEOF(PSException):
+ pass
+
+
+class PSSyntaxError(PSException):
+ pass
+
+
+class PSTypeError(PSException):
+ pass
+
+
+class PSValueError(PSException):
+ pass
diff --git a/babeldoc/pdfminer/psparser.py b/babeldoc/pdfminer/psparser.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c11dd0b7437078560da0c8d92cb62b856f0c089
--- /dev/null
+++ b/babeldoc/pdfminer/psparser.py
@@ -0,0 +1,659 @@
+#!/usr/bin/env python3
+import io
+import logging
+import re
+from collections.abc import Iterator
+from typing import Any
+from typing import BinaryIO
+from typing import Generic
+from typing import TypeVar
+from typing import Union
+
+from babeldoc.pdfminer.utils import choplist
+from babeldoc.pdfminer import psexceptions
+from babeldoc.pdfminer import settings
+
+log = logging.getLogger(__name__)
+
+
+# Adding aliases for these exceptions for backwards compatibility
+PSException = psexceptions.PSException
+PSEOF = psexceptions.PSEOF
+PSSyntaxError = psexceptions.PSSyntaxError
+PSTypeError = psexceptions.PSTypeError
+PSValueError = psexceptions.PSValueError
+
+
+class PSObject:
+ """Base class for all PS or PDF-related data types."""
+
+
+class PSLiteral(PSObject):
+ """A class that represents a PostScript literal.
+
+ Postscript literals are used as identifiers, such as
+ variable names, property names and dictionary keys.
+ Literals are case sensitive and denoted by a preceding
+ slash sign (e.g. "/Name")
+
+ Note: Do not create an instance of PSLiteral directly.
+ Always use PSLiteralTable.intern().
+ """
+
+ NameType = Union[str, bytes]
+
+ def __init__(self, name: NameType) -> None:
+ self.name = name
+
+ def __repr__(self) -> str:
+ name = self.name
+ return "/%r" % name
+
+
+class PSKeyword(PSObject):
+ """A class that represents a PostScript keyword.
+
+ PostScript keywords are a dozen of predefined words.
+ Commands and directives in PostScript are expressed by keywords.
+ They are also used to denote the content boundaries.
+
+ Note: Do not create an instance of PSKeyword directly.
+ Always use PSKeywordTable.intern().
+ """
+
+ def __init__(self, name: bytes) -> None:
+ self.name = name
+
+ def __repr__(self) -> str:
+ name = self.name
+ return "/%r" % name
+
+
+_SymbolT = TypeVar("_SymbolT", PSLiteral, PSKeyword)
+
+
+class PSSymbolTable(Generic[_SymbolT]):
+ """A utility class for storing PSLiteral/PSKeyword objects.
+
+ Interned objects can be checked its identity with "is" operator.
+ """
+
+ def __init__(self, klass: type[_SymbolT]) -> None:
+ self.dict: dict[PSLiteral.NameType, _SymbolT] = {}
+ self.klass: type[_SymbolT] = klass
+
+ def intern(self, name: PSLiteral.NameType) -> _SymbolT:
+ if name in self.dict:
+ lit = self.dict[name]
+ else:
+ # Type confusion issue: PSKeyword always takes bytes as name
+ # PSLiteral uses either str or bytes
+ lit = self.klass(name) # type: ignore[arg-type]
+ self.dict[name] = lit
+ return lit
+
+
+PSLiteralTable = PSSymbolTable(PSLiteral)
+PSKeywordTable = PSSymbolTable(PSKeyword)
+LIT = PSLiteralTable.intern
+KWD = PSKeywordTable.intern
+KEYWORD_PROC_BEGIN = KWD(b"{")
+KEYWORD_PROC_END = KWD(b"}")
+KEYWORD_ARRAY_BEGIN = KWD(b"[")
+KEYWORD_ARRAY_END = KWD(b"]")
+KEYWORD_DICT_BEGIN = KWD(b"<<")
+KEYWORD_DICT_END = KWD(b">>")
+
+
+def literal_name(x: Any) -> str:
+ if isinstance(x, PSLiteral):
+ if isinstance(x.name, str):
+ return x.name
+ try:
+ return str(x.name, "utf-8")
+ except UnicodeDecodeError:
+ return str(x.name)
+ else:
+ if settings.STRICT:
+ raise PSTypeError(f"Literal required: {x!r}")
+ return str(x)
+
+
+def keyword_name(x: Any) -> Any:
+ if not isinstance(x, PSKeyword):
+ if settings.STRICT:
+ raise PSTypeError("Keyword required: %r" % x)
+ else:
+ name = x
+ else:
+ name = str(x.name, "utf-8", "ignore")
+ return name
+
+
+EOL = re.compile(rb"[\r\n]")
+SPC = re.compile(rb"\s")
+NONSPC = re.compile(rb"\S")
+HEX = re.compile(rb"[0-9a-fA-F]")
+END_LITERAL = re.compile(rb"[#/%\[\]()<>{}\s]")
+END_HEX_STRING = re.compile(rb"[^\s0-9a-fA-F]")
+HEX_PAIR = re.compile(rb"[0-9a-fA-F]{2}|.")
+END_NUMBER = re.compile(rb"[^0-9]")
+END_KEYWORD = re.compile(rb"[#/%\[\]()<>{}\s]")
+END_STRING = re.compile(rb"[()\134]")
+OCT_STRING = re.compile(rb"[0-7]")
+ESC_STRING = {
+ b"b": 8,
+ b"t": 9,
+ b"n": 10,
+ b"f": 12,
+ b"r": 13,
+ b"(": 40,
+ b")": 41,
+ b"\\": 92,
+}
+
+
+PSBaseParserToken = Union[float, bool, PSLiteral, PSKeyword, bytes]
+
+
+class PSBaseParser:
+ """Most basic PostScript parser that performs only tokenization."""
+
+ BUFSIZ = 4096
+
+ def __init__(self, fp: BinaryIO) -> None:
+ self.fp = fp
+ self.eof = False
+ self.seek(0)
+
+ def __repr__(self) -> str:
+ return "<%s: %r, bufpos=%d>" % (self.__class__.__name__, self.fp, self.bufpos)
+
+ def flush(self) -> None:
+ pass
+
+ def close(self) -> None:
+ self.flush()
+
+ def tell(self) -> int:
+ return self.bufpos + self.charpos
+
+ def poll(self, pos: int | None = None, n: int = 80) -> None:
+ pos0 = self.fp.tell()
+ if not pos:
+ pos = self.bufpos + self.charpos
+ self.fp.seek(pos)
+ log.debug("poll(%d): %r", pos, self.fp.read(n))
+ self.fp.seek(pos0)
+
+ def seek(self, pos: int) -> None:
+ """Seeks the parser to the given position."""
+ log.debug("seek: %r", pos)
+ self.fp.seek(pos)
+ # reset the status for nextline()
+ self.bufpos = pos
+ self.buf = b""
+ self.charpos = 0
+ # reset the status for nexttoken()
+ self._parse1 = self._parse_main
+ self._curtoken = b""
+ self._curtokenpos = 0
+ self._tokens: list[tuple[int, PSBaseParserToken]] = []
+ self.eof = False
+
+ def fillbuf(self) -> None:
+ if self.charpos < len(self.buf):
+ return
+ # fetch next chunk.
+ self.bufpos = self.fp.tell()
+ self.buf = self.fp.read(self.BUFSIZ)
+ if not self.buf:
+ raise PSEOF("Unexpected EOF")
+ self.charpos = 0
+
+ def nextline(self) -> tuple[int, bytes]:
+ """Fetches a next line that ends either with \\r or \\n."""
+ linebuf = b""
+ linepos = self.bufpos + self.charpos
+ eol = False
+ while 1:
+ self.fillbuf()
+ if eol:
+ c = self.buf[self.charpos : self.charpos + 1]
+ # handle b'\r\n'
+ if c == b"\n":
+ linebuf += c
+ self.charpos += 1
+ break
+ m = EOL.search(self.buf, self.charpos)
+ if m:
+ linebuf += self.buf[self.charpos : m.end(0)]
+ self.charpos = m.end(0)
+ if linebuf[-1:] == b"\r":
+ eol = True
+ else:
+ break
+ else:
+ linebuf += self.buf[self.charpos :]
+ self.charpos = len(self.buf)
+ log.debug("nextline: %r, %r", linepos, linebuf)
+
+ return (linepos, linebuf)
+
+ def revreadlines(self) -> Iterator[bytes]:
+ """Fetches a next line backword.
+
+ This is used to locate the trailers at the end of a file.
+ """
+ self.fp.seek(0, io.SEEK_END)
+ pos = self.fp.tell()
+ buf = b""
+ while pos > 0:
+ prevpos = pos
+ pos = max(0, pos - self.BUFSIZ)
+ self.fp.seek(pos)
+ s = self.fp.read(prevpos - pos)
+ if not s:
+ break
+ while 1:
+ n = max(s.rfind(b"\r"), s.rfind(b"\n"))
+ if n == -1:
+ buf = s + buf
+ break
+ yield s[n:] + buf
+ s = s[:n]
+ buf = b""
+
+ def _parse_main(self, s: bytes, i: int) -> int:
+ m = NONSPC.search(s, i)
+ if not m:
+ return len(s)
+ j = m.start(0)
+ c = s[j : j + 1]
+ self._curtokenpos = self.bufpos + j
+ if c == b"%":
+ self._curtoken = b"%"
+ self._parse1 = self._parse_comment
+ return j + 1
+ elif c == b"/":
+ self._curtoken = b""
+ self._parse1 = self._parse_literal
+ return j + 1
+ elif c in b"-+" or c.isdigit():
+ self._curtoken = c
+ self._parse1 = self._parse_number
+ return j + 1
+ elif c == b".":
+ self._curtoken = c
+ self._parse1 = self._parse_float
+ return j + 1
+ elif c.isalpha():
+ self._curtoken = c
+ self._parse1 = self._parse_keyword
+ return j + 1
+ elif c == b"(":
+ self._curtoken = b""
+ self.paren = 1
+ self._parse1 = self._parse_string
+ return j + 1
+ elif c == b"<":
+ self._curtoken = b""
+ self._parse1 = self._parse_wopen
+ return j + 1
+ elif c == b">":
+ self._curtoken = b""
+ self._parse1 = self._parse_wclose
+ return j + 1
+ elif c == b"\x00":
+ return j + 1
+ else:
+ self._add_token(KWD(c))
+ return j + 1
+
+ def _add_token(self, obj: PSBaseParserToken) -> None:
+ self._tokens.append((self._curtokenpos, obj))
+
+ def _parse_comment(self, s: bytes, i: int) -> int:
+ m = EOL.search(s, i)
+ if not m:
+ self._curtoken += s[i:]
+ return len(s)
+ j = m.start(0)
+ self._curtoken += s[i:j]
+ self._parse1 = self._parse_main
+ # We ignore comments.
+ # self._tokens.append(self._curtoken)
+ return j
+
+ def _parse_literal(self, s: bytes, i: int) -> int:
+ m = END_LITERAL.search(s, i)
+ if not m:
+ self._curtoken += s[i:]
+ return len(s)
+ j = m.start(0)
+ self._curtoken += s[i:j]
+ c = s[j : j + 1]
+ if c == b"#":
+ self.hex = b""
+ self._parse1 = self._parse_literal_hex
+ return j + 1
+ try:
+ name: str | bytes = str(self._curtoken, "utf-8")
+ except Exception:
+ name = self._curtoken
+ self._add_token(LIT(name))
+ self._parse1 = self._parse_main
+ return j
+
+ def _parse_literal_hex(self, s: bytes, i: int) -> int:
+ c = s[i : i + 1]
+ if HEX.match(c) and len(self.hex) < 2:
+ self.hex += c
+ return i + 1
+ if self.hex:
+ self._curtoken += bytes((int(self.hex, 16),))
+ self._parse1 = self._parse_literal
+ return i
+
+ def _parse_number(self, s: bytes, i: int) -> int:
+ m = END_NUMBER.search(s, i)
+ if not m:
+ self._curtoken += s[i:]
+ return len(s)
+ j = m.start(0)
+ self._curtoken += s[i:j]
+ c = s[j : j + 1]
+ if c == b".":
+ self._curtoken += c
+ self._parse1 = self._parse_float
+ return j + 1
+ try:
+ self._add_token(int(self._curtoken))
+ except ValueError:
+ pass
+ self._parse1 = self._parse_main
+ return j
+
+ def _parse_float(self, s: bytes, i: int) -> int:
+ m = END_NUMBER.search(s, i)
+ if not m:
+ self._curtoken += s[i:]
+ return len(s)
+ j = m.start(0)
+ self._curtoken += s[i:j]
+ try:
+ self._add_token(float(self._curtoken))
+ except ValueError:
+ pass
+ self._parse1 = self._parse_main
+ return j
+
+ def _parse_keyword(self, s: bytes, i: int) -> int:
+ m = END_KEYWORD.search(s, i)
+ if m:
+ j = m.start(0)
+ self._curtoken += s[i:j]
+ else:
+ self._curtoken += s[i:]
+ return len(s)
+ if self._curtoken == b"true":
+ token: bool | PSKeyword = True
+ elif self._curtoken == b"false":
+ token = False
+ else:
+ token = KWD(self._curtoken)
+ self._add_token(token)
+ self._parse1 = self._parse_main
+ return j
+
+ def _parse_string(self, s: bytes, i: int) -> int:
+ m = END_STRING.search(s, i)
+ if not m:
+ self._curtoken += s[i:]
+ return len(s)
+ j = m.start(0)
+ self._curtoken += s[i:j]
+ c = s[j : j + 1]
+ if c == b"\\":
+ self.oct = b""
+ self._parse1 = self._parse_string_1
+ return j + 1
+ if c == b"(":
+ self.paren += 1
+ self._curtoken += c
+ return j + 1
+ if c == b")":
+ self.paren -= 1
+ if self.paren:
+ # WTF, they said balanced parens need no special treatment.
+ self._curtoken += c
+ return j + 1
+ self._add_token(self._curtoken)
+ self._parse1 = self._parse_main
+ return j + 1
+
+ def _parse_string_1(self, s: bytes, i: int) -> int:
+ """Parse literal strings
+
+ PDF Reference 3.2.3
+ """
+ c = s[i : i + 1]
+ if OCT_STRING.match(c) and len(self.oct) < 3:
+ self.oct += c
+ return i + 1
+
+ elif self.oct:
+ chrcode = int(self.oct, 8)
+ assert chrcode < 256, "Invalid octal %s (%d)" % (repr(self.oct), chrcode)
+ self._curtoken += bytes((chrcode,))
+ self._parse1 = self._parse_string
+ return i
+
+ elif c in ESC_STRING:
+ self._curtoken += bytes((ESC_STRING[c],))
+
+ elif c == b"\r" and len(s) > i + 1 and s[i + 1 : i + 2] == b"\n":
+ # If current and next character is \r\n skip both because enters
+ # after a \ are ignored
+ i += 1
+
+ # default action
+ self._parse1 = self._parse_string
+ return i + 1
+
+ def _parse_wopen(self, s: bytes, i: int) -> int:
+ c = s[i : i + 1]
+ if c == b"<":
+ self._add_token(KEYWORD_DICT_BEGIN)
+ self._parse1 = self._parse_main
+ i += 1
+ else:
+ self._parse1 = self._parse_hexstring
+ return i
+
+ def _parse_wclose(self, s: bytes, i: int) -> int:
+ c = s[i : i + 1]
+ if c == b">":
+ self._add_token(KEYWORD_DICT_END)
+ i += 1
+ self._parse1 = self._parse_main
+ return i
+
+ def _parse_hexstring(self, s: bytes, i: int) -> int:
+ m = END_HEX_STRING.search(s, i)
+ if not m:
+ self._curtoken += s[i:]
+ return len(s)
+ j = m.start(0)
+ self._curtoken += s[i:j]
+ token = HEX_PAIR.sub(
+ lambda m: bytes((int(m.group(0), 16),)),
+ SPC.sub(b"", self._curtoken),
+ )
+ self._add_token(token)
+ self._parse1 = self._parse_main
+ return j
+
+ def nexttoken(self) -> tuple[int, PSBaseParserToken]:
+ if self.eof:
+ # It's not really unexpected, come on now...
+ raise PSEOF("Unexpected EOF")
+ while not self._tokens:
+ try:
+ self.fillbuf()
+ self.charpos = self._parse1(self.buf, self.charpos)
+ except PSEOF:
+ # If we hit EOF in the middle of a token, try to parse
+ # it by tacking on whitespace, and delay raising PSEOF
+ # until next time around
+ self.charpos = self._parse1(b"\n", 0)
+ self.eof = True
+ # Oh, so there wasn't actually a token there? OK.
+ if not self._tokens:
+ raise
+ token = self._tokens.pop(0)
+ log.debug("nexttoken: %r", token)
+ return token
+
+
+# Stack slots may by occupied by any of:
+# * the name of a literal
+# * the PSBaseParserToken types
+# * list (via KEYWORD_ARRAY)
+# * dict (via KEYWORD_DICT)
+# * subclass-specific extensions (e.g. PDFStream, PDFObjRef) via ExtraT
+ExtraT = TypeVar("ExtraT")
+PSStackType = Union[str, float, bool, PSLiteral, bytes, list, dict, ExtraT]
+PSStackEntry = tuple[int, PSStackType[ExtraT]]
+
+
+class PSStackParser(PSBaseParser, Generic[ExtraT]):
+ def __init__(self, fp: BinaryIO) -> None:
+ PSBaseParser.__init__(self, fp)
+ self.reset()
+
+ def reset(self) -> None:
+ self.context: list[tuple[int, str | None, list[PSStackEntry[ExtraT]]]] = []
+ self.curtype: str | None = None
+ self.curstack: list[PSStackEntry[ExtraT]] = []
+ self.results: list[PSStackEntry[ExtraT]] = []
+
+ def seek(self, pos: int) -> None:
+ PSBaseParser.seek(self, pos)
+ self.reset()
+
+ def push(self, *objs: PSStackEntry[ExtraT]) -> None:
+ self.curstack.extend(objs)
+
+ def pop(self, n: int) -> list[PSStackEntry[ExtraT]]:
+ objs = self.curstack[-n:]
+ self.curstack[-n:] = []
+ return objs
+
+ def popall(self) -> list[PSStackEntry[ExtraT]]:
+ objs = self.curstack
+ self.curstack = []
+ return objs
+
+ def add_results(self, *objs: PSStackEntry[ExtraT]) -> None:
+ try:
+ log.debug("add_results: %r", objs)
+ except Exception:
+ log.debug("add_results: (unprintable object)")
+ self.results.extend(objs)
+
+ def start_type(self, pos: int, type: str) -> None:
+ self.context.append((pos, self.curtype, self.curstack))
+ (self.curtype, self.curstack) = (type, [])
+ log.debug("start_type: pos=%r, type=%r", pos, type)
+
+ def end_type(self, type: str) -> tuple[int, list[PSStackType[ExtraT]]]:
+ if self.curtype != type:
+ raise PSTypeError(f"Type mismatch: {self.curtype!r} != {type!r}")
+ objs = [obj for (_, obj) in self.curstack]
+ (pos, self.curtype, self.curstack) = self.context.pop()
+ log.debug("end_type: pos=%r, type=%r, objs=%r", pos, type, objs)
+ return (pos, objs)
+
+ def do_keyword(self, pos: int, token: PSKeyword) -> None:
+ pass
+
+ def nextobject(self) -> PSStackEntry[ExtraT]:
+ """Yields a list of objects.
+
+ Arrays and dictionaries are represented as Python lists and
+ dictionaries.
+
+ :return: keywords, literals, strings, numbers, arrays and dictionaries.
+ """
+ while not self.results:
+ (pos, token) = self.nexttoken()
+ if isinstance(token, (int, float, bool, str, bytes, PSLiteral)):
+ # normal token
+ self.push((pos, token))
+ elif token == KEYWORD_ARRAY_BEGIN:
+ # begin array
+ self.start_type(pos, "a")
+ elif token == KEYWORD_ARRAY_END:
+ # end array
+ try:
+ self.push(self.end_type("a"))
+ except PSTypeError:
+ if settings.STRICT:
+ raise
+ elif token == KEYWORD_DICT_BEGIN:
+ # begin dictionary
+ self.start_type(pos, "d")
+ elif token == KEYWORD_DICT_END:
+ # end dictionary
+ try:
+ (pos, objs) = self.end_type("d")
+ if len(objs) % 2 != 0:
+ error_msg = "Invalid dictionary construct: %r" % objs
+ raise PSSyntaxError(error_msg)
+ d = {
+ literal_name(k): v
+ for (k, v) in choplist(2, objs)
+ if v is not None
+ }
+ self.push((pos, d))
+ except PSTypeError:
+ if settings.STRICT:
+ raise
+ elif token == KEYWORD_PROC_BEGIN:
+ # begin proc
+ self.start_type(pos, "p")
+ elif token == KEYWORD_PROC_END:
+ # end proc
+ try:
+ self.push(self.end_type("p"))
+ except PSTypeError:
+ if settings.STRICT:
+ raise
+ elif isinstance(token, PSKeyword):
+ log.debug(
+ "do_keyword: pos=%r, token=%r, stack=%r",
+ pos,
+ token,
+ self.curstack,
+ )
+ self.do_keyword(pos, token)
+ else:
+ log.error(
+ "unknown token: pos=%r, token=%r, stack=%r",
+ pos,
+ token,
+ self.curstack,
+ )
+ self.do_keyword(pos, token)
+ raise PSException
+ if self.context:
+ continue
+ else:
+ self.flush()
+ obj = self.results.pop(0)
+ try:
+ log.debug("nextobject: %r", obj)
+ except Exception:
+ log.debug("nextobject: (unprintable object)")
+ return obj
diff --git a/babeldoc/pdfminer/py.typed b/babeldoc/pdfminer/py.typed
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/babeldoc/pdfminer/runlength.py b/babeldoc/pdfminer/runlength.py
new file mode 100644
index 0000000000000000000000000000000000000000..c821e7cc2416fa0751d17b5ec2c2e459c27b649c
--- /dev/null
+++ b/babeldoc/pdfminer/runlength.py
@@ -0,0 +1,36 @@
+#
+# RunLength decoder (Adobe version) implementation based on PDF Reference
+# version 1.4 section 3.3.4.
+#
+# * public domain *
+#
+
+
+def rldecode(data: bytes) -> bytes:
+ """RunLength decoder (Adobe version) implementation based on PDF Reference
+ version 1.4 section 3.3.4:
+ The RunLengthDecode filter decodes data that has been encoded in a
+ simple byte-oriented format based on run length. The encoded data
+ is a sequence of runs, where each run consists of a length byte
+ followed by 1 to 128 bytes of data. If the length byte is in the
+ range 0 to 127, the following length + 1 (1 to 128) bytes are
+ copied literally during decompression. If length is in the range
+ 129 to 255, the following single byte is to be copied 257 - length
+ (2 to 128) times during decompression. A length value of 128
+ denotes EOD.
+ """
+ decoded_array: list[int] = []
+ data_iter = iter(data)
+
+ while True:
+ length = next(data_iter, 128)
+ if length == 128:
+ break
+
+ if 0 <= length < 128:
+ decoded_array.extend(next(data_iter) for _ in range(length + 1))
+
+ if length > 128:
+ run = [next(data_iter)] * (257 - length)
+ decoded_array.extend(run)
+ return bytes(decoded_array)
diff --git a/babeldoc/pdfminer/settings.py b/babeldoc/pdfminer/settings.py
new file mode 100644
index 0000000000000000000000000000000000000000..810077a0718f9fa29556b23009550b37a29cafab
--- /dev/null
+++ b/babeldoc/pdfminer/settings.py
@@ -0,0 +1 @@
+STRICT = False
diff --git a/babeldoc/pdfminer/utils.py b/babeldoc/pdfminer/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..5161ee7839ea2b5f77557cf068ec44da340ba6ae
--- /dev/null
+++ b/babeldoc/pdfminer/utils.py
@@ -0,0 +1,799 @@
+"""Miscellaneous Routines."""
+
+import io
+import pathlib
+import string
+from collections.abc import Callable
+from collections.abc import Iterable
+from collections.abc import Iterator
+from html import escape
+from typing import TYPE_CHECKING
+from typing import Any
+from typing import BinaryIO
+from typing import Generic
+from typing import TextIO
+from typing import TypeVar
+from typing import Union
+from typing import cast
+
+from babeldoc.pdfminer.pdfexceptions import PDFTypeError
+from babeldoc.pdfminer.pdfexceptions import PDFValueError
+
+if TYPE_CHECKING:
+ from babeldoc.pdfminer.layout import LTComponent
+
+import charset_normalizer # For str encoding detection
+
+# from sys import maxint as INF doesn't work anymore under Python3, but PDF
+# still uses 32 bits ints
+INF = (1 << 31) - 1
+
+
+FileOrName = Union[pathlib.PurePath, str, io.IOBase]
+AnyIO = Union[TextIO, BinaryIO]
+
+
+class open_filename:
+ """Context manager that allows opening a filename
+ (str or pathlib.PurePath type is supported) and closes it on exit,
+ (just like `open`), but does nothing for file-like objects.
+ """
+
+ def __init__(self, filename: FileOrName, *args: Any, **kwargs: Any) -> None:
+ if isinstance(filename, pathlib.PurePath):
+ filename = str(filename)
+ if isinstance(filename, str):
+ self.file_handler: AnyIO = open(filename, *args, **kwargs)
+ self.closing = True
+ elif isinstance(filename, io.IOBase):
+ self.file_handler = cast(AnyIO, filename)
+ self.closing = False
+ else:
+ raise PDFTypeError("Unsupported input type: %s" % type(filename))
+
+ def __enter__(self) -> AnyIO:
+ return self.file_handler
+
+ def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
+ if self.closing:
+ self.file_handler.close()
+
+
+def make_compat_bytes(in_str: str) -> bytes:
+ """Converts to bytes, encoding to unicode."""
+ assert isinstance(in_str, str), str(type(in_str))
+ return in_str.encode()
+
+
+def make_compat_str(o: object) -> str:
+ """Converts everything to string, if bytes guessing the encoding."""
+ if isinstance(o, bytes):
+ enc = charset_normalizer.detect(o)
+ try:
+ return o.decode(enc["encoding"])
+ except UnicodeDecodeError:
+ return str(o)
+ else:
+ return str(o)
+
+
+def shorten_str(s: str, size: int) -> str:
+ if size < 7:
+ return s[:size]
+ if len(s) > size:
+ length = (size - 5) // 2
+ return f"{s[:length]} ... {s[-length:]}"
+ else:
+ return s
+
+
+def compatible_encode_method(
+ bytesorstring: bytes | str,
+ encoding: str = "utf-8",
+ erraction: str = "ignore",
+) -> str:
+ """When Py2 str.encode is called, it often means bytes.encode in Py3.
+
+ This does either.
+ """
+ if isinstance(bytesorstring, str):
+ return bytesorstring
+ assert isinstance(bytesorstring, bytes), str(type(bytesorstring))
+ return bytesorstring.decode(encoding, erraction)
+
+
+def paeth_predictor(left: int, above: int, upper_left: int) -> int:
+ # From http://www.libpng.org/pub/png/spec/1.2/PNG-Filters.html
+ # Initial estimate
+ p = left + above - upper_left
+ # Distances to a,b,c
+ pa = abs(p - left)
+ pb = abs(p - above)
+ pc = abs(p - upper_left)
+
+ # Return nearest of a,b,c breaking ties in order a,b,c
+ if pa <= pb and pa <= pc:
+ return left
+ elif pb <= pc:
+ return above
+ else:
+ return upper_left
+
+
+def apply_png_predictor(
+ pred: int,
+ colors: int,
+ columns: int,
+ bitspercomponent: int,
+ data: bytes,
+) -> bytes:
+ """Reverse the effect of the PNG predictor
+
+ Documentation: http://www.libpng.org/pub/png/spec/1.2/PNG-Filters.html
+ """
+ if bitspercomponent not in [8, 1]:
+ msg = "Unsupported `bitspercomponent': %d" % bitspercomponent
+ raise PDFValueError(msg)
+
+ nbytes = colors * columns * bitspercomponent // 8
+ bpp = colors * bitspercomponent // 8 # number of bytes per complete pixel
+ buf = []
+ line_above = list(b"\x00" * columns)
+ for scanline_i in range(0, len(data), nbytes + 1):
+ filter_type = data[scanline_i]
+ line_encoded = data[scanline_i + 1 : scanline_i + 1 + nbytes]
+ raw = []
+
+ if filter_type == 0:
+ # Filter type 0: None
+ raw = list(line_encoded)
+
+ elif filter_type == 1:
+ # Filter type 1: Sub
+ # To reverse the effect of the Sub() filter after decompression,
+ # output the following value:
+ # Raw(x) = Sub(x) + Raw(x - bpp)
+ # (computed mod 256), where Raw() refers to the bytes already
+ # decoded.
+ for j, sub_x in enumerate(line_encoded):
+ if j - bpp < 0:
+ raw_x_bpp = 0
+ else:
+ raw_x_bpp = int(raw[j - bpp])
+ raw_x = (sub_x + raw_x_bpp) & 255
+ raw.append(raw_x)
+
+ elif filter_type == 2:
+ # Filter type 2: Up
+ # To reverse the effect of the Up() filter after decompression,
+ # output the following value:
+ # Raw(x) = Up(x) + Prior(x)
+ # (computed mod 256), where Prior() refers to the decoded bytes of
+ # the prior scanline.
+ for up_x, prior_x in zip(line_encoded, line_above, strict=False):
+ raw_x = (up_x + prior_x) & 255
+ raw.append(raw_x)
+
+ elif filter_type == 3:
+ # Filter type 3: Average
+ # To reverse the effect of the Average() filter after
+ # decompression, output the following value:
+ # Raw(x) = Average(x) + floor((Raw(x-bpp)+Prior(x))/2)
+ # where the result is computed mod 256, but the prediction is
+ # calculated in the same way as for encoding. Raw() refers to the
+ # bytes already decoded, and Prior() refers to the decoded bytes of
+ # the prior scanline.
+ for j, average_x in enumerate(line_encoded):
+ if j - bpp < 0:
+ raw_x_bpp = 0
+ else:
+ raw_x_bpp = int(raw[j - bpp])
+ prior_x = int(line_above[j])
+ raw_x = (average_x + (raw_x_bpp + prior_x) // 2) & 255
+ raw.append(raw_x)
+
+ elif filter_type == 4:
+ # Filter type 4: Paeth
+ # To reverse the effect of the Paeth() filter after decompression,
+ # output the following value:
+ # Raw(x) = Paeth(x)
+ # + PaethPredictor(Raw(x-bpp), Prior(x), Prior(x-bpp))
+ # (computed mod 256), where Raw() and Prior() refer to bytes
+ # already decoded. Exactly the same PaethPredictor() function is
+ # used by both encoder and decoder.
+ for j, paeth_x in enumerate(line_encoded):
+ if j - bpp < 0:
+ raw_x_bpp = 0
+ prior_x_bpp = 0
+ else:
+ raw_x_bpp = int(raw[j - bpp])
+ prior_x_bpp = int(line_above[j - bpp])
+ prior_x = int(line_above[j])
+ paeth = paeth_predictor(raw_x_bpp, prior_x, prior_x_bpp)
+ raw_x = (paeth_x + paeth) & 255
+ raw.append(raw_x)
+
+ else:
+ raise PDFValueError("Unsupported predictor value: %d" % filter_type)
+
+ buf.extend(raw)
+ line_above = raw
+ return bytes(buf)
+
+
+Point = tuple[float, float]
+Rect = tuple[float, float, float, float]
+Matrix = tuple[float, float, float, float, float, float]
+PathSegment = Union[
+ tuple[str], # Literal['h']
+ tuple[str, float, float], # Literal['m', 'l']
+ tuple[str, float, float, float, float], # Literal['v', 'y']
+ tuple[str, float, float, float, float, float, float],
+] # Literal['c']
+
+# Matrix operations
+MATRIX_IDENTITY: Matrix = (1, 0, 0, 1, 0, 0)
+
+
+def parse_rect(o: Any) -> Rect:
+ try:
+ (x0, y0, x1, y1) = o
+ return float(x0), float(y0), float(x1), float(y1)
+ except ValueError:
+ raise PDFValueError("Could not parse rectangle")
+
+
+def mult_matrix(m1: Matrix, m0: Matrix) -> Matrix:
+ (a1, b1, c1, d1, e1, f1) = m1
+ (a0, b0, c0, d0, e0, f0) = m0
+ """Returns the multiplication of two matrices."""
+ return (
+ a0 * a1 + c0 * b1,
+ b0 * a1 + d0 * b1,
+ a0 * c1 + c0 * d1,
+ b0 * c1 + d0 * d1,
+ a0 * e1 + c0 * f1 + e0,
+ b0 * e1 + d0 * f1 + f0,
+ )
+
+
+def translate_matrix(m: Matrix, v: Point) -> Matrix:
+ """Translates a matrix by (x, y)."""
+ (a, b, c, d, e, f) = m
+ (x, y) = v
+ return a, b, c, d, x * a + y * c + e, x * b + y * d + f
+
+
+def apply_matrix_pt(m: Matrix, v: Point) -> Point:
+ (a, b, c, d, e, f) = m
+ (x, y) = v
+ """Applies a matrix to a point."""
+ return a * x + c * y + e, b * x + d * y + f
+
+
+def apply_matrix_norm(m: Matrix, v: Point) -> Point:
+ """Equivalent to apply_matrix_pt(M, (p,q)) - apply_matrix_pt(M, (0,0))"""
+ (a, b, c, d, e, f) = m
+ (p, q) = v
+ return a * p + c * q, b * p + d * q
+
+
+# Utility functions
+
+
+def isnumber(x: object) -> bool:
+ return isinstance(x, (int, float))
+
+
+_T = TypeVar("_T")
+
+
+def uniq(objs: Iterable[_T]) -> Iterator[_T]:
+ """Eliminates duplicated elements."""
+ done = set()
+ for obj in objs:
+ if obj in done:
+ continue
+ done.add(obj)
+ yield obj
+
+
+def fsplit(pred: Callable[[_T], bool], objs: Iterable[_T]) -> tuple[list[_T], list[_T]]:
+ """Split a list into two classes according to the predicate."""
+ t = []
+ f = []
+ for obj in objs:
+ if pred(obj):
+ t.append(obj)
+ else:
+ f.append(obj)
+ return t, f
+
+
+def drange(v0: float, v1: float, d: int) -> range:
+ """Returns a discrete range."""
+ return range(int(v0) // d, int(v1 + d) // d)
+
+
+def get_bound(pts: Iterable[Point]) -> Rect:
+ """Compute a minimal rectangle that covers all the points."""
+ limit: Rect = (INF, INF, -INF, -INF)
+ (x0, y0, x1, y1) = limit
+ for x, y in pts:
+ x0 = min(x0, x)
+ y0 = min(y0, y)
+ x1 = max(x1, x)
+ y1 = max(y1, y)
+ return x0, y0, x1, y1
+
+
+def pick(
+ seq: Iterable[_T],
+ func: Callable[[_T], float],
+ maxobj: _T | None = None,
+) -> _T | None:
+ """Picks the object obj where func(obj) has the highest value."""
+ maxscore = None
+ for obj in seq:
+ score = func(obj)
+ if maxscore is None or maxscore < score:
+ (maxscore, maxobj) = (score, obj)
+ return maxobj
+
+
+def choplist(n: int, seq: Iterable[_T]) -> Iterator[tuple[_T, ...]]:
+ """Groups every n elements of the list."""
+ r = []
+ for x in seq:
+ r.append(x)
+ if len(r) == n:
+ yield tuple(r)
+ r = []
+
+
+def nunpack(s: bytes, default: int = 0) -> int:
+ """Unpacks variable-length unsigned integers (big endian)."""
+ length = len(s)
+ if not length:
+ return default
+ else:
+ return int.from_bytes(s, byteorder="big", signed=False)
+
+
+PDFDocEncoding = "".join(
+ chr(x)
+ for x in (
+ 0x0000,
+ 0x0001,
+ 0x0002,
+ 0x0003,
+ 0x0004,
+ 0x0005,
+ 0x0006,
+ 0x0007,
+ 0x0008,
+ 0x0009,
+ 0x000A,
+ 0x000B,
+ 0x000C,
+ 0x000D,
+ 0x000E,
+ 0x000F,
+ 0x0010,
+ 0x0011,
+ 0x0012,
+ 0x0013,
+ 0x0014,
+ 0x0015,
+ 0x0017,
+ 0x0017,
+ 0x02D8,
+ 0x02C7,
+ 0x02C6,
+ 0x02D9,
+ 0x02DD,
+ 0x02DB,
+ 0x02DA,
+ 0x02DC,
+ 0x0020,
+ 0x0021,
+ 0x0022,
+ 0x0023,
+ 0x0024,
+ 0x0025,
+ 0x0026,
+ 0x0027,
+ 0x0028,
+ 0x0029,
+ 0x002A,
+ 0x002B,
+ 0x002C,
+ 0x002D,
+ 0x002E,
+ 0x002F,
+ 0x0030,
+ 0x0031,
+ 0x0032,
+ 0x0033,
+ 0x0034,
+ 0x0035,
+ 0x0036,
+ 0x0037,
+ 0x0038,
+ 0x0039,
+ 0x003A,
+ 0x003B,
+ 0x003C,
+ 0x003D,
+ 0x003E,
+ 0x003F,
+ 0x0040,
+ 0x0041,
+ 0x0042,
+ 0x0043,
+ 0x0044,
+ 0x0045,
+ 0x0046,
+ 0x0047,
+ 0x0048,
+ 0x0049,
+ 0x004A,
+ 0x004B,
+ 0x004C,
+ 0x004D,
+ 0x004E,
+ 0x004F,
+ 0x0050,
+ 0x0051,
+ 0x0052,
+ 0x0053,
+ 0x0054,
+ 0x0055,
+ 0x0056,
+ 0x0057,
+ 0x0058,
+ 0x0059,
+ 0x005A,
+ 0x005B,
+ 0x005C,
+ 0x005D,
+ 0x005E,
+ 0x005F,
+ 0x0060,
+ 0x0061,
+ 0x0062,
+ 0x0063,
+ 0x0064,
+ 0x0065,
+ 0x0066,
+ 0x0067,
+ 0x0068,
+ 0x0069,
+ 0x006A,
+ 0x006B,
+ 0x006C,
+ 0x006D,
+ 0x006E,
+ 0x006F,
+ 0x0070,
+ 0x0071,
+ 0x0072,
+ 0x0073,
+ 0x0074,
+ 0x0075,
+ 0x0076,
+ 0x0077,
+ 0x0078,
+ 0x0079,
+ 0x007A,
+ 0x007B,
+ 0x007C,
+ 0x007D,
+ 0x007E,
+ 0x0000,
+ 0x2022,
+ 0x2020,
+ 0x2021,
+ 0x2026,
+ 0x2014,
+ 0x2013,
+ 0x0192,
+ 0x2044,
+ 0x2039,
+ 0x203A,
+ 0x2212,
+ 0x2030,
+ 0x201E,
+ 0x201C,
+ 0x201D,
+ 0x2018,
+ 0x2019,
+ 0x201A,
+ 0x2122,
+ 0xFB01,
+ 0xFB02,
+ 0x0141,
+ 0x0152,
+ 0x0160,
+ 0x0178,
+ 0x017D,
+ 0x0131,
+ 0x0142,
+ 0x0153,
+ 0x0161,
+ 0x017E,
+ 0x0000,
+ 0x20AC,
+ 0x00A1,
+ 0x00A2,
+ 0x00A3,
+ 0x00A4,
+ 0x00A5,
+ 0x00A6,
+ 0x00A7,
+ 0x00A8,
+ 0x00A9,
+ 0x00AA,
+ 0x00AB,
+ 0x00AC,
+ 0x0000,
+ 0x00AE,
+ 0x00AF,
+ 0x00B0,
+ 0x00B1,
+ 0x00B2,
+ 0x00B3,
+ 0x00B4,
+ 0x00B5,
+ 0x00B6,
+ 0x00B7,
+ 0x00B8,
+ 0x00B9,
+ 0x00BA,
+ 0x00BB,
+ 0x00BC,
+ 0x00BD,
+ 0x00BE,
+ 0x00BF,
+ 0x00C0,
+ 0x00C1,
+ 0x00C2,
+ 0x00C3,
+ 0x00C4,
+ 0x00C5,
+ 0x00C6,
+ 0x00C7,
+ 0x00C8,
+ 0x00C9,
+ 0x00CA,
+ 0x00CB,
+ 0x00CC,
+ 0x00CD,
+ 0x00CE,
+ 0x00CF,
+ 0x00D0,
+ 0x00D1,
+ 0x00D2,
+ 0x00D3,
+ 0x00D4,
+ 0x00D5,
+ 0x00D6,
+ 0x00D7,
+ 0x00D8,
+ 0x00D9,
+ 0x00DA,
+ 0x00DB,
+ 0x00DC,
+ 0x00DD,
+ 0x00DE,
+ 0x00DF,
+ 0x00E0,
+ 0x00E1,
+ 0x00E2,
+ 0x00E3,
+ 0x00E4,
+ 0x00E5,
+ 0x00E6,
+ 0x00E7,
+ 0x00E8,
+ 0x00E9,
+ 0x00EA,
+ 0x00EB,
+ 0x00EC,
+ 0x00ED,
+ 0x00EE,
+ 0x00EF,
+ 0x00F0,
+ 0x00F1,
+ 0x00F2,
+ 0x00F3,
+ 0x00F4,
+ 0x00F5,
+ 0x00F6,
+ 0x00F7,
+ 0x00F8,
+ 0x00F9,
+ 0x00FA,
+ 0x00FB,
+ 0x00FC,
+ 0x00FD,
+ 0x00FE,
+ 0x00FF,
+ )
+)
+
+
+def decode_text(s: bytes) -> str:
+ """Decodes a PDFDocEncoding string to Unicode."""
+ if s.startswith(b"\xfe\xff"):
+ return str(s[2:], "utf-16be", "ignore")
+ else:
+ return "".join(PDFDocEncoding[c] for c in s)
+
+
+def enc(x: str) -> str:
+ """Encodes a string for SGML/XML/HTML"""
+ if isinstance(x, bytes):
+ return ""
+ return escape(x)
+
+
+def bbox2str(bbox: Rect) -> str:
+ (x0, y0, x1, y1) = bbox
+ return f"{x0:.3f},{y0:.3f},{x1:.3f},{y1:.3f}"
+
+
+def matrix2str(m: Matrix) -> str:
+ (a, b, c, d, e, f) = m
+ return f"[{a:.2f},{b:.2f},{c:.2f},{d:.2f}, ({e:.2f},{f:.2f})]"
+
+
+def vecBetweenBoxes(obj1: "LTComponent", obj2: "LTComponent") -> Point:
+ """A distance function between two TextBoxes.
+
+ Consider the bounding rectangle for obj1 and obj2.
+ Return vector between 2 boxes boundaries if they don't overlap, otherwise
+ returns vector betweeen boxes centers
+
+ +------+..........+ (x1, y1)
+ | obj1 | :
+ +------+www+------+
+ : | obj2 |
+ (x0, y0) +..........+------+
+ """
+ (x0, y0) = (min(obj1.x0, obj2.x0), min(obj1.y0, obj2.y0))
+ (x1, y1) = (max(obj1.x1, obj2.x1), max(obj1.y1, obj2.y1))
+ (ow, oh) = (x1 - x0, y1 - y0)
+ (iw, ih) = (ow - obj1.width - obj2.width, oh - obj1.height - obj2.height)
+ if iw < 0 and ih < 0:
+ # if one is inside another we compute euclidean distance
+ (xc1, yc1) = ((obj1.x0 + obj1.x1) / 2, (obj1.y0 + obj1.y1) / 2)
+ (xc2, yc2) = ((obj2.x0 + obj2.x1) / 2, (obj2.y0 + obj2.y1) / 2)
+ return xc1 - xc2, yc1 - yc2
+ else:
+ return max(0, iw), max(0, ih)
+
+
+LTComponentT = TypeVar("LTComponentT", bound="LTComponent")
+
+
+class Plane(Generic[LTComponentT]):
+ """A set-like data structure for objects placed on a plane.
+
+ Can efficiently find objects in a certain rectangular area.
+ It maintains two parallel lists of objects, each of
+ which is sorted by its x or y coordinate.
+ """
+
+ def __init__(self, bbox: Rect, gridsize: int = 50) -> None:
+ self._seq: list[LTComponentT] = [] # preserve the object order.
+ self._objs: set[LTComponentT] = set()
+ self._grid: dict[Point, list[LTComponentT]] = {}
+ self.gridsize = gridsize
+ (self.x0, self.y0, self.x1, self.y1) = bbox
+
+ def __repr__(self) -> str:
+ return "" % list(self)
+
+ def __iter__(self) -> Iterator[LTComponentT]:
+ return (obj for obj in self._seq if obj in self._objs)
+
+ def __len__(self) -> int:
+ return len(self._objs)
+
+ def __contains__(self, obj: object) -> bool:
+ return obj in self._objs
+
+ def _getrange(self, bbox: Rect) -> Iterator[Point]:
+ (x0, y0, x1, y1) = bbox
+ if x1 <= self.x0 or self.x1 <= x0 or y1 <= self.y0 or self.y1 <= y0:
+ return
+ x0 = max(self.x0, x0)
+ y0 = max(self.y0, y0)
+ x1 = min(self.x1, x1)
+ y1 = min(self.y1, y1)
+ for grid_y in drange(y0, y1, self.gridsize):
+ for grid_x in drange(x0, x1, self.gridsize):
+ yield (grid_x, grid_y)
+
+ def extend(self, objs: Iterable[LTComponentT]) -> None:
+ for obj in objs:
+ self.add(obj)
+
+ def add(self, obj: LTComponentT) -> None:
+ """Place an object."""
+ for k in self._getrange((obj.x0, obj.y0, obj.x1, obj.y1)):
+ if k not in self._grid:
+ r: list[LTComponentT] = []
+ self._grid[k] = r
+ else:
+ r = self._grid[k]
+ r.append(obj)
+ self._seq.append(obj)
+ self._objs.add(obj)
+
+ def remove(self, obj: LTComponentT) -> None:
+ """Displace an object."""
+ for k in self._getrange((obj.x0, obj.y0, obj.x1, obj.y1)):
+ try:
+ self._grid[k].remove(obj)
+ except (KeyError, ValueError):
+ pass
+ self._objs.remove(obj)
+
+ def find(self, bbox: Rect) -> Iterator[LTComponentT]:
+ """Finds objects that are in a certain area."""
+ (x0, y0, x1, y1) = bbox
+ done = set()
+ for k in self._getrange(bbox):
+ if k not in self._grid:
+ continue
+ for obj in self._grid[k]:
+ if obj in done:
+ continue
+ done.add(obj)
+ if obj.x1 <= x0 or x1 <= obj.x0 or obj.y1 <= y0 or y1 <= obj.y0:
+ continue
+ yield obj
+
+
+ROMAN_ONES = ["i", "x", "c", "m"]
+ROMAN_FIVES = ["v", "l", "d"]
+
+
+def format_int_roman(value: int) -> str:
+ """Format a number as lowercase Roman numerals."""
+ assert 0 < value < 4000
+ result: list[str] = []
+ index = 0
+
+ while value != 0:
+ value, remainder = divmod(value, 10)
+ if remainder == 9:
+ result.insert(0, ROMAN_ONES[index])
+ result.insert(1, ROMAN_ONES[index + 1])
+ elif remainder == 4:
+ result.insert(0, ROMAN_ONES[index])
+ result.insert(1, ROMAN_FIVES[index])
+ else:
+ over_five = remainder >= 5
+ if over_five:
+ result.insert(0, ROMAN_FIVES[index])
+ remainder -= 5
+ result.insert(1 if over_five else 0, ROMAN_ONES[index] * remainder)
+ index += 1
+
+ return "".join(result)
+
+
+def format_int_alpha(value: int) -> str:
+ """Format a number as lowercase letters a-z, aa-zz, etc."""
+ assert value > 0
+ result: list[str] = []
+
+ while value != 0:
+ value, remainder = divmod(value - 1, len(string.ascii_lowercase))
+ result.append(string.ascii_lowercase[remainder])
+
+ result.reverse()
+ return "".join(result)
diff --git a/babeldoc/progress_monitor.py b/babeldoc/progress_monitor.py
new file mode 100644
index 0000000000000000000000000000000000000000..555afe4586e343db0dbe378823582c0eaa357b68
--- /dev/null
+++ b/babeldoc/progress_monitor.py
@@ -0,0 +1,315 @@
+import asyncio
+import logging
+import threading
+import time
+from asyncio import CancelledError
+from collections.abc import Callable
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+class ProgressMonitor:
+ def __init__(
+ self,
+ stages: list[tuple[str, float]],
+ progress_change_callback: Callable | None = None,
+ finish_callback: Callable | None = None,
+ report_interval: float = 0.1,
+ finish_event: asyncio.Event | None = None,
+ cancel_event: threading.Event | None = None,
+ loop: asyncio.AbstractEventLoop | None = None,
+ parent_monitor: Optional["ProgressMonitor"] = None,
+ part_index: int | None = 0,
+ total_parts: int | None = 1,
+ ):
+ self.lock = threading.Lock()
+ self.parent_monitor = parent_monitor
+ self.part_index = part_index
+ self.total_parts = total_parts
+ self.raw_stages = stages
+ self.part_results = {}
+
+ # Convert stages list to dict with name and weight
+ self.stage = {}
+ total_weight = sum(weight for _, weight in stages)
+ for name, weight in stages:
+ normalized_weight = weight / total_weight
+ self.stage[name] = TranslationStage(
+ name,
+ 0,
+ self,
+ normalized_weight,
+ self.lock,
+ )
+
+ self.progress_change_callback = progress_change_callback
+ self.finish_callback = finish_callback
+ self.report_interval = report_interval
+ logger.debug(f"report_interval: {self.report_interval}")
+ self.last_report_time = 0
+ self.finish_stage_count = 0
+ self.finish_event = finish_event
+ self.cancel_event = cancel_event
+ self.loop = loop
+ self.disable = False
+ if finish_event and not loop:
+ raise ValueError("finish_event requires a loop")
+ if self.progress_change_callback:
+ self.progress_change_callback(
+ type="stage_summary",
+ stages=[
+ {
+ "name": name,
+ "percent": self.stage[name].weight,
+ }
+ for name, _ in stages
+ ],
+ part_index=self.part_index,
+ total_parts=self.total_parts,
+ )
+
+ def create_part_monitor(
+ self, part_index: int, total_parts: int
+ ) -> "ProgressMonitor":
+ """Create a new progress monitor for a document part"""
+ return ProgressMonitor(
+ stages=self.raw_stages,
+ progress_change_callback=self._handle_part_progress,
+ finish_callback=self._handle_part_finish,
+ report_interval=self.report_interval,
+ cancel_event=self.cancel_event,
+ loop=self.loop,
+ parent_monitor=self,
+ part_index=part_index,
+ total_parts=total_parts,
+ )
+
+ def _handle_part_progress(self, **kwargs):
+ """Handle progress updates from part monitors"""
+ if self.progress_change_callback and not self.disable:
+ # Add part information to progress update
+ kwargs["part_index"] = kwargs.get("part_index")
+ kwargs["total_parts"] = kwargs.get("total_parts")
+ self.progress_change_callback(**kwargs)
+
+ def _handle_part_finish(self, **kwargs):
+ """Handle completion of a part translation"""
+ if kwargs["type"] == "error":
+ logger.info(f"progress_monitor handle_part_finish: {kwargs['error']}")
+ self.finish_callback(type="error", error=kwargs["error"])
+ return
+ if "translate_result" in kwargs:
+ part_index = kwargs.get("part_index")
+ if part_index is not None:
+ self.part_results[part_index] = kwargs["translate_result"]
+
+ # if self.finish_callback and not self.disable:
+ # self.finish_callback(**kwargs)
+
+ def stage_start(self, stage_name: str, total: int):
+ if self.disable or self.parent_monitor and self.parent_monitor.disable:
+ return DummyTranslationStage(stage_name, total, self, 0)
+ stage = self.stage[stage_name]
+ stage.run_time += 1
+ stage.name = stage_name
+ stage.display_name = f"{stage_name}" if stage.run_time > 1 else stage_name
+ stage.current = 0
+ stage.total = total
+ if self.progress_change_callback:
+ self.progress_change_callback(
+ type="progress_start",
+ stage=stage.display_name,
+ stage_progress=0.0,
+ stage_current=0,
+ stage_total=total,
+ overall_progress=self.calculate_current_progress(),
+ part_index=self.part_index + 1,
+ total_parts=self.total_parts,
+ )
+ self.last_report_time = 0.0
+ return stage
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ logger.debug("ProgressMonitor __exit__")
+
+ def on_finish(self):
+ if self.disable or self.parent_monitor and self.parent_monitor.disable:
+ return
+ if self.cancel_event:
+ self.cancel_event.set()
+ if self.finish_event and self.loop:
+ self.loop.call_soon_threadsafe(self.finish_event.set)
+ if self.cancel_event and self.cancel_event.is_set():
+ self.finish_callback(type="error", error=CancelledError)
+
+ def stage_done(self, stage):
+ if self.disable or self.parent_monitor and self.parent_monitor.disable:
+ return
+ self.last_report_time = 0.0
+ self.finish_stage_count += 1
+ if (
+ stage.current != stage.total
+ and self.cancel_event is not None
+ and not self.cancel_event.is_set()
+ ):
+ logger.warning(
+ f"Stage {stage.name} completed with {stage.current}/{stage.total} items",
+ )
+ return
+ if self.progress_change_callback:
+ self.progress_change_callback(
+ type="progress_end",
+ stage=stage.display_name,
+ stage_progress=100.0,
+ stage_current=stage.total,
+ stage_total=stage.total,
+ overall_progress=self.calculate_current_progress(),
+ part_index=self.part_index + 1,
+ total_parts=self.total_parts,
+ )
+
+ def calculate_current_progress(self, stage=None):
+ if self.disable or self.parent_monitor and self.parent_monitor.disable:
+ return 100
+ part_weight = 1 / self.total_parts
+ if self.parent_monitor:
+ part_offset = self.part_index * part_weight
+ else:
+ part_offset = len(self.part_results) * part_weight
+ part_offset *= 100
+ progress = self._calculate_current_progress(stage) * part_weight + part_offset
+ return progress
+
+ def _calculate_current_progress(self, stage=None):
+ """Calculate overall progress including part progress"""
+ # Count completed stages
+ completed_stages = sum(
+ 1 for s in self.stage.values() if s.run_time > 0 and s.current == s.total
+ )
+
+ # If all stages are complete, return exactly 100
+ if completed_stages == len(self.stage):
+ return 100
+
+ # Calculate progress based on weights
+ progress = sum(
+ s.weight * 100
+ for s in self.stage.values()
+ if s.run_time > 0 and s.current == s.total
+ )
+ if stage is not None and 0 < stage.total != stage.current:
+ progress += stage.weight * stage.current * 100 / stage.total
+
+ # If this is a part monitor (has parent_monitor), return the progress as is
+ if hasattr(self, "parent_monitor") and self.parent_monitor:
+ return progress
+
+ # Otherwise return the standard progress
+ return progress
+
+ def stage_update(self, stage, n: int):
+ if self.disable or self.parent_monitor and self.parent_monitor.disable:
+ return
+ report_time_delta = time.time() - self.last_report_time
+ if report_time_delta < self.report_interval and stage.total > 3:
+ return
+ if self.progress_change_callback:
+ if stage.total != 0:
+ stage_progress = stage.current * 100 / stage.total
+ else:
+ stage_progress = 100
+ self.progress_change_callback(
+ type="progress_update",
+ stage=stage.display_name,
+ stage_progress=stage_progress,
+ stage_current=stage.current,
+ stage_total=stage.total,
+ overall_progress=self.calculate_current_progress(stage),
+ part_index=self.part_index + 1,
+ total_parts=self.total_parts,
+ )
+ self.last_report_time = time.time()
+
+ def translate_done(self, translate_result):
+ if self.disable or self.parent_monitor and self.parent_monitor.disable:
+ return
+ if self.finish_callback:
+ self.finish_callback(type="finish", translate_result=translate_result)
+
+ def translate_error(self, error):
+ if self.disable or self.parent_monitor and self.parent_monitor.disable:
+ return
+ if self.finish_callback:
+ logger.info(f"progress_monitor handle translate_error: {error}")
+ self.finish_callback(type="error", error=error)
+
+ def raise_if_cancelled(self):
+ if self.cancel_event and self.cancel_event.is_set():
+ raise asyncio.CancelledError
+
+ def cancel(self):
+ if self.disable or self.parent_monitor and self.parent_monitor.disable:
+ return
+ if self.cancel_event:
+ logger.info("Translation canceled")
+ self.cancel_event.set()
+
+
+class TranslationStage:
+ def __init__(
+ self,
+ name: str,
+ total: int,
+ pm: ProgressMonitor,
+ weight: float,
+ lock: threading.Lock,
+ ):
+ self.name = name
+ self.display_name = name
+ self.current = 0
+ self.total = total
+ self.pm = pm
+ self.run_time = 0
+ self.weight = weight
+ self.lock = lock
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ with self.lock:
+ diff = self.total - self.current
+ if diff > 0:
+ logger.info(
+ f"Stage {self.name} completed with {self.current}/{self.total} items"
+ )
+ self.pm.stage_update(self, diff)
+ self.current = self.total
+ self.pm.stage_done(self)
+
+ def advance(self, n: int = 1):
+ with self.lock:
+ self.current += n
+ self.pm.stage_update(self, n)
+
+
+class DummyTranslationStage:
+ def __init__(self, name: str, total: int, pm: ProgressMonitor, weight: float):
+ self.name = name
+ self.display_name = name
+ self.current = 0
+ self.total = total
+ self.pm = pm
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ pass
+
+ def advance(self, n: int = 1):
+ pass
diff --git a/babeldoc/server.py b/babeldoc/server.py
new file mode 100644
index 0000000000000000000000000000000000000000..808fe4cbb148428d4986cb67776218fdceb60066
--- /dev/null
+++ b/babeldoc/server.py
@@ -0,0 +1,345 @@
+"""BabelDOC FastAPI Server - Production Ready"""
+import asyncio
+import logging
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+from fastapi import FastAPI, File, Form, HTTPException, UploadFile
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+
+# Import BabelDOC modules
+from babeldoc.format.pdf.high_level import async_translate, init
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from babeldoc.progress_monitor import ProgressMonitor
+from babeldoc.translator.translator import OpenAITranslator
+
+# Configure logging
+logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Suppress verbose logs
+logging.getLogger("httpx").setLevel("CRITICAL")
+logging.getLogger("openai").setLevel("CRITICAL")
+
+# Initialize FastAPI app
+app = FastAPI(
+ title="BabelDOC Translation API",
+ description="Intelligent PDF Translation with Layout Preservation",
+ version="1.0.0"
+)
+
+# Configure CORS
+app.add_middleware(
+ CORSMiddleware,
+ allow_origins=["*"], # Change in production
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+)
+
+# Serve frontend static files
+try:
+ app.mount("/static", StaticFiles(directory="frontend"), name="static")
+except RuntimeError:
+ logger.warning("Frontend directory not found, skipping static file serving")
+
+# Temporary directory for file processing
+TEMP_DIR = Path(tempfile.gettempdir()) / "babeldoc_api"
+TEMP_DIR.mkdir(exist_ok=True)
+
+# Language code mapping
+LANGUAGE_CODES = {
+ 'en': 'en',
+ 'ar': 'en-ar',
+ 'es': 'es',
+ 'fr': 'fr',
+ 'de': 'de',
+ 'zh': 'zh',
+ 'ja': 'ja',
+ 'ko': 'ko',
+ 'pt': 'pt',
+ 'ru': 'ru',
+ 'it': 'it',
+}
+
+# Initialize BabelDOC on startup
+@app.on_event("startup")
+async def startup_event():
+ """Initialize BabelDOC resources"""
+ logger.info("Initializing BabelDOC...")
+ try:
+ init()
+ logger.info("BabelDOC initialized successfully")
+ except Exception as e:
+ logger.error(f"Failed to initialize BabelDOC: {e}")
+
+
+@app.get("/")
+@app.head("/")
+async def root():
+ """Serve the frontend HTML"""
+ try:
+ with open("frontend/index.html", "r", encoding="utf-8") as f:
+ return HTMLResponse(content=f.read())
+ except FileNotFoundError:
+ return JSONResponse({
+ "name": "BabelDOC API",
+ "version": "1.0.0",
+ "status": "running",
+ "endpoints": {
+ "health": "/health",
+ "languages": "/languages",
+ "translate": "/translate"
+ }
+ })
+
+
+@app.get("/health")
+async def health_check():
+ """Health check endpoint"""
+ return {
+ "status": "healthy",
+ "service": "babeldoc-api",
+ "version": "1.0.0"
+ }
+
+
+@app.get("/languages")
+async def get_supported_languages():
+ """Get list of supported languages"""
+ return {
+ "supported_languages": {
+ "en": "English",
+ "ar": "Arabic",
+ "es": "Spanish",
+ "fr": "French",
+ "de": "German",
+ "zh": "Chinese",
+ "ja": "Japanese",
+ "ko": "Korean",
+ "pt": "Portuguese",
+ "ru": "Russian",
+ "it": "Italian",
+ },
+ "count": len(LANGUAGE_CODES)
+ }
+
+
+@app.post("/translate")
+async def translate_document(
+ file: UploadFile = File(...),
+ source_lang: str = Form(...),
+ target_lang: str = Form(...),
+ model: Optional[str] = Form("gpt-4o-mini"),
+):
+ """
+ Translate a PDF document from source language to target language
+
+ Args:
+ file: PDF file to translate
+ source_lang: Source language code (e.g., 'en')
+ target_lang: Target language code (e.g., 'ar')
+ model: OpenAI model to use (default: gpt-4o-mini)
+
+ Returns:
+ Translated PDF file
+ """
+
+ # Validate file type
+ if not file.filename.lower().endswith('.pdf'):
+ raise HTTPException(
+ status_code=400,
+ detail="Only PDF files are supported"
+ )
+
+ # Validate languages
+ if source_lang not in LANGUAGE_CODES:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Unsupported source language: {source_lang}. Supported: {list(LANGUAGE_CODES.keys())}"
+ )
+
+ if target_lang not in LANGUAGE_CODES:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Unsupported target language: {target_lang}. Supported: {list(LANGUAGE_CODES.keys())}"
+ )
+
+ if source_lang == target_lang:
+ raise HTTPException(
+ status_code=400,
+ detail="Source and target languages must be different"
+ )
+
+ # Create session directory
+ session_id = f"session_{os.urandom(8).hex()}"
+ session_dir = TEMP_DIR / session_id
+ session_dir.mkdir(exist_ok=True)
+
+ input_path = session_dir / file.filename
+ output_directory = session_dir / "output"
+ output_directory.mkdir(exist_ok=True)
+
+ try:
+ # Save uploaded file
+ logger.info(f"Processing translation: {file.filename}")
+ logger.info(f"Language pair: {source_lang} -> {target_lang}")
+ logger.info(f"Model: {model}")
+
+ with open(input_path, "wb") as buffer:
+ shutil.copyfileobj(file.file, buffer)
+
+ # Verify API key
+ openai_api_key = os.getenv("OPENAI_API_KEY")
+ if not openai_api_key:
+ raise HTTPException(
+ status_code=500,
+ detail="OPENAI_API_KEY not configured on server"
+ )
+
+ # Create translator
+ translator = OpenAITranslator(
+ lang_in=LANGUAGE_CODES[source_lang],
+ lang_out=LANGUAGE_CODES[target_lang],
+ model=model,
+ api_key=openai_api_key,
+ ignore_cache=True
+ )
+
+ # Configure translation
+ config = TranslationConfig(
+ translator=translator,
+ input_file=str(input_path),
+ lang_in=LANGUAGE_CODES[source_lang],
+ lang_out=LANGUAGE_CODES[target_lang],
+ output_dir=str(output_directory),
+ doc_layout_model= None,
+ pages=None, # Translate all pages
+ skip_clean=False, # Clean temp files
+ )
+
+ # Perform translation asynchronously
+ logger.info("Starting translation process...")
+
+ translate_result = None
+ async for event in async_translate(config):
+ if event["type"] == "progress_update":
+ logger.debug(
+ f"Progress: {event['stage']} - "
+ f"{event['stage_current']}/{event['stage_total']} "
+ f"(Overall: {event['overall_progress']}%)"
+ )
+ elif event["type"] == "finish":
+ translate_result = event["translate_result"]
+ logger.info("Translation completed successfully")
+ break
+ elif event["type"] == "error":
+ error_msg = event.get("error", "Unknown error")
+ logger.error(f"Translation error: {error_msg}")
+ raise HTTPException(
+ status_code=500,
+ detail=f"Translation failed: {error_msg}"
+ )
+
+ if translate_result is None:
+ raise HTTPException(
+ status_code=500,
+ detail="Translation completed but no result returned"
+ )
+
+ # Find the output PDF
+ output_pdf = None
+
+ # Check if translate_result has the expected attributes
+ try:
+ if hasattr(translate_result, 'mono_pdf_path') and translate_result.mono_pdf_path:
+ output_pdf = translate_result.mono_pdf_path
+ except:
+ pass
+
+ if not output_pdf:
+ try:
+ if hasattr(translate_result, 'no_watermark_mono_pdf_path') and translate_result.no_watermark_mono_pdf_path:
+ output_pdf = translate_result.no_watermark_mono_pdf_path
+ except:
+ pass
+
+ # Fallback: search output directory
+ if not output_pdf or not Path(output_pdf).exists():
+ pdf_files = list(output_directory.glob("*.pdf"))
+ if pdf_files:
+ output_pdf = pdf_files[0]
+
+ if not output_pdf:
+ raise HTTPException(
+ status_code=500,
+ detail="Translation completed but output file not found"
+ )
+
+ # Convert to Path if it's a string
+ if isinstance(output_pdf, str):
+ output_pdf = Path(output_pdf)
+
+ if not output_pdf.exists():
+ raise HTTPException(
+ status_code=500,
+ detail=f"Translation completed but output file does not exist: {output_pdf}"
+ )
+
+ logger.info(f"Translation successful: {output_pdf}")
+
+ # Return the translated file
+ output_filename = f"translated_{file.filename}"
+
+ return FileResponse(
+ path=str(output_pdf),
+ filename=output_filename,
+ media_type="application/pdf",
+ headers={
+ "Content-Disposition": f"attachment; filename={output_filename}"
+ }
+ )
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Translation error: {str(e)}", exc_info=True)
+ raise HTTPException(
+ status_code=500,
+ detail=f"Translation failed: {str(e)}"
+ )
+
+ finally:
+ # Cleanup temporary files after a delay to allow file download
+ # Comment out for debugging
+ pass
+ # try:
+ # if session_dir.exists():
+ # shutil.rmtree(session_dir)
+ # logger.info(f"Cleaned up session: {session_id}")
+ # except Exception as e:
+ # logger.warning(f"Failed to cleanup session {session_id}: {e}")
+
+
+if __name__ == "__main__":
+ import uvicorn
+
+ port = int(os.getenv("PORT", 8000))
+
+ logger.info(f"Starting BabelDOC API server on port {port}")
+
+ uvicorn.run(
+ "server:app",
+ host="0.0.0.0",
+ port=port,
+ log_level="info",
+ reload=False # Set to True for development
+ )
diff --git a/babeldoc/tools/generate_font_metadata.py b/babeldoc/tools/generate_font_metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab153f454582a79272511cc754a3c2523293716a
--- /dev/null
+++ b/babeldoc/tools/generate_font_metadata.py
@@ -0,0 +1,117 @@
+# This script is used to automatically generate the following files:
+# https://github.com/funstory-ai/BabelDOC-Assets/blob/main/font_metadata.json
+
+
+import argparse
+import hashlib
+import io
+import logging
+import re
+from pathlib import Path
+
+import babeldoc.format.pdf.high_level
+import babeldoc.format.pdf.translation_config
+import orjson
+import pymupdf
+from babeldoc.format.pdf.document_il import PdfFont
+from rich.logging import RichHandler
+
+logger = logging.getLogger(__name__)
+
+serif_keywords = [
+ "serif",
+]
+sans_serif_keywords = ["sans", "GoNotoKurrent"]
+serif_regex = "|".join(serif_keywords)
+sans_serif_regex = "|".join(sans_serif_keywords)
+
+
+def get_font_metadata(font_path) -> PdfFont:
+ doc = pymupdf.open()
+ page = doc.new_page(width=1000, height=1000)
+ page.insert_font("test_font", font_path)
+ translation_config = babeldoc.format.pdf.translation_config.TranslationConfig(
+ *[None for _ in range(4)], doc_layout_model=1
+ )
+ translation_config.progress_monitor = (
+ babeldoc.format.pdf.high_level.ProgressMonitor(
+ babeldoc.format.pdf.high_level.get_translation_stage(translation_config)
+ )
+ )
+ translation_config.font = font_path
+ il_creater = babeldoc.format.pdf.high_level.ILCreater(translation_config)
+ il_creater.mupdf = doc
+ buffer = io.BytesIO()
+ doc.save(buffer)
+ babeldoc.format.pdf.high_level.start_parse_il(
+ buffer,
+ doc_zh=doc,
+ resfont="test_font",
+ il_creater=il_creater,
+ translation_config=translation_config,
+ )
+
+ il = il_creater.create_il()
+ il_page = il.page[0]
+ font_metadata = il_page.pdf_font[0]
+ return font_metadata
+
+
+def main():
+ logging.basicConfig(level=logging.INFO, handlers=[RichHandler()])
+ parser = argparse.ArgumentParser(description="Get font metadata.")
+ parser.add_argument("assets_repo_path", type=str, help="Path to the font file.")
+ args = parser.parse_args()
+ repo_path = Path(args.assets_repo_path)
+ assert repo_path.exists(), f"Assets repo path {repo_path} does not exist."
+ assert (repo_path / "README.md").exists(), (
+ f"Assets repo path {repo_path} does not contain a README.md file."
+ )
+ assert (repo_path / "fonts").exists(), (
+ f"Assets repo path {repo_path} does not contain a fonts folder."
+ )
+ logger.info(f"Getting font metadata for {repo_path}")
+
+ metadatas = {}
+ for font_path in list((repo_path / "fonts").glob("**/*.ttf")):
+ logger.info(f"Getting font metadata for {font_path}")
+ with Path(font_path).open("rb") as f:
+ # Read the file in chunks to handle large files efficiently
+ hash_ = hashlib.sha3_256()
+ while True:
+ chunk = f.read(1024 * 1024)
+ if not chunk:
+ break
+ hash_.update(chunk)
+ extracted_metadata = get_font_metadata(font_path)
+
+ if re.search(serif_regex, extracted_metadata.name, re.IGNORECASE):
+ serif = 1
+ else:
+ serif = 0
+
+ metadata = {
+ "file_name": font_path.name,
+ "font_name": extracted_metadata.name,
+ "encoding_length": extracted_metadata.encoding_length,
+ "bold": extracted_metadata.bold,
+ "italic": extracted_metadata.italic,
+ "monospace": extracted_metadata.monospace,
+ "serif": serif,
+ "ascent": extracted_metadata.ascent,
+ "descent": extracted_metadata.descent,
+ "sha3_256": hash_.hexdigest(),
+ "size": font_path.stat().st_size,
+ }
+ metadatas[font_path.name] = metadata
+ metadatas = orjson.dumps(
+ metadatas,
+ option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS,
+ ).decode()
+ print(f"FONT METADATA: {metadatas}")
+ with (repo_path / "font_metadata.json").open("w") as f:
+ f.write(metadatas)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/babeldoc/tools/italic_assistance.py b/babeldoc/tools/italic_assistance.py
new file mode 100644
index 0000000000000000000000000000000000000000..43b18d6ca1269ffe5ac9a1ada12b8a63719a9d88
--- /dev/null
+++ b/babeldoc/tools/italic_assistance.py
@@ -0,0 +1,294 @@
+import argparse
+import json
+import re
+from pathlib import Path
+
+import orjson
+from babeldoc.const import CACHE_FOLDER
+from babeldoc.format.pdf.document_il.utils.formular_helper import is_formulas_font
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from rich.console import Console
+from rich.table import Table
+
+WORKING_FOLDER = Path(CACHE_FOLDER) / "working"
+
+
+def find_latest_il_json() -> Path | None:
+ """
+ Find the latest il_translated.json file in ~/.cache/babeldoc/ subdirectories.
+
+ Returns:
+ Path to the most recently modified il_translated.json file, or None if not found.
+ """
+ base_dir = Path(WORKING_FOLDER)
+ json_files = list(base_dir.glob("*/il_translated.json"))
+
+ if not json_files:
+ return None
+
+ # Sort by modification time (newest first)
+ json_files.sort(key=lambda p: p.stat().st_mtime, reverse=True)
+ return json_files[0]
+
+
+def extract_fonts_from_paragraph(
+ paragraph: dict, page_font_map: dict[str, tuple[str, str]]
+) -> set[tuple[str, str]]:
+ """
+ Extract all font_ids and names used in a paragraph.
+
+ Args:
+ paragraph: The paragraph dictionary
+ page_font_map: Dictionary mapping font_id to (font_id, name) tuples
+
+ Returns:
+ Set of (font_id, name) tuples
+ """
+ fonts = set()
+
+ # Check if paragraph has a pdfStyle with font_id
+ if (
+ "pdf_style" in paragraph
+ and paragraph["pdf_style"]
+ and "font_id" in paragraph["pdf_style"]
+ ):
+ font_id = paragraph["pdf_style"]["font_id"]
+ if font_id in page_font_map:
+ fonts.add(page_font_map[font_id])
+
+ # Process paragraph compositions if present
+ if "pdf_paragraph_composition" in paragraph:
+ for comp in paragraph["pdf_paragraph_composition"]:
+ # Check different composition types that might contain font information
+
+ # Direct pdfCharacter in composition
+ if "pdf_character" in comp and comp["pdf_character"]:
+ char = comp["pdf_character"]
+ if "pdf_style" in char and "font_id" in char["pdf_style"]:
+ font_id = char["pdf_style"]["font_id"]
+ if font_id in page_font_map:
+ fonts.add(page_font_map[font_id])
+
+ # PdfLine in composition
+ elif "pdf_line" in comp and comp["pdf_line"]:
+ line = comp["pdf_line"]
+ if "pdf_character" in line:
+ for char in line["pdf_character"]:
+ if "pdf_style" in char and "font_id" in char["pdf_style"]:
+ font_id = char["pdf_style"]["font_id"]
+ if font_id in page_font_map:
+ fonts.add(page_font_map[font_id])
+
+ # PdfFormula in composition
+ elif "pdf_formula" in comp and comp["pdf_formula"]:
+ formula = comp["pdf_formula"]
+ if "pdf_character" in formula:
+ for char in formula["pdf_character"]:
+ if "pdf_style" in char and "font_id" in char["pdf_style"]:
+ font_id = char["pdf_style"]["font_id"]
+ if font_id in page_font_map:
+ fonts.add(page_font_map[font_id])
+
+ # PdfSameStyleCharacters in composition
+ elif (
+ "pdf_same_style_characters" in comp
+ and comp["pdf_same_style_characters"]
+ ):
+ same_style = comp["pdf_same_style_characters"]
+ if "pdf_style" in same_style and "font_id" in same_style["pdf_style"]:
+ font_id = same_style["pdf_style"]["font_id"]
+ if font_id in page_font_map:
+ fonts.add(page_font_map[font_id])
+
+ # PdfSameStyleUnicodeCharacters in composition
+ elif (
+ "pdf_same_style_unicode_characters" in comp
+ and comp["pdf_same_style_unicode_characters"]
+ ):
+ same_style_unicode = comp["pdf_same_style_unicode_characters"]
+ if (
+ "pdf_style" in same_style_unicode
+ and same_style_unicode["pdf_style"] is not None
+ and "font_id" in same_style_unicode["pdf_style"]
+ ):
+ font_id = same_style_unicode["pdf_style"]["font_id"]
+ if font_id in page_font_map:
+ fonts.add(page_font_map[font_id])
+
+ return fonts
+
+
+def find_fonts_by_debug_id(json_path: Path, debug_id_regex: str) -> dict[str, str]:
+ """
+ Find all fonts used in paragraphs with matching debug_id.
+
+ Args:
+ json_path: Path to the il_translated.json file
+ debug_id_regex: Regular expression to match debug_id values
+
+ Returns:
+ Dictionary mapping font_ids to font names
+ """
+ # Load and parse JSON
+ with json_path.open("rb") as f:
+ doc_data = orjson.loads(f.read())
+
+ # Compile regex pattern (case insensitive)
+ pattern = re.compile(debug_id_regex.strip(" \"'"), re.IGNORECASE)
+
+ # Set to collect all found font information
+ found_fonts = set()
+
+ # Process each page
+ for page in doc_data.get("page", []):
+ # Create a mapping of font_id to (font_id, name) tuples for this page
+ page_font_map = {}
+ for font in page.get("pdf_font", []):
+ if "font_id" in font and "name" in font:
+ page_font_map[font["font_id"]] = (font["font_id"], font["name"])
+
+ # Check each paragraph
+ for paragraph in page.get("pdf_paragraph", []):
+ # Check if paragraph has debug_id and if it matches the pattern
+ debug_id = paragraph.get("debug_id")
+ if debug_id and pattern.search(debug_id):
+ # Get all fonts used in this paragraph
+ paragraph_fonts = extract_fonts_from_paragraph(paragraph, page_font_map)
+ found_fonts.update(paragraph_fonts)
+
+ # Convert set of tuples to dictionary
+ return dict(found_fonts)
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Extract fonts from paragraphs with matching debug_id"
+ )
+ parser.add_argument(
+ "debug_id_regex", nargs="+", help="Regular expression to match debug_id values"
+ )
+ parser.add_argument(
+ "--json-path",
+ help="Path to il_translated.json (if not provided, will use the latest file)",
+ )
+ parser.add_argument(
+ "--working-folder",
+ help="Path to the working folder containing il_translated.json files",
+ )
+
+ args = parser.parse_args()
+
+ if args.working_folder:
+ global WORKING_FOLDER
+ WORKING_FOLDER = Path(args.working_folder)
+ if not WORKING_FOLDER.exists():
+ print(f"Error: Working folder does not exist: {WORKING_FOLDER}")
+ return 1
+
+ # Determine JSON file path
+ json_path = None
+ if args.json_path:
+ json_path = Path(args.json_path)
+ if not json_path.exists():
+ print(f"Error: File not found: {json_path}")
+ return 1
+ else:
+ json_path = find_latest_il_json()
+ if not json_path:
+ print("Error: Could not find any il_translated.json file")
+ return 1
+
+ print(f"Using JSON file: {json_path}")
+
+ # Find fonts matching the debug_id pattern
+ fonts = find_fonts_by_debug_id(json_path, "|".join(args.debug_id_regex))
+
+ # Output the results
+ if fonts:
+ print(
+ f"Found {len(fonts)} fonts in paragraphs matching debug_id pattern: {args.debug_id_regex}"
+ )
+ print(json.dumps(fonts, indent=2, ensure_ascii=False))
+ else:
+ print(
+ f"No fonts found for paragraphs matching debug_id pattern: {args.debug_id_regex}"
+ )
+
+ fonts = []
+
+ # Read intermediate representation
+ with json_path.open(encoding="utf-8") as f:
+ pdf_data = json.load(f)
+
+ for page_index, page in enumerate(pdf_data["page"]):
+ for paragraph_index, paragraph_content in enumerate(page["pdf_paragraph"]):
+ font_debug_id = paragraph_content["debug_id"]
+ if font_debug_id:
+ # Create page font mapping
+ page_font_map = {}
+ for font in page["pdf_font"]:
+ if "font_id" in font and "name" in font:
+ page_font_map[font["font_id"]] = (font["font_id"], font["name"])
+
+ # Extract fonts from paragraph
+ name_list = []
+ paragraph_fonts = extract_fonts_from_paragraph(
+ paragraph_content, page_font_map
+ )
+ for _font_id, font_name in paragraph_fonts:
+ name_list.append(font_name)
+
+ font_list = []
+ for each in fonts:
+ font_list.append(each[1])
+
+ for each_name in name_list:
+ if each_name not in font_list:
+ fonts.append(
+ (page_index, each_name, paragraph_index, font_debug_id)
+ )
+
+ # Initialize checker
+ translation_config = TranslationConfig(
+ *[None for _ in range(3)], lang_out="zh_cn", doc_layout_model=1
+ )
+
+ # Create table
+ table = Table(title="Font Recognition Results")
+ table.add_column("Page #", justify="center", style="cyan")
+ table.add_column("Paragraph #", justify="center", style="cyan")
+ table.add_column("DEBUG_ID", justify="center", style="cyan")
+ table.add_column("Font Name", style="magenta")
+ table.add_column("Recognition Result", justify="center")
+
+ # Output results
+ for each_font in fonts:
+ page_index, font_name, paragraph_index, font_debug_id = each_font
+
+ if is_formulas_font(font_name, None):
+ table.add_row(
+ str(page_index),
+ str(paragraph_index),
+ str(font_debug_id),
+ font_name,
+ "[bold red]Formula Font[/bold red]",
+ )
+ else:
+ table.add_row(
+ str(page_index),
+ str(paragraph_index),
+ str(font_debug_id),
+ font_name,
+ "[bold blue]Non-Formula Font[/bold blue]",
+ )
+
+ # Print table
+ console = Console()
+
+ console.print(table)
+
+ return 0
+
+
+if __name__ == "__main__":
+ exit(main())
diff --git a/babeldoc/tools/italic_recognize_tool.py b/babeldoc/tools/italic_recognize_tool.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e8b199a98db4814a9d69ab11d8ea6e00202d17c
--- /dev/null
+++ b/babeldoc/tools/italic_recognize_tool.py
@@ -0,0 +1,85 @@
+# Identify non-formula italic fonts that were incorrectly classified as formulas in BableDOC translation results (intermediate)
+
+import json
+
+import babeldoc.tools.italic_assistance as italic_assistance
+from babeldoc.format.pdf.document_il.midend.styles_and_formulas import StylesAndFormulas
+from babeldoc.format.pdf.translation_config import TranslationConfig
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+json_path = italic_assistance.find_latest_il_json()
+
+fonts = []
+
+# Read intermediate representation
+with json_path.open(encoding="utf-8") as f:
+ pdf_data = json.load(f)
+
+for page_index, page in enumerate(pdf_data["page"]):
+ for paragraph_index, paragraph_content in enumerate(page["pdf_paragraph"]):
+ font_debug_id = paragraph_content["debug_id"]
+ if font_debug_id:
+ # Create page font mapping
+ page_font_map = {}
+ for font in page["pdf_font"]:
+ if "font_id" in font and "name" in font:
+ page_font_map[font["font_id"]] = (font["font_id"], font["name"])
+
+ # Extract fonts from paragraph
+ name_list = []
+ paragraph_fonts = italic_assistance.extract_fonts_from_paragraph(
+ paragraph_content, page_font_map
+ )
+ for _font_id, font_name in paragraph_fonts:
+ name_list.append(font_name)
+
+ font_list = []
+ for each in fonts:
+ font_list.append(each[1])
+
+ for each_name in name_list:
+ if each_name not in font_list:
+ fonts.append(
+ (page_index, each_name, paragraph_index, font_debug_id)
+ )
+
+# Initialize checker
+translation_config = TranslationConfig(
+ *[None for _ in range(3)], lang_out="zh_cn", doc_layout_model=1
+)
+checker = StylesAndFormulas(translation_config)
+
+# Create table
+table = Table(title="Font Recognition Results")
+table.add_column("Page #", justify="center", style="cyan")
+table.add_column("Paragraph #", justify="center", style="cyan")
+table.add_column("DEBUG_ID", justify="center", style="cyan")
+table.add_column("Font Name", style="magenta")
+table.add_column("Recognition Result", justify="center")
+
+# Output results
+for each_font in fonts:
+ page_index, font_name, paragraph_index, font_debug_id = each_font
+
+ if checker.is_formulas_font(font_name):
+ table.add_row(
+ str(page_index),
+ str(paragraph_index),
+ str(font_debug_id),
+ font_name,
+ "[bold red]Formula Font[/bold red]",
+ )
+ else:
+ table.add_row(
+ str(page_index),
+ str(paragraph_index),
+ str(font_debug_id),
+ font_name,
+ "[bold blue]Non-Formula Font[/bold blue]",
+ )
+
+# Print table
+console.print(table)
diff --git a/babeldoc/translator/__init__.py b/babeldoc/translator/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/babeldoc/translator/cache.py b/babeldoc/translator/cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..d482ee2274013a3d5e73d2cb20c6a197b7f3864b
--- /dev/null
+++ b/babeldoc/translator/cache.py
@@ -0,0 +1,199 @@
+import json
+import logging
+import random
+import threading
+from pathlib import Path
+
+import peewee
+from peewee import SQL
+from peewee import AutoField
+from peewee import CharField
+from peewee import Model
+from peewee import SqliteDatabase
+from peewee import TextField
+from peewee import fn # For aggregation functions
+
+from babeldoc.const import CACHE_FOLDER
+
+logger = logging.getLogger(__name__)
+
+# we don't init the database here
+db = SqliteDatabase(None)
+
+# Cleanup configuration
+CLEAN_PROBABILITY = 0.001 # 0.1% chance to trigger cleanup
+MAX_CACHE_ROWS = 50_000 # Keep only the latest 50,000 rows
+
+# Thread-level mutex to ensure only one cleanup runs at a time within the process
+_cleanup_lock = threading.Lock()
+
+
+class _TranslationCache(Model):
+ id = AutoField()
+ translate_engine = CharField(max_length=20)
+ translate_engine_params = TextField()
+ original_text = TextField()
+ translation = TextField()
+
+ class Meta:
+ database = db
+ constraints = [
+ SQL(
+ """
+ UNIQUE (
+ translate_engine,
+ translate_engine_params,
+ original_text
+ )
+ ON CONFLICT REPLACE
+ """,
+ ),
+ ]
+
+
+class TranslationCache:
+ @staticmethod
+ def _sort_dict_recursively(obj):
+ if isinstance(obj, dict):
+ return {
+ k: TranslationCache._sort_dict_recursively(v)
+ for k in sorted(obj.keys())
+ for v in [obj[k]]
+ }
+ elif isinstance(obj, list):
+ return [TranslationCache._sort_dict_recursively(item) for item in obj]
+ return obj
+
+ def __init__(self, translate_engine: str, translate_engine_params: dict = None):
+ self.translate_engine = translate_engine
+ self.replace_params(translate_engine_params)
+
+ # The program typically starts multi-threaded translation
+ # only after cache parameters are fully configured,
+ # so thread safety doesn't need to be considered here.
+ def replace_params(self, params: dict = None):
+ if params is None:
+ params = {}
+ self.params = params
+ params = self._sort_dict_recursively(params)
+ self.translate_engine_params = json.dumps(params)
+
+ def update_params(self, params: dict = None):
+ if params is None:
+ params = {}
+ self.params.update(params)
+ self.replace_params(self.params)
+
+ def add_params(self, k: str, v):
+ self.params[k] = v
+ self.replace_params(self.params)
+
+ # Since peewee and the underlying sqlite are thread-safe,
+ # get and set operations don't need locks.
+ def get(self, original_text: str) -> str | None:
+ try:
+ result = _TranslationCache.get_or_none(
+ translate_engine=self.translate_engine,
+ translate_engine_params=self.translate_engine_params,
+ original_text=original_text,
+ )
+ # Trigger cache cleanup with a small probability.
+ if result and random.random() < CLEAN_PROBABILITY: # noqa: S311
+ self._cleanup()
+ return result.translation if result else None
+ except peewee.OperationalError as e:
+ if "database is locked" in str(e):
+ logger.debug("Cache is locked")
+ return None
+ else:
+ raise
+
+ def set(self, original_text: str, translation: str):
+ try:
+ _TranslationCache.create(
+ translate_engine=self.translate_engine,
+ translate_engine_params=self.translate_engine_params,
+ original_text=original_text,
+ translation=translation,
+ )
+ # Trigger cache cleanup with a small probability.
+ if random.random() < CLEAN_PROBABILITY: # noqa: S311
+ self._cleanup()
+ except peewee.OperationalError as e:
+ if "database is locked" in str(e):
+ logger.debug("Cache is locked")
+ else:
+ raise
+
+ def _cleanup(self) -> None:
+ """Remove old cache entries, keeping only the latest MAX_CACHE_ROWS records."""
+ # Quick exit if another thread is already performing cleanup.
+ if not _cleanup_lock.acquire(blocking=False):
+ return
+ try:
+ logger.info("Cleaning up translation cache...")
+ max_id = _TranslationCache.select(fn.MAX(_TranslationCache.id)).scalar()
+ # Nothing to do if table is empty or below threshold
+ if not max_id or max_id <= MAX_CACHE_ROWS:
+ return
+ threshold = max_id - MAX_CACHE_ROWS
+ # Delete rows with id *less than or equal* to threshold so that at most MAX_CACHE_ROWS remain.
+ _TranslationCache.delete().where(
+ _TranslationCache.id <= threshold
+ ).execute()
+ finally:
+ _cleanup_lock.release()
+
+
+def init_db(remove_exists=False):
+ CACHE_FOLDER.mkdir(parents=True, exist_ok=True)
+ # The current version does not support database migration, so add the version number to the file name.
+ cache_db_path = CACHE_FOLDER / "cache.v1.db"
+ logger.info(f"Initializing cache database at {cache_db_path}")
+ if remove_exists and cache_db_path.exists():
+ cache_db_path.unlink()
+ db.init(
+ cache_db_path,
+ pragmas={
+ "journal_mode": "wal",
+ "busy_timeout": 1000,
+ },
+ )
+ db.create_tables([_TranslationCache], safe=True)
+
+
+def init_test_db():
+ import tempfile
+
+ temp_file = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
+ cache_db_path = temp_file.name
+ temp_file.close()
+
+ test_db = SqliteDatabase(
+ cache_db_path,
+ pragmas={
+ "journal_mode": "wal",
+ "busy_timeout": 1000,
+ },
+ )
+ test_db.bind([_TranslationCache], bind_refs=False, bind_backrefs=False)
+ test_db.connect()
+ test_db.create_tables([_TranslationCache], safe=True)
+ return test_db
+
+
+def clean_test_db(test_db):
+ test_db.drop_tables([_TranslationCache])
+ test_db.close()
+ db_path = Path(test_db.database)
+ if db_path.exists():
+ db_path.unlink()
+ wal_path = Path(str(db_path) + "-wal")
+ if wal_path.exists():
+ wal_path.unlink()
+ shm_path = Path(str(db_path) + "-shm")
+ if shm_path.exists():
+ shm_path.unlink()
+
+
+init_db()
diff --git a/babeldoc/translator/translator.py b/babeldoc/translator/translator.py
new file mode 100644
index 0000000000000000000000000000000000000000..258f00591baa02e4fa9d5c318fb6f71c677865e1
--- /dev/null
+++ b/babeldoc/translator/translator.py
@@ -0,0 +1,360 @@
+import contextlib
+import logging
+import threading
+import time
+import unicodedata
+from abc import ABC
+from abc import abstractmethod
+
+import httpx
+import openai
+from tenacity import before_sleep_log
+from tenacity import retry
+from tenacity import retry_if_exception_type
+from tenacity import stop_after_attempt
+from tenacity import wait_exponential
+
+from babeldoc.babeldoc_exception.BabelDOCException import ContentFilterError
+from babeldoc.translator.cache import TranslationCache
+from babeldoc.utils.atomic_integer import AtomicInteger
+
+logger = logging.getLogger(__name__)
+
+
+def remove_control_characters(s):
+ return "".join(ch for ch in s if unicodedata.category(ch)[0] != "C")
+
+
+class RateLimiter:
+ """
+ A rate limiter using the leaky bucket algorithm to ensure a smooth, constant rate of requests.
+ This implementation is thread-safe and robust against system clock changes.
+ """
+
+ def __init__(self, max_qps: int):
+ if max_qps <= 0:
+ raise ValueError("max_qps must be a positive number")
+ self.max_qps = max_qps
+ self.min_interval = 1.0 / max_qps
+ self.lock = threading.Lock()
+ # Use monotonic time to prevent issues with system time changes
+ self.next_request_time = time.monotonic()
+
+ def wait(self, _rate_limit_params: dict = None):
+ """
+ Blocks until the next request can be processed, ensuring the rate limit is not exceeded.
+ """
+ with self.lock:
+ now = time.monotonic()
+
+ wait_duration = self.next_request_time - now
+ if wait_duration > 0:
+ time.sleep(wait_duration)
+
+ # Update the next allowed request time.
+ # If the limiter has been idle, the next request should start from 'now'.
+ now = time.monotonic()
+ self.next_request_time = (
+ max(self.next_request_time, now) + self.min_interval
+ )
+
+ def set_max_qps(self, max_qps: int):
+ """
+ Updates the maximum queries per second. This operation is thread-safe.
+ """
+ if max_qps <= 0:
+ raise ValueError("max_qps must be a positive number")
+ with self.lock:
+ self.max_qps = max_qps
+ self.min_interval = 1.0 / max_qps
+
+
+_translate_rate_limiter = RateLimiter(5)
+
+
+def set_translate_rate_limiter(max_qps):
+ _translate_rate_limiter.set_max_qps(max_qps)
+
+
+class BaseTranslator(ABC):
+ # Due to cache limitations, name should be within 20 characters.
+ # cache.py: translate_engine = CharField(max_length=20)
+ name = "base"
+ lang_map = {}
+
+ def __init__(self, lang_in, lang_out, ignore_cache):
+ self.ignore_cache = ignore_cache
+ lang_in = self.lang_map.get(lang_in.lower(), lang_in)
+ lang_out = self.lang_map.get(lang_out.lower(), lang_out)
+ self.lang_in = lang_in
+ self.lang_out = lang_out
+
+ self.cache = TranslationCache(
+ self.name,
+ {
+ "lang_in": lang_in,
+ "lang_out": lang_out,
+ },
+ )
+
+ self.translate_call_count = 0
+ self.translate_cache_call_count = 0
+
+ def __del__(self):
+ with contextlib.suppress(Exception):
+ logger.info(
+ f"{self.name} translate call count: {self.translate_call_count}"
+ )
+ logger.info(
+ f"{self.name} translate cache call count: {self.translate_cache_call_count}",
+ )
+
+ def add_cache_impact_parameters(self, k: str, v):
+ """
+ Add parameters that affect the translation quality to distinguish the translation effects under different parameters.
+ :param k: key
+ :param v: value
+ """
+ self.cache.add_params(k, v)
+
+ def translate(self, text, ignore_cache=False, rate_limit_params: dict = None):
+ """
+ Translate the text, and the other part should call this method.
+ :param text: text to translate
+ :return: translated text
+ """
+ self.translate_call_count += 1
+ if not (self.ignore_cache or ignore_cache):
+ try:
+ cache = self.cache.get(text)
+ if cache is not None:
+ self.translate_cache_call_count += 1
+ return cache
+ except Exception as e:
+ logger.debug(f"try get cache failed, ignore it: {e}")
+ _translate_rate_limiter.wait()
+ translation = self.do_translate(text, rate_limit_params)
+ if not (self.ignore_cache or ignore_cache):
+ self.cache.set(text, translation)
+ return translation
+
+ def llm_translate(self, text, ignore_cache=False, rate_limit_params: dict = None):
+ """
+ Translate the text, and the other part should call this method.
+ :param text: text to translate
+ :return: translated text
+ """
+ self.translate_call_count += 1
+ if not (self.ignore_cache or ignore_cache):
+ try:
+ cache = self.cache.get(text)
+ if cache is not None:
+ self.translate_cache_call_count += 1
+ return cache
+ except Exception as e:
+ logger.debug(f"try get cache failed, ignore it: {e}")
+ _translate_rate_limiter.wait()
+ translation = self.do_llm_translate(text, rate_limit_params)
+ if not (self.ignore_cache or ignore_cache):
+ try:
+ self.cache.set(text, translation)
+ except Exception as e:
+ logger.debug(
+ f"try set cache failed, ignore it: {e}, text: {text}, translation: {translation}"
+ )
+ return translation
+
+ @abstractmethod
+ def do_llm_translate(self, text, rate_limit_params: dict = None):
+ """
+ Actual translate text, override this method
+ :param text: text to translate
+ :return: translated text
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def do_translate(self, text, rate_limit_params: dict = None):
+ """
+ Actual translate text, override this method
+ :param text: text to translate
+ :return: translated text
+ """
+ logger.critical(
+ f"Do not call BaseTranslator.do_translate. "
+ f"Translator: {self}. "
+ f"Text: {text}. ",
+ )
+ raise NotImplementedError
+
+ def __str__(self):
+ return f"{self.name} {self.lang_in} {self.lang_out} {self.model}"
+
+ def get_rich_text_left_placeholder(self, placeholder_id: int):
+ return f""
+
+ def get_rich_text_right_placeholder(self, placeholder_id: int):
+ return f""
+
+ def get_formular_placeholder(self, placeholder_id: int):
+ return self.get_rich_text_left_placeholder(placeholder_id)
+
+
+class OpenAITranslator(BaseTranslator):
+ # https://github.com/openai/openai-python
+ name = "openai"
+
+ def __init__(
+ self,
+ lang_in,
+ lang_out,
+ model,
+ base_url=None,
+ api_key=None,
+ ignore_cache=False,
+ enable_json_mode_if_requested=False,
+ send_dashscope_header=False,
+ send_temperature=True,
+ ):
+ super().__init__(lang_in, lang_out, ignore_cache)
+ self.options = {"temperature": 0} # 随机采样可能会打断公式标记
+ self.extra_body = {}
+ # if 'gpt-5' in model and 'gpt-5-chat' not in model:
+ # self.extra_body['reasoning'] = {
+ # "effort": "minimal"
+ # }
+ # self.add_cache_impact_parameters("reasoning-effort", 'minimal')
+ self.client = openai.OpenAI(
+ base_url=base_url,
+ api_key=api_key,
+ http_client=httpx.Client(
+ limits=httpx.Limits(
+ max_connections=None, max_keepalive_connections=None
+ ),
+ timeout=60, # Set a reasonable timeout
+ ),
+ )
+ if send_temperature:
+ self.add_cache_impact_parameters("temperature", self.options["temperature"])
+ self.model = model
+ self.enable_json_mode_if_requested = enable_json_mode_if_requested
+ self.send_dashscope_header = send_dashscope_header
+ self.send_temperature = send_temperature
+ self.add_cache_impact_parameters("model", self.model)
+ self.add_cache_impact_parameters("prompt", self.prompt(""))
+ if self.enable_json_mode_if_requested:
+ self.add_cache_impact_parameters(
+ "enable_json_mode_if_requested", self.enable_json_mode_if_requested
+ )
+ self.token_count = AtomicInteger()
+ self.prompt_token_count = AtomicInteger()
+ self.completion_token_count = AtomicInteger()
+ self.cache_hit_prompt_token_count = AtomicInteger()
+
+ @retry(
+ retry=retry_if_exception_type(openai.RateLimitError),
+ stop=stop_after_attempt(100),
+ wait=wait_exponential(multiplier=1, min=1, max=15),
+ before_sleep=before_sleep_log(logger, logging.WARNING),
+ )
+ def do_translate(self, text, rate_limit_params: dict = None) -> str:
+ options = {}
+ if self.send_temperature:
+ options.update(self.options)
+
+ response = self.client.chat.completions.create(
+ model=self.model,
+ **options,
+ messages=self.prompt(text),
+ extra_body=self.extra_body,
+ )
+ self.update_token_count(response)
+ return response.choices[0].message.content.strip()
+
+ def prompt(self, text):
+ return [
+ {
+ "role": "system",
+ "content": "You are a professional,authentic machine translation engine.",
+ },
+ {
+ "role": "user",
+ "content": f";; Treat next line as plain text input and translate it into {self.lang_out}, output translation ONLY. If translation is unnecessary (e.g. proper nouns, codes, {'{{1}}, etc. '}), return the original text. NO explanations. NO notes. Input:\n\n{text}",
+ },
+ ]
+
+ @retry(
+ retry=retry_if_exception_type(openai.RateLimitError),
+ stop=stop_after_attempt(100),
+ wait=wait_exponential(multiplier=1, min=1, max=15),
+ before_sleep=before_sleep_log(logger, logging.WARNING),
+ )
+ def do_llm_translate(self, text, rate_limit_params: dict = None):
+ if text is None:
+ return None
+
+ options = {}
+ if self.send_temperature:
+ options.update(self.options)
+ if self.enable_json_mode_if_requested and rate_limit_params.get(
+ "request_json_mode", False
+ ):
+ options["response_format"] = {"type": "json_object"}
+
+ extra_headers = {}
+ if self.send_dashscope_header:
+ extra_headers["X-DashScope-DataInspection"] = (
+ '{"input": "disable", "output": "disable"}'
+ )
+ try:
+ response = self.client.chat.completions.create(
+ model=self.model,
+ **options,
+ max_tokens=2048,
+ messages=[
+ {
+ "role": "user",
+ "content": text,
+ },
+ ],
+ extra_headers=extra_headers,
+ extra_body=self.extra_body,
+ )
+ self.update_token_count(response)
+ return response.choices[0].message.content.strip()
+ except openai.BadRequestError as e:
+ if (
+ "系统检测到输入或生成内容可能包含不安全或敏感内容，请您避免输入易产生敏感内容的提示语，感谢您的配合。"
+ in e.message
+ ):
+ raise ContentFilterError(e.message) from e
+ else:
+ raise
+
+ def update_token_count(self, response):
+ try:
+ if response.usage and response.usage.total_tokens:
+ self.token_count.inc(response.usage.total_tokens)
+ if response.usage and response.usage.prompt_tokens:
+ self.prompt_token_count.inc(response.usage.prompt_tokens)
+ if response.usage and response.usage.completion_tokens:
+ self.completion_token_count.inc(response.usage.completion_tokens)
+ if response.usage and (
+ hit_count := getattr(response.usage, "prompt_cache_hit_tokens", 0)
+ ):
+ self.cache_hit_prompt_token_count.inc(hit_count)
+ except Exception as e:
+ logger.exception("Error updating token count")
+
+ def get_formular_placeholder(self, placeholder_id: int):
+ return "{v" + str(placeholder_id) + "}", f"{{\\s*v\\s*{placeholder_id}\\s*}}"
+ return "{{" + str(placeholder_id) + "}}"
+
+ def get_rich_text_left_placeholder(self, placeholder_id: int):
+ return (
+ f"", r"<\s*\/\s*style\s*>"
diff --git a/babeldoc/utils/__init__.py b/babeldoc/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/babeldoc/utils/atomic_integer.py b/babeldoc/utils/atomic_integer.py
new file mode 100644
index 0000000000000000000000000000000000000000..76c27eefc39dbb4c46b6c9ad733daac960990696
--- /dev/null
+++ b/babeldoc/utils/atomic_integer.py
@@ -0,0 +1,26 @@
+import threading
+
+
+class AtomicInteger:
+ def __init__(self, value=0):
+ self._value = int(value)
+ self._lock = threading.Lock()
+
+ def inc(self, d=1):
+ with self._lock:
+ self._value += int(d)
+ return self._value
+
+ def dec(self, d=1):
+ return self.inc(-d)
+
+ @property
+ def value(self):
+ with self._lock:
+ return self._value
+
+ @value.setter
+ def value(self, v):
+ with self._lock:
+ self._value = int(v)
+ return self._value
diff --git a/babeldoc/utils/memory.py b/babeldoc/utils/memory.py
new file mode 100644
index 0000000000000000000000000000000000000000..04d548b6e92d22b5537ca56f268c5e22f68ddde3
--- /dev/null
+++ b/babeldoc/utils/memory.py
@@ -0,0 +1,251 @@
+import os
+import sys
+import time
+from pathlib import Path
+
+try:
+ import psutil
+except ImportError:
+ psutil = None
+
+
+def _parse_pss_from_smaps_rollup(pid: int) -> int | None:
+ """
+ Try to read PSS from /proc//smaps_rollup.
+ Returns PSS in bytes, or None if not available/readable.
+ """
+ try:
+ smaps_rollup_path = Path(f"/proc/{pid}/smaps_rollup")
+ with smaps_rollup_path.open() as f:
+ for line in f:
+ if line.startswith("Pss:"):
+ # Format: "Pss: 1234 kB"
+ parts = line.split()
+ if len(parts) >= 2:
+ pss_kb = int(parts[1])
+ return pss_kb * 1024 # Convert to bytes
+ return None
+ except (FileNotFoundError, PermissionError, ValueError, OSError):
+ return None
+
+
+def _parse_pss_from_smaps(pid: int) -> int | None:
+ """
+ Try to read PSS from /proc//smaps and sum all Pss entries.
+ Returns PSS in bytes, or None if not available/readable.
+ """
+ try:
+ smaps_path = Path(f"/proc/{pid}/smaps")
+ total_pss_kb = 0
+ with smaps_path.open() as f:
+ for line in f:
+ if line.startswith("Pss:"):
+ # Format: "Pss: 1234 kB"
+ parts = line.split()
+ if len(parts) >= 2:
+ total_pss_kb += int(parts[1])
+ if total_pss_kb > 0:
+ return total_pss_kb * 1024 # Convert to bytes
+ return None
+ except (FileNotFoundError, PermissionError, ValueError, OSError):
+ return None
+
+
+def _get_pss_linux(pid: int) -> int | None:
+ """
+ Try to get PSS on Linux.
+ Priority: smaps_rollup -> smaps -> None
+ Returns PSS in bytes, or None if not available.
+ """
+ # Try smaps_rollup first (lightweight)
+ pss = _parse_pss_from_smaps_rollup(pid)
+ if pss is not None:
+ return pss
+
+ # Fallback to smaps (heavier)
+ pss = _parse_pss_from_smaps(pid)
+ if pss is not None:
+ return pss
+
+ return None
+
+
+def _get_rss_psutil(pid: int) -> int | None:
+ """
+ Get RSS using psutil for a single process.
+ Returns RSS in bytes, or None if psutil unavailable or process not found.
+ """
+ if psutil is None:
+ return None
+
+ try:
+ process = psutil.Process(pid)
+ return process.memory_info().rss
+ except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.TimeoutExpired):
+ return None
+
+
+def _get_single_process_memory(
+ pid: int, prefer_pss: bool = True, use_smaps_rollup_only: bool = False
+) -> int | None:
+ """
+ Get memory usage for a single process (no children).
+
+ Args:
+ pid: Process ID
+ prefer_pss: If True and on Linux, try PSS first; otherwise use RSS
+ use_smaps_rollup_only: If True, only try smaps_rollup (faster), fallback to RSS if not available
+
+ Returns:
+ Memory usage in bytes, or None if all methods fail
+ """
+ if sys.platform == "linux":
+ if prefer_pss:
+ if use_smaps_rollup_only:
+ # Only try smaps_rollup, then fallback to RSS
+ pss = _parse_pss_from_smaps_rollup(pid)
+ if pss is not None:
+ return pss
+ else:
+ # Try full PSS (smaps_rollup -> smaps)
+ pss = _get_pss_linux(pid)
+ if pss is not None:
+ return pss
+
+ # Fallback to RSS
+ return _get_rss_psutil(pid)
+
+
+def get_memory_usage_bytes(
+ pid: int | None = None,
+ include_children: bool = True,
+ prefer_pss: bool = True,
+) -> int:
+ """
+ Get memory usage of a process (and optionally its children).
+
+ On Linux with prefer_pss=True:
+ - Tries /proc//smaps_rollup first (lightweight)
+ - Falls back to /proc//smaps if smaps_rollup unavailable (heavier)
+ - Falls back to psutil RSS if smaps unavailable
+
+ On non-Linux systems or prefer_pss=False:
+ - Uses psutil RSS
+
+ Args:
+ pid: Process ID to monitor. If None, uses current process.
+ include_children: If True, also includes memory of child processes.
+ prefer_pss: If True on Linux, attempts to use PSS; otherwise uses RSS.
+
+ Returns:
+ Total memory usage in bytes (guaranteed non-negative).
+ """
+ if pid is None:
+ pid = os.getpid()
+
+ total_memory = 0
+
+ # Determine if we're using smaps (heavier) vs smaps_rollup (lighter)
+ use_smaps_rollup_only = False
+ if sys.platform == "linux" and prefer_pss:
+ # If we can read smaps_rollup, use rollup-only mode
+ test_rollup = _parse_pss_from_smaps_rollup(pid)
+ use_smaps_rollup_only = test_rollup is not None
+
+ # Get current process memory
+ memory = _get_single_process_memory(
+ pid, prefer_pss=prefer_pss, use_smaps_rollup_only=use_smaps_rollup_only
+ )
+ if memory is not None:
+ total_memory += memory
+
+ # Get children memory if requested
+ if include_children:
+ if psutil is None:
+ # Cannot get children without psutil
+ return total_memory
+
+ try:
+ parent_process = psutil.Process(pid)
+ children = parent_process.children(recursive=True)
+ except (psutil.NoSuchProcess, psutil.AccessDenied):
+ # Parent process not found or no permission
+ return total_memory
+
+ for child in children:
+ try:
+ child_pid = child.pid
+ child_memory = _get_single_process_memory(
+ child_pid,
+ prefer_pss=prefer_pss,
+ use_smaps_rollup_only=use_smaps_rollup_only,
+ )
+ if child_memory is not None:
+ total_memory += child_memory
+ except (psutil.NoSuchProcess, psutil.AccessDenied):
+ # Child process died or no permission; skip it
+ pass
+
+ return max(0, total_memory)
+
+
+def get_memory_usage_with_throttle(
+ pid: int | None = None,
+ include_children: bool = True,
+ prefer_pss: bool = True,
+ last_pss_check_time: float | None = None,
+ pss_throttle_seconds: float = 2.0,
+) -> tuple[int, float | None]:
+ """
+ Get memory usage with throttling for PSS checks on Linux.
+
+ When PSS is not available via smaps_rollup and must read smaps (expensive),
+ this throttles checks to at most once per pss_throttle_seconds.
+
+ Args:
+ pid: Process ID. If None, uses current process.
+ include_children: If True, includes child process memory.
+ prefer_pss: If True on Linux, attempts to use PSS.
+ last_pss_check_time: Timestamp of last PSS check. For throttling logic.
+ pss_throttle_seconds: Minimum interval (seconds) between smaps reads.
+
+ Returns:
+ Tuple of (memory_bytes, new_check_time).
+ If throttled, returns cached estimate (0) and original check time.
+ """
+ current_time = time.time()
+
+ # Check if we should throttle
+ if (
+ prefer_pss
+ and sys.platform == "linux"
+ and last_pss_check_time is not None
+ and (current_time - last_pss_check_time) < pss_throttle_seconds
+ ):
+ # Throttled: use RSS only as a fast estimate
+ memory = 0
+ pid_to_check = pid if pid is not None else os.getpid()
+ rss = _get_rss_psutil(pid_to_check)
+ if rss is not None:
+ memory += rss
+
+ if include_children and psutil is not None:
+ try:
+ parent_process = psutil.Process(pid_to_check)
+ for child in parent_process.children(recursive=True):
+ try:
+ child_rss = _get_rss_psutil(child.pid)
+ if child_rss is not None:
+ memory += child_rss
+ except (psutil.NoSuchProcess, psutil.AccessDenied):
+ pass
+ except (psutil.NoSuchProcess, psutil.AccessDenied):
+ pass
+
+ return memory, last_pss_check_time
+
+ # Not throttled: do full check
+ memory = get_memory_usage_bytes(
+ pid=pid, include_children=include_children, prefer_pss=prefer_pss
+ )
+ return memory, current_time
diff --git a/babeldoc/utils/priority_thread_pool_executor.py b/babeldoc/utils/priority_thread_pool_executor.py
new file mode 100644
index 0000000000000000000000000000000000000000..bdc56292a1aad8857231e4bda33a92d9cba9bef9
--- /dev/null
+++ b/babeldoc/utils/priority_thread_pool_executor.py
@@ -0,0 +1,269 @@
+# thanks to:
+# https://github.com/oleglpts/PriorityThreadPoolExecutor/blob/master/PriorityThreadPoolExecutor/__init__.py
+# https://github.com/oleglpts/PriorityThreadPoolExecutor/issues/4
+
+import atexit
+import itertools
+import logging
+import queue
+import random
+import sys
+import threading
+import weakref
+from concurrent.futures import _base
+from concurrent.futures.thread import BrokenThreadPool
+from concurrent.futures.thread import ThreadPoolExecutor
+from concurrent.futures.thread import _python_exit
+from concurrent.futures.thread import _threads_queues
+from concurrent.futures.thread import _WorkItem
+from heapq import heappop
+from heapq import heappush
+
+logger = logging.getLogger(__name__)
+
+########################################################################################################################
+# Global variables #
+########################################################################################################################
+
+NULL_ENTRY = (sys.maxsize, _WorkItem(None, None, (), {}))
+_shutdown = False
+
+########################################################################################################################
+# Before system exit procedure #
+########################################################################################################################
+
+
+def python_exit():
+ """
+
+ Cleanup before system exit
+
+ """
+ global _shutdown
+ _shutdown = True
+ items = list(_threads_queues.items())
+ for _t, q in items:
+ q.put(NULL_ENTRY)
+ for t, _q in items:
+ t.join()
+
+
+# change default cleanup
+
+
+atexit.unregister(_python_exit)
+atexit.register(python_exit)
+
+
+class PriorityQueue(queue.Queue):
+ """Variant of Queue that retrieves open entries in priority order (lowest first).
+
+ Entries are typically tuples of the form: (priority number, data).
+ """
+
+ REMOVED = ""
+ DEFAULT_PRIORITY = 100
+
+ def _init(self, maxsize):
+ self.queue = []
+ self.entry_finder = {}
+ self.counter = itertools.count()
+
+ def _qsize(self):
+ return len(self.queue)
+
+ def _put(self, item):
+ # heappush(self.queue, item)
+ try:
+ if item[1] in self.entry_finder:
+ self.remove(item[1])
+ count = next(self.counter)
+ entry = [item[0], count, item[1]]
+ self.entry_finder[item[1]] = entry
+ heappush(self.queue, entry)
+ except TypeError: # handle item==None
+ self._put((self.DEFAULT_PRIORITY, None))
+
+ def remove(self, task):
+ """
+ This simply replaces the data with the REMOVED value,
+ which will get cleared out once _get reaches it.
+ """
+ entry = self.entry_finder.pop(task)
+ entry[-1] = self.REMOVED
+
+ def _get(self):
+ while self.queue:
+ entry = heappop(self.queue)
+ if entry[2] is not self.REMOVED:
+ del self.entry_finder[entry[2]]
+ return entry
+ return None
+
+
+def _worker(executor_reference, work_queue, initializer, initargs):
+ if initializer is not None:
+ try:
+ initializer(*initargs)
+ except BaseException:
+ _base.LOGGER.critical("Exception in initializer:", exc_info=True)
+ executor = executor_reference()
+ if executor is not None:
+ executor._initializer_failed()
+ return
+ try:
+ while True:
+ work_item = work_queue.get(block=True)
+ try:
+ if work_item[2] is not None:
+ work_item[2].run()
+ # Delete references to object. See issue16284
+ del work_item
+
+ # attempt to increment idle count
+ executor = executor_reference()
+ if executor is not None:
+ executor._idle_semaphore.release()
+ del executor
+ continue
+
+ executor = executor_reference()
+ # Exit if:
+ # - The interpreter is shutting down OR
+ # - The executor that owns the worker has been collected OR
+ # - The executor that owns the worker has been shutdown.
+ if _shutdown or executor is None or executor._shutdown:
+ # Flag the executor as shutting down as early as possible if it
+ # is not gc-ed yet.
+ if executor is not None:
+ executor._shutdown = True
+ # Notice other workers
+ work_queue.put(None)
+ return
+ del executor
+ finally:
+ work_queue.task_done()
+ except BaseException:
+ _base.LOGGER.critical("Exception in worker", exc_info=True)
+
+
+class PriorityThreadPoolExecutor(ThreadPoolExecutor):
+ """
+ Thread pool executor with priority queue (priorities must be different, lowest first)
+ """
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ # change work queue type to queue.PriorityQueue
+ self._work_queue: PriorityQueue = PriorityQueue()
+ self._all_future = []
+
+ def submit(self, fn, *args, **kwargs):
+ """
+
+ Sending the function to the execution queue
+
+ :param fn: function being executed
+ :type fn: callable
+ :param args: function's positional arguments
+ :param kwargs: function's keywords arguments
+ :return: future instance
+ :rtype: _base.Future
+
+ Added keyword:
+
+ - priority (integer later sys.maxsize)
+
+ """
+ with self._shutdown_lock:
+ if self._broken:
+ raise BrokenThreadPool(self._broken)
+
+ if self._shutdown:
+ raise RuntimeError("cannot schedule new futures after shutdown")
+ if _shutdown:
+ raise RuntimeError(
+ "cannot schedule new futures after interpreter shutdown"
+ )
+
+ priority = kwargs.get("priority", random.randint(0, sys.maxsize - 1)) # noqa: S311
+ if "priority" in kwargs:
+ del kwargs["priority"]
+
+ f = _base.Future()
+ w = _WorkItem(f, fn, args, kwargs)
+
+ self._work_queue.put((priority, w))
+ self._adjust_thread_count()
+ self._all_future.append(f)
+ return f
+
+ def _adjust_thread_count(self):
+ # if idle threads are available, don't spin new threads
+ if self._idle_semaphore.acquire(timeout=0):
+ return
+
+ # When the executor gets lost, the weakref callback will wake up
+ # the worker threads.
+ def weakref_cb(_, q=self._work_queue):
+ q.put(None)
+
+ num_threads = len(self._threads)
+ if num_threads < self._max_workers:
+ thread_name = f"{self._thread_name_prefix or self}_{num_threads:d}"
+ t = threading.Thread(
+ name=thread_name,
+ target=_worker,
+ args=(
+ weakref.ref(self, weakref_cb),
+ self._work_queue,
+ self._initializer,
+ self._initargs,
+ ),
+ )
+ t.start()
+ self._threads.add(t)
+ _threads_queues[t] = self._work_queue
+
+ def shutdown(self, wait=True, *, cancel_futures=False):
+ logger.debug("Shutting down executor %s", self._thread_name_prefix or self)
+ if wait:
+ logger.debug(
+ "Waiting for all tasks done %s", self._thread_name_prefix or self
+ )
+ self._work_queue.join()
+ logger.debug("All tasks done %s", self._thread_name_prefix or self)
+
+ with self._shutdown_lock:
+ self._shutdown = True
+ if cancel_futures:
+ # Drain all work items from the queue, and then cancel their
+ # associated futures.
+ while True:
+ try:
+ work_item = self._work_queue.get_nowait()
+ except queue.Empty:
+ break
+ if work_item is not None:
+ work_item.future.cancel()
+
+ # Send a wake-up to prevent threads calling
+ # _work_queue.get(block=True) from permanently blocking.
+ self._work_queue.put(None)
+ if wait:
+ logger.debug(
+ "Waiting for all thread done %s", self._thread_name_prefix or self
+ )
+ for t in self._threads:
+ self._work_queue.put(None)
+ t.join()
+ logger.debug("shutdown finish %s", self._thread_name_prefix or self)
+
+ def __del__(self):
+ for f in self._all_future:
+ if f.done() and not f.cancelled():
+ try:
+ f.result()
+ except Exception as e:
+ logger.warning("Exception in future %s: %s", f, e, exc_info=True)
diff --git a/frontend/index.html b/frontend/index.html
new file mode 100644
index 0000000000000000000000000000000000000000..96a8a28557cc08cdd6d0e174f3b4d6bef074371e
--- /dev/null
+++ b/frontend/index.html
@@ -0,0 +1,518 @@
+
+
+
+
+
+ BabelDOC - Intelligent PDF Translation
+
+
+
+
+
+
+
+
📤
+
Drop your PDF here or click to browse
+
Supports PDF files up to 50MB
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Processing your document...
+
+
+
+
✅
+
Translation completed successfully!
+
+
+
+
+