aiflow-math-ink-06-intermediate / src /structure_relations.py
cwLeeDev's picture
Add AIFlow Math Ink 0.6 intermediate research snapshot
2948983 verified
Raw
History Blame Contribute Delete
9.67 kB
"""๊ธฐํ˜ธ ์ธ์‹๊ณผ ๋…๋ฆฝ์ ์œผ๋กœ ์ˆ˜์‹์˜ 2D ๋ฐฐ์น˜ ๊ด€๊ณ„๋ฅผ ์ถ”๋ก ํ•˜๋Š” ์—ฐ๊ตฌ์šฉ ๋ ˆ์ด์–ด๋‹ค."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Sequence
import numpy as np
from .math_tray import infer_math_trays
RELATION_TYPES = ("RIGHT", "SUPERSCRIPT", "SUBSCRIPT", "ABOVE", "BELOW", "CONTAINS")
STRUCTURAL_RELATION_TYPES = frozenset({"SUPERSCRIPT", "SUBSCRIPT", "ABOVE", "BELOW", "CONTAINS"})
@dataclass(frozen=True, slots=True)
class RelationConfig:
"""ํ•„์š” ๋ณ€์ˆ˜: ์ฒจ์ž ๊ธฐํ•˜ ์ž„๊ณ„๊ฐ’. ์ž‘๋™ ์›๋ฆฌ: writer validation์—์„œ ๊ณ ๋ฅผ ์ˆ˜ ์žˆ๋„๋ก ๊ด€๊ณ„ ํŒ์ •๊ฐ’์„ ์™ธ๋ถ€ํ™”ํ•œ๋‹ค."""
script_min_height_ratio: float = 0.18
script_max_height_ratio: float = 0.82
script_vertical_shift_ratio: float = 0.18
script_max_horizontal_gap_ratio: float = 0.75
def _label(segment: dict[str, Any]) -> str:
"""ํ•„์š” ๋ณ€์ˆ˜: ์ธ์‹๋œ segment. ์ž‘๋™ ์›๋ฆฌ: ๊ตฌ์กฐ ํžŒํŠธ๋ฅผ ์šฐ์„ ํ•˜๊ณ  ์ฒซ ์ „๋ฌธ๊ฐ€์˜ top-1์„ ๋ณด์กฐ ๋ผ๋ฒจ๋กœ ์ฝ๋Š”๋‹ค."""
hint = segment.get("structural_hint")
if hint:
return str(hint)
for result in (segment.get("results") or {}).values():
candidates = result.get("candidates") or []
if candidates:
return str(candidates[0]["label"])
return ""
def _geometry(segment: dict[str, Any]) -> dict[str, float]:
"""ํ•„์š” ๋ณ€์ˆ˜: segment bbox. ์ž‘๋™ ์›๋ฆฌ: ๊ด€๊ณ„ ํŒ์ •์— ํ•„์š”ํ•œ ํญยท๋†’์ดยท์ค‘์‹ฌ ์ขŒํ‘œ๋ฅผ ํ•œ ๋ฒˆ ๊ณ„์‚ฐํ•œ๋‹ค."""
box = segment["box"]
left, top = float(box["left"]), float(box["top"])
right, bottom = float(box["right"]), float(box["bottom"])
return {
"left": left, "top": top, "right": right, "bottom": bottom,
"width": max(right - left, 1.0), "height": max(bottom - top, 1.0),
"cx": (left + right) * 0.5, "cy": (top + bottom) * 0.5,
}
def infer_spatial_relations(
segments: Sequence[dict[str, Any]], config: RelationConfig | None = None,
*, trays: Sequence[dict[str, Any]] | None = None,
) -> list[dict[str, Any]]:
"""ํ•„์š” ๋ณ€์ˆ˜: geometry group๊ณผ ์„ ํƒ์  OCR ๊ฒฐ๊ณผ. ์ž‘๋™ ์›๋ฆฌ: ๊ฐ ๊ธฐํ˜ธ๋ฅผ ํ•œ ๋ฒˆ๋งŒ ์ž์‹์œผ๋กœ ์†Œ๋น„ํ•ด 2D ๊ด€๊ณ„ ๊ทธ๋ž˜ํ”„๋ฅผ ๋งŒ๋“ ๋‹ค."""
relation_config = config or RelationConfig()
if len(segments) < 2:
return []
boxes = [_geometry(segment) for segment in segments]
labels = [_label(segment) for segment in segments]
maximum_height = max(box["height"] for box in boxes)
body_heights = [box["height"] for box in boxes if box["height"] >= maximum_height * 0.25]
reference_height = float(np.median(body_heights))
relations: list[dict[str, Any]] = []
assigned: set[int] = set()
structural_parents: set[int] = set()
# ๊ทผํ˜ธ๋Š” ๋‚ด๋ถ€ ๊ธฐํ˜ธ๋ฅผ symbol group์— ํ•ฉ์น˜์ง€ ์•Š๊ณ  CONTAINS edge๋กœ๋งŒ ์†Œ์œ ํ•œ๋‹ค.
for parent, (box, label) in enumerate(zip(boxes, labels, strict=True)):
root_like = label in {r"\sqrt", r"\sqrt{}"}
if not root_like:
continue
structural_parents.add(parent)
for child, inner in enumerate(boxes):
if child == parent or child in assigned:
continue
inside_x = box["left"] + box["width"] * 0.20 <= inner["cx"] <= box["right"] + reference_height * 0.15
inside_y = inner["cy"] >= box["top"] and inner["top"] <= box["bottom"] + reference_height * 0.35
if inside_x and inside_y:
relations.append({"parent": parent, "child": child, "type": "CONTAINS", "confidence": 0.95})
assigned.add(child)
# ๋ถ„์ˆ˜๋Š” bar ๋‹จ๋…์ด ์•„๋‹ˆ๋ผ ๋ถ„์žยท๋ถ„๋ชจ๊ฐ€ ๋ชจ๋‘ ์ ์œ ๋œ ์›์ž์  Tray์ผ ๋•Œ๋งŒ ๊ด€๊ณ„๋ฅผ ๋งŒ๋“ ๋‹ค.
tray_rows = list(trays) if trays is not None else infer_math_trays(segments)
fraction_trays = [tray for tray in tray_rows if tray["type"] == "FRACTION"]
for tray in fraction_trays:
parent = int(tray["anchor"])
structural_parents.add(parent)
for slot, relation in (("numerator", "ABOVE"), ("denominator", "BELOW")):
for child in tray["slots"].get(slot, []):
if child in assigned:
continue
relations.append({
"parent": parent, "child": child, "type": relation,
"confidence": float(tray["constraint_score"]),
})
assigned.add(child)
# ์ ๋ถ„ยทํ•ฉยท๊ณฑยท๊ทนํ•œ์˜ ์ƒํ•˜ํ•œ์€ ์ผ๋ฐ˜ ์ตœ๊ทผ์ ‘ ์ฒจ์ž๋ณด๋‹ค ์ˆ˜ํ•™ Tray์˜ anchor ์†Œ์œ ๊ถŒ์„ ์šฐ์„ ํ•œ๋‹ค.
operator_trays = [tray for tray in tray_rows if tray["type"] in {"INTEGRAL", "SUM", "PRODUCT", "LIM"}]
for tray in operator_trays:
parent = int(tray["anchor"])
structural_parents.add(parent)
for slot, relation in (("lower", "SUBSCRIPT"), ("upper", "SUPERSCRIPT")):
for child in tray["slots"].get(slot, []):
if child in assigned:
continue
relations.append({
"parent": parent, "child": int(child), "type": relation,
"confidence": float(tray["constraint_score"]),
})
assigned.add(int(child))
# ๋‚จ์€ ๊ธฐํ˜ธ๋Š” ๊ฐ€์žฅ ๊ฐ€๊นŒ์šด ์™ผ์ชฝ ๊ธฐํ˜ธ๋ฅผ ๊ธฐ์ค€์œผ๋กœ ์ฒจ์ž ๋˜๋Š” RIGHT ๊ด€๊ณ„๋ฅผ ๊ฐ–๋Š”๋‹ค.
for child, box in enumerate(boxes):
if child in assigned or child in structural_parents:
continue
candidates = [index for index, other in enumerate(boxes) if index != child and other["cx"] < box["cx"]]
if not candidates:
continue
parent = min(candidates, key=lambda index: max(0.0, box["left"] - boxes[index]["right"]))
base = boxes[parent]
horizontal_gap = max(0.0, box["left"] - base["right"])
script_sized = (
reference_height * relation_config.script_min_height_ratio
<= box["height"]
<= base["height"] * relation_config.script_max_height_ratio
)
script_near = horizontal_gap <= reference_height * relation_config.script_max_horizontal_gap_ratio
vertical_shift = base["height"] * relation_config.script_vertical_shift_ratio
if script_sized and script_near and box["cy"] < base["cy"] - vertical_shift:
relation = "SUPERSCRIPT"
confidence = 0.82
elif script_sized and script_near and box["cy"] > base["cy"] + vertical_shift:
relation = "SUBSCRIPT"
confidence = 0.82
else:
relation = "RIGHT"
confidence = 0.75
relations.append({"parent": parent, "child": child, "type": relation, "confidence": confidence})
assigned.add(child)
return sorted(relations, key=lambda item: (item["child"], item["parent"], item["type"]))
def serialize_relation_graph(
labels: Sequence[str], segments: Sequence[dict[str, Any]], relations: Sequence[dict[str, Any]],
) -> str:
"""ํ•„์š” ๋ณ€์ˆ˜: group๋ณ„ ๋ผ๋ฒจยทbboxยท๊ด€๊ณ„ edge. ์ž‘๋™ ์›๋ฆฌ: ๊ตฌ์กฐ ์ž์‹์„ ํ•œ ๋ฒˆ๋งŒ ์†Œ๋น„ํ•ด 2D graph๋ฅผ LaTeX๋กœ ์ง๋ ฌํ™”ํ•œ๋‹ค."""
if len(labels) != len(segments):
raise ValueError("๊ด€๊ณ„ serializer์˜ label๊ณผ segment ์ˆ˜๊ฐ€ ๋‹ค๋ฆ…๋‹ˆ๋‹ค.")
children: dict[int, dict[str, list[int]]] = {}
structural_children: set[int] = set()
for relation in relations:
relation_type = str(relation["type"])
if relation_type not in STRUCTURAL_RELATION_TYPES:
continue
parent, child = int(relation["parent"]), int(relation["child"])
if parent < 0 or child < 0 or parent >= len(labels) or child >= len(labels) or parent == child:
raise ValueError("๊ด€๊ณ„ graph์— ์œ ํšจํ•˜์ง€ ์•Š์€ node index๊ฐ€ ์žˆ์Šต๋‹ˆ๋‹ค.")
children.setdefault(parent, {}).setdefault(relation_type, []).append(child)
structural_children.add(child)
def left(index: int) -> float:
"""ํ•„์š” ๋ณ€์ˆ˜: node index. ์ž‘๋™ ์›๋ฆฌ: ๊ฐ™์€ ๊ตฌ์กฐ ์Šฌ๋กฏ ์•ˆ์˜ ๊ธฐํ˜ธ๋ฅผ ์›๋ž˜ x ์ˆœ์„œ๋กœ ์ •๋ ฌํ•œ๋‹ค."""
return float(segments[index]["box"]["left"])
def render_sequence(indices: Sequence[int], active: frozenset[int]) -> str:
"""ํ•„์š” ๋ณ€์ˆ˜: node index ์—ดยท์žฌ๊ท€ ๊ฒฝ๋กœ. ์ž‘๋™ ์›๋ฆฌ: ๊ฐ™์€ ์Šฌ๋กฏ์˜ node๋ฅผ ์ขŒโ†’์šฐ๋กœ ํ•œ ๋ฒˆ์”ฉ ์ถœ๋ ฅํ•œ๋‹ค."""
return "".join(render_node(index, active) for index in sorted(set(indices), key=left))
def render_node(index: int, active: frozenset[int]) -> str:
"""ํ•„์š” ๋ณ€์ˆ˜: ํ˜„์žฌ nodeยท์žฌ๊ท€ ๊ฒฝ๋กœ. ์ž‘๋™ ์›๋ฆฌ: root/fraction/script ๊ตฌ์กฐ๋ฅผ ์ค‘์ฒฉ ๊ฐ€๋Šฅํ•œ LaTeX node๋กœ ๋งŒ๋“ ๋‹ค."""
if index in active:
raise ValueError("๊ด€๊ณ„ graph์— cycle์ด ์žˆ์Šต๋‹ˆ๋‹ค.")
next_active = active | {index}
slots = children.get(index, {})
above = slots.get("ABOVE", [])
below = slots.get("BELOW", [])
contained = slots.get("CONTAINS", [])
if above and below:
base = rf"\frac{{{render_sequence(above, next_active)}}}{{{render_sequence(below, next_active)}}}"
elif contained and labels[index] in {r"\sqrt", r"\sqrt{}"}:
base = rf"\sqrt{{{render_sequence(contained, next_active)}}}"
else:
base = labels[index]
superscript = slots.get("SUPERSCRIPT", [])
subscript = slots.get("SUBSCRIPT", [])
if subscript:
base += rf"_{{{render_sequence(subscript, next_active)}}}"
if superscript:
base += rf"^{{{render_sequence(superscript, next_active)}}}"
return base
roots = [index for index in range(len(labels)) if index not in structural_children]
return render_sequence(roots, frozenset())