|
|
|
|
|
|
|
|
from __future__ import annotations |
|
|
|
|
|
from typing import Dict, List, Any, Optional |
|
|
|
|
|
from mecari.utils.signature import signature_key |
|
|
|
|
|
|
|
|
def dedup_morphemes(morphs: List[Dict]) -> List[Dict]: |
|
|
seen = set() |
|
|
out: List[Dict] = [] |
|
|
for m in morphs: |
|
|
key = signature_key(m) |
|
|
if key in seen: |
|
|
continue |
|
|
seen.add(key) |
|
|
out.append(m) |
|
|
out.sort(key=lambda m: ( |
|
|
m.get("start_pos", 0), |
|
|
-(m.get("end_pos", 0) - m.get("start_pos", 0)), |
|
|
m.get("surface", ""), |
|
|
m.get("reading", ""), |
|
|
m.get("pos", "*"), |
|
|
)) |
|
|
return out |
|
|
|
|
|
|
|
|
def build_adjacent_edges(morphs: List[Dict]) -> List[Dict]: |
|
|
edges: List[Dict] = [] |
|
|
for i, s in enumerate(morphs): |
|
|
for j, t in enumerate(morphs): |
|
|
if i >= j: |
|
|
continue |
|
|
if s.get("end_pos", 0) == t.get("start_pos", 0): |
|
|
edges.append({"source_idx": i, "target_idx": j, "edge_type": "forward"}) |
|
|
return edges |
|
|
|
|
|
|
|
|
def normalize_mecab_candidates(candidates: List[Dict]) -> List[Dict]: |
|
|
"""Normalize MeCab candidates consistently for preprocessing/inference. |
|
|
|
|
|
- If surface is digit-only and base_form is empty/missing, set base_form = surface. |
|
|
Modifies candidates in place and returns the list for convenience. |
|
|
""" |
|
|
for c in candidates: |
|
|
surf = c.get("surface", "") |
|
|
bf = c.get("base_form") |
|
|
if (bf is None or bf == "") and surf and surf.isdigit(): |
|
|
c["base_form"] = surf |
|
|
return candidates |
|
|
|
