import re from typing import Any, Dict, List, Tuple import unicodedata SPAN_RE = re.compile(r"(.*?)") _ALLOWED_EMOJI_GLUE = { "\u200d", # ZWJ "\ufe0f", # variation selector-16 "\u20e3", # combining enclosing keycap } def validate_emoji_only(value: str) -> Tuple[bool, str]: if not isinstance(value, str) or not value.strip(): return False, "empty emoji string" s = value.strip() for ch in s: if unicodedata.category(ch).startswith("L"): return False, f"contains letters: {repr(ch)}" try: import emoji except ImportError: return False, "missing dependency: pip install emoji" tokens = emoji.emoji_list(s) if not tokens: return False, "no emoji found" covered = [False] * len(s) for t in tokens: start, end = t["match_start"], t["match_end"] for i in range(start, end): covered[i] = True for i, ch in enumerate(s): if covered[i]: continue if ch in _ALLOWED_EMOJI_GLUE: continue return False, f"contains non-emoji character: {repr(ch)}" return True, "ok" def extract_marked_spans(marked_sentence: str) -> List[str]: return [s.strip() for s in SPAN_RE.findall(marked_sentence)] def validate_marked_matches_original(marked: str, original: str) -> Tuple[bool, str]: stripped = marked.replace("", "").replace("", "") if stripped != original: return False, f"stripped marked != original" return True, "ok" def validate_annotation_json(obj: Dict[str, Any], original_sentence: str) -> Tuple[bool, str]: if not isinstance(obj, dict): return False, "annotation is not a dict" marked = obj.get("marked") if not isinstance(marked, str): return False, "missing/invalid 'marked' field" annotations = obj.get("annotations") if not isinstance(annotations, list): return False, "missing/invalid 'annotations' field" # Check that stripping tags recovers the original sentence ok, reason = validate_marked_matches_original(marked, original_sentence) if not ok: return False, reason # Extract spans positionally from marked text (one entry per occurrence, in order) marked_spans = extract_marked_spans(marked) # Zero spans is valid (e.g. all stopwords or named entities) if not marked_spans: if annotations: return False, "annotations list is non-empty but no spans found in marked text" return True, "ok" # annotations must be a positional list matching marked_spans exactly if len(annotations) != len(marked_spans): return False, ( f"annotations length ({len(annotations)}) != " f"number of marked spans ({len(marked_spans)})" ) for i, (item, expected_span) in enumerate(zip(annotations, marked_spans)): if not isinstance(item, dict): return False, f"annotation entry {i} is not a dict" span = item.get("span") emojis = item.get("emojis") if not isinstance(span, str) or not span.strip(): return False, f"annotation entry {i} has empty/missing 'span'" if span.strip() != expected_span: return False, ( f"annotation entry {i} span {span.strip()!r} " f"does not match marked span {expected_span!r}" ) if not isinstance(emojis, str) or not emojis.strip(): return False, f"annotation entry {i} has empty/missing 'emojis' for span: {span}" ok_emoji, why = validate_emoji_only(emojis.strip()) if not ok_emoji: return False, f"non-emoji content for span '{span}': {why}" return True, "ok"