""" Structural syntax analysis for word complexity project. Provides: - analyze_syntax(sentence): structural metrics for a sentence - find_hardest_span(sentence): the most syntactically tangled span (for faithfulness edits) """ from __future__ import annotations import re from dataclasses import dataclass from functools import lru_cache from typing import Any import spacy from spacy.tokens import Doc, Span, Token @lru_cache(maxsize=1) def _load_nlp(): try: return spacy.load("en_core_web_sm") except OSError as exc: raise RuntimeError( "spaCy model 'en_core_web_sm' not found. Run: python -m spacy download en_core_web_sm" ) from exc @dataclass class SyntaxMetrics: token_count: int max_tree_depth: int avg_dependency_length: float subordinate_clause_count: int passive_voice_count: int punctuation_density: float complexity_score: float def to_dict(self) -> dict[str, Any]: return { "token_count": self.token_count, "max_tree_depth": self.max_tree_depth, "avg_dependency_length": self.avg_dependency_length, "subordinate_clause_count": self.subordinate_clause_count, "passive_voice_count": self.passive_voice_count, "punctuation_density": self.punctuation_density, "complexity_score": self.complexity_score, } def _token_depth(token: Token) -> int: depth = 0 while token.head != token: depth += 1 token = token.head return depth def _subtree_depth(token: Token) -> int: children = list(token.subtree) if not children: return 0 return 1 + max(_subtree_depth(child) for child in token.children) if token.children else 1 def _dependency_length(token: Token) -> int: if token.head == token: return 0 return abs(token.i - token.head.i) def _count_subordinate_clauses(doc: Doc) -> int: markers = {"ccomp", "xcomp", "advcl", "relcl", "acl"} return sum(1 for tok in doc if tok.dep_ in markers) def _count_passive(doc: Doc) -> int: return sum( 1 for tok in doc if tok.dep_ in {"nsubjpass", "auxpass"} or (tok.tag_ == "VBN" and tok.dep_ == "ROOT") ) def analyze_syntax(sentence: str) -> SyntaxMetrics: """Return structural complexity metrics for a sentence.""" nlp = _load_nlp() doc = nlp(sentence or "") tokens = [t for t in doc if not t.is_space] if not tokens: return SyntaxMetrics(0, 0, 0.0, 0, 0, 0.0, 0.0) depths = [_token_depth(t) for t in tokens] dep_lengths = [_dependency_length(t) for t in tokens] max_depth = max(depths) avg_dep = sum(dep_lengths) / len(dep_lengths) sub_clauses = _count_subordinate_clauses(doc) passive = _count_passive(doc) punct = len(re.findall(r"[,;:()\[\]{}\"']", sentence)) punct_density = punct / max(len(sentence), 1) complexity_score = ( 0.35 * max_depth + 0.25 * avg_dep + 0.20 * sub_clauses + 0.10 * passive + 0.10 * punct_density * 20 + 0.05 * (len(tokens) / 30.0) ) return SyntaxMetrics( token_count=len(tokens), max_tree_depth=max_depth, avg_dependency_length=avg_dep, subordinate_clause_count=sub_clauses, passive_voice_count=passive, punctuation_density=punct_density, complexity_score=complexity_score, ) def _span_complexity(span: Span) -> float: text = span.text.strip() if not text: return 0.0 metrics = analyze_syntax(text) return metrics.complexity_score def find_hardest_span(sentence: str, min_tokens: int = 3) -> tuple[str, int, int]: """ Find the most syntactically tangled contiguous span in the sentence. Returns (span_text, start_char, end_char). """ nlp = _load_nlp() doc = nlp(sentence or "") tokens = [t for t in doc if not t.is_space] if len(tokens) < min_tokens: return sentence, 0, len(sentence) best_score = -1.0 best_span: Span | None = None for start in range(len(tokens)): for end in range(start + min_tokens, min(start + 25, len(tokens)) + 1): span = doc[tokens[start].i : tokens[end - 1].i + 1] score = _span_complexity(span) if score > best_score: best_score = score best_span = span if best_span is None: return sentence, 0, len(sentence) return best_span.text, best_span.start_char, best_span.end_char def syntax_signal_high(sentence: str, percentile_threshold: float = 0.6) -> bool: """Heuristic: sentence is structurally complex relative to typical text.""" score = analyze_syntax(sentence).complexity_score return score >= percentile_threshold * 8.0