Spaces:
Runtime error
Runtime error
File size: 4,801 Bytes
fc7db98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 | """
Structural syntax analysis for word complexity project.
Provides:
- analyze_syntax(sentence): structural metrics for a sentence
- find_hardest_span(sentence): the most syntactically tangled span (for faithfulness edits)
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from functools import lru_cache
from typing import Any
import spacy
from spacy.tokens import Doc, Span, Token
@lru_cache(maxsize=1)
def _load_nlp():
try:
return spacy.load("en_core_web_sm")
except OSError as exc:
raise RuntimeError(
"spaCy model 'en_core_web_sm' not found. Run: python -m spacy download en_core_web_sm"
) from exc
@dataclass
class SyntaxMetrics:
token_count: int
max_tree_depth: int
avg_dependency_length: float
subordinate_clause_count: int
passive_voice_count: int
punctuation_density: float
complexity_score: float
def to_dict(self) -> dict[str, Any]:
return {
"token_count": self.token_count,
"max_tree_depth": self.max_tree_depth,
"avg_dependency_length": self.avg_dependency_length,
"subordinate_clause_count": self.subordinate_clause_count,
"passive_voice_count": self.passive_voice_count,
"punctuation_density": self.punctuation_density,
"complexity_score": self.complexity_score,
}
def _token_depth(token: Token) -> int:
depth = 0
while token.head != token:
depth += 1
token = token.head
return depth
def _subtree_depth(token: Token) -> int:
children = list(token.subtree)
if not children:
return 0
return 1 + max(_subtree_depth(child) for child in token.children) if token.children else 1
def _dependency_length(token: Token) -> int:
if token.head == token:
return 0
return abs(token.i - token.head.i)
def _count_subordinate_clauses(doc: Doc) -> int:
markers = {"ccomp", "xcomp", "advcl", "relcl", "acl"}
return sum(1 for tok in doc if tok.dep_ in markers)
def _count_passive(doc: Doc) -> int:
return sum(
1
for tok in doc
if tok.dep_ in {"nsubjpass", "auxpass"} or (tok.tag_ == "VBN" and tok.dep_ == "ROOT")
)
def analyze_syntax(sentence: str) -> SyntaxMetrics:
"""Return structural complexity metrics for a sentence."""
nlp = _load_nlp()
doc = nlp(sentence or "")
tokens = [t for t in doc if not t.is_space]
if not tokens:
return SyntaxMetrics(0, 0, 0.0, 0, 0, 0.0, 0.0)
depths = [_token_depth(t) for t in tokens]
dep_lengths = [_dependency_length(t) for t in tokens]
max_depth = max(depths)
avg_dep = sum(dep_lengths) / len(dep_lengths)
sub_clauses = _count_subordinate_clauses(doc)
passive = _count_passive(doc)
punct = len(re.findall(r"[,;:()\[\]{}\"']", sentence))
punct_density = punct / max(len(sentence), 1)
complexity_score = (
0.35 * max_depth
+ 0.25 * avg_dep
+ 0.20 * sub_clauses
+ 0.10 * passive
+ 0.10 * punct_density * 20
+ 0.05 * (len(tokens) / 30.0)
)
return SyntaxMetrics(
token_count=len(tokens),
max_tree_depth=max_depth,
avg_dependency_length=avg_dep,
subordinate_clause_count=sub_clauses,
passive_voice_count=passive,
punctuation_density=punct_density,
complexity_score=complexity_score,
)
def _span_complexity(span: Span) -> float:
text = span.text.strip()
if not text:
return 0.0
metrics = analyze_syntax(text)
return metrics.complexity_score
def find_hardest_span(sentence: str, min_tokens: int = 3) -> tuple[str, int, int]:
"""
Find the most syntactically tangled contiguous span in the sentence.
Returns (span_text, start_char, end_char).
"""
nlp = _load_nlp()
doc = nlp(sentence or "")
tokens = [t for t in doc if not t.is_space]
if len(tokens) < min_tokens:
return sentence, 0, len(sentence)
best_score = -1.0
best_span: Span | None = None
for start in range(len(tokens)):
for end in range(start + min_tokens, min(start + 25, len(tokens)) + 1):
span = doc[tokens[start].i : tokens[end - 1].i + 1]
score = _span_complexity(span)
if score > best_score:
best_score = score
best_span = span
if best_span is None:
return sentence, 0, len(sentence)
return best_span.text, best_span.start_char, best_span.end_char
def syntax_signal_high(sentence: str, percentile_threshold: float = 0.6) -> bool:
"""Heuristic: sentence is structurally complex relative to typical text."""
score = analyze_syntax(sentence).complexity_score
return score >= percentile_threshold * 8.0
|