File size: 3,458 Bytes
11c7d9f 8f6d79d 39cfcd1 8f6d79d 11c7d9f 8f6d79d 31bce5e 39cfcd1 8f6d79d 39cfcd1 8f6d79d 39cfcd1 8f6d79d 31bce5e 8f6d79d 11c7d9f 8f6d79d 11c7d9f 8f6d79d 11c7d9f 8f6d79d 11c7d9f 8f6d79d 11c7d9f 8f6d79d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | """Sentence type classification — spaCy-driven when available."""
from __future__ import annotations
import re
from functools import lru_cache
from app.pipeline.nlp import get_nlp
_REWRITEABLE = frozenset(
{"simple_declarative", "compound", "because_clause"}
)
_CITATION = re.compile(
r"(?:\[[0-9]+(?:\s*[-,]\s*[0-9]+)*\]"
r"|\([A-Z][A-Za-z'-]+(?:\s+et\s+al\.)?,?\s+(?:19|20)\d{2}[a-z]?\)"
r"|\bdoi:\s*10\.\d{4,9}/\S+)",
re.I,
)
@lru_cache(maxsize=4096)
def classify_sentence(text: str) -> str:
"""Tag sentence type for rewrite eligibility."""
t = (text or "").strip()
if not t:
return "empty"
if re.match(r"^#{1,6}\s", t) or (len(t.split()) <= 6 and t.isupper()):
return "heading"
if re.match(r"^(\d+[\.\)]\s+|[-*•]\s+)", t):
return "list_item"
if t.startswith(('"', "'", "\u201c", "\u2018")) and (
t.count('"') >= 2 or t.count("\u201c") or t.count("'") >= 2
):
return "quoted"
if _CITATION.search(t):
return "citation"
if "?" in t or t.endswith("?"):
return "question"
if len(t.split()) < 3:
return "too_short"
# Allow longer prose through paraphrase/lexical; only extreme length is skipped.
if len(t.split()) > 70:
return "too_long"
nlp = get_nlp()
if nlp is not None:
return _classify_spacy(t, nlp)
return _classify_regex(t)
def _classify_spacy(text: str, nlp) -> str:
doc = nlp(text)
# because as mark / SCONJ
for t in doc:
if t.lemma_.lower() == "because" and t.pos_ in {"SCONJ", "ADP"}:
return "because_clause"
# Subordinate clauses via mark / advcl / SCONJ
for t in doc:
if t.dep_ == "mark" and t.head.dep_ in {"advcl", "acl"}:
if t.lemma_.lower() != "because":
return "complex"
if t.pos_ == "SCONJ" and t.lemma_.lower() != "because":
if t.head.dep_ in {"advcl", "acl", "ROOT"} or t.dep_ == "mark":
return "complex"
# Relative clauses on longer sentences are unsafe to slot-rebuild
if any(t.dep_ == "relcl" for t in doc) and len(text.split()) >= 10:
return "complex"
# Coordinating compound with comma
if "," in text and any(t.dep_ == "cc" and t.head.dep_ in {"conj", "ROOT"} for t in doc):
if len(text.split()) <= 28:
return "compound"
return "complex"
return "simple_declarative"
def _classify_regex(text: str) -> str:
low = text.lower()
if re.search(r"\bbecause\b", low):
return "because_clause"
if re.search(r"\b(and|but|or|so|yet)\b", low) and "," in text:
if re.search(
r"\b(although|though|while|whilst|whereas|unless|until|since|if|when|"
r"whenever|wherever|whether|before|after)\b",
low,
):
return "complex"
return "compound" if len(text.split()) <= 28 else "complex"
if re.search(
r"\b(although|though|while|whilst|whereas|unless|until|since|if|when|"
r"whenever|wherever|whether|before|after)\b",
low,
):
return "complex"
if re.search(r"\b(who|whom|whose|which|that)\b", low) and len(text.split()) > 12:
return "complex"
return "simple_declarative"
def is_rewriteable_type(sentence_type: str) -> bool:
return sentence_type in _REWRITEABLE
|