idnameraj commited on
Commit
3100e33
·
verified ·
1 Parent(s): 7f7383e

Upload 34 files

Browse files
app/data/content_swaps.json CHANGED
@@ -192,5 +192,211 @@
192
  "alone": "solitary",
193
  "first": "initial",
194
  "second": "next",
195
- "third": "another"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
196
  }
 
192
  "alone": "solitary",
193
  "first": "initial",
194
  "second": "next",
195
+ "third": "another",
196
+
197
+ "change": "shift",
198
+ "changes": "shifts",
199
+ "changed": "shifted",
200
+ "changing": "shifting",
201
+ "many": "numerous",
202
+ "much": "considerable",
203
+ "world": "globe",
204
+ "regions": "areas",
205
+ "region": "area",
206
+ "methods": "approaches",
207
+ "method": "approach",
208
+ "approach": "method",
209
+ "approaches": "methods",
210
+ "adapt": "adjust",
211
+ "adapts": "adjusts",
212
+ "adapted": "adjusted",
213
+ "survive": "endure",
214
+ "survives": "endures",
215
+ "survived": "endured",
216
+ "longer": "extended",
217
+ "unpredictable": "erratic",
218
+ "rainfall": "rain",
219
+ "droughts": "dry spells",
220
+ "drought": "dry spell",
221
+ "governments": "authorities",
222
+ "government": "authority",
223
+ "companies": "firms",
224
+ "company": "firm",
225
+ "better": "stronger",
226
+ "policies": "rules",
227
+ "policy": "rule",
228
+ "protect": "safeguard",
229
+ "protects": "safeguards",
230
+ "protected": "safeguarded",
231
+ "people": "communities",
232
+ "environment": "ecosystem",
233
+ "agriculture": "farming",
234
+ "farmers": "growers",
235
+ "farmer": "grower",
236
+ "reshaping": "remaking",
237
+ "reshape": "remake",
238
+ "across": "throughout",
239
+ "support": "aid",
240
+ "supports": "aids",
241
+ "supported": "aided",
242
+ "problem": "issue",
243
+ "problems": "issues",
244
+ "solution": "answer",
245
+ "solutions": "answers",
246
+ "important": "vital",
247
+ "crucial": "key",
248
+ "critical": "key",
249
+ "major": "primary",
250
+ "main": "primary",
251
+ "primary": "main",
252
+ "effect": "impact",
253
+ "effects": "impacts",
254
+ "impact": "effect",
255
+ "impacts": "effects",
256
+ "create": "build",
257
+ "creates": "builds",
258
+ "created": "built",
259
+ "develop": "build",
260
+ "develops": "builds",
261
+ "developed": "built",
262
+ "increase": "raise",
263
+ "increases": "raises",
264
+ "increased": "raised",
265
+ "decrease": "reduce",
266
+ "decreases": "reduces",
267
+ "decreased": "reduced",
268
+ "reduce": "lower",
269
+ "reduces": "lowers",
270
+ "reduced": "lowered",
271
+ "allow": "let",
272
+ "allows": "lets",
273
+ "allowed": "let",
274
+ "prevent": "stop",
275
+ "prevents": "stops",
276
+ "prevented": "stopped",
277
+ "include": "cover",
278
+ "includes": "covers",
279
+ "included": "covered",
280
+ "consider": "weigh",
281
+ "considers": "weighs",
282
+ "considered": "weighed",
283
+ "believe": "think",
284
+ "believes": "thinks",
285
+ "believed": "thought",
286
+ "understand": "grasp",
287
+ "understands": "grasps",
288
+ "understood": "grasped",
289
+ "explain": "clarify",
290
+ "explains": "clarifies",
291
+ "explained": "clarified",
292
+ "describe": "outline",
293
+ "describes": "outlines",
294
+ "described": "outlined",
295
+ "research": "study",
296
+ "studies": "research",
297
+ "study": "review",
298
+ "data": "evidence",
299
+ "information": "details",
300
+ "knowledge": "insight",
301
+ "system": "framework",
302
+ "systems": "frameworks",
303
+ "process": "procedure",
304
+ "processes": "procedures",
305
+ "workers": "staff",
306
+ "worker": "employee",
307
+ "team": "group",
308
+ "teams": "groups",
309
+ "project": "initiative",
310
+ "projects": "initiatives",
311
+ "goal": "aim",
312
+ "goals": "aims",
313
+ "success": "progress",
314
+ "successful": "effective",
315
+ "failure": "setback",
316
+ "challenge": "obstacle",
317
+ "challenges": "obstacles",
318
+ "opportunity": "chance",
319
+ "opportunities": "chances",
320
+ "benefit": "advantage",
321
+ "benefits": "advantages",
322
+ "risk": "hazard",
323
+ "risks": "hazards",
324
+ "quality": "standard",
325
+ "level": "degree",
326
+ "levels": "degrees",
327
+ "role": "part",
328
+ "roles": "parts",
329
+ "factor": "element",
330
+ "factors": "elements",
331
+ "example": "instance",
332
+ "examples": "instances",
333
+ "today": "nowadays",
334
+ "nowadays": "today",
335
+ "often": "frequently",
336
+ "rarely": "seldom",
337
+ "always": "consistently",
338
+ "clearly": "plainly",
339
+ "simply": "merely",
340
+ "really": "truly",
341
+ "very": "highly",
342
+ "quite": "fairly",
343
+ "rather": "somewhat",
344
+ "especially": "particularly",
345
+ "particularly": "especially",
346
+ "generally": "usually",
347
+ "usually": "typically",
348
+ "typically": "generally",
349
+ "global": "worldwide",
350
+ "local": "regional",
351
+ "public": "civic",
352
+ "private": "personal",
353
+ "social": "communal",
354
+ "economic": "financial",
355
+ "political": "civic",
356
+ "scientific": "technical",
357
+ "technology": "tech",
358
+ "digital": "online",
359
+ "online": "digital",
360
+ "communication": "dialogue",
361
+ "community": "society",
362
+ "communities": "societies",
363
+ "society": "community",
364
+ "education": "learning",
365
+ "health": "wellbeing",
366
+ "safety": "security",
367
+ "security": "safety",
368
+ "growth": "expansion",
369
+ "progress": "advancement",
370
+ "development": "growth",
371
+ "innovation": "novelty",
372
+ "strategy": "plan",
373
+ "strategies": "plans",
374
+ "decision": "choice",
375
+ "decisions": "choices",
376
+ "action": "step",
377
+ "actions": "steps",
378
+ "focus": "emphasis",
379
+ "focused": "centered",
380
+ "based": "grounded",
381
+ "related": "linked",
382
+ "connected": "linked",
383
+ "similar": "alike",
384
+ "various": "assorted",
385
+ "common": "usual",
386
+ "unique": "distinct",
387
+ "complex": "intricate",
388
+ "simple": "basic",
389
+ "basic": "simple",
390
+ "advanced": "sophisticated",
391
+ "traditional": "conventional",
392
+ "conventional": "traditional",
393
+ "efficient": "productive",
394
+ "useful": "helpful",
395
+ "helpful": "useful",
396
+ "harmful": "damaging",
397
+ "positive": "favorable",
398
+ "negative": "adverse",
399
+ "serious": "severe",
400
+ "severe": "serious",
401
+ "strong": "sturdy"
402
  }
app/pipeline/__pycache__/mechanics.cpython-311.pyc CHANGED
Binary files a/app/pipeline/__pycache__/mechanics.cpython-311.pyc and b/app/pipeline/__pycache__/mechanics.cpython-311.pyc differ
 
app/pipeline/__pycache__/orchestrator.cpython-311.pyc CHANGED
Binary files a/app/pipeline/__pycache__/orchestrator.cpython-311.pyc and b/app/pipeline/__pycache__/orchestrator.cpython-311.pyc differ
 
app/pipeline/__pycache__/synonym.cpython-311.pyc CHANGED
Binary files a/app/pipeline/__pycache__/synonym.cpython-311.pyc and b/app/pipeline/__pycache__/synonym.cpython-311.pyc differ
 
app/pipeline/__pycache__/syntax_rewrite.cpython-311.pyc CHANGED
Binary files a/app/pipeline/__pycache__/syntax_rewrite.cpython-311.pyc and b/app/pipeline/__pycache__/syntax_rewrite.cpython-311.pyc differ
 
app/pipeline/mechanics.py CHANGED
@@ -70,7 +70,10 @@ def tidy(text: str) -> str:
70
 
71
 
72
  def enforce_length_budget(original: str, rewritten: str, preserve_length: bool) -> str:
73
- """Trim only when rewrite clearly ballooned past the original."""
 
 
 
74
  o = len(original.split())
75
  r = len(rewritten.split())
76
  if o == 0:
@@ -80,13 +83,25 @@ def enforce_length_budget(original: str, rewritten: str, preserve_length: bool)
80
  if r <= max_words:
81
  return rewritten
82
 
83
- parts = re.split(r"(?<=[.!?])\s+", rewritten.strip())
84
- kept: list[str] = []
 
 
 
85
  count = 0
86
- for p in parts:
87
- w = len(p.split())
88
- if kept and count + w > max_words:
 
 
 
 
 
 
 
 
 
 
 
89
  break
90
- kept.append(p)
91
- count += w
92
- return " ".join(kept).strip() or rewritten
 
70
 
71
 
72
  def enforce_length_budget(original: str, rewritten: str, preserve_length: bool) -> str:
73
+ """Trim only when rewrite clearly ballooned past the original.
74
+
75
+ Preserves paragraph breaks (\\n\\n) — never flattens the document.
76
+ """
77
  o = len(original.split())
78
  r = len(rewritten.split())
79
  if o == 0:
 
83
  if r <= max_words:
84
  return rewritten
85
 
86
+ paragraphs = [p.strip() for p in re.split(r"\n\s*\n", rewritten.strip()) if p.strip()]
87
+ if not paragraphs:
88
+ return rewritten
89
+
90
+ kept_paras: list[str] = []
91
  count = 0
92
+ for para in paragraphs:
93
+ parts = re.split(r"(?<=[.!?])\s+", para.strip())
94
+ kept_sents: list[str] = []
95
+ for p in parts:
96
+ w = len(p.split())
97
+ if kept_sents and count + w > max_words:
98
+ break
99
+ if not kept_sents and kept_paras and count + w > max_words:
100
+ break
101
+ kept_sents.append(p)
102
+ count += w
103
+ if kept_sents:
104
+ kept_paras.append(" ".join(kept_sents))
105
+ if count >= max_words:
106
  break
107
+ return "\n\n".join(kept_paras).strip() or rewritten
 
 
app/pipeline/orchestrator.py CHANGED
@@ -74,16 +74,26 @@ def _rewrite_paragraph(paragraph: str, tone: str, strength: int, rng: random.Ran
74
 
75
 
76
  def _force_more_changes(text: str, tone: str, strength: int, rng: random.Random) -> str:
77
- """Second pass if first pass left text nearly identical tone phrases, not verb WordNet."""
78
- bumped = apply_tone_style(text, tone, min(2, strength + 1), rng)
79
  bump = min(2, strength + 1)
80
- nlp = get_nlp()
81
- if nlp is not None:
82
- sents = [s.text.strip() for s in nlp(bumped).sents if s.text.strip()]
83
- else:
84
- sents = split_sentences_regex(bumped)
85
- out = [rewrite_sentence_synonyms(s, bump, rng, tone=tone) for s in sents]
86
- return scrub_phrases(" ".join(out))
 
 
 
 
 
 
 
 
 
 
 
87
 
88
 
89
  def rewrite_text(
@@ -111,11 +121,9 @@ def rewrite_text(
111
 
112
  ratio = _similarity_ratio(original, rewritten)
113
  notes = ""
114
- if ratio > 0.88:
115
  logger.info("Low change detected (ratio=%.3f); running stronger second pass", ratio)
116
  rewritten = _force_more_changes(rewritten, tone, strength, rng)
117
- rewritten = apply_tone_style(rewritten, tone, 2, rng)
118
- rewritten = apply_tone_contractions(rewritten, tone)
119
  rewritten = enforce_length_budget(original, rewritten, preserve_length)
120
  rewritten = tidy(rewritten)
121
  notes = "Applied second pass (input was too similar after first rewrite)."
 
74
 
75
 
76
  def _force_more_changes(text: str, tone: str, strength: int, rng: random.Random) -> str:
77
+ """Stronger per-paragraph pass — preserves \\n\\n breaks."""
 
78
  bump = min(2, strength + 1)
79
+ paras = split_paragraphs(text)
80
+ out_paras: list[str] = []
81
+ for para in paras:
82
+ bumped = apply_tone_style(para, tone, bump, rng)
83
+ nlp = get_nlp()
84
+ if nlp is not None:
85
+ sents = [s.text.strip() for s in nlp(bumped).sents if s.text.strip()]
86
+ else:
87
+ sents = split_sentences_regex(bumped)
88
+ rewritten = [
89
+ rewrite_sentence_synonyms(s, bump, rng, tone=tone, force_all_lexicon=True)
90
+ for s in sents
91
+ ]
92
+ joined = scrub_phrases(" ".join(rewritten))
93
+ joined = apply_tone_style(joined, tone, bump, rng)
94
+ joined = apply_tone_contractions(joined, tone)
95
+ out_paras.append(joined.strip())
96
+ return "\n\n".join(p for p in out_paras if p)
97
 
98
 
99
  def rewrite_text(
 
121
 
122
  ratio = _similarity_ratio(original, rewritten)
123
  notes = ""
124
+ if ratio > 0.85:
125
  logger.info("Low change detected (ratio=%.3f); running stronger second pass", ratio)
126
  rewritten = _force_more_changes(rewritten, tone, strength, rng)
 
 
127
  rewritten = enforce_length_budget(original, rewritten, preserve_length)
128
  rewritten = tidy(rewritten)
129
  notes = "Applied second pass (input was too similar after first rewrite)."
app/pipeline/synonym.py CHANGED
@@ -5,6 +5,7 @@ from __future__ import annotations
5
  import json
6
  import random
7
  import re
 
8
  from functools import lru_cache
9
 
10
  from app.config import DATA_DIR
@@ -27,6 +28,29 @@ _STOP_SWAP = {
27
  "across", "around", "without", "within", "beside", "behind", "above", "below",
28
  }
29
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  _OPENERS = {
31
  "Academic": {
32
  "but": "However",
@@ -110,7 +134,7 @@ def _lexicon_for(tone: str, strength: int) -> dict[str, str]:
110
  return {**content, **load_elevate_swaps()}
111
  # preferred first, content overrides for everyday wording
112
  merged = {**load_preferred_swaps(), **content}
113
- if tone_l == "Neutral" and strength >= 2:
114
  # Mild elevate fill-ins when content didn't cover the token
115
  for k, v in load_elevate_swaps().items():
116
  merged.setdefault(k, v)
@@ -134,42 +158,52 @@ def _looks_inflected_verb(word: str) -> bool:
134
  return w in _IRREGULAR_PAST or w.endswith(("ing", "ed"))
135
 
136
 
137
- def _same_ending_shape(original: str, candidate: str) -> bool:
 
138
  o, c = original.lower(), candidate.lower()
139
- for end in ("ing", "ed", "ies", "es", "ly", "s"):
140
- if o.endswith(end):
141
- return c.endswith(end)
142
- if c.endswith(("ing", "ed")):
143
- return False
144
- return True
 
145
 
146
 
147
  def _looks_safe_synonym(original: str, candidate: str) -> bool:
 
 
 
 
 
148
  o, c = original.lower(), candidate.lower()
149
  if c in _STOP_SWAP or c == o:
150
  return False
 
 
151
  if not c.isalpha() or " " in c:
152
  return False
153
- if len(o) < 4 or len(c) < 4 or abs(len(c) - len(o)) > 5:
154
  return False
155
- if c.endswith(("ish", "ness", "ment", "ly")) and not o.endswith(
156
- ("ish", "ness", "ment", "ly")
157
- ):
158
  return False
159
  if o in {"first", "second", "third", "fourth", "fifth", "last", "next"}:
160
  return False
161
- if not _same_ending_shape(o, c):
162
  return False
163
- from difflib import SequenceMatcher
164
-
165
- return SequenceMatcher(None, o, c).ratio() >= 0.58
166
 
167
 
168
  @lru_cache(maxsize=4096)
169
  def _wordnet_synonyms(word: str, pos: str | None = None) -> tuple[str, ...]:
170
  if not _ensure_wordnet():
171
  return ()
172
- if word.lower() in _STOP_SWAP or _looks_inflected_verb(word):
 
 
 
173
  return ()
174
  from nltk.corpus import wordnet as wn
175
 
@@ -178,14 +212,18 @@ def _wordnet_synonyms(word: str, pos: str | None = None) -> tuple[str, ...]:
178
  elif pos in {"a", "r", "n"}:
179
  order = [pos]
180
  else:
181
- order = ["a", "r", "n"]
 
182
  out: list[str] = []
183
  for p in order:
184
  synsets = wn.synsets(word, pos=p)
185
  if not synsets:
186
  continue
 
187
  for lemma in synsets[0].lemmas():
188
  name = lemma.name().replace("_", " ").lower()
 
 
189
  if _looks_safe_synonym(word, name) and name not in out:
190
  out.append(name)
191
  if out:
@@ -193,6 +231,14 @@ def _wordnet_synonyms(word: str, pos: str | None = None) -> tuple[str, ...]:
193
  return tuple(out[:5])
194
 
195
 
 
 
 
 
 
 
 
 
196
  def _match_case(original: str, replacement: str) -> str:
197
  if " " in replacement:
198
  parts = replacement.split()
@@ -246,6 +292,7 @@ def rewrite_sentence_synonyms(
246
  rng: random.Random,
247
  *,
248
  tone: str = "Neutral",
 
249
  ) -> str:
250
  """
251
  Dynamic per-token rewrite for any input:
@@ -254,12 +301,17 @@ def rewrite_sentence_synonyms(
254
  lexicon = _lexicon_for(tone, strength)
255
  tone_l = (tone or "").strip()
256
  openers = _OPENERS.get(tone_l, {})
257
- allow_wordnet = tone_l not in {"Academic", "Formal"}
258
- nlp = get_nlp()
259
- max_wn = {0: 1, 1: 3, 2: 5}.get(strength, 3) if allow_wordnet else 0
260
- wn_rate = {0: 0.45, 1: 0.75, 2: 1.0}.get(strength, 0.75)
261
- # Lexicon hits always apply at Normal+; Light still usually applies
262
- lex_rate = {0: 0.7, 1: 1.0, 2: 1.0}.get(strength, 1.0)
 
 
 
 
 
263
 
264
  def transform(
265
  raw: str,
@@ -267,7 +319,7 @@ def rewrite_sentence_synonyms(
267
  wn_budget: list[int],
268
  *,
269
  sentence_start: bool,
270
- force_lex: bool = False,
271
  ) -> str:
272
  low = raw.lower()
273
  if not raw.isalpha():
@@ -276,13 +328,19 @@ def rewrite_sentence_synonyms(
276
  return openers[low]
277
  if low in _STOP_SWAP or len(low) < 4:
278
  return raw
279
- if low in lexicon and (force_lex or rng.random() < lex_rate):
280
  return _match_case(raw, lexicon[low])
281
- if not allow_wordnet or _looks_inflected_verb(low):
282
- return raw
283
  if wn_budget[0] <= 0 or rng.random() > wn_rate:
284
  return raw
285
- cands = list(_wordnet_synonyms(low, pos))
 
 
 
 
 
 
 
 
286
  if not cands:
287
  return raw
288
  wn_budget[0] -= 1
@@ -296,7 +354,7 @@ def rewrite_sentence_synonyms(
296
  if not tok.isalpha():
297
  continue
298
  if out[i] != tok:
299
- continue # already rewritten
300
  low = tok.lower()
301
  if low in _STOP_SWAP or len(low) < 4 or low not in lexicon:
302
  continue
@@ -304,8 +362,30 @@ def rewrite_sentence_synonyms(
304
  changed += 1
305
  if changed >= min_changes:
306
  break
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
307
  return out
308
 
 
 
309
  if nlp is None:
310
  tokens = re.findall(r"\w+|[^\w\s]", sentence, flags=re.UNICODE)
311
  content_n = sum(
@@ -313,9 +393,11 @@ def rewrite_sentence_synonyms(
313
  for t in tokens
314
  if t.isalpha() and t.lower() not in _STOP_SWAP and len(t) >= 4
315
  )
316
- min_ch = {0: 1, 1: max(2, content_n // 5), 2: max(3, content_n // 4)}.get(
317
  strength, 2
318
  )
 
 
319
  budget = [max_wn]
320
  out = [
321
  transform(tok, None, budget, sentence_start=_is_sentence_start(tokens, i))
@@ -337,8 +419,13 @@ def rewrite_sentence_synonyms(
337
  start = i == 0 or prev_text in {".", "!", "?", "…"}
338
  if token.ent_type_ or token.pos_ in {"PROPN", "PRON", "DET", "ADP", "PART"}:
339
  low = raw.lower()
340
- # Still allow lexicon on adjectives mis-tagged? Keep ADP locked.
341
- pieces.append(openers[low] if start and low in openers else raw)
 
 
 
 
 
342
  continue
343
  if token.pos_ in {"CCONJ", "SCONJ"}:
344
  low = raw.lower()
@@ -351,14 +438,30 @@ def rewrite_sentence_synonyms(
351
  continue
352
  if token.pos_ in {"VERB", "AUX"}:
353
  low = raw.lower()
354
- if low in lexicon:
355
  pieces.append(_match_case(raw, lexicon[low]))
356
  elif not _looks_inflected_verb(low):
357
- pieces.append(transform(raw, "v", budget, sentence_start=start))
 
 
 
 
 
 
 
 
358
  else:
359
  pieces.append(raw)
360
  continue
361
- pieces.append(transform(raw, _wn_pos(token.tag_), budget, sentence_start=start))
 
 
 
 
 
 
 
 
362
 
363
  # Force lexicon fills if barely changed
364
  raws = [t.text for t in toks]
@@ -367,6 +470,8 @@ def rewrite_sentence_synonyms(
367
  for t in toks
368
  if t.is_alpha and t.text.lower() not in _STOP_SWAP and len(t.text) >= 4
369
  )
370
- min_ch = {0: 1, 1: max(2, content_n // 5), 2: max(3, content_n // 4)}.get(strength, 2)
 
 
371
  pieces = _force_min_changes(raws, pieces, min_ch)
372
  return "".join(p + t.whitespace_ for t, p in zip(doc, pieces)).strip()
 
5
  import json
6
  import random
7
  import re
8
+ from difflib import SequenceMatcher
9
  from functools import lru_cache
10
 
11
  from app.config import DATA_DIR
 
28
  "across", "around", "without", "within", "beside", "behind", "above", "below",
29
  }
30
 
31
+ # Known bad WordNet lookalikes / truncated / wrong-sense forms
32
+ _BLOCKED_SYNONYMS = {
33
+ "clime",
34
+ "climates",
35
+ "erstwhile",
36
+ "downhearted",
37
+ "reason", # as synonym for ground
38
+ "late", # as synonym for deep
39
+ "betimes",
40
+ "perceive",
41
+ "perceived",
42
+ "perceives",
43
+ "mood", # climate.n.02 wrong sense
44
+ "moods",
45
+ }
46
+
47
+ # Prefer lexicon; avoid WordNet on short/concrete nouns that often misfire
48
+ _RISKY_WN_NOUNS = {
49
+ "ground", "blue", "sky", "time", "day", "way", "thing", "part", "place",
50
+ "man", "woman", "boy", "girl", "seed", "once", "climate", "change",
51
+ "world", "people", "water", "fire", "earth", "light", "dark", "home",
52
+ }
53
+
54
  _OPENERS = {
55
  "Academic": {
56
  "but": "However",
 
134
  return {**content, **load_elevate_swaps()}
135
  # preferred first, content overrides for everyday wording
136
  merged = {**load_preferred_swaps(), **content}
137
+ if tone_l == "Neutral" and strength >= 1:
138
  # Mild elevate fill-ins when content didn't cover the token
139
  for k, v in load_elevate_swaps().items():
140
  merged.setdefault(k, v)
 
158
  return w in _IRREGULAR_PAST or w.endswith(("ing", "ed"))
159
 
160
 
161
+ def _is_morph_cousin(original: str, candidate: str) -> bool:
162
+ """Reject truncated / lookalike forms (climate→clime) that are not real paraphrases."""
163
  o, c = original.lower(), candidate.lower()
164
+ if o.startswith(c) or c.startswith(o):
165
+ if abs(len(o) - len(c)) <= 3 and min(len(o), len(c)) >= 3:
166
+ return True
167
+ # Very high string overlap usually means same stem, not a paraphrase
168
+ if SequenceMatcher(None, o, c).ratio() >= 0.82:
169
+ return True
170
+ return False
171
 
172
 
173
  def _looks_safe_synonym(original: str, candidate: str) -> bool:
174
+ """Allow real paraphrases; reject stopwords, junk, and lookalike truncations.
175
+
176
+ NOTE: Do NOT require high character similarity — that rejects happy→glad
177
+ and accepts climate→clime.
178
+ """
179
  o, c = original.lower(), candidate.lower()
180
  if c in _STOP_SWAP or c == o:
181
  return False
182
+ if c in _BLOCKED_SYNONYMS:
183
+ return False
184
  if not c.isalpha() or " " in c:
185
  return False
186
+ if len(o) < 4 or len(c) < 4:
187
  return False
188
+ if abs(len(c) - len(o)) > 8:
189
+ return False
190
+ if c.endswith(("ish", "ness", "ment")) and not o.endswith(("ish", "ness", "ment")):
191
  return False
192
  if o in {"first", "second", "third", "fourth", "fifth", "last", "next"}:
193
  return False
194
+ if _is_morph_cousin(o, c):
195
  return False
196
+ return True
 
 
197
 
198
 
199
  @lru_cache(maxsize=4096)
200
  def _wordnet_synonyms(word: str, pos: str | None = None) -> tuple[str, ...]:
201
  if not _ensure_wordnet():
202
  return ()
203
+ if word.lower() in _STOP_SWAP:
204
+ return ()
205
+ # Skip heavily inflected surface forms — lemma path handles verbs via spaCy
206
+ if _looks_inflected_verb(word):
207
  return ()
208
  from nltk.corpus import wordnet as wn
209
 
 
212
  elif pos in {"a", "r", "n"}:
213
  order = [pos]
214
  else:
215
+ # Prefer adjectives/adverbs before nouns (noun senses are often wrong)
216
+ order = ["a", "r", "v", "n"]
217
  out: list[str] = []
218
  for p in order:
219
  synsets = wn.synsets(word, pos=p)
220
  if not synsets:
221
  continue
222
+ # Only the most common sense — extra synsets cause ground→reason, blue→sad
223
  for lemma in synsets[0].lemmas():
224
  name = lemma.name().replace("_", " ").lower()
225
+ if " " in name:
226
+ continue
227
  if _looks_safe_synonym(word, name) and name not in out:
228
  out.append(name)
229
  if out:
 
231
  return tuple(out[:5])
232
 
233
 
234
+ @lru_cache(maxsize=4096)
235
+ def _wordnet_synonyms_lemma(lemma: str, pos: str | None = None) -> tuple[str, ...]:
236
+ """Synonyms for a lemma (used for verbs after spaCy lemmatization)."""
237
+ if not lemma or len(lemma) < 3:
238
+ return ()
239
+ return _wordnet_synonyms(lemma, pos)
240
+
241
+
242
  def _match_case(original: str, replacement: str) -> str:
243
  if " " in replacement:
244
  parts = replacement.split()
 
292
  rng: random.Random,
293
  *,
294
  tone: str = "Neutral",
295
+ force_all_lexicon: bool = False,
296
  ) -> str:
297
  """
298
  Dynamic per-token rewrite for any input:
 
301
  lexicon = _lexicon_for(tone, strength)
302
  tone_l = (tone or "").strip()
303
  openers = _OPENERS.get(tone_l, {})
304
+ # Lexicon carries Formal/Academic register; WordNet often picks wrong senses there
305
+ if tone_l in {"Academic", "Formal"}:
306
+ max_wn = 0
307
+ wn_rate = 0.0
308
+ else:
309
+ max_wn = {0: 1, 1: 3, 2: 5}.get(strength, 3)
310
+ wn_rate = {0: 0.4, 1: 0.7, 2: 0.9}.get(strength, 0.7)
311
+ # Lexicon hits always apply at Normal+; Light usually applies; second pass forces all
312
+ lex_rate = 1.0 if force_all_lexicon else {0: 0.85, 1: 1.0, 2: 1.0}.get(strength, 1.0)
313
+
314
+ # Prefer lexicon; avoid WordNet on short/concrete nouns that often misfire
315
 
316
  def transform(
317
  raw: str,
 
319
  wn_budget: list[int],
320
  *,
321
  sentence_start: bool,
322
+ lemma: str | None = None,
323
  ) -> str:
324
  low = raw.lower()
325
  if not raw.isalpha():
 
328
  return openers[low]
329
  if low in _STOP_SWAP or len(low) < 4:
330
  return raw
331
+ if low in lexicon and rng.random() < lex_rate:
332
  return _match_case(raw, lexicon[low])
 
 
333
  if wn_budget[0] <= 0 or rng.random() > wn_rate:
334
  return raw
335
+ if low in _RISKY_WN_NOUNS:
336
+ return raw
337
+ if lemma and _looks_inflected_verb(low) and pos == "v":
338
+ return raw
339
+ lookup = low
340
+ lookup_pos = pos
341
+ cands = list(_wordnet_synonyms(lookup, lookup_pos))
342
+ if not cands and lemma and lemma.lower() == low:
343
+ cands = list(_wordnet_synonyms_lemma(lemma, lookup_pos))
344
  if not cands:
345
  return raw
346
  wn_budget[0] -= 1
 
354
  if not tok.isalpha():
355
  continue
356
  if out[i] != tok:
357
+ continue
358
  low = tok.lower()
359
  if low in _STOP_SWAP or len(low) < 4 or low not in lexicon:
360
  continue
 
362
  changed += 1
363
  if changed >= min_changes:
364
  break
365
+ # If still short, try WordNet on remaining content words (Neutral/Casual only)
366
+ if changed < min_changes and max_wn > 0:
367
+ for i, tok in enumerate(tokens):
368
+ if not tok.isalpha() or out[i] != tok:
369
+ continue
370
+ low = tok.lower()
371
+ if (
372
+ low in _STOP_SWAP
373
+ or len(low) < 4
374
+ or _looks_inflected_verb(low)
375
+ or low in _RISKY_WN_NOUNS
376
+ ):
377
+ continue
378
+ cands = list(_wordnet_synonyms(low, None))
379
+ if not cands:
380
+ continue
381
+ out[i] = _match_case(tok, cands[0])
382
+ changed += 1
383
+ if changed >= min_changes:
384
+ break
385
  return out
386
 
387
+ nlp = get_nlp()
388
+
389
  if nlp is None:
390
  tokens = re.findall(r"\w+|[^\w\s]", sentence, flags=re.UNICODE)
391
  content_n = sum(
 
393
  for t in tokens
394
  if t.isalpha() and t.lower() not in _STOP_SWAP and len(t) >= 4
395
  )
396
+ min_ch = {0: 1, 1: max(2, content_n // 4), 2: max(3, content_n // 3)}.get(
397
  strength, 2
398
  )
399
+ if force_all_lexicon:
400
+ min_ch = max(min_ch, max(2, content_n // 3))
401
  budget = [max_wn]
402
  out = [
403
  transform(tok, None, budget, sentence_start=_is_sentence_start(tokens, i))
 
419
  start = i == 0 or prev_text in {".", "!", "?", "…"}
420
  if token.ent_type_ or token.pos_ in {"PROPN", "PRON", "DET", "ADP", "PART"}:
421
  low = raw.lower()
422
+ # Allow lexicon even on locked POS when we have an explicit swap
423
+ if low in lexicon and token.pos_ not in {"PROPN", "PRON", "DET"} and (
424
+ force_all_lexicon or rng.random() < lex_rate
425
+ ):
426
+ pieces.append(_match_case(raw, lexicon[low]))
427
+ else:
428
+ pieces.append(openers[low] if start and low in openers else raw)
429
  continue
430
  if token.pos_ in {"CCONJ", "SCONJ"}:
431
  low = raw.lower()
 
438
  continue
439
  if token.pos_ in {"VERB", "AUX"}:
440
  low = raw.lower()
441
+ if low in lexicon and (force_all_lexicon or rng.random() < lex_rate):
442
  pieces.append(_match_case(raw, lexicon[low]))
443
  elif not _looks_inflected_verb(low):
444
+ pieces.append(
445
+ transform(
446
+ raw,
447
+ "v",
448
+ budget,
449
+ sentence_start=start,
450
+ lemma=token.lemma_,
451
+ )
452
+ )
453
  else:
454
  pieces.append(raw)
455
  continue
456
+ pieces.append(
457
+ transform(
458
+ raw,
459
+ _wn_pos(token.tag_),
460
+ budget,
461
+ sentence_start=start,
462
+ lemma=token.lemma_,
463
+ )
464
+ )
465
 
466
  # Force lexicon fills if barely changed
467
  raws = [t.text for t in toks]
 
470
  for t in toks
471
  if t.is_alpha and t.text.lower() not in _STOP_SWAP and len(t.text) >= 4
472
  )
473
+ min_ch = {0: 1, 1: max(2, content_n // 4), 2: max(3, content_n // 3)}.get(strength, 2)
474
+ if force_all_lexicon:
475
+ min_ch = max(min_ch, max(2, content_n // 3))
476
  pieces = _force_min_changes(raws, pieces, min_ch)
477
  return "".join(p + t.whitespace_ for t, p in zip(doc, pieces)).strip()
app/pipeline/syntax_rewrite.py CHANGED
@@ -17,31 +17,47 @@ def strip_hedges(sentence: str) -> str:
17
  def maybe_split_long(sentence: str, strength: int, rng: random.Random) -> list[str]:
18
  """Split long sentences on clause-like joins — based on length, not fixed content."""
19
  words = sentence.split()
20
- min_len = {0: 28, 1: 22, 2: 18}.get(strength, 22)
21
- chance = {0: 0.35, 1: 0.55, 2: 0.75}.get(strength, 0.55)
22
  if len(words) < min_len or rng.random() > chance:
23
  return [sentence]
24
  if '"' in sentence or "'" in sentence or "“" in sentence or "”" in sentence:
25
  return [sentence]
26
 
27
- for sep in ["; ", ", and ", ", but ", ", so ", ", which "]:
28
- if sep in sentence:
29
- left, right = sentence.split(sep, 1)
30
- left, right = left.strip(), right.strip()
31
- if len(left.split()) >= 6 and len(right.split()) >= 6:
32
- if right and right[0].islower():
33
- right = right[0].upper() + right[1:]
34
- if not left.endswith((".", "!", "?")):
35
- left += "."
36
- if not right.endswith((".", "!", "?")):
37
- right += "."
38
- return [left, right]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  return [sentence]
40
 
41
 
42
  def maybe_passive_to_active(sentence: str, strength: int, rng: random.Random) -> str:
43
  """Light-touch: 'X was VERBed by Y' → 'Y VERBed X' when parse looks safe."""
44
- if strength < 1 or rng.random() > 0.25 * strength:
45
  return sentence
46
  nlp = get_nlp()
47
  if nlp is None:
@@ -61,7 +77,7 @@ def maybe_passive_to_active(sentence: str, strength: int, rng: random.Random) ->
61
  break
62
  if agent is None or verb.tag_ not in {"VBN"}:
63
  continue
64
- if len(doc) > 18:
65
  return sentence
66
  agent_span = agent.text
67
  for chunk in doc.noun_chunks:
 
17
  def maybe_split_long(sentence: str, strength: int, rng: random.Random) -> list[str]:
18
  """Split long sentences on clause-like joins — based on length, not fixed content."""
19
  words = sentence.split()
20
+ min_len = {0: 22, 1: 16, 2: 12}.get(strength, 16)
21
+ chance = {0: 0.45, 1: 0.7, 2: 0.9}.get(strength, 0.7)
22
  if len(words) < min_len or rng.random() > chance:
23
  return [sentence]
24
  if '"' in sentence or "'" in sentence or "“" in sentence or "”" in sentence:
25
  return [sentence]
26
 
27
+ for sep in [
28
+ "; ",
29
+ ", and ",
30
+ ", but ",
31
+ ", so ",
32
+ ", which ",
33
+ ", although ",
34
+ ", because ",
35
+ ", while ",
36
+ ", whereas ",
37
+ " although ",
38
+ " because ",
39
+ " whereas ",
40
+ ]:
41
+ if sep not in sentence:
42
+ continue
43
+ left, right = sentence.split(sep, 1)
44
+ left, right = left.strip(), right.strip()
45
+ # Avoid splitting tiny fragments
46
+ if len(left.split()) < 5 or len(right.split()) < 5:
47
+ continue
48
+ if right and right[0].islower():
49
+ right = right[0].upper() + right[1:]
50
+ if not left.endswith((".", "!", "?")):
51
+ left += "."
52
+ if not right.endswith((".", "!", "?")):
53
+ right += "."
54
+ return [left, right]
55
  return [sentence]
56
 
57
 
58
  def maybe_passive_to_active(sentence: str, strength: int, rng: random.Random) -> str:
59
  """Light-touch: 'X was VERBed by Y' → 'Y VERBed X' when parse looks safe."""
60
+ if strength < 1 or rng.random() > 0.4 * strength:
61
  return sentence
62
  nlp = get_nlp()
63
  if nlp is None:
 
77
  break
78
  if agent is None or verb.tag_ not in {"VBN"}:
79
  continue
80
+ if len(doc) > 22:
81
  return sentence
82
  agent_span = agent.text
83
  for chunk in doc.noun_chunks: