complexity-levels-api / src /reason_interventions.py
uzzam2121
Sync API: alter endpoint and decrease OpenAI edits.
ff0e5c5
Raw
History Blame Contribute Delete
9.22 kB
"""
Sentence edits for reason faithfulness tests (UI + 07_faithfulness.py).
Each reason type has one targeted edit; re-predict after edit to see if hardness drops.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from syntax_complexity import find_hardest_span
from utils import REASON_ORDER
@dataclass
class InterventionResult:
reason: str
original_sentence: str
edited_sentence: str
edit_method: str # "openai", "gemini", or "rule"
def _call_llm(prompt: str) -> tuple[str | None, str]:
if os.environ.get("OPENAI_API_KEY"):
try:
from openai import OpenAI
client = OpenAI()
resp = client.chat.completions.create(
model=os.environ.get("FAITHFULNESS_LLM", "gpt-4o-mini"),
messages=[{"role": "user", "content": prompt}],
temperature=0,
max_tokens=256,
)
return resp.choices[0].message.content.strip(), "openai"
except Exception:
pass
if os.environ.get("GOOGLE_API_KEY"):
try:
import google.generativeai as genai
genai.configure(api_key=os.environ["GOOGLE_API_KEY"])
model = genai.GenerativeModel(os.environ.get("FAITHFULNESS_GEMINI", "gemini-1.5-flash"))
return model.generate_content(prompt).text.strip(), "gemini"
except Exception:
pass
return None, "rule"
def _rule_synonym_swap(sentence: str, target_word: str) -> str:
simple = {
"frankincense": "incense",
"astonishment": "surprise",
"partiality": "bias",
"dominion": "rule",
"assemblies": "meetings",
"ewe": "sheep",
"scribe": "writer",
"scrivener": "writer",
"inflammation": "swelling",
"treaty": "deal",
}
rep = simple.get(target_word.lower(), target_word)
if target_word in sentence:
return sentence.replace(target_word, rep, 1)
return sentence
def edit_lexical_rarity(sentence: str, target_word: str, use_llm: bool = True) -> InterventionResult:
prompt = (
f"Replace ONLY the word '{target_word}' in this sentence with a common, simple synonym "
f"of the same meaning and part of speech. Change nothing else.\n"
f"Sentence: {sentence}\n"
f"Output only the edited sentence."
)
method = "rule"
edited = _rule_synonym_swap(sentence, target_word)
if use_llm:
llm_out, method = _call_llm(prompt)
if llm_out:
edited = llm_out
return InterventionResult("Lexical Rarity", sentence, edited, method)
def edit_contextual_ambiguity(sentence: str, target_word: str, use_llm: bool = True) -> InterventionResult:
prompt = (
f"Add at most 3 words to this sentence to make the meaning of '{target_word}' clear. "
f"Do NOT change or remove '{target_word}'.\n"
f"Sentence: {sentence}\n"
f"Output only the edited sentence."
)
method = "rule"
edited = f"{sentence} (meaning: {target_word})"
if use_llm:
llm_out, method = _call_llm(prompt)
if llm_out:
edited = llm_out
return InterventionResult("Contextual Ambiguity", sentence, edited, method)
def edit_syntactic_complexity(sentence: str, target_word: str, use_llm: bool = True) -> InterventionResult:
span, start, end = find_hardest_span(sentence)
method = "rule"
simplified = span
if use_llm and span.strip() != sentence.strip():
prompt = (
f"Simplify ONLY this phrase to plain English. Keep all key words including '{target_word}'.\n"
f"Phrase: {span}\n"
f"Output only the simplified phrase."
)
llm_out, method = _call_llm(prompt)
if llm_out:
simplified = llm_out
edited = sentence[:start] + simplified + sentence[end:]
return InterventionResult("Syntactic Complexity", sentence, edited, method)
EDIT_FNS = {
"Lexical Rarity": edit_lexical_rarity,
"Contextual Ambiguity": edit_contextual_ambiguity,
"Syntactic Complexity": edit_syntactic_complexity,
}
def apply_intervention(reason: str, sentence: str, target_word: str, use_llm: bool = True) -> InterventionResult:
if reason not in EDIT_FNS:
raise ValueError(f"Unknown reason: {reason}. Choose from {REASON_ORDER}")
return EDIT_FNS[reason](sentence, target_word, use_llm=use_llm)
def apply_all_interventions(sentence: str, target_word: str, use_llm: bool = True) -> list[InterventionResult]:
return [fn(sentence, target_word, use_llm=use_llm) for fn in EDIT_FNS.values()]
@dataclass
class AlterComplexityResult:
reason: str
direction: str # "increase" | "decrease"
original_sentence: str
original_target: str
new_sentence: str
new_target: str
edit_method: str
explanation: str
def _require_llm(prompt: str) -> tuple[str, str]:
text, method = _call_llm(prompt)
if not text or method == "rule":
raise RuntimeError(
"OpenAI edit failed. Set OPENAI_API_KEY on the backend (HF Space secrets)."
)
return text, method
def _parse_json_block(raw: str) -> dict:
import json
import re
raw = raw.strip()
if raw.startswith("```"):
raw = re.sub(r"^```(?:json)?\s*", "", raw)
raw = re.sub(r"\s*```$", "", raw)
return json.loads(raw)
def alter_complexity(
reason: str,
sentence: str,
target_word: str,
direction: str,
) -> AlterComplexityResult:
"""User-guided edit: increase or decrease one complexity dimension via OpenAI."""
if reason not in EDIT_FNS:
raise ValueError(f"Unknown reason: {reason}. Choose from {REASON_ORDER}")
if direction not in ("increase", "decrease"):
raise ValueError("direction must be 'increase' or 'decrease'")
if reason == "Lexical Rarity":
if direction == "decrease":
task = (
f"Replace ONLY '{target_word}' with a MORE COMMON, simpler synonym "
f"(same meaning and part of speech). Keep the rest of the sentence unchanged."
)
else:
task = (
f"Replace ONLY '{target_word}' with a RARER, more formal or technical synonym "
f"(same meaning and part of speech). Keep the rest of the sentence unchanged."
)
prompt = (
f"{task}\n"
f"Sentence: {sentence}\n"
f"Target word: {target_word}\n"
f'Return ONLY valid JSON: {{"sentence": "full sentence with new word", '
f'"target_word": "the new target word", "explanation": "one short sentence"}}'
)
raw, method = _require_llm(prompt)
data = _parse_json_block(raw)
new_sentence = str(data["sentence"]).strip()
new_target = str(data["target_word"]).strip()
explanation = str(data.get("explanation", "Lexical rarity adjusted."))
elif reason == "Contextual Ambiguity":
if direction == "decrease":
task = (
f"Rewrite the sentence so the meaning of '{target_word}' is CLEARER and less ambiguous. "
f"You MUST keep the exact word '{target_word}' unchanged (same spelling)."
)
else:
task = (
f"Rewrite the sentence so '{target_word}' is MORE AMBIGUOUS in context "
f"(multiple plausible meanings). You MUST keep the exact word '{target_word}' unchanged."
)
prompt = (
f"{task}\n"
f"Sentence: {sentence}\n"
f'Return ONLY valid JSON: {{"sentence": "new sentence", '
f'"explanation": "one short sentence"}}'
)
raw, method = _require_llm(prompt)
data = _parse_json_block(raw)
new_sentence = str(data["sentence"]).strip()
new_target = target_word
explanation = str(data.get("explanation", "Contextual ambiguity adjusted."))
else: # Syntactic Complexity
if direction == "decrease":
task = (
f"Simplify the sentence structure to plain, short English. "
f"You MUST keep the exact word '{target_word}' unchanged."
)
else:
task = (
f"Make the sentence syntactically MORE COMPLEX (subordinate clauses, passive, etc.). "
f"You MUST keep the exact word '{target_word}' unchanged."
)
prompt = (
f"{task}\n"
f"Sentence: {sentence}\n"
f'Return ONLY valid JSON: {{"sentence": "new sentence", '
f'"explanation": "one short sentence"}}'
)
raw, method = _require_llm(prompt)
data = _parse_json_block(raw)
new_sentence = str(data["sentence"]).strip()
new_target = target_word
explanation = str(data.get("explanation", "Syntactic complexity adjusted."))
return AlterComplexityResult(
reason=reason,
direction=direction,
original_sentence=sentence,
original_target=target_word,
new_sentence=new_sentence,
new_target=new_target,
edit_method=method,
explanation=explanation,
)