research/framing.py

"""Framing / language forensics (journalism suite layer 3).

Framing = selection + salience (Entman 1993). Deterministic proxies we can
measure without a model:
  - passive voice (agency hidden: "was ordered" vs "X ordered")
  - loaded/emotive terms (charged vocabulary)
  - hedges (plausible deniability: "appears", "reportedly", "may")
  - nominalization (actions turned into nouns: "the decision" hides who decided)
  - agency: who performs the action in active-verb clauses
  - omission: which sources NEVER mention a topic the others cover

These are heuristics (suit logic), not a trained detector. The suite flags;
the human decides.

Usage:
    from research.framing import FramingAnalyzer
    f = FramingAnalyzer()
    f.add_doc("s1", "The memo was destroyed. Officials reportedly decided...")
    f.report()
"""
import re
from collections import defaultdict

LOADED = [
    "secret", "cover-up", "conspiracy", "plot", "scandal", "corrupt",
    "fraud", "shocking", "outrage", "horrific", "brutal", "crisis",
    "cover", "smear", "whistleblower", "leak", "collusion", "betrayal",
    "liar", "hoax", "traitor", "unprecedented", "catastrophic",
]
HEDGES = [
    "appears", "apparently", "reportedly", "allegedly", "seems", "seem",
    "may", "might", "could", "possibly", "perhaps", "suggest", "claims to",
    "is said to", "it is believed", "sources say", "not clear",
]
PASSIVE_RE = re.compile(r"\b(was|were|been|being|is|are)\s+(?:\w+ly\s+)?"
                        r"(\w+ed|torn|broken|hidden|destroyed|taken|given|"
                        r"made|held|filed)\b", re.IGNORECASE)
NOMINAL = re.compile(r"\b\w+(?:tion|sion|ment|ness|ity|ence|ance)\b", re.IGNORECASE)
ACTIVE_VERBS = ("said", "announced", "ordered", "admitted", "denied", "claimed",
                "confirmed", "reported", "released", "disclosed", "wrote",
                "testified", "warned", "decided", "approved")
_ACTIVE_RE = re.compile(r"\b([A-Z][a-zA-Z]{2,30}(?:\s+[A-Z][a-zA-Z]{2,30}){0,2})"
                        r"\s+(?:" + "|".join(ACTIVE_VERBS) + r")\b")


class FramingAnalyzer:
    def __init__(self):
        self.docs = {}          # source_id -> text

    def add_doc(self, source_id, text):
        self.docs[source_id] = text

    @staticmethod
    def passive_ratio(text):
        clauses = len(re.findall(r"[.!?]", text)) + 1
        hits = len(PASSIVE_RE.findall(text))
        return round(hits / max(clauses, 1), 3), hits

    @staticmethod
    def loaded_terms(text):
        low = text.lower()
        return [(w, low.count(w)) for w in LOADED if w in low]

    @staticmethod
    def hedges(text):
        low = text.lower()
        return [(w, low.count(w)) for w in HEDGES if w in low]

    @staticmethod
    def nominalizations(text):
        out = defaultdict(int)
        for m in NOMINAL.finditer(text):
            w = m.group(0).lower()
            if len(w) > 6:
                out[w] += 1
        return sorted(out.items(), key=lambda kv: -kv[1])[:12]

    @staticmethod
    def agency(text):
        """Who performs actions: leading noun phrases before active verbs."""
        return [m.group(1) for m in _ACTIVE_RE.finditer(text)][:10]

    def omissions(self, topics):
        """Sources that never mention a topic other sources cover."""
        flags = []
        for topic in topics:
            low_t = topic.lower()
            mentioned = [sid for sid, t in self.docs.items() if low_t in t.lower()]
            if 1 <= len(mentioned) < len(self.docs):
                for sid, t in self.docs.items():
                    if low_t not in t.lower():
                        flags.append({
                            "topic": topic,
                            "source_id": sid,
                            "flag": f"source {sid} never mentions '{topic}' "
                                    f"while {len(mentioned)} source(s) do",
                        })
        return flags

    def doc_card(self, source_id):
        text = self.docs.get(source_id, "")
        if not text:
            return None
        ratio, passive = self.passive_ratio(text)
        return {
            "source_id": source_id,
            "passive_ratio": ratio,
            "passive_hits": passive,
            "loaded": self.loaded_terms(text),
            "hedges": self.hedges(text),
            "nominalizations": self.nominalizations(text),
            "agency": self.agency(text),
        }

    def report(self, topics=()):
        lines = ["# Framing / Language Forensics", ""]
        for sid in self.docs:
            c = self.doc_card(sid)
            if not c:
                continue
            lines.append(f"## {sid}")
            lines.append(f"- passive ratio: {c['passive_ratio']} "
                         f"({c['passive_hits']} hits) — agency hidden where?")
            if c["loaded"]:
                lines.append("- loaded terms: " + ", ".join(
                    f"{w} x{n}" for w, n in c["loaded"]))
            if c["hedges"]:
                lines.append("- hedges: " + ", ".join(
                    f"{w} x{n}" for w, n in c["hedges"]))
            if c["nominalizations"]:
                lines.append("- nominalizations: " + ", ".join(
                    f"{w} x{n}" for w, n in c["nominalizations"][:6]))
            if c["agency"]:
                lines.append("- agency: " + ", ".join(c["agency"][:6]))
            else:
                lines.append("- agency: none found (fully passive?)")
            lines.append("")
        if topics:
            lines.append("## Omissions (what a source does NOT say)")
            for o in self.omissions(topics):
                lines.append(f"- {o['flag']}")
            if not self.omissions(topics):
                lines.append("- all sources mention all topics, or only one source exists")
        return "\n".join(lines)