| from ..models import PageContext, Channel, RiskLevel |
|
|
| SEVERITY = {RiskLevel.low: 1, RiskLevel.medium: 2, RiskLevel.high: 3, RiskLevel.blocked: 4} |
| BLOCKED_TERMS = ["kill yourself", "dox"] |
| HEALTH_ESCALATION_TERMS = ["chest pain", "suicidal", "overdose", "can't breathe", "cant breathe", "stroke", "911", "emergency", "urgent medical"] |
| CIVIC_HIGH_RISK_TERMS = ["where do i vote", "am i registered", "absentee ballot", "mail ballot", "polling place", "opponent committed", "fraud", "illegal", "deepfake"] |
|
|
| def max_risk(*levels: RiskLevel) -> RiskLevel: |
| return max(levels, key=lambda r: SEVERITY[r]) |
|
|
| def classify_risk(page_context: PageContext, channel: Channel, text: str | None, is_public: bool) -> RiskLevel: |
| body = (text or "").lower() |
| if any(term in body for term in BLOCKED_TERMS): |
| return RiskLevel.blocked |
| if page_context == PageContext.healthcare: |
| if any(term in body for term in HEALTH_ESCALATION_TERMS): |
| return RiskLevel.high |
| if any(term in body for term in ["medication", "diagnosis", "patient", "symptoms"]): |
| return RiskLevel.high |
| if page_context == PageContext.civic: |
| if any(term in body for term in CIVIC_HIGH_RISK_TERMS): |
| return RiskLevel.high |
| if is_public and channel == Channel.comment: |
| return RiskLevel.medium |
| if is_public and channel in {Channel.comment, Channel.page_post}: |
| return RiskLevel.medium |
| return RiskLevel.low |
|
|
| def requires_human_approval(page_context: PageContext, channel: Channel, risk_level: RiskLevel, is_public: bool) -> bool: |
| if risk_level in {RiskLevel.medium, RiskLevel.high, RiskLevel.blocked}: |
| return True |
| if page_context == PageContext.civic and is_public: |
| return True |
| if channel in {Channel.comment, Channel.page_post}: |
| return True |
| return False |
|
|
