File size: 1,847 Bytes
0e84a1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | from ..models import PageContext, Channel, RiskLevel
SEVERITY = {RiskLevel.low: 1, RiskLevel.medium: 2, RiskLevel.high: 3, RiskLevel.blocked: 4}
BLOCKED_TERMS = ["kill yourself", "dox"]
HEALTH_ESCALATION_TERMS = ["chest pain", "suicidal", "overdose", "can't breathe", "cant breathe", "stroke", "911", "emergency", "urgent medical"]
CIVIC_HIGH_RISK_TERMS = ["where do i vote", "am i registered", "absentee ballot", "mail ballot", "polling place", "opponent committed", "fraud", "illegal", "deepfake"]
def max_risk(*levels: RiskLevel) -> RiskLevel:
return max(levels, key=lambda r: SEVERITY[r])
def classify_risk(page_context: PageContext, channel: Channel, text: str | None, is_public: bool) -> RiskLevel:
body = (text or "").lower()
if any(term in body for term in BLOCKED_TERMS):
return RiskLevel.blocked
if page_context == PageContext.healthcare:
if any(term in body for term in HEALTH_ESCALATION_TERMS):
return RiskLevel.high
if any(term in body for term in ["medication", "diagnosis", "patient", "symptoms"]):
return RiskLevel.high
if page_context == PageContext.civic:
if any(term in body for term in CIVIC_HIGH_RISK_TERMS):
return RiskLevel.high
if is_public and channel == Channel.comment:
return RiskLevel.medium
if is_public and channel in {Channel.comment, Channel.page_post}:
return RiskLevel.medium
return RiskLevel.low
def requires_human_approval(page_context: PageContext, channel: Channel, risk_level: RiskLevel, is_public: bool) -> bool:
if risk_level in {RiskLevel.medium, RiskLevel.high, RiskLevel.blocked}:
return True
if page_context == PageContext.civic and is_public:
return True
if channel in {Channel.comment, Channel.page_post}:
return True
return False
|