ai-redes-rubertuito / app /application /threshold_mapper.py
Carlos Ojeda
initial FastAPI Robertuito space
2391f1d
Raw
History Blame Contribute Delete
4.94 kB
from dataclasses import dataclass
import re
import unicodedata
from app.domain.entities import ClassificationLabel, ClassificationResult, ModelScores
HATER_PATTERNS = [
r"\bmu[eé]rete\b",
r"\bmu[eé]ranse\b",
r"\bmuere\b",
r"\bque\s+te\s+mueras\b",
r"\bojal[aá]\s+(?:que\s+)?(?:te\s+)?mueras\b",
r"\bte\s+voy\s+a\s+matar\b",
r"\bdeber[ií]as\s+morir\b",
r"\bmaldito(?:s|as)?\b",
r"\bescoria\b",
]
CRITICAL_PATTERNS = [
r"\bhorrible\b",
r"\basqueros[oa]s?\b",
r"\bapesta\b",
r"\binfumable\b",
r"\bbasura\b",
r"\bp[eé]sim[oa]\b",
r"\bmal[ií]sim[oa]\b",
r"\bmalo\s+totalmente\b",
r"\bno\s+me\s+gusta\b",
r"\bdecepcionante\b",
r"\bdesastre\b",
r"\bvergonzoso\b",
]
@dataclass(frozen=True)
class ThresholdConfig:
hater_hs: float = 0.70
hater_ag: float = 0.80
hater_borderline_hs: float = 0.55
hater_borderline_ag: float = 0.55
critical_hs: float = 0.25
critical_ag: float = 0.25
critical_tr: float = 0.25
class HateSpeechThresholdMapper:
def __init__(self, config: ThresholdConfig):
self._config = config
def map(self, scores: ModelScores) -> ClassificationResult:
hs = self._clamp(scores.hs)
tr = self._clamp(scores.tr)
ag = self._clamp(scores.ag)
text_match = self._match_text_rules(scores.text)
hater_score = max(hs, ag)
critical_score = max(hs, ag, tr)
neutral_score = self._clamp(1 - critical_score)
if text_match and text_match["label"] == ClassificationLabel.HATER:
label = ClassificationLabel.HATER
confidence = max(hater_score, 0.95)
hater_score = max(hater_score, 0.95)
critical_score = max(critical_score, 0.95)
neutral_score = self._clamp(1 - hater_score)
elif text_match and text_match["label"] == ClassificationLabel.CRITICAL:
label = ClassificationLabel.CRITICAL
confidence = max(critical_score, 0.85)
critical_score = max(critical_score, 0.85)
neutral_score = self._clamp(1 - critical_score)
elif self._is_hater(hs, ag):
label = ClassificationLabel.HATER
confidence = hater_score
elif self._is_critical(hs, tr, ag):
label = ClassificationLabel.CRITICAL
confidence = critical_score
else:
label = ClassificationLabel.NEUTRAL
confidence = neutral_score
return ClassificationResult(
predicted_label=label,
confidence=round(confidence, 4),
hater_score=round(hater_score, 4),
critical_score=round(critical_score, 4),
neutral_score=round(neutral_score, 4),
raw_response={
"model_scores": {"HS": hs, "TR": tr, "AG": ag},
"model_output": scores.raw,
"thresholds": self._config.__dict__,
"rule_match": self._serialize_rule_match(text_match),
},
)
def _is_hater(self, hs: float, ag: float) -> bool:
return (
hs >= self._config.hater_hs
or ag >= self._config.hater_ag
or (
hs >= self._config.hater_borderline_hs
and ag >= self._config.hater_borderline_ag
)
)
def _is_critical(self, hs: float, tr: float, ag: float) -> bool:
return (
hs >= self._config.critical_hs
or ag >= self._config.critical_ag
or tr >= self._config.critical_tr
)
@staticmethod
def _clamp(value: float) -> float:
return max(0.0, min(1.0, float(value or 0.0)))
@classmethod
def _match_text_rules(cls, text: str):
normalized = cls._normalize_text(text)
if not normalized:
return None
for pattern in HATER_PATTERNS:
if re.search(pattern, normalized):
return {
"label": ClassificationLabel.HATER,
"pattern": pattern,
"source": "text_rule",
}
for pattern in CRITICAL_PATTERNS:
if re.search(pattern, normalized):
return {
"label": ClassificationLabel.CRITICAL,
"pattern": pattern,
"source": "text_rule",
}
return None
@staticmethod
def _normalize_text(text: str) -> str:
text = unicodedata.normalize("NFKD", text or "")
text = "".join(char for char in text if not unicodedata.combining(char))
text = text.lower()
return re.sub(r"\s+", " ", text).strip()
@staticmethod
def _serialize_rule_match(match):
if not match:
return None
return {
"label": match["label"].value,
"pattern": match["pattern"],
"source": match["source"],
}