Vineet commited on
Commit ·
990d2ad
1
Parent(s): ab2ab6c
Fix stray div in UI and wire MongoDB auto-logging for all moderation results
Browse files
ml-service/app/services/moderation_service.py
CHANGED
|
@@ -4,12 +4,15 @@ SafeChat — Moderation Service
|
|
| 4 |
Orchestrates the full moderation pipeline:
|
| 5 |
1. Classify text for toxicity (fine-tuned HingBERT)
|
| 6 |
2. Generate polite alternative via LLM if toxic
|
| 7 |
-
3.
|
|
|
|
| 8 |
|
| 9 |
This is the main entry point called by the API routes.
|
| 10 |
"""
|
| 11 |
|
| 12 |
import time
|
|
|
|
|
|
|
| 13 |
from typing import Dict, List, Optional
|
| 14 |
|
| 15 |
from loguru import logger
|
|
@@ -18,6 +21,33 @@ from app.models.model_manager import model_manager
|
|
| 18 |
from app.schemas.moderation import ModerationResponse
|
| 19 |
|
| 20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
class ModerationService:
|
| 22 |
"""
|
| 23 |
Orchestrates toxicity classification + LLM detoxification.
|
|
@@ -68,7 +98,7 @@ class ModerationService:
|
|
| 68 |
|
| 69 |
total_time_ms = int((time.perf_counter() - start_time) * 1000)
|
| 70 |
|
| 71 |
-
|
| 72 |
is_toxic=classification["is_toxic"],
|
| 73 |
overall_score=classification["overall_score"],
|
| 74 |
severity=classification["severity"],
|
|
@@ -79,6 +109,11 @@ class ModerationService:
|
|
| 79 |
inference_time_ms=total_time_ms,
|
| 80 |
)
|
| 81 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
@staticmethod
|
| 83 |
async def moderate_batch(
|
| 84 |
texts: List[str],
|
|
|
|
| 4 |
Orchestrates the full moderation pipeline:
|
| 5 |
1. Classify text for toxicity (fine-tuned HingBERT)
|
| 6 |
2. Generate polite alternative via LLM if toxic
|
| 7 |
+
3. Auto-log every result to MongoDB for continuous learning
|
| 8 |
+
4. Return combined result
|
| 9 |
|
| 10 |
This is the main entry point called by the API routes.
|
| 11 |
"""
|
| 12 |
|
| 13 |
import time
|
| 14 |
+
import uuid
|
| 15 |
+
from datetime import datetime, timezone
|
| 16 |
from typing import Dict, List, Optional
|
| 17 |
|
| 18 |
from loguru import logger
|
|
|
|
| 21 |
from app.schemas.moderation import ModerationResponse
|
| 22 |
|
| 23 |
|
| 24 |
+
async def _log_to_mongodb(text: str, result: ModerationResponse) -> None:
|
| 25 |
+
"""Fire-and-forget: save every moderation result to MongoDB for training data."""
|
| 26 |
+
try:
|
| 27 |
+
from app.services.feedback_service import feedback_service
|
| 28 |
+
if feedback_service.collection is None:
|
| 29 |
+
return
|
| 30 |
+
doc = {
|
| 31 |
+
"message_id": str(uuid.uuid4()),
|
| 32 |
+
"text": text,
|
| 33 |
+
"is_toxic": result.is_toxic,
|
| 34 |
+
"overall_score": result.overall_score,
|
| 35 |
+
"severity": result.severity,
|
| 36 |
+
"categories": result.categories,
|
| 37 |
+
"detected_language": result.detected_language,
|
| 38 |
+
"suggestion": result.suggestion,
|
| 39 |
+
"model_version": result.model_version,
|
| 40 |
+
"inference_time_ms": result.inference_time_ms,
|
| 41 |
+
"logged_at": datetime.now(timezone.utc),
|
| 42 |
+
# Moderator correction fields (filled later via /feedback endpoint)
|
| 43 |
+
"model_prediction_was_correct": None,
|
| 44 |
+
}
|
| 45 |
+
await feedback_service.collection.insert_one(doc)
|
| 46 |
+
except Exception as e:
|
| 47 |
+
logger.warning(f"MongoDB auto-log failed (non-critical): {e}")
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
|
| 51 |
class ModerationService:
|
| 52 |
"""
|
| 53 |
Orchestrates toxicity classification + LLM detoxification.
|
|
|
|
| 98 |
|
| 99 |
total_time_ms = int((time.perf_counter() - start_time) * 1000)
|
| 100 |
|
| 101 |
+
response = ModerationResponse(
|
| 102 |
is_toxic=classification["is_toxic"],
|
| 103 |
overall_score=classification["overall_score"],
|
| 104 |
severity=classification["severity"],
|
|
|
|
| 109 |
inference_time_ms=total_time_ms,
|
| 110 |
)
|
| 111 |
|
| 112 |
+
# Auto-log to MongoDB for continuous learning (non-blocking)
|
| 113 |
+
await _log_to_mongodb(text, response)
|
| 114 |
+
|
| 115 |
+
return response
|
| 116 |
+
|
| 117 |
@staticmethod
|
| 118 |
async def moderate_batch(
|
| 119 |
texts: List[str],
|
python-frontend/app.py
CHANGED
|
@@ -385,9 +385,9 @@ for idx, msg in enumerate(st.session_state.messages):
|
|
| 385 |
<div style="font-size:0.78rem; color:#9ca3af; margin-bottom:6px; font-weight:500;">
|
| 386 |
CATEGORY PROBABILITIES
|
| 387 |
</div>
|
| 388 |
-
{render_categories(mod.get('categories', dict()))}
|
| 389 |
</div>
|
| 390 |
""", unsafe_allow_html=True)
|
|
|
|
| 391 |
|
| 392 |
# Show suggestion
|
| 393 |
suggestion = mod.get("suggestion") or msg.get("detoxified")
|
|
|
|
| 385 |
<div style="font-size:0.78rem; color:#9ca3af; margin-bottom:6px; font-weight:500;">
|
| 386 |
CATEGORY PROBABILITIES
|
| 387 |
</div>
|
|
|
|
| 388 |
</div>
|
| 389 |
""", unsafe_allow_html=True)
|
| 390 |
+
st.markdown(render_categories(mod.get('categories', dict())), unsafe_allow_html=True)
|
| 391 |
|
| 392 |
# Show suggestion
|
| 393 |
suggestion = mod.get("suggestion") or msg.get("detoxified")
|