Spaces:
Sleeping
Sleeping
mothy-08 commited on
Commit ·
e62960c
1
Parent(s): c603a60
Updated
Browse files- DockerFile +21 -0
- api/__init__.py +1 -6
- api/app.py +3 -8
- api/config.py +0 -4
- api/content_moderator.py +4 -26
- api/schemas.py +0 -5
- requirements.txt +4 -0
- tests/__pycache__/test_moderator.cpython-313-pytest-9.0.1.pyc +0 -0
- tests/test_moderator.py +1 -20
DockerFile
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.10-slim
|
| 2 |
+
|
| 3 |
+
ENV PYTHONUNBUFFERED=1 \
|
| 4 |
+
PYTHONDONTWRITEBYTECODE=1
|
| 5 |
+
|
| 6 |
+
WORKDIR /app
|
| 7 |
+
|
| 8 |
+
RUN useradd -m -u 1000 user
|
| 9 |
+
|
| 10 |
+
COPY requirements.txt .
|
| 11 |
+
RUN pip install --no-cache-dir --upgrade -r requirements.txt
|
| 12 |
+
|
| 13 |
+
COPY api ./api
|
| 14 |
+
|
| 15 |
+
RUN chown -R user:user /app
|
| 16 |
+
|
| 17 |
+
USER user
|
| 18 |
+
|
| 19 |
+
EXPOSE 7860
|
| 20 |
+
|
| 21 |
+
CMD ["uvicorn", "api.app:app", "--host", "0.0.0.0", "--port", "7860"]
|
api/__init__.py
CHANGED
|
@@ -2,9 +2,4 @@ from .content_moderator import ContentModerator
|
|
| 2 |
from .schemas import Item
|
| 3 |
from .config import get_classifier
|
| 4 |
|
| 5 |
-
|
| 6 |
-
__all__ = [
|
| 7 |
-
"ContentModerator",
|
| 8 |
-
"Item",
|
| 9 |
-
"get_classifier",
|
| 10 |
-
]
|
|
|
|
| 2 |
from .schemas import Item
|
| 3 |
from .config import get_classifier
|
| 4 |
|
| 5 |
+
__all__ = ["ContentModerator", "Item", "get_classifier"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
api/app.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
from contextlib import asynccontextmanager
|
| 2 |
-
from fastapi import FastAPI, Depends
|
| 3 |
from . import Item, ContentModerator, get_classifier
|
| 4 |
|
| 5 |
|
|
@@ -13,8 +13,7 @@ app = FastAPI(title="Content Moderator API", lifespan=lifespan)
|
|
| 13 |
|
| 14 |
|
| 15 |
def get_moderator():
|
| 16 |
-
|
| 17 |
-
return ContentModerator(classifier)
|
| 18 |
|
| 19 |
|
| 20 |
@app.get("/health")
|
|
@@ -24,8 +23,4 @@ def health():
|
|
| 24 |
|
| 25 |
@app.post("/predict")
|
| 26 |
def predict(item: Item, moderator: ContentModerator = Depends(get_moderator)):
|
| 27 |
-
|
| 28 |
-
return moderator.predict_text(item.text)
|
| 29 |
-
except Exception as e:
|
| 30 |
-
print(f"Error: {e}")
|
| 31 |
-
raise HTTPException(status_code=500, detail="Internal Model Error")
|
|
|
|
| 1 |
from contextlib import asynccontextmanager
|
| 2 |
+
from fastapi import FastAPI, Depends
|
| 3 |
from . import Item, ContentModerator, get_classifier
|
| 4 |
|
| 5 |
|
|
|
|
| 13 |
|
| 14 |
|
| 15 |
def get_moderator():
|
| 16 |
+
return ContentModerator(get_classifier())
|
|
|
|
| 17 |
|
| 18 |
|
| 19 |
@app.get("/health")
|
|
|
|
| 23 |
|
| 24 |
@app.post("/predict")
|
| 25 |
def predict(item: Item, moderator: ContentModerator = Depends(get_moderator)):
|
| 26 |
+
return moderator.predict_text(item.text)
|
|
|
|
|
|
|
|
|
|
|
|
api/config.py
CHANGED
|
@@ -4,8 +4,4 @@ from transformers import pipeline
|
|
| 4 |
|
| 5 |
@lru_cache(maxsize=1)
|
| 6 |
def get_classifier():
|
| 7 |
-
"""
|
| 8 |
-
Lazy-loads the Hugging Face text-classification pipeline.
|
| 9 |
-
Ensures a single instance is used across the service.
|
| 10 |
-
"""
|
| 11 |
return pipeline("text-classification", "mothy-08/drbftcm")
|
|
|
|
| 4 |
|
| 5 |
@lru_cache(maxsize=1)
|
| 6 |
def get_classifier():
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
return pipeline("text-classification", "mothy-08/drbftcm")
|
api/content_moderator.py
CHANGED
|
@@ -1,42 +1,20 @@
|
|
| 1 |
-
from typing import Dict
|
| 2 |
import unicodedata
|
| 3 |
|
| 4 |
|
| 5 |
class ContentModerator:
|
| 6 |
-
"""
|
| 7 |
-
Handles text moderation by validating, preprocessing, and classifying
|
| 8 |
-
"""
|
| 9 |
-
|
| 10 |
def __init__(self, classifier):
|
| 11 |
self.classifier = classifier
|
| 12 |
|
| 13 |
-
def predict_text(self, text: str) ->
|
| 14 |
-
"""
|
| 15 |
-
Main method to classify text.
|
| 16 |
-
Returns a dictionary with label and confidence score.
|
| 17 |
-
"""
|
| 18 |
-
|
| 19 |
text = self._preprocess_text(text)
|
| 20 |
result = self.classifier(text)[0]
|
| 21 |
|
| 22 |
-
if (
|
| 23 |
-
not isinstance(result, dict)
|
| 24 |
-
or "label" not in result
|
| 25 |
-
or "score" not in result
|
| 26 |
-
):
|
| 27 |
-
raise ValueError("Malformed model output")
|
| 28 |
-
|
| 29 |
-
label, score = result["label"].lower(), result["score"]
|
| 30 |
-
|
| 31 |
return {
|
| 32 |
-
"label": label,
|
| 33 |
-
"score": score,
|
| 34 |
}
|
| 35 |
|
| 36 |
def _preprocess_text(self, text: str) -> str:
|
| 37 |
-
"""Sanitizes text to be model-ready."""
|
| 38 |
-
|
| 39 |
text = unicodedata.normalize("NFKC", text)
|
| 40 |
text = "".join(ch for ch in text if unicodedata.category(ch)[0] != "C")
|
| 41 |
-
|
| 42 |
-
return "".join(text.split())
|
|
|
|
|
|
|
| 1 |
import unicodedata
|
| 2 |
|
| 3 |
|
| 4 |
class ContentModerator:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
def __init__(self, classifier):
|
| 6 |
self.classifier = classifier
|
| 7 |
|
| 8 |
+
def predict_text(self, text: str) -> dict:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
text = self._preprocess_text(text)
|
| 10 |
result = self.classifier(text)[0]
|
| 11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
return {
|
| 13 |
+
"label": result["label"],
|
| 14 |
+
"score": result["score"],
|
| 15 |
}
|
| 16 |
|
| 17 |
def _preprocess_text(self, text: str) -> str:
|
|
|
|
|
|
|
| 18 |
text = unicodedata.normalize("NFKC", text)
|
| 19 |
text = "".join(ch for ch in text if unicodedata.category(ch)[0] != "C")
|
| 20 |
+
return " ".join(text.split())
|
|
|
api/schemas.py
CHANGED
|
@@ -3,11 +3,6 @@ from pydantic import BaseModel, StringConstraints
|
|
| 3 |
|
| 4 |
|
| 5 |
class Item(BaseModel):
|
| 6 |
-
"""
|
| 7 |
-
Schema for incoming text to be moderated.
|
| 8 |
-
Ensures text is not empty and within max token length of 512 (approximately 2500 characters)
|
| 9 |
-
"""
|
| 10 |
-
|
| 11 |
text: Annotated[
|
| 12 |
str,
|
| 13 |
StringConstraints(
|
|
|
|
| 3 |
|
| 4 |
|
| 5 |
class Item(BaseModel):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
text: Annotated[
|
| 7 |
str,
|
| 8 |
StringConstraints(
|
requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi
|
| 2 |
+
uvicorn[standard]
|
| 3 |
+
transformers
|
| 4 |
+
torch --index-url https://download.pytorch.org/whl/cpu
|
tests/__pycache__/test_moderator.cpython-313-pytest-9.0.1.pyc
CHANGED
|
Binary files a/tests/__pycache__/test_moderator.cpython-313-pytest-9.0.1.pyc and b/tests/__pycache__/test_moderator.cpython-313-pytest-9.0.1.pyc differ
|
|
|
tests/test_moderator.py
CHANGED
|
@@ -14,53 +14,34 @@ def get_mock_moderator():
|
|
| 14 |
|
| 15 |
|
| 16 |
app.dependency_overrides[get_moderator] = get_mock_moderator
|
| 17 |
-
|
| 18 |
client = TestClient(app)
|
| 19 |
|
| 20 |
|
| 21 |
def test_health():
|
| 22 |
-
"""Ensure the app is alive"""
|
| 23 |
response = client.get("/health")
|
| 24 |
assert response.status_code == 200
|
| 25 |
assert response.json() == {"status": "ok"}
|
| 26 |
|
| 27 |
|
| 28 |
def test_predict_valid_text():
|
| 29 |
-
"""Test the happy path with the mock."""
|
| 30 |
response = client.post("/predict", json={"text": "Hello"})
|
| 31 |
assert response.status_code == 200
|
| 32 |
data = response.json()
|
| 33 |
assert data["label"] == "safe"
|
| 34 |
assert data["score"] == 0.99
|
| 35 |
-
assert data["confidence_text"] == "high"
|
| 36 |
|
| 37 |
|
| 38 |
def test_predict_normalization():
|
| 39 |
-
"""
|
| 40 |
-
CRITICAL TEST: Verify that full-width characters are normalized.
|
| 41 |
-
We send 'Hello' (Full width).
|
| 42 |
-
If your normalization logic works, the classifier sees 'Hello' and returns 'clean'.
|
| 43 |
-
If it fails, the classifier sees 'Hello' (unknown) and returns 'toxic' (default fallback in our mock).
|
| 44 |
-
"""
|
| 45 |
response = client.post("/predict", json={"text": "Hello"})
|
| 46 |
assert response.status_code == 200
|
| 47 |
assert response.json()["label"] == "safe"
|
| 48 |
|
| 49 |
|
| 50 |
-
def
|
| 51 |
-
"""
|
| 52 |
-
Test the Pydantic Bouncer
|
| 53 |
-
Should fail with 422 Unprocessable Entity, NOT 500 Server Error
|
| 54 |
-
"""
|
| 55 |
-
|
| 56 |
response = client.post("/predict", json={"text": ""})
|
| 57 |
assert response.status_code == 422
|
| 58 |
|
| 59 |
|
| 60 |
def test_predict_whitespace_only():
|
| 61 |
-
"""
|
| 62 |
-
Test Pydantic stripping whitespace.
|
| 63 |
-
' ' becomes '' -> min_length=1 fails.
|
| 64 |
-
"""
|
| 65 |
response = client.post("/predict", json={"text": " "})
|
| 66 |
assert response.status_code == 422
|
|
|
|
| 14 |
|
| 15 |
|
| 16 |
app.dependency_overrides[get_moderator] = get_mock_moderator
|
|
|
|
| 17 |
client = TestClient(app)
|
| 18 |
|
| 19 |
|
| 20 |
def test_health():
|
|
|
|
| 21 |
response = client.get("/health")
|
| 22 |
assert response.status_code == 200
|
| 23 |
assert response.json() == {"status": "ok"}
|
| 24 |
|
| 25 |
|
| 26 |
def test_predict_valid_text():
|
|
|
|
| 27 |
response = client.post("/predict", json={"text": "Hello"})
|
| 28 |
assert response.status_code == 200
|
| 29 |
data = response.json()
|
| 30 |
assert data["label"] == "safe"
|
| 31 |
assert data["score"] == 0.99
|
|
|
|
| 32 |
|
| 33 |
|
| 34 |
def test_predict_normalization():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
response = client.post("/predict", json={"text": "Hello"})
|
| 36 |
assert response.status_code == 200
|
| 37 |
assert response.json()["label"] == "safe"
|
| 38 |
|
| 39 |
|
| 40 |
+
def test_predict_empty_text():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
response = client.post("/predict", json={"text": ""})
|
| 42 |
assert response.status_code == 422
|
| 43 |
|
| 44 |
|
| 45 |
def test_predict_whitespace_only():
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
response = client.post("/predict", json={"text": " "})
|
| 47 |
assert response.status_code == 422
|