Upload 9 files

models/content_moderation_api.py

@@ -0,0 +1,75 @@
+"""
+Content Moderation API
+Generated on: 2025-10-03 07:26:24
+"""
+
+import joblib
+import json
+import re
+from typing import Dict, List, Tuple
+
+class ContentModeratorAPI:
+    def __init__(self, model_path: str, config_path: str):
+        # Load model
+        self.model = joblib.load(model_path)
+
+        # Load configuration
+        with open(config_path, 'r') as f:
+            config = json.load(f)
+
+        self.blocked_words = config['blocked_words']
+        self.sensitivity_threshold = config['sensitivity_threshold']
+        self.model_type = config['embedding_type']
+
+        # Compile blocked word patterns
+        self.patterns = [re.compile(r'' + re.escape(word) + r'', re.IGNORECASE) 
+                        for word in self.blocked_words]
+
+    def detect_blocked_words(self, text: str) -> List[str]:
+        """Detect blocked words in text"""
+        found_words = []
+        for word, pattern in zip(self.blocked_words, self.patterns):
+            if pattern.search(text):
+                found_words.append(word)
+        return found_words
+
+    def censor_text(self, text: str, replacement: str = "***") -> str:
+        """Censor inappropriate content"""
+        censored = text
+        for word, pattern in zip(self.blocked_words, self.patterns):
+            censored = pattern.sub(replacement, censored)
+        return censored
+
+    def moderate(self, text: str) -> Dict:
+        """Moderate a single text"""
+        # Rule-based detection
+        blocked_words = self.detect_blocked_words(text)
+        rule_inappropriate = len(blocked_words) > 0
+
+        # ML-based detection
+        ml_confidence = 0.0
+        ml_inappropriate = False
+
+        if self.model_type == 'tfidf':
+            # This would need the vectorizer loaded separately
+            pass
+        elif self.model_type == 'sentence_transformer':
+            # This would need the embedder loaded separately
+            pass
+
+        # Combine predictions
+        is_inappropriate = rule_inappropriate or ml_inappropriate
+        censored_text = self.censor_text(text) if is_inappropriate else text
+
+        return {"text": text, "is_inappropriate": is_inappropriate,
+                "blocked_words": blocked_words, "censored_text": censored_text,
+                "confidence": ml_confidence}
+
+    def moderate_batch(self, texts: List[str]) -> List[Dict]:
+        """Moderate a batch of texts"""
+        return [self.moderate(text) for text in texts]
+
+# Usage example:
+# moderator = ContentModeratorAPI('models/model.pkl', 'models/config.json')
+# result = moderator.moderate("Test text")
+# print(result)

models/content_moderation_api_cuda.py

@@ -0,0 +1,75 @@
+"""
+Content Moderation API
+Generated on: 2025-10-03 04:10:32
+"""
+
+import joblib
+import json
+import re
+from typing import Dict, List, Tuple
+
+class ContentModeratorAPI:
+    def __init__(self, model_path: str, config_path: str):
+        # Load model
+        self.model = joblib.load(model_path)
+        
+        # Load configuration
+        with open(config_path, 'r') as f:
+            config = json.load(f)
+        
+        self.blocked_words = config['blocked_words']
+        self.sensitivity_threshold = config['sensitivity_threshold']
+        self.model_type = config['embedding_type']
+        
+        # Compile blocked word patterns
+        self.patterns = [re.compile(r'' + re.escape(word) + r'', re.IGNORECASE) 
+                        for word in self.blocked_words]
+    
+    def detect_blocked_words(self, text: str) -> List[str]:
+        """Detect blocked words in text"""
+        found_words = []
+        for word, pattern in zip(self.blocked_words, self.patterns):
+            if pattern.search(text):
+                found_words.append(word)
+        return found_words
+    
+    def censor_text(self, text: str, replacement: str = "***") -> str:
+        """Censor inappropriate content"""
+        censored = text
+        for word, pattern in zip(self.blocked_words, self.patterns):
+            censored = pattern.sub(replacement, censored)
+        return censored
+    
+    def moderate(self, text: str) -> Dict:
+        """Moderate a single text"""
+        # Rule-based detection
+        blocked_words = self.detect_blocked_words(text)
+        rule_inappropriate = len(blocked_words) > 0
+        
+        # ML-based detection
+        ml_confidence = 0.0
+        ml_inappropriate = False
+        
+        if self.model_type == 'tfidf':
+            # This would need the vectorizer loaded separately
+            pass
+        elif self.model_type == 'sentence_transformer':
+            # This would need the embedder loaded separately
+            pass
+        
+        # Combine predictions
+        is_inappropriate = rule_inappropriate or ml_inappropriate
+        censored_text = self.censor_text(text) if is_inappropriate else text
+        
+        return {"text": text, "is_inappropriate": is_inappropriate,
+                "blocked_words": blocked_words, "censored_text": censored_text,
+                "confidence": ml_confidence}
+    
+    def moderate_batch(self, texts: List[str]) -> List[Dict]:
+        """Moderate a batch of texts"""
+        return [self.moderate(text) for text in texts]
+
+# Usage example:
+# moderator = ContentModeratorAPI('models/model.pkl', 'models/config.json')
+# result = moderator.moderate("Test text")
+# print(result)

models/content_moderation_config.json

@@ -0,0 +1,66 @@
+{
+  "timestamp": "20251003_072624",
+  "models": {
+    "sentence_transformer": {
+      "model": "models/content_moderation_sentence_transformer_20251003_072624.pkl",
+      "embedder": "models/content_moderation_sentence_transformer_embedder_20251003_072624.pkl",
+      "type": "sentence_transformer"
+    }
+  },
+  "blocked_words": [
+    "hate",
+    "kill",
+    "murder",
+    "attack",
+    "threaten",
+    "bully",
+    "harass",
+    "abuse",
+    "violent",
+    "weapon",
+    "bomb",
+    "gun",
+    "knife",
+    "fight",
+    "war",
+    "death",
+    "sex",
+    "porn",
+    "nude",
+    "adult",
+    "explicit",
+    "xxx",
+    "nsfw",
+    "erotic",
+    "strip",
+    "escort",
+    "hooker",
+    "prostitute",
+    "condom",
+    "vibrator",
+    "damn",
+    "hell",
+    "bastard",
+    "bitch",
+    "ass",
+    "fuck",
+    "shit",
+    "cunt",
+    "piss",
+    "dick",
+    "cock",
+    "pussy",
+    "tits",
+    "boobs",
+    "slut",
+    "whore",
+    "racist",
+    "sexist",
+    "homophobic",
+    "transphobic",
+    "xenophobic"
+  ],
+  "sensitivity_threshold": 0.7,
+  "embedding_type": "sentence_transformers",
+  "model_name": "all-mpnet-base-v2"
+}

models/content_moderation_config_cuda.json

@@ -0,0 +1,66 @@
+{
+  "timestamp": "20251003_041031",
+  "models": {
+    "sentence_transformer": {
+      "model": "models/content_moderation_sentence_transformer_20251003_041031.pkl",
+      "embedder": "models/content_moderation_sentence_transformer_embedder_20251003_041031.pkl",
+      "type": "sentence_transformer"
+    }
+  },
+  "blocked_words": [
+    "hate",
+    "kill",
+    "murder",
+    "attack",
+    "threaten",
+    "bully",
+    "harass",
+    "abuse",
+    "violent",
+    "weapon",
+    "bomb",
+    "gun",
+    "knife",
+    "fight",
+    "war",
+    "death",
+    "sex",
+    "porn",
+    "nude",
+    "adult",
+    "explicit",
+    "xxx",
+    "nsfw",
+    "erotic",
+    "strip",
+    "escort",
+    "hooker",
+    "prostitute",
+    "condom",
+    "vibrator",
+    "damn",
+    "hell",
+    "bastard",
+    "bitch",
+    "ass",
+    "fuck",
+    "shit",
+    "cunt",
+    "piss",
+    "dick",
+    "cock",
+    "pussy",
+    "tits",
+    "boobs",
+    "slut",
+    "whore",
+    "racist",
+    "sexist",
+    "homophobic",
+    "transphobic",
+    "xenophobic"
+  ],
+  "sensitivity_threshold": 0.7,
+  "embedding_type": "sentence_transformers",
+  "model_name": "all-mpnet-base-v2"
+}

models/content_moderation_sentence_transformer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9747086d0f1d6e0c6262d6388db23b6b89ada16858d7e8bc06ce3a028990c8ff
+size 7007

models/content_moderation_sentence_transformer_cuda.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a024f3b5cdaafc6d0f34de926f7819474a4bef503a0df03875f98f4d82fe76c3
+size 7007

models/content_moderation_sentence_transformer_embedder.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dea50d157098740447cb1d4a1e6bc2acc4367f1969bedaa6c3fba2ae404ab98
+size 438525899

models/content_moderation_sentence_transformer_embedder_cuda.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a52e0ef7d9ae61ac0bcd6369a0e4ca50ad263043e30bc55df37b1498a91ad66b
+size 438526502