Spaces:
Paused
Paused
File size: 1,916 Bytes
ef73937 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | import Groq from 'groq-sdk';
import { config } from '@config/index.js';
interface AIResult {
safe: boolean;
modelUsed: string;
}
export class GroqModerator {
private client: Groq;
private models: string[];
constructor() {
this.client = new Groq({ apiKey: config.GROQ_API_KEY });
this.models = config.groqModels;
}
async moderate(description: string): Promise<AIResult> {
const prompt = this.buildPrompt(description);
for (const model of this.models) {
try {
const completion = await this.client.chat.completions.create({
model,
messages: [{ role: 'user', content: prompt }],
temperature: 0,
max_tokens: 10,
});
const response = completion.choices[0]?.message?.content?.trim().toUpperCase();
if (response === 'SAFE' || response === 'NOT SAFE') {
return {
safe: response === 'SAFE',
modelUsed: model,
};
}
// Unexpected response, try next model
console.warn(`Unexpected AI response from ${model}: ${response}`);
} catch (error) {
console.warn(`Model ${model} failed:`, error);
// Continue to next model
}
}
// All models failed - default to NOT SAFE for safety
console.error('All Groq models failed, defaulting to NOT SAFE');
return { safe: false, modelUsed: 'none' };
}
private buildPrompt(description: string): string {
return `Analyze this WhatsApp channel description for safety. Reply ONLY with "SAFE" or "NOT SAFE".
Description: "${description}"
Rules:
- NOT SAFE: illegal content, hate speech, violence, sexual content, spam, scams, malware, phishing, drugs, weapons, terrorism, child exploitation
- SAFE: legitimate channels for communities, businesses, education, news, hobbies, support groups, etc.
Response:`;
}
}
export const groqModerator = new GroqModerator(); |