NathMen12's picture
Upload 56 files
ef73937 verified
Raw
History Blame Contribute Delete
1.92 kB
import Groq from 'groq-sdk';
import { config } from '@config/index.js';
interface AIResult {
safe: boolean;
modelUsed: string;
}
export class GroqModerator {
private client: Groq;
private models: string[];
constructor() {
this.client = new Groq({ apiKey: config.GROQ_API_KEY });
this.models = config.groqModels;
}
async moderate(description: string): Promise<AIResult> {
const prompt = this.buildPrompt(description);
for (const model of this.models) {
try {
const completion = await this.client.chat.completions.create({
model,
messages: [{ role: 'user', content: prompt }],
temperature: 0,
max_tokens: 10,
});
const response = completion.choices[0]?.message?.content?.trim().toUpperCase();
if (response === 'SAFE' || response === 'NOT SAFE') {
return {
safe: response === 'SAFE',
modelUsed: model,
};
}
// Unexpected response, try next model
console.warn(`Unexpected AI response from ${model}: ${response}`);
} catch (error) {
console.warn(`Model ${model} failed:`, error);
// Continue to next model
}
}
// All models failed - default to NOT SAFE for safety
console.error('All Groq models failed, defaulting to NOT SAFE');
return { safe: false, modelUsed: 'none' };
}
private buildPrompt(description: string): string {
return `Analyze this WhatsApp channel description for safety. Reply ONLY with "SAFE" or "NOT SAFE".
Description: "${description}"
Rules:
- NOT SAFE: illegal content, hate speech, violence, sexual content, spam, scams, malware, phishing, drugs, weapons, terrorism, child exploitation
- SAFE: legitimate channels for communities, businesses, education, news, hobbies, support groups, etc.
Response:`;
}
}
export const groqModerator = new GroqModerator();