File size: 1,916 Bytes
ef73937
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import Groq from 'groq-sdk';
import { config } from '@config/index.js';

interface AIResult {
  safe: boolean;
  modelUsed: string;
}

export class GroqModerator {
  private client: Groq;
  private models: string[];

  constructor() {
    this.client = new Groq({ apiKey: config.GROQ_API_KEY });
    this.models = config.groqModels;
  }

  async moderate(description: string): Promise<AIResult> {
    const prompt = this.buildPrompt(description);

    for (const model of this.models) {
      try {
        const completion = await this.client.chat.completions.create({
          model,
          messages: [{ role: 'user', content: prompt }],
          temperature: 0,
          max_tokens: 10,
        });

        const response = completion.choices[0]?.message?.content?.trim().toUpperCase();

        if (response === 'SAFE' || response === 'NOT SAFE') {
          return {
            safe: response === 'SAFE',
            modelUsed: model,
          };
        }

        // Unexpected response, try next model
        console.warn(`Unexpected AI response from ${model}: ${response}`);
      } catch (error) {
        console.warn(`Model ${model} failed:`, error);
        // Continue to next model
      }
    }

    // All models failed - default to NOT SAFE for safety
    console.error('All Groq models failed, defaulting to NOT SAFE');
    return { safe: false, modelUsed: 'none' };
  }

  private buildPrompt(description: string): string {
    return `Analyze this WhatsApp channel description for safety. Reply ONLY with "SAFE" or "NOT SAFE".

Description: "${description}"

Rules:
- NOT SAFE: illegal content, hate speech, violence, sexual content, spam, scams, malware, phishing, drugs, weapons, terrorism, child exploitation
- SAFE: legitimate channels for communities, businesses, education, news, hobbies, support groups, etc.

Response:`;
  }
}

export const groqModerator = new GroqModerator();