feat(ai): implement multimodal vision and dynamic data for business context

apps/api/src/routes/ai.ts

@@ -207,19 +207,24 @@ export async function aiRoutes(fastify: FastifyInstance) {
             previousResponses: z.array(z.object({
                 day: z.number(),
                 response: z.string()
-            })).optional()
+            })).optional(),
+            totalDays: z.number().optional().default(1),
+            isDeepDive: z.boolean().optional().default(false),
+            iterationCount: z.number().optional().default(0),
+            imageUrl: z.string().optional()
         });
         const {
             answers, lessonText, exercisePrompt, userLanguage, businessProfile, exerciseCriteria,
-            userActivity, userRegion, dayNumber, previousResponses
+            userActivity, userRegion, dayNumber, previousResponses, isDeepDive, iterationCount, imageUrl
         } = bodySchema.parse(request.body);
 
-        console.log(`[AI] Generating expert feedback for answers (Lang: ${userLanguage}, Day: ${dayNumber || '?'}).`);
+        console.log(`[AI] Generating feedback for user... (Lang: ${userLanguage}, DeepDive: ${isDeepDive}, Iteration: ${iterationCount}, Image: ${!!imageUrl})`);
 
         try {
             const feedback = await aiService.generateFeedback(
                 answers, exercisePrompt, lessonText, userLanguage, businessProfile ?? undefined, exerciseCriteria ?? undefined,
-                userActivity ?? undefined, userRegion ?? undefined, dayNumber ?? undefined, previousResponses ?? undefined
+                userActivity ?? undefined, userRegion ?? undefined, dayNumber ?? undefined, previousResponses ?? undefined,
+                isDeepDive, iterationCount, imageUrl ?? undefined
             );
 
             // 🌟 Standard Feedback UX: 3 lines 🌟

apps/api/src/services/ai/gemini-provider.ts

@@ -14,18 +14,40 @@ export class GeminiProvider implements LLMProvider {
         this.proModel = this.genAI.getGenerativeModel({ model: 'gemini-1.5-pro' });
     }
 
-    async generateStructuredData<T>(prompt: string, _schema: z.ZodSchema<T>, temperature?: number): Promise<T> {
+    async generateStructuredData<T>(prompt: string, _schema: z.ZodSchema<T>, temperature?: number, imageUrl?: string): Promise<T> {
         // Use Flash for standard feedback/chat (fast)
         // Use Pro for complex docs (OnePager/PitchDeck) - detected by prompt length or keyword
         const isComplex = prompt.includes('PITCH_DECK') || prompt.includes('ONE_PAGER') || prompt.length > 2000;
         const model = isComplex ? this.proModel : this.flashModel;
         const modelName = isComplex ? 'gemini-1.5-pro' : 'gemini-1.5-flash';
 
-        console.log(`[GEMINI] Generating structured data with ${modelName}...`);
+        console.log(`[GEMINI] Generating structured data with ${modelName}... (Vision: ${!!imageUrl})`);
 
         try {
+            const parts: any[] = [{ text: prompt }];
+
+            if (imageUrl) {
+                try {
+                    console.log(`[GEMINI] Fetching image from: ${imageUrl}`);
+                    const response = await fetch(imageUrl);
+                    const buffer = await response.arrayBuffer();
+                    const base64 = Buffer.from(buffer).toString('base64');
+                    const mimeType = response.headers.get('content-type') || 'image/png';
+
+                    parts.push({
+                        inlineData: {
+                            data: base64,
+                            mimeType
+                        }
+                    });
+                } catch (imgErr) {
+                    console.error('[GEMINI] Failed to fetch image for vision:', imgErr);
+                    // Fallback to text-only if image fetch fails rather than crashing
+                }
+            }
+
             const result = await model.generateContent({
-                contents: [{ role: 'user', parts: [{ text: prompt }] }],
+                contents: [{ role: 'user', parts }],
                 generationConfig: {
                     responseMimeType: 'application/json',
                     temperature: temperature ?? 0.2, // Default to 0.2

apps/api/src/services/ai/index.ts

@@ -45,18 +45,19 @@ class AIService {
     private async callWithFailover<T>(
         prompt: string,
         schema: z.ZodSchema<T>,
-        temperature?: number
+        temperature?: number,
+        imageUrl?: string
     ): Promise<{ data: T, source: string }> {
         try {
-            const data = await this.primaryProvider.generateStructuredData(prompt, schema, temperature);
+            const data = await this.primaryProvider.generateStructuredData(prompt, schema, temperature, imageUrl);
             const source = (this.primaryProvider instanceof GeminiProvider) ? 'GEMINI' :
                 (this.primaryProvider instanceof OpenAIProvider) ? 'OPENAI' : 'MOCK';
-            console.log(`[AI_INFO] ${source} used successfully.`);
+            console.log(`[AI_INFO] ${source} used successfully. (Vision: ${!!imageUrl})`);
             return { data, source };
         } catch (err) {
             if (this.fallbackProvider) {
                 console.warn('[AI_WARNING] Primary provider failed, falling back to OpenAI...', (err as Error).message);
-                const data = await this.fallbackProvider.generateStructuredData(prompt, schema, temperature);
+                const data = await this.fallbackProvider.generateStructuredData(prompt, schema, temperature, imageUrl);
                 console.log('[AI_INFO] OPENAI used as fallback.');
                 return { data, source: 'OPENAI' };
             }
@@ -81,7 +82,7 @@ class AIService {
         ${marketDataInjected}
 
         STRICTES CONTRAINTES DE QUALITÉ "PREMIUM V4" :
-        - DENSITÉ RÉDACTIONNELLE : Chaque section (Problem, Solution, Target, Business Model) doit être un paragraphe détaillé, articulé et stratégique (minimum 3 phrases analytiques, 15-25 mots). Les réponses courtes de l'utilisateur DOIVENT être enrichies avec ton 'Knowledge Base' métier (ex: importance de l'hygiène pour l'agro, délais pour la couture).
+        - DENSITÉ RÉDACTIONNELLE : Chaque section (Problem, Solution, Target, Business Model) doit être un paragraphe détaillé, articulé et stratégique (minimum 3 phrases analytiques, 15-25 mots). Les réponses courtes de l'utilisateur DOIVENT être enrichies avec ton 'Knowledge Base' métier (ex: enjeux de distribution pour ${businessProfile?.activityLabel || 'ce secteur'}, délais de livraison locaux).
         - ANTI-JARGON SAAS : INTERDICTION FORMELLE d'utiliser les mots "Premium", "Trial", "Subscription", "Sign up" ou "SaaS". Adapte le Modèle Économique au secteur réel (Vente directe, prestation de service, acompte, etc).
         - SOURCES (marketSources) : Si tu utilises les données de marché injectées, cite explicitement la source (ex: "Source: ANSD 2024").
         - ANALYSE vs DESCRIPTION : Ne te contente pas d'énumérer. Explique l'impact business et le positionnement luxe.
@@ -164,7 +165,8 @@ class AIService {
         dayNumber?: number,
         previousResponses?: Array<{ day: number; response: string }>,
         isDeepDive: boolean = false,
-        iterationCount: number = 0
+        iterationCount: number = 0,
+        imageUrl?: string
     ): Promise<FeedbackData & { searchResults?: any[] }> {
         // 🚀 Question Detection Logic (Lead AI Engineer Requirement)
         const questionKeywords = ['?', 'avis', 'penses', 'conseil', 'aider', 'comment', 'pourquoi', 'idée', 'prix', 'standard'];
@@ -205,13 +207,21 @@ class AIService {
 
         // Remove hallucinatory generic fallback words
         const cleanActivity = activityLabel.replace(/non précisé|e-commerce/i, '').trim() || 'Entrepreneuriat';
-        const query = `${cleanActivity} ${region} Sénégal marché chiffres statistiques data`;
+        let query = `${cleanActivity} ${region} Sénégal marché chiffres statistiques data`;
+
+        // 🚀 Diversification des recherches (Lead AI Architect Requirement)
+        if (dayNumber === 10) {
+            query = `startups concurrents ${cleanActivity} ${region} Sénégal solutions paiement UEMOA`;
+        } else if (dayNumber === 11) {
+            query = `benchmarks marges rentabilité ${cleanActivity} Afrique de l'Ouest tech business model`;
+        }
+
         try {
             const results = await searchService.search(query);
             if (results && results.length > 0) {
                 searchResults = results;
                 searchContext = `\n🌐 DONNÉES DE MARCHÉ RÉELLES (Google Search) :\n${results.map(r => `- ${r.title}: ${r.snippet}`).join('\n')}\n`;
-                console.log(`[AI_SERVICE] ✅ Search enrichment added (${results.length} results).`);
+                console.log(`[AI_SERVICE] ✅ Search enrichment added for Day ${dayNumber} (Query: ${query}).`);
             }
         } catch (err) {
             console.error('[AI_SERVICE] Search enrichment failed:', err);
@@ -267,8 +277,15 @@ class AIService {
 
             LES 3 PILIERS OBLIGATOIRES DU FEEDBACK :
             1. 🌟 Validation (Pilier 1) : Félicite et valide l'idée de l'utilisateur avec l'enthousiasme d'un investisseur.
-            2. 🚀 Version Enrichie (Pilier 2) : Réécris sa phrase de manière exécutive en y intégrant OBLIGATOIREMENT des données chiffrées réelles trouvées dans ta recherche (ex: taille du marché ANSD, nombre de boutiques) et des termes stratégiques (supply chain, B2B, rétention). 
-            3. 💡 Conseil Actionnable (Pilier 3) : Donne un conseil de terrain hyper concret basé là aussi sur la recherche web (ex: "Conseil : Le créneau 6h-8h est critique dans les boutiques de Dakar...").
+            2. 🚀 Version Enrichie (Pilier 2) : Réécris sa phrase de manière exécutive en y intégrant OBLIGATOIREMENT des données chiffrées réelles trouvées dans ta recherche (ex: taille du marché selon la recherche, nombre de clients potentiels) et des termes stratégiques (supply chain, B2B, rétention). 
+            3. 💡 Conseil Actionnable (Pilier 3) : Donne un conseil de terrain hyper concret basé là aussi sur la recherche web (ex: "Conseil : Dans le secteur de ${activityLabel} à ${region}, il est crucial de...").
+
+            ${imageUrl ? `
+            📸 ANALYSE VISUELLE (MULTIMODAL) :
+            - L'utilisateur a envoyé une image comme preuve ou illustration.
+            - TU DOIS analyser visuellement cette image et intégrer ton constat dans le feedback.
+            - Si c'est un trophée (ex: Blue Ocean Awards), un diplôme ou un logo, souligne sa valeur pour la crédibilité et la 'Slide Confiance'.
+            - Si l'image contient des chiffres ou des contrats, extrais-les pour confirmer les données financières.` : ''}
 
             ⚠️ INTERDICTION ABSOLUE (Anti-Remediation Loop) :
             - Tu NE DOIS PLUS JAMAIS demander à l'utilisateur de s'expliquer davantage (ex: "Quel est l'âge exact ?", "Combien gagnes-tu ?").
@@ -321,7 +338,7 @@ class AIService {
                 `ALERTE MAXIMALE : Tu as l'INTERDICTION FORMELLE d'utiliser du Wolof. Reste dans un Français institutionnel et pragmatique.`}
         `;
 
-        const { data, source } = await this.callWithFailover(prompt, FeedbackSchema, 0.7);
+        const { data, source } = await this.callWithFailover(prompt, FeedbackSchema, 0.7, imageUrl);
         return {
             ...data,
             searchResults,

apps/api/src/services/ai/openai-provider.ts

@@ -26,7 +26,7 @@ export class OpenAIProvider implements LLMProvider {
         });
     }
 
-    async generateStructuredData<T>(prompt: string, schema: z.ZodSchema<T>, temperature?: number): Promise<T> {
+    async generateStructuredData<T>(prompt: string, schema: z.ZodSchema<T>, temperature?: number, _imageUrl?: string): Promise<T> {
         console.log('[OPENAI] Generating structured data...');
 
         const timeout = new Promise<never>((_, reject) =>

apps/api/src/services/ai/types.ts

@@ -7,7 +7,7 @@ export interface TranscriptionResult {
 
 // Base interface for all LLM Providers
 export interface LLMProvider {
-    generateStructuredData<T>(prompt: string, schema: z.ZodSchema<T>, temperature?: number): Promise<T>;
+    generateStructuredData<T>(prompt: string, schema: z.ZodSchema<T>, temperature?: number, imageUrl?: string): Promise<T>;
     transcribeAudio(audioBuffer: Buffer, filename: string, language?: string): Promise<TranscriptionResult>;
     generateSpeech(text: string): Promise<Buffer>;
     generateImage(prompt: string): Promise<string>;

apps/api/src/services/whatsapp.ts

@@ -53,10 +53,10 @@ export class WhatsAppService {
         return similarity >= threshold;
     }
 
-    static async handleIncomingMessage(phone: string, text: string, audioUrl?: string) {
-        const traceId = audioUrl ? `[STT-FLOW-${phone.slice(-4)}]` : `[TXT-FLOW-${phone.slice(-4)}]`;
+    static async handleIncomingMessage(phone: string, text: string, audioUrl?: string, imageUrl?: string) {
+        const traceId = audioUrl ? `[STT-FLOW-${phone.slice(-4)}]` : imageUrl ? `[IMG-FLOW-${phone.slice(-4)}]` : `[TXT-FLOW-${phone.slice(-4)}]`;
         const normalizedText = this.normalizeCommand(text);
-        console.log(`${traceId} Received: ${normalizedText} (Audio: ${audioUrl || 'N/A'})`);
+        console.log(`${traceId} Received: ${normalizedText} (Audio: ${audioUrl || 'N/A'}, Image: ${imageUrl || 'N/A'})`);
 
         // 1. Find or Create User
         let user = await prisma.user.findUnique({ where: { phone } });
@@ -92,7 +92,7 @@ export class WhatsAppService {
             await prisma.message.create({
                 data: {
                     content: text,
-                    mediaUrl: audioUrl,
+                    mediaUrl: audioUrl || imageUrl,
                     direction: 'INBOUND',
                     userId: user.id
                 }
@@ -503,7 +503,7 @@ export class WhatsAppService {
                         userId: user.id, text, trackId: activeEnrollment.trackId, trackDayId: trackDay.id,
                         exercisePrompt: trackDay.exercisePrompt || '', lessonText: trackDay.lessonText || '',
                         exerciseCriteria: trackDay.exerciseCriteria, pendingProgressId: pendingProgress.id,
-                        enrollmentId: activeEnrollment.id, currentDay: activeEnrollment.currentDay,
+                        dayNumber: activeEnrollment.currentDay,
                         totalDays: activeEnrollment.track.duration, language: user.language,
                         // NEW EXPERT CONTEXT
                         userActivity: user.activity,
@@ -511,7 +511,8 @@ export class WhatsAppService {
                         previousResponses,
                         // DEEP DIVE PARAMETERS
                         isDeepDive: isDeepDiveAction,
-                        iterationCount: currentIterationCount
+                        iterationCount: currentIterationCount,
+                        imageUrl: imageUrl // Pass the image URL to the AI Coach
                     }, { attempts: 3, backoff: { type: 'exponential', delay: 2000 } });
                     return;
                 }
@@ -569,7 +570,8 @@ export class WhatsAppService {
                     // NEW EXPERT CONTEXT
                     userActivity: user.activity,
                     userRegion: user.city,
-                    previousResponses
+                    previousResponses,
+                    imageUrl: imageUrl // Pass the image URL to the AI Coach (fallback case)
                 });
                 return;
             }

apps/whatsapp-worker/src/index.ts

@@ -48,7 +48,7 @@ const worker = new Worker('whatsapp-queue', async (job: Job) => {
             await sendTextMessage(phone, text);
         }
         else if (job.name === 'generate-feedback') {
-            const { userId, text, trackId, exercisePrompt, lessonText, exerciseCriteria, currentDay, totalDays, language, userActivity, userRegion, previousResponses, isDeepDive, iterationCount } = job.data;
+            const { userId, text, trackId, exercisePrompt, lessonText, exerciseCriteria, currentDay, totalDays, language, userActivity, userRegion, previousResponses, isDeepDive, iterationCount, imageUrl } = job.data;
             const user = await prisma.user.findUnique({
                 where: { id: userId },
                 include: { businessProfile: true } as any
@@ -86,7 +86,8 @@ const worker = new Worker('whatsapp-queue', async (job: Job) => {
                         previousResponses,
                         // DEEP DIVE CONTEXT
                         isDeepDive: isDeepDive || false,
-                        iterationCount: iterationCount || 0
+                        iterationCount: iterationCount || 0,
+                        imageUrl: imageUrl
                     })
                 });
 
@@ -521,8 +522,9 @@ const worker = new Worker('whatsapp-queue', async (job: Job) => {
                     }
                 }
 
-                // Transcribe with Whisper via API
-                console.log(`${traceId} Transcribe start calling ${AI_API_BASE_URL.replace(/\/$/, "")}/v1/ai/transcribe`);
+                // ─── Routing: Transcribe if Audio, Forward if Image ─────────
+                if (mimeType.startsWith('audio/')) {
+                    console.log(`${traceId} Transcribe start calling ${AI_API_BASE_URL.replace(/\/$/, "")}/v1/ai/transcribe`);
                 const transcribeRes = await fetch(`${AI_API_BASE_URL.replace(/\/$/, "")}/v1/ai/transcribe`, {
                     method: 'POST',
                     headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${apiKey}` },
@@ -669,7 +671,6 @@ const worker = new Worker('whatsapp-queue', async (job: Job) => {
                         }
                         console.log(`${traceId} API handle-message success.`);
                     }
-
                 } else if (transcribeRes.status === 429) {
                     // OpenAI quota exceeded — send fallback and do NOT requeue
                     console.warn(`[WORKER] 429 Error during transcription`);
@@ -686,11 +687,41 @@ const worker = new Worker('whatsapp-queue', async (job: Job) => {
                     console.error(`[WORKER] /v1/ai/transcribe failed with HTTP ${transcribeRes.status}: ${errText}`);
                     throw new Error(`Transcription failed HTTP ${transcribeRes.status}`); // throw so BullMQ retries
                 }
+            } else if (mimeType.startsWith('image/')) {
+                // 📸 VISION FLOW (Lead AI Architect Requirement)
+                console.log(`${traceId} Image detected. Forwarding to API handle-message...`);
+                
+                // ─── Port Hardening: Handle HF Space 7860 vs Local 8080 ─────────
+                let finalApiUrl = AI_API_BASE_URL.replace(/\/$/, "");
+                if (finalApiUrl.includes('localhost:8080')) {
+                    try {
+                        const pingRes = await fetch('http://localhost:8080/health').catch(() => null);
+                        if (!pingRes || !pingRes.ok) {
+                            console.log(`${traceId} Local port 8080 not responding, trying 7860 (HF Default)...`);
+                            finalApiUrl = finalApiUrl.replace('8080', '7860');
+                        }
+                    } catch (e) {
+                        finalApiUrl = finalApiUrl.replace('8080', '7860');
+                    }
+                }
 
-            } catch (err: unknown) {
-                console.error(`[WORKER] handle-inbound-audio failed:`, err);
+                const handleRes = await fetch(`${finalApiUrl}/v1/internal/handle-message`, {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${apiKey}` },
+                    body: JSON.stringify({ phone, text: 'Image reçue', imageUrl: audioUrl }) // text: 'Image reçue' to pass gibberish guardrail
+                });
+
+                if (!handleRes.ok) {
+                    const errText = await handleRes.text();
+                    console.error(`${traceId} API handle-message (IMAGE) failed: ${errText}`);
+                    throw new Error(`API handle-message failed: ${handleRes.status}`);
+                }
+                console.log(`${traceId} API handle-message (IMAGE) success.`);
             }
+        } catch (err: unknown) {
+            console.error(`[WORKER] download-media failed:`, err);
         }
+    }
         else if (job.name === 'send-image') {
             const { to, imageUrl, caption } = job.data;
             try {