import { FastifyInstance } from 'fastify'; import { prisma } from '../services/prisma'; import { whatsappQueue } from '../services/queue'; import { z } from 'zod'; // ─── Zod Schemas ─────────────────────────────────────────────────────────────── const TrackSchema = z.object({ title: z.string().min(1), description: z.string().optional(), duration: z.number().int().positive(), language: z.enum(['FR', 'WOLOF']).default('FR'), isPremium: z.boolean().default(false), priceAmount: z.number().int().optional(), stripePriceId: z.string().optional(), }); const TrackDaySchema = z.object({ dayNumber: z.number().int().positive(), title: z.string().optional(), lessonText: z.string().optional(), audioUrl: z.string().url().optional().or(z.literal('')), exerciseType: z.enum(['TEXT', 'AUDIO', 'BUTTON']).default('TEXT'), exercisePrompt: z.string().optional(), validationKeyword: z.string().optional(), buttonsJson: z.array(z.object({ id: z.string(), title: z.string() })).optional(), unlockCondition: z.string().optional(), }); const OverrideFeedbackSchema = z.object({ userId: z.string(), trackId: z.string(), transcription: z.string().min(1), overrideAudioUrl: z.string().url(), adminId: z.string() }); export async function adminRoutes(fastify: FastifyInstance) { // ── Dashboard Stats ──────────────────────────────────────────────────────── fastify.get('/stats', async () => { const [totalUsers, activeEnrollments, completedEnrollments, totalTracks, totalRevenue] = await Promise.all([ prisma.user.count(), prisma.enrollment.count({ where: { status: 'ACTIVE' } }), prisma.enrollment.count({ where: { status: 'COMPLETED' } }), prisma.track.count(), prisma.payment.aggregate({ where: { status: 'COMPLETED' }, _sum: { amount: true } }), ]); return { totalUsers, activeEnrollments, completedEnrollments, totalTracks, totalRevenue: totalRevenue._sum.amount || 0 }; }); // ── Users ────────────────────────────────────────────────────────────────── fastify.get('/users', async (req) => { const query = req.query as { page?: string; limit?: string }; const page = Math.max(1, parseInt(query.page || '1')); const limit = Math.min(100, parseInt(query.limit || '50')); const [users, total] = await Promise.all([ prisma.user.findMany({ orderBy: { createdAt: 'desc' }, skip: (page - 1) * limit, take: limit, include: { enrollments: { include: { track: true }, orderBy: { startedAt: 'desc' }, take: 1 }, _count: { select: { enrollments: true, responses: true } } } }), prisma.user.count() ]); return { users, total, page, limit }; }); fastify.get('/users/:userId/messages', async (req, reply) => { const { userId } = req.params as { userId: string }; const messages = await prisma.message.findMany({ where: { userId }, orderBy: { createdAt: 'asc' }, }); const user = await prisma.user.findUnique({ where: { id: userId }, select: { id: true, name: true, phone: true } }); if (!user) return reply.status(404).send({ error: 'User not found' }); return { user, messages }; }); // ── Enrollments ──────────────────────────────────────────────────────────── fastify.get('/enrollments', async () => { const enrollments = await prisma.enrollment.findMany({ include: { user: true, track: true }, orderBy: { startedAt: 'desc' }, take: 100, }); return enrollments; }); // ── Human-in-the-Loop / Audio Overdrive ──────────────────────────────────── // LIVE FEED : Students blocked waiting for manual review // LIVE FEED : Students blocked waiting for manual review fastify.get('/live-feed', async () => { const pendingReviews = await prisma.userProgress.findMany({ where: { exerciseStatus: 'PENDING_REVIEW', user: { language: 'WOLOF' } // Currently only focusing on Wolof interceptions }, include: { user: { select: { id: true, name: true, phone: true, activity: true, language: true } }, track: { select: { id: true, title: true } } }, orderBy: { updatedAt: 'asc' } }); // Map the raw payload to find the actual response audio for each pending review const liveFeed = await Promise.all(pendingReviews.map(async (progress) => { const enrollment = await prisma.enrollment.findFirst({ where: { userId: progress.userId, trackId: progress.trackId, status: 'ACTIVE' } }); // If no active enrollment found, fallback gracefully if (!enrollment) return { ...progress, lastResponse: null }; // Find the most recent response from this user for this enrollment const lastResponse = await prisma.response.findFirst({ where: { userId: progress.userId, enrollmentId: enrollment.id }, orderBy: { createdAt: 'desc' } }); return { ...progress, audioUrl: lastResponse?.mediaUrl || null, content: lastResponse?.content || null, dayNumber: lastResponse?.dayNumber || Math.floor(enrollment.currentDay) }; })); return liveFeed; }); // OVERRIDE ACTION : Admin posts the manual review fastify.post('/override-feedback', async (req, reply) => { const body = OverrideFeedbackSchema.safeParse(req.body); if (!body.success) return reply.code(400).send({ error: body.error.flatten() }); const { userId, trackId, transcription, overrideAudioUrl, adminId } = body.data; // 1. Update UserProgress status & logs const progress = await prisma.userProgress.update({ where: { userId_trackId: { userId, trackId } }, data: { exerciseStatus: 'COMPLETED', adminTranscription: transcription, overrideAudioUrl: overrideAudioUrl, reviewedBy: adminId } }); // 2. Update BusinessProfile with human-cleaned transcription const enrollment = await prisma.enrollment.findFirst({ where: { userId, trackId, status: 'ACTIVE' } }); const currentDay = enrollment ? Math.floor(enrollment.currentDay) : 0; await prisma.businessProfile.upsert({ where: { userId }, update: { lastUpdatedFromDay: currentDay }, create: { userId, lastUpdatedFromDay: currentDay } }); // 3. Dispatch Background Job (Audio Delivery + Next Day Increment) await whatsappQueue.add('send-admin-audio-override', { userId, trackId, overrideAudioUrl, adminId }); return reply.code(200).send({ ok: true, progress }); }); // ══════════════════════════════════════════════════════════════════════════ // TRACKS CRUD // ══════════════════════════════════════════════════════════════════════════ // List tracks fastify.get('/tracks', async () => { return prisma.track.findMany({ include: { _count: { select: { days: true, enrollments: true } } }, orderBy: { createdAt: 'desc' } }); }); // Get single track with all days fastify.get<{ Params: { id: string } }>('/tracks/:id', async (req, reply) => { const track = await prisma.track.findUnique({ where: { id: req.params.id }, include: { days: { orderBy: { dayNumber: 'asc' } } } }); if (!track) return reply.code(404).send({ error: 'Track not found' }); return track; }); // Create track fastify.post('/tracks', async (req, reply) => { const body = TrackSchema.safeParse(req.body); if (!body.success) return reply.code(400).send({ error: body.error.flatten() }); const track = await prisma.track.create({ data: body.data }); return reply.code(201).send(track); }); // Update track fastify.put<{ Params: { id: string } }>('/tracks/:id', async (req, reply) => { const body = TrackSchema.partial().safeParse(req.body); if (!body.success) return reply.code(400).send({ error: body.error.flatten() }); try { const track = await prisma.track.update({ where: { id: req.params.id }, data: body.data }); return track; } catch { return reply.code(404).send({ error: 'Track not found' }); } }); // Delete track fastify.delete<{ Params: { id: string } }>('/tracks/:id', async (req, reply) => { try { await prisma.trackDay.deleteMany({ where: { trackId: req.params.id } }); await prisma.track.delete({ where: { id: req.params.id } }); return { ok: true }; } catch { return reply.code(404).send({ error: 'Track not found' }); } }); // ── STT Quality Calibration Endpoint ─────────────────────────────────────── fastify.get('/stats/confidence-distribution', async (_req, reply) => { const fs = require('fs'); const path = require('path'); const statsPath = path.join(__dirname, '../../data/calibration_stats.json'); try { if (fs.existsSync(statsPath)) { const data = JSON.parse(fs.readFileSync(statsPath, 'utf8')); return data; } else { return reply.code(404).send({ error: "Calibration not run yet", message: "Le fichier calibration_stats.json est manquant. Lancez runCalibration()." }); } } catch (err: unknown) { return reply.code(500).send({ error: (err instanceof Error ? (err instanceof Error ? err.message : String(err)) : String(err)) }); } }); // ══════════════════════════════════════════════════════════════════════════ // TRACK DAYS CRUD // ══════════════════════════════════════════════════════════════════════════ // List days for a track fastify.get<{ Params: { trackId: string } }>('/tracks/:trackId/days', async (req) => { return prisma.trackDay.findMany({ where: { trackId: req.params.trackId }, orderBy: { dayNumber: 'asc' } }); }); // Create day fastify.post<{ Params: { trackId: string } }>('/tracks/:trackId/days', async (req, reply) => { const body = TrackDaySchema.safeParse(req.body); if (!body.success) return reply.code(400).send({ error: body.error.flatten() }); const day = await prisma.trackDay.create({ data: { ...body.data, trackId: req.params.trackId, audioUrl: body.data.audioUrl || null, buttonsJson: body.data.buttonsJson ? body.data.buttonsJson : undefined } }); return reply.code(201).send(day); }); // Update day fastify.put<{ Params: { trackId: string; dayId: string } }>('/tracks/:trackId/days/:dayId', async (req, reply) => { const body = TrackDaySchema.partial().safeParse(req.body); if (!body.success) return reply.code(400).send({ error: body.error.flatten() }); try { const day = await prisma.trackDay.update({ where: { id: req.params.dayId }, data: { ...body.data, audioUrl: body.data.audioUrl === '' ? null : body.data.audioUrl } }); return day; } catch { return reply.code(404).send({ error: 'Day not found' }); } }); // Delete day fastify.delete<{ Params: { trackId: string; dayId: string } }>('/tracks/:trackId/days/:dayId', async (req, reply) => { try { await prisma.trackDay.delete({ where: { id: req.params.dayId } }); return { ok: true }; } catch { return reply.code(404).send({ error: 'Day not found' }); } }); // ── Training Lab Endpoints ─────────────────────────────────────────────── // Get pending audios for training fastify.get('/training/audios', async (_req, reply) => { const pending = await prisma.trainingData.findMany({ where: { status: 'PENDING' }, orderBy: { createdAt: 'desc' } }); return reply.send(pending); }); // Submit a manual correction fastify.post('/training/submit', async (req, reply) => { const schema = z.object({ id: z.string().uuid().optional(), audioUrl: z.string(), transcription: z.string(), manualCorrection: z.string() }); const body = schema.safeParse(req.body); if (!body.success) return reply.code(400).send({ error: body.error.flatten() }); const calculateWER = (reference: string, hypothesis: string): number => { const levenshtein = require('fast-levenshtein'); const refWords = reference.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter(w => w); const hypWords = hypothesis.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter(w => w); if (refWords.length === 0) return 0; const wordMap = new Map(); let charCode = 0xE000; const getChar = (word: string) => { if (!wordMap.has(word)) wordMap.set(word, String.fromCharCode(charCode++)); return wordMap.get(word)!; }; const refChars = refWords.map(getChar).join(''); const hypChars = hypWords.map(getChar).join(''); return levenshtein.get(refChars, hypChars) / refWords.length; }; const { normalizeWolof } = require('../scripts/normalizeWolof'); const normResult = normalizeWolof(body.data.transcription); const rawWER = calculateWER(body.data.manualCorrection, body.data.transcription); const normalizedWER = calculateWER(body.data.manualCorrection, normResult.normalizedText); // Analyze missing words const manualWords = body.data.manualCorrection.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter(w => w); const normWords = normResult.normalizedText.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter((w: string) => w); // missingWords = elements in manual that are completely absent in normalized const missingWords = Array.from(new Set(manualWords.filter((w: string) => !normWords.includes(w)))); const data = await prisma.trainingData.upsert({ where: { id: body.data.id || '00000000-0000-0000-0000-000000000000' }, update: { manualCorrection: body.data.manualCorrection, rawWER, normalizedWER, status: 'REVIEWED' }, create: { audioUrl: body.data.audioUrl, transcription: body.data.transcription, manualCorrection: body.data.manualCorrection, rawWER, normalizedWER, status: 'REVIEWED' } }); return reply.send({ data, missingWords, rawWER, normalizedWER }); }); // Suggest new dictionary rules from TrainingData fastify.get('/training/suggestions', async (_req, reply) => { const diff = require('diff'); const trainingData = await prisma.trainingData.findMany({ where: { status: 'REVIEWED' } }); const substitutionCounts: Record = {}; trainingData.forEach(item => { if (!item.manualCorrection) return; const changes = diff.diffWords(item.transcription, item.manualCorrection); // Look for adjacent pairs of [removed] then [added] for (let i = 0; i < changes.length - 1; i++) { if (changes[i].removed && changes[i + 1].added) { const original = changes[i].value.trim().toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, ""); const replacement = changes[i + 1].value.trim().toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, ""); if (original && replacement && original !== replacement && original.split(' ').length === 1 && replacement.split(' ').length === 1) { const key = `${original}->${replacement}`; if (!substitutionCounts[key]) substitutionCounts[key] = { original, replacement, count: 0 }; substitutionCounts[key].count++; } } } }); const suggestions = Object.values(substitutionCounts) .sort((a, b) => b.count - a.count) .slice(0, 20); return reply.send(suggestions); }); // Apply suggestions directly into normalizeWolof.ts files fastify.post('/training/apply-suggestions', async (req, reply) => { const schema = z.object({ suggestions: z.array(z.object({ original: z.string(), replacement: z.string() })) }); const body = schema.safeParse(req.body); if (!body.success) return reply.code(400).send({ error: body.error.flatten() }); if (body.data.suggestions.length === 0) return reply.send({ ok: true, message: "No suggestions provided" }); const fs = require('fs'); const path = require('path'); const targetFiles = [ path.join(__dirname, '../scripts/normalizeWolof.ts'), path.join(__dirname, '../../../whatsapp-worker/src/normalizeWolof.ts') ]; let rulesToInject = ""; body.data.suggestions.forEach(s => { rulesToInject += ` "${s.original}": "${s.replacement}",\n`; }); for (const file of targetFiles) { if (fs.existsSync(file)) { let content = fs.readFileSync(file, 'utf8'); const insertPos = content.indexOf('const NORMALIZATION_RULES: Record = {\n'); if (insertPos !== -1) { const offset = insertPos + 'const NORMALIZATION_RULES: Record = {\n'.length; content = content.slice(0, offset) + rulesToInject + content.slice(offset); fs.writeFileSync(file, content, 'utf8'); } } } // Auto-recalculate WER after injection return reply.send({ ok: true, injectedCount: body.data.suggestions.length }); }); // Recalculate WER across all reviewed TrainingData with the current dictionary fastify.post('/training/recalculate-wer', async (_req, reply) => { const trainingData = await prisma.trainingData.findMany({ where: { status: 'REVIEWED' } }); const calculateWER = (reference: string, hypothesis: string): number => { const levenshtein = require('fast-levenshtein'); const refWords = reference.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter(w => w); const hypWords = hypothesis.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter(w => w); if (refWords.length === 0) return 0; const wordMap = new Map(); let charCode = 0xE000; const getChar = (word: string) => { if (!wordMap.has(word)) wordMap.set(word, String.fromCharCode(charCode++)); return wordMap.get(word)!; }; const refChars = refWords.map(getChar).join(''); const hypChars = hypWords.map(getChar).join(''); return levenshtein.get(refChars, hypChars) / refWords.length; }; // We need to bust the require cache to load the newly written normalizeWolof.ts const normalizeWolofPath = require.resolve('../scripts/normalizeWolof'); delete require.cache[normalizeWolofPath]; const { normalizeWolof } = require('../scripts/normalizeWolof'); let totalRawWER = 0; let totalNormalizedWER = 0; let count = 0; for (const item of trainingData) { if (!item.manualCorrection) continue; const rawWER = calculateWER(item.manualCorrection, item.transcription); const normResult = normalizeWolof(item.transcription); const normalizedWER = calculateWER(item.manualCorrection, normResult.normalizedText); await prisma.trainingData.update({ where: { id: item.id }, data: { rawWER, normalizedWER } }); totalRawWER += rawWER; totalNormalizedWER += normalizedWER; count++; } const avgRaw = count > 0 ? totalRawWER / count : 0; const avgNorm = count > 0 ? totalNormalizedWER / count : 0; return reply.send({ processed: count, avgRawWER: avgRaw, avgNormalizedWER: avgNorm, improvementPercent: count > 0 && avgRaw > 0 ? ((avgRaw - avgNorm) / avgRaw) * 100 : 0 }); }); fastify.post('/training/upload', async (_req, reply) => { // Just a placeholder until full R2 integration for standalone uploads return reply.code(501).send({ error: "Not Implemented Yet" }); }); }