CognxSafeTrack
chore: execute Sprint 38 technical debt resolution (Type Safety, Zod validation, Vitest, Mock LLM extracted)
d9879cf
import { FastifyInstance } from 'fastify';
import { prisma } from '../services/prisma';
import { whatsappQueue } from '../services/queue';
import { z } from 'zod';
// ─── Zod Schemas ───────────────────────────────────────────────────────────────
const TrackSchema = z.object({
title: z.string().min(1),
description: z.string().optional(),
duration: z.number().int().positive(),
language: z.enum(['FR', 'WOLOF']).default('FR'),
isPremium: z.boolean().default(false),
priceAmount: z.number().int().optional(),
stripePriceId: z.string().optional(),
});
const TrackDaySchema = z.object({
dayNumber: z.number().int().positive(),
title: z.string().optional(),
lessonText: z.string().optional(),
audioUrl: z.string().url().optional().or(z.literal('')),
exerciseType: z.enum(['TEXT', 'AUDIO', 'BUTTON']).default('TEXT'),
exercisePrompt: z.string().optional(),
validationKeyword: z.string().optional(),
buttonsJson: z.array(z.object({ id: z.string(), title: z.string() })).optional(),
unlockCondition: z.string().optional(),
});
const OverrideFeedbackSchema = z.object({
userId: z.string(),
trackId: z.string(),
transcription: z.string().min(1),
overrideAudioUrl: z.string().url(),
adminId: z.string()
});
export async function adminRoutes(fastify: FastifyInstance) {
// ── Dashboard Stats ────────────────────────────────────────────────────────
fastify.get('/stats', async () => {
const [totalUsers, activeEnrollments, completedEnrollments, totalTracks, totalRevenue] = await Promise.all([
prisma.user.count(),
prisma.enrollment.count({ where: { status: 'ACTIVE' } }),
prisma.enrollment.count({ where: { status: 'COMPLETED' } }),
prisma.track.count(),
prisma.payment.aggregate({ where: { status: 'COMPLETED' }, _sum: { amount: true } }),
]);
return { totalUsers, activeEnrollments, completedEnrollments, totalTracks, totalRevenue: totalRevenue._sum.amount || 0 };
});
// ── Users ──────────────────────────────────────────────────────────────────
fastify.get('/users', async (req) => {
const query = req.query as { page?: string; limit?: string };
const page = Math.max(1, parseInt(query.page || '1'));
const limit = Math.min(100, parseInt(query.limit || '50'));
const [users, total] = await Promise.all([
prisma.user.findMany({
orderBy: { createdAt: 'desc' },
skip: (page - 1) * limit,
take: limit,
include: {
enrollments: { include: { track: true }, orderBy: { startedAt: 'desc' }, take: 1 },
_count: { select: { enrollments: true, responses: true } }
}
}),
prisma.user.count()
]);
return { users, total, page, limit };
});
fastify.get('/users/:userId/messages', async (req, reply) => {
const { userId } = req.params as { userId: string };
const messages = await prisma.message.findMany({
where: { userId },
orderBy: { createdAt: 'asc' },
});
const user = await prisma.user.findUnique({
where: { id: userId },
select: { id: true, name: true, phone: true }
});
if (!user) return reply.status(404).send({ error: 'User not found' });
return { user, messages };
});
// ── Enrollments ────────────────────────────────────────────────────────────
fastify.get('/enrollments', async () => {
const enrollments = await prisma.enrollment.findMany({
include: { user: true, track: true },
orderBy: { startedAt: 'desc' },
take: 100,
});
return enrollments;
});
// ── Human-in-the-Loop / Audio Overdrive ────────────────────────────────────
// LIVE FEED : Students blocked waiting for manual review
// LIVE FEED : Students blocked waiting for manual review
fastify.get('/live-feed', async () => {
const pendingReviews = await prisma.userProgress.findMany({
where: {
exerciseStatus: 'PENDING_REVIEW',
user: { language: 'WOLOF' } // Currently only focusing on Wolof interceptions
},
include: {
user: { select: { id: true, name: true, phone: true, activity: true, language: true } },
track: { select: { id: true, title: true } }
},
orderBy: { updatedAt: 'asc' }
});
// Map the raw payload to find the actual response audio for each pending review
const liveFeed = await Promise.all(pendingReviews.map(async (progress) => {
const enrollment = await prisma.enrollment.findFirst({
where: { userId: progress.userId, trackId: progress.trackId, status: 'ACTIVE' }
});
// If no active enrollment found, fallback gracefully
if (!enrollment) return { ...progress, lastResponse: null };
// Find the most recent response from this user for this enrollment
const lastResponse = await prisma.response.findFirst({
where: { userId: progress.userId, enrollmentId: enrollment.id },
orderBy: { createdAt: 'desc' }
});
return {
...progress,
audioUrl: lastResponse?.mediaUrl || null,
content: lastResponse?.content || null,
dayNumber: lastResponse?.dayNumber || Math.floor(enrollment.currentDay)
};
}));
return liveFeed;
});
// OVERRIDE ACTION : Admin posts the manual review
fastify.post('/override-feedback', async (req, reply) => {
const body = OverrideFeedbackSchema.safeParse(req.body);
if (!body.success) return reply.code(400).send({ error: body.error.flatten() });
const { userId, trackId, transcription, overrideAudioUrl, adminId } = body.data;
// 1. Update UserProgress status & logs
const progress = await prisma.userProgress.update({
where: { userId_trackId: { userId, trackId } },
data: {
exerciseStatus: 'COMPLETED',
adminTranscription: transcription,
overrideAudioUrl: overrideAudioUrl,
reviewedBy: adminId
}
});
// 2. Update BusinessProfile with human-cleaned transcription
const enrollment = await prisma.enrollment.findFirst({
where: { userId, trackId, status: 'ACTIVE' }
});
const currentDay = enrollment ? Math.floor(enrollment.currentDay) : 0;
await prisma.businessProfile.upsert({
where: { userId },
update: { lastUpdatedFromDay: currentDay },
create: { userId, lastUpdatedFromDay: currentDay }
});
// 3. Dispatch Background Job (Audio Delivery + Next Day Increment)
await whatsappQueue.add('send-admin-audio-override', {
userId,
trackId,
overrideAudioUrl,
adminId
});
return reply.code(200).send({ ok: true, progress });
});
// ══════════════════════════════════════════════════════════════════════════
// TRACKS CRUD
// ══════════════════════════════════════════════════════════════════════════
// List tracks
fastify.get('/tracks', async () => {
return prisma.track.findMany({
include: { _count: { select: { days: true, enrollments: true } } },
orderBy: { createdAt: 'desc' }
});
});
// Get single track with all days
fastify.get<{ Params: { id: string } }>('/tracks/:id', async (req, reply) => {
const track = await prisma.track.findUnique({
where: { id: req.params.id },
include: { days: { orderBy: { dayNumber: 'asc' } } }
});
if (!track) return reply.code(404).send({ error: 'Track not found' });
return track;
});
// Create track
fastify.post('/tracks', async (req, reply) => {
const body = TrackSchema.safeParse(req.body);
if (!body.success) return reply.code(400).send({ error: body.error.flatten() });
const track = await prisma.track.create({ data: body.data });
return reply.code(201).send(track);
});
// Update track
fastify.put<{ Params: { id: string } }>('/tracks/:id', async (req, reply) => {
const body = TrackSchema.partial().safeParse(req.body);
if (!body.success) return reply.code(400).send({ error: body.error.flatten() });
try {
const track = await prisma.track.update({ where: { id: req.params.id }, data: body.data });
return track;
} catch {
return reply.code(404).send({ error: 'Track not found' });
}
});
// Delete track
fastify.delete<{ Params: { id: string } }>('/tracks/:id', async (req, reply) => {
try {
await prisma.trackDay.deleteMany({ where: { trackId: req.params.id } });
await prisma.track.delete({ where: { id: req.params.id } });
return { ok: true };
} catch {
return reply.code(404).send({ error: 'Track not found' });
}
});
// ── STT Quality Calibration Endpoint ───────────────────────────────────────
fastify.get('/stats/confidence-distribution', async (_req, reply) => {
const fs = require('fs');
const path = require('path');
const statsPath = path.join(__dirname, '../../data/calibration_stats.json');
try {
if (fs.existsSync(statsPath)) {
const data = JSON.parse(fs.readFileSync(statsPath, 'utf8'));
return data;
} else {
return reply.code(404).send({ error: "Calibration not run yet", message: "Le fichier calibration_stats.json est manquant. Lancez runCalibration()." });
}
} catch (err: unknown) {
return reply.code(500).send({ error: (err instanceof Error ? (err instanceof Error ? err.message : String(err)) : String(err)) });
}
});
// ══════════════════════════════════════════════════════════════════════════
// TRACK DAYS CRUD
// ══════════════════════════════════════════════════════════════════════════
// List days for a track
fastify.get<{ Params: { trackId: string } }>('/tracks/:trackId/days', async (req) => {
return prisma.trackDay.findMany({
where: { trackId: req.params.trackId },
orderBy: { dayNumber: 'asc' }
});
});
// Create day
fastify.post<{ Params: { trackId: string } }>('/tracks/:trackId/days', async (req, reply) => {
const body = TrackDaySchema.safeParse(req.body);
if (!body.success) return reply.code(400).send({ error: body.error.flatten() });
const day = await prisma.trackDay.create({
data: {
...body.data,
trackId: req.params.trackId,
audioUrl: body.data.audioUrl || null,
buttonsJson: body.data.buttonsJson ? body.data.buttonsJson : undefined
}
});
return reply.code(201).send(day);
});
// Update day
fastify.put<{ Params: { trackId: string; dayId: string } }>('/tracks/:trackId/days/:dayId', async (req, reply) => {
const body = TrackDaySchema.partial().safeParse(req.body);
if (!body.success) return reply.code(400).send({ error: body.error.flatten() });
try {
const day = await prisma.trackDay.update({
where: { id: req.params.dayId },
data: { ...body.data, audioUrl: body.data.audioUrl === '' ? null : body.data.audioUrl }
});
return day;
} catch {
return reply.code(404).send({ error: 'Day not found' });
}
});
// Delete day
fastify.delete<{ Params: { trackId: string; dayId: string } }>('/tracks/:trackId/days/:dayId', async (req, reply) => {
try {
await prisma.trackDay.delete({ where: { id: req.params.dayId } });
return { ok: true };
} catch {
return reply.code(404).send({ error: 'Day not found' });
}
});
// ── Training Lab Endpoints ───────────────────────────────────────────────
// Get pending audios for training
fastify.get('/training/audios', async (_req, reply) => {
const pending = await prisma.trainingData.findMany({
where: { status: 'PENDING' },
orderBy: { createdAt: 'desc' }
});
return reply.send(pending);
});
// Submit a manual correction
fastify.post('/training/submit', async (req, reply) => {
const schema = z.object({
id: z.string().uuid().optional(),
audioUrl: z.string(),
transcription: z.string(),
manualCorrection: z.string()
});
const body = schema.safeParse(req.body);
if (!body.success) return reply.code(400).send({ error: body.error.flatten() });
const calculateWER = (reference: string, hypothesis: string): number => {
const levenshtein = require('fast-levenshtein');
const refWords = reference.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter(w => w);
const hypWords = hypothesis.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter(w => w);
if (refWords.length === 0) return 0;
const wordMap = new Map<string, string>();
let charCode = 0xE000;
const getChar = (word: string) => {
if (!wordMap.has(word)) wordMap.set(word, String.fromCharCode(charCode++));
return wordMap.get(word)!;
};
const refChars = refWords.map(getChar).join('');
const hypChars = hypWords.map(getChar).join('');
return levenshtein.get(refChars, hypChars) / refWords.length;
};
const { normalizeWolof } = require('../scripts/normalizeWolof');
const normResult = normalizeWolof(body.data.transcription);
const rawWER = calculateWER(body.data.manualCorrection, body.data.transcription);
const normalizedWER = calculateWER(body.data.manualCorrection, normResult.normalizedText);
// Analyze missing words
const manualWords = body.data.manualCorrection.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter(w => w);
const normWords = normResult.normalizedText.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter((w: string) => w);
// missingWords = elements in manual that are completely absent in normalized
const missingWords = Array.from(new Set(manualWords.filter((w: string) => !normWords.includes(w))));
const data = await prisma.trainingData.upsert({
where: { id: body.data.id || '00000000-0000-0000-0000-000000000000' },
update: {
manualCorrection: body.data.manualCorrection,
rawWER,
normalizedWER,
status: 'REVIEWED'
},
create: {
audioUrl: body.data.audioUrl,
transcription: body.data.transcription,
manualCorrection: body.data.manualCorrection,
rawWER,
normalizedWER,
status: 'REVIEWED'
}
});
return reply.send({ data, missingWords, rawWER, normalizedWER });
});
// Suggest new dictionary rules from TrainingData
fastify.get('/training/suggestions', async (_req, reply) => {
const diff = require('diff');
const trainingData = await prisma.trainingData.findMany({
where: { status: 'REVIEWED' }
});
const substitutionCounts: Record<string, { original: string, replacement: string, count: number }> = {};
trainingData.forEach(item => {
if (!item.manualCorrection) return;
const changes = diff.diffWords(item.transcription, item.manualCorrection);
// Look for adjacent pairs of [removed] then [added]
for (let i = 0; i < changes.length - 1; i++) {
if (changes[i].removed && changes[i + 1].added) {
const original = changes[i].value.trim().toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "");
const replacement = changes[i + 1].value.trim().toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "");
if (original && replacement && original !== replacement && original.split(' ').length === 1 && replacement.split(' ').length === 1) {
const key = `${original}->${replacement}`;
if (!substitutionCounts[key]) substitutionCounts[key] = { original, replacement, count: 0 };
substitutionCounts[key].count++;
}
}
}
});
const suggestions = Object.values(substitutionCounts)
.sort((a, b) => b.count - a.count)
.slice(0, 20);
return reply.send(suggestions);
});
// Apply suggestions directly into normalizeWolof.ts files
fastify.post('/training/apply-suggestions', async (req, reply) => {
const schema = z.object({
suggestions: z.array(z.object({
original: z.string(),
replacement: z.string()
}))
});
const body = schema.safeParse(req.body);
if (!body.success) return reply.code(400).send({ error: body.error.flatten() });
if (body.data.suggestions.length === 0) return reply.send({ ok: true, message: "No suggestions provided" });
const fs = require('fs');
const path = require('path');
const targetFiles = [
path.join(__dirname, '../scripts/normalizeWolof.ts'),
path.join(__dirname, '../../../whatsapp-worker/src/normalizeWolof.ts')
];
let rulesToInject = "";
body.data.suggestions.forEach(s => {
rulesToInject += ` "${s.original}": "${s.replacement}",\n`;
});
for (const file of targetFiles) {
if (fs.existsSync(file)) {
let content = fs.readFileSync(file, 'utf8');
const insertPos = content.indexOf('const NORMALIZATION_RULES: Record<string, string> = {\n');
if (insertPos !== -1) {
const offset = insertPos + 'const NORMALIZATION_RULES: Record<string, string> = {\n'.length;
content = content.slice(0, offset) + rulesToInject + content.slice(offset);
fs.writeFileSync(file, content, 'utf8');
}
}
}
// Auto-recalculate WER after injection
return reply.send({ ok: true, injectedCount: body.data.suggestions.length });
});
// Recalculate WER across all reviewed TrainingData with the current dictionary
fastify.post('/training/recalculate-wer', async (_req, reply) => {
const trainingData = await prisma.trainingData.findMany({
where: { status: 'REVIEWED' }
});
const calculateWER = (reference: string, hypothesis: string): number => {
const levenshtein = require('fast-levenshtein');
const refWords = reference.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter(w => w);
const hypWords = hypothesis.toLowerCase().replace(/[.,/#!$%^&*;:{}=\-_`~()]/g, "").split(/\s+/).filter(w => w);
if (refWords.length === 0) return 0;
const wordMap = new Map<string, string>();
let charCode = 0xE000;
const getChar = (word: string) => {
if (!wordMap.has(word)) wordMap.set(word, String.fromCharCode(charCode++));
return wordMap.get(word)!;
};
const refChars = refWords.map(getChar).join('');
const hypChars = hypWords.map(getChar).join('');
return levenshtein.get(refChars, hypChars) / refWords.length;
};
// We need to bust the require cache to load the newly written normalizeWolof.ts
const normalizeWolofPath = require.resolve('../scripts/normalizeWolof');
delete require.cache[normalizeWolofPath];
const { normalizeWolof } = require('../scripts/normalizeWolof');
let totalRawWER = 0;
let totalNormalizedWER = 0;
let count = 0;
for (const item of trainingData) {
if (!item.manualCorrection) continue;
const rawWER = calculateWER(item.manualCorrection, item.transcription);
const normResult = normalizeWolof(item.transcription);
const normalizedWER = calculateWER(item.manualCorrection, normResult.normalizedText);
await prisma.trainingData.update({
where: { id: item.id },
data: { rawWER, normalizedWER }
});
totalRawWER += rawWER;
totalNormalizedWER += normalizedWER;
count++;
}
const avgRaw = count > 0 ? totalRawWER / count : 0;
const avgNorm = count > 0 ? totalNormalizedWER / count : 0;
return reply.send({
processed: count,
avgRawWER: avgRaw,
avgNormalizedWER: avgNorm,
improvementPercent: count > 0 && avgRaw > 0 ? ((avgRaw - avgNorm) / avgRaw) * 100 : 0
});
});
fastify.post('/training/upload', async (_req, reply) => {
// Just a placeholder until full R2 integration for standalone uploads
return reply.code(501).send({ error: "Not Implemented Yet" });
});
}