Spaces:
Sleeping
Sleeping
Upload 62 files
Browse files- ai-routes.js +45 -172
- components/ai/AssessmentPanel.tsx +43 -204
- components/ai/ChatPanel.tsx +109 -101
ai-routes.js
CHANGED
|
@@ -29,7 +29,7 @@ async function recordUsage(model, provider) {
|
|
| 29 |
} catch (e) { console.error("Failed to record AI usage stats:", e); }
|
| 30 |
}
|
| 31 |
|
| 32 |
-
// Fallback Speech-to-Text using Hugging Face
|
| 33 |
async function transcribeAudioWithHF(audioBase64) {
|
| 34 |
const token = await getHFToken();
|
| 35 |
if (!token) {
|
|
@@ -38,42 +38,33 @@ async function transcribeAudioWithHF(audioBase64) {
|
|
| 38 |
}
|
| 39 |
|
| 40 |
try {
|
| 41 |
-
console.log("[AI] 🎤 Using Hugging Face
|
| 42 |
-
const response = await fetch(
|
| 43 |
-
"https://api-inference.huggingface.co/models/openai/whisper-large-v3",
|
| 44 |
-
{
|
| 45 |
-
headers: {
|
| 46 |
-
Authorization: `Bearer ${token}`,
|
| 47 |
-
"Content-Type": "application/json",
|
| 48 |
-
},
|
| 49 |
-
method: "POST",
|
| 50 |
-
body: JSON.stringify({
|
| 51 |
-
inputs: audioBase64, // HF Inference often accepts base64 direct in inputs or raw bytes. For stability with free tier, simple base64 JSON payload often works for some wrappers, but raw bytes are safer for 'audio/...' types.
|
| 52 |
-
// Actually, let's convert to buffer then send raw bytes for better compatibility.
|
| 53 |
-
}),
|
| 54 |
-
}
|
| 55 |
-
);
|
| 56 |
-
|
| 57 |
-
// Retry with raw bytes if JSON failed or for standard audio models
|
| 58 |
const buffer = Buffer.from(audioBase64, 'base64');
|
| 59 |
-
|
|
|
|
|
|
|
| 60 |
"https://api-inference.huggingface.co/models/openai/whisper-large-v3",
|
| 61 |
{
|
| 62 |
headers: {
|
| 63 |
Authorization: `Bearer ${token}`,
|
| 64 |
-
"Content-Type": "audio/webm", // Assuming webm from frontend
|
| 65 |
},
|
| 66 |
method: "POST",
|
| 67 |
body: buffer,
|
| 68 |
}
|
| 69 |
);
|
| 70 |
|
| 71 |
-
if (!
|
| 72 |
-
const errText = await
|
| 73 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
}
|
| 75 |
|
| 76 |
-
const result = await
|
|
|
|
| 77 |
return result.text;
|
| 78 |
} catch (e) {
|
| 79 |
console.error("[AI] HF STT Failed:", e.message);
|
|
@@ -84,16 +75,12 @@ async function transcribeAudioWithHF(audioBase64) {
|
|
| 84 |
function convertGeminiToOpenAI(baseParams) {
|
| 85 |
const messages = [];
|
| 86 |
if (baseParams.config?.systemInstruction) messages.push({ role: 'system', content: baseParams.config.systemInstruction });
|
| 87 |
-
|
| 88 |
let contents = baseParams.contents;
|
| 89 |
-
if (contents && !Array.isArray(contents))
|
| 90 |
-
contents = [contents];
|
| 91 |
-
}
|
| 92 |
|
| 93 |
if (contents && Array.isArray(contents)) {
|
| 94 |
contents.forEach(content => {
|
| 95 |
let role = (content.role === 'model' || content.role === 'assistant') ? 'assistant' : 'user';
|
| 96 |
-
|
| 97 |
const messageContent = [];
|
| 98 |
if (content.parts) {
|
| 99 |
content.parts.forEach(p => {
|
|
@@ -102,13 +89,11 @@ function convertGeminiToOpenAI(baseParams) {
|
|
| 102 |
if (p.inlineData.mimeType.startsWith('image/')) {
|
| 103 |
messageContent.push({ type: 'image_url', image_url: { url: `data:${p.inlineData.mimeType};base64,${p.inlineData.data}` } });
|
| 104 |
} else if (p.inlineData.mimeType.startsWith('audio/')) {
|
| 105 |
-
// Temporary marker for audio, will be resolved before sending if possible
|
| 106 |
messageContent.push({ type: 'audio_base64', data: p.inlineData.data });
|
| 107 |
}
|
| 108 |
}
|
| 109 |
});
|
| 110 |
}
|
| 111 |
-
|
| 112 |
if (messageContent.length > 0) {
|
| 113 |
if (messageContent.length === 1 && messageContent[0].type === 'text') {
|
| 114 |
messages.push({ role: role, content: messageContent[0].text });
|
|
@@ -124,14 +109,11 @@ function convertGeminiToOpenAI(baseParams) {
|
|
| 124 |
const PROVIDERS = { GEMINI: 'GEMINI', OPENROUTER: 'OPENROUTER', GEMMA: 'GEMMA' };
|
| 125 |
const DEFAULT_OPENROUTER_MODELS = ['qwen/qwen3-coder:free', 'openai/gpt-oss-120b:free', 'qwen/qwen3-235b-a22b:free', 'tngtech/deepseek-r1t-chimera:free'];
|
| 126 |
|
| 127 |
-
// Runtime override logic
|
| 128 |
let runtimeProviderOrder = [];
|
| 129 |
|
| 130 |
function deprioritizeProvider(providerName) {
|
| 131 |
if (runtimeProviderOrder.length > 0 && runtimeProviderOrder[runtimeProviderOrder.length - 1] === providerName) return;
|
| 132 |
-
console.log(`[AI System] ⚠️ Deprioritizing ${providerName} due to errors. Moving to end of queue.`);
|
| 133 |
runtimeProviderOrder = runtimeProviderOrder.filter(p => p !== providerName).concat(providerName);
|
| 134 |
-
console.log(`[AI System] 🔄 New Priority Order: ${runtimeProviderOrder.join(' -> ')}`);
|
| 135 |
}
|
| 136 |
|
| 137 |
function isQuotaError(e) {
|
|
@@ -139,7 +121,6 @@ function isQuotaError(e) {
|
|
| 139 |
return e.status === 429 || e.status === 503 || msg.includes('quota') || msg.includes('overloaded') || msg.includes('resource_exhausted') || msg.includes('rate limit') || msg.includes('credits');
|
| 140 |
}
|
| 141 |
|
| 142 |
-
// Streaming Helpers
|
| 143 |
async function streamGemini(baseParams, res) {
|
| 144 |
const { GoogleGenAI } = await import("@google/genai");
|
| 145 |
const models = ['gemini-2.5-flash', 'gemini-2.5-flash-lite'];
|
|
@@ -150,35 +131,21 @@ async function streamGemini(baseParams, res) {
|
|
| 150 |
const client = new GoogleGenAI({ apiKey });
|
| 151 |
for (const modelName of models) {
|
| 152 |
try {
|
| 153 |
-
console.log(`[AI] 🚀 Attempting Gemini Model: ${modelName} (Key ends with ...${apiKey.slice(-4)})`);
|
| 154 |
const result = await client.models.generateContentStream({ ...baseParams, model: modelName });
|
| 155 |
-
|
| 156 |
let hasStarted = false;
|
| 157 |
let fullText = "";
|
| 158 |
-
|
| 159 |
for await (const chunk of result) {
|
| 160 |
-
if (!hasStarted) {
|
| 161 |
-
console.log(`[AI] ✅ Connected to Gemini: ${modelName}`);
|
| 162 |
-
recordUsage(modelName, PROVIDERS.GEMINI);
|
| 163 |
-
hasStarted = true;
|
| 164 |
-
}
|
| 165 |
if (chunk.text) {
|
| 166 |
fullText += chunk.text;
|
| 167 |
res.write(`data: ${JSON.stringify({ text: chunk.text })}\n\n`);
|
| 168 |
-
if (res.flush) res.flush();
|
| 169 |
}
|
| 170 |
}
|
| 171 |
return fullText;
|
| 172 |
-
} catch (e) {
|
| 173 |
-
console.warn(`[AI] ⚠️ Gemini ${modelName} Error: ${e.message}`);
|
| 174 |
-
if (isQuotaError(e)) {
|
| 175 |
-
continue;
|
| 176 |
-
}
|
| 177 |
-
throw e;
|
| 178 |
-
}
|
| 179 |
}
|
| 180 |
}
|
| 181 |
-
throw new Error("Gemini
|
| 182 |
}
|
| 183 |
|
| 184 |
async function streamOpenRouter(baseParams, res) {
|
|
@@ -188,22 +155,17 @@ async function streamOpenRouter(baseParams, res) {
|
|
| 188 |
const keys = await getKeyPool('openrouter');
|
| 189 |
if (keys.length === 0) throw new Error("No OpenRouter API keys");
|
| 190 |
|
| 191 |
-
// --- FALLBACK STT LOGIC ---
|
| 192 |
-
// Check if there are any pending audio parts in messages
|
| 193 |
-
let hasAudio = false;
|
| 194 |
for (let msg of messages) {
|
| 195 |
if (Array.isArray(msg.content)) {
|
| 196 |
for (let part of msg.content) {
|
| 197 |
if (part.type === 'audio_base64') {
|
| 198 |
-
hasAudio = true;
|
| 199 |
-
// Try to transcribe
|
| 200 |
const text = await transcribeAudioWithHF(part.data);
|
| 201 |
if (text) {
|
| 202 |
part.type = 'text';
|
| 203 |
-
part.text = `[
|
| 204 |
-
delete part.data;
|
| 205 |
} else {
|
| 206 |
-
throw new Error("
|
| 207 |
}
|
| 208 |
}
|
| 209 |
}
|
|
@@ -214,125 +176,68 @@ async function streamOpenRouter(baseParams, res) {
|
|
| 214 |
for (const modelName of models) {
|
| 215 |
const modelConfig = config?.openRouterModels?.find(m => m.id === modelName);
|
| 216 |
const baseURL = modelConfig?.apiUrl ? modelConfig.apiUrl : "https://openrouter.ai/api/v1";
|
| 217 |
-
const
|
| 218 |
-
|
| 219 |
-
const client = new OpenAI({ baseURL, apiKey, defaultHeaders: { "HTTP-Referer": "https://smart.com", "X-Title": "Smart School" } });
|
| 220 |
-
|
| 221 |
try {
|
| 222 |
-
console.log(`[AI] 🚀 Attempting ${providerLabel} Model: ${modelName}`);
|
| 223 |
-
|
| 224 |
const stream = await client.chat.completions.create({ model: modelName, messages, stream: true });
|
| 225 |
-
|
| 226 |
-
console.log(`[AI] ✅ Connected to ${providerLabel}: ${modelName}`);
|
| 227 |
recordUsage(modelName, PROVIDERS.OPENROUTER);
|
| 228 |
-
|
| 229 |
let fullText = '';
|
| 230 |
for await (const chunk of stream) {
|
| 231 |
const text = chunk.choices[0]?.delta?.content || '';
|
| 232 |
if (text) {
|
| 233 |
fullText += text;
|
| 234 |
res.write(`data: ${JSON.stringify({ text: text })}\n\n`);
|
| 235 |
-
if (res.flush) res.flush();
|
| 236 |
}
|
| 237 |
}
|
| 238 |
return fullText;
|
| 239 |
-
} catch (e) {
|
| 240 |
-
console.warn(`[AI] ⚠️ ${providerLabel} ${modelName} Error: ${e.message}`);
|
| 241 |
-
if (isQuotaError(e)) {
|
| 242 |
-
break;
|
| 243 |
-
}
|
| 244 |
-
}
|
| 245 |
}
|
| 246 |
}
|
| 247 |
-
throw new Error("OpenRouter
|
| 248 |
}
|
| 249 |
|
| 250 |
async function streamGemma(baseParams, res) {
|
| 251 |
const { GoogleGenAI } = await import("@google/genai");
|
| 252 |
const models = ['gemma-3-27b-it', 'gemma-3-12b-it'];
|
| 253 |
const keys = await getKeyPool('gemini');
|
| 254 |
-
if (keys.length === 0) throw new Error("No keys for Gemma");
|
| 255 |
-
|
| 256 |
for (const apiKey of keys) {
|
| 257 |
const client = new GoogleGenAI({ apiKey });
|
| 258 |
for (const modelName of models) {
|
| 259 |
try {
|
| 260 |
-
console.log(`[AI] 🚀 Attempting Gemma Model: ${modelName}`);
|
| 261 |
const result = await client.models.generateContentStream({ ...baseParams, model: modelName });
|
| 262 |
-
|
| 263 |
let hasStarted = false;
|
| 264 |
let fullText = "";
|
| 265 |
for await (const chunk of result) {
|
| 266 |
-
if (!hasStarted) {
|
| 267 |
-
console.log(`[AI] ✅ Connected to Gemma: ${modelName}`);
|
| 268 |
-
recordUsage(modelName, PROVIDERS.GEMMA);
|
| 269 |
-
hasStarted = true;
|
| 270 |
-
}
|
| 271 |
if (chunk.text) {
|
| 272 |
fullText += chunk.text;
|
| 273 |
res.write(`data: ${JSON.stringify({ text: chunk.text })}\n\n`);
|
| 274 |
-
if (res.flush) res.flush();
|
| 275 |
}
|
| 276 |
}
|
| 277 |
return fullText;
|
| 278 |
-
} catch (e) {
|
| 279 |
-
console.warn(`[AI] ⚠️ Gemma ${modelName} Error: ${e.message}`);
|
| 280 |
-
if (isQuotaError(e)) continue;
|
| 281 |
-
}
|
| 282 |
}
|
| 283 |
}
|
| 284 |
-
throw new Error("Gemma
|
| 285 |
}
|
| 286 |
|
| 287 |
async function streamContentWithSmartFallback(baseParams, res) {
|
| 288 |
-
let hasAudio = false;
|
| 289 |
-
const contentsArray = Array.isArray(baseParams.contents) ? baseParams.contents : [baseParams.contents];
|
| 290 |
-
|
| 291 |
-
contentsArray.forEach(c => {
|
| 292 |
-
if (c && c.parts) {
|
| 293 |
-
c.parts.forEach(p => { if (p.inlineData && p.inlineData.mimeType.startsWith('audio/')) hasAudio = true; });
|
| 294 |
-
}
|
| 295 |
-
});
|
| 296 |
-
|
| 297 |
-
// FETCH CONFIG AND SET PROVIDER ORDER
|
| 298 |
const config = await ConfigModel.findOne({ key: 'main' });
|
| 299 |
-
const configuredOrder = config?.aiProviderOrder && config.aiProviderOrder.length > 0
|
| 300 |
-
|
| 301 |
-
: [PROVIDERS.GEMINI, PROVIDERS.OPENROUTER, PROVIDERS.GEMMA];
|
| 302 |
-
|
| 303 |
-
const runtimeSet = new Set(runtimeProviderOrder);
|
| 304 |
-
if (runtimeProviderOrder.length === 0 || runtimeProviderOrder.length !== configuredOrder.length || !configuredOrder.every(p => runtimeSet.has(p))) {
|
| 305 |
-
runtimeProviderOrder = [...configuredOrder];
|
| 306 |
-
}
|
| 307 |
|
| 308 |
let finalError = null;
|
| 309 |
-
|
| 310 |
for (const provider of runtimeProviderOrder) {
|
| 311 |
try {
|
| 312 |
-
console.log(`[AI] 👉 Trying Provider: ${provider}...`);
|
| 313 |
-
|
| 314 |
-
// Note: Unlike before, we allow Audio to pass to OpenRouter/Gemma,
|
| 315 |
-
// because those functions now have internal Fallback STT logic.
|
| 316 |
-
|
| 317 |
if (provider === PROVIDERS.GEMINI) return await streamGemini(baseParams, res);
|
| 318 |
else if (provider === PROVIDERS.OPENROUTER) return await streamOpenRouter(baseParams, res);
|
| 319 |
else if (provider === PROVIDERS.GEMMA) return await streamGemma(baseParams, res);
|
| 320 |
-
|
| 321 |
} catch (e) {
|
| 322 |
-
console.error(`[AI] ❌ Provider ${provider} Failed: ${e.message}`);
|
| 323 |
finalError = e;
|
| 324 |
-
|
| 325 |
-
if (isQuotaError(e)) {
|
| 326 |
-
console.log(`[AI] 📉 Quota/Rate Limit detected. Switching provider...`);
|
| 327 |
-
deprioritizeProvider(provider);
|
| 328 |
-
continue;
|
| 329 |
-
}
|
| 330 |
continue;
|
| 331 |
}
|
| 332 |
}
|
| 333 |
-
|
| 334 |
-
console.error(`[AI] 💀 All providers failed.`);
|
| 335 |
-
throw finalError || new Error('All streaming models unavailable.');
|
| 336 |
}
|
| 337 |
|
| 338 |
const checkAIAccess = async (req, res, next) => {
|
|
@@ -347,7 +252,6 @@ const checkAIAccess = async (req, res, next) => {
|
|
| 347 |
next();
|
| 348 |
};
|
| 349 |
|
| 350 |
-
// NEW: Endpoint to provide a temporary key for Client-Side Live API
|
| 351 |
router.get('/live-access', checkAIAccess, async (req, res) => {
|
| 352 |
try {
|
| 353 |
const keys = await getKeyPool('gemini');
|
|
@@ -374,11 +278,7 @@ router.get('/stats', checkAIAccess, async (req, res) => {
|
|
| 374 |
} catch (e) { res.status(500).json({ error: e.message }); }
|
| 375 |
});
|
| 376 |
|
| 377 |
-
router.post('/reset-pool', checkAIAccess, (req, res) => {
|
| 378 |
-
runtimeProviderOrder = [];
|
| 379 |
-
console.log('[AI] 🔄 Provider priority pool reset.');
|
| 380 |
-
res.json({ success: true });
|
| 381 |
-
});
|
| 382 |
|
| 383 |
router.post('/chat', checkAIAccess, async (req, res) => {
|
| 384 |
const { text, audio, history } = req.body;
|
|
@@ -399,7 +299,7 @@ router.post('/chat', checkAIAccess, async (req, res) => {
|
|
| 399 |
|
| 400 |
const answerText = await streamContentWithSmartFallback({
|
| 401 |
contents: fullContents,
|
| 402 |
-
config: { systemInstruction: "
|
| 403 |
}, res);
|
| 404 |
|
| 405 |
if (answerText) {
|
|
@@ -425,12 +325,10 @@ router.post('/chat', checkAIAccess, async (req, res) => {
|
|
| 425 |
}
|
| 426 |
res.write('data: [DONE]\n\n'); res.end();
|
| 427 |
} catch (e) {
|
| 428 |
-
console.error("[AI Chat Route Error]", e);
|
| 429 |
res.write(`data: ${JSON.stringify({ error: true, message: e.message })}\n\n`); res.end();
|
| 430 |
}
|
| 431 |
});
|
| 432 |
|
| 433 |
-
// STREAMING ASSESSMENT ENDPOINT
|
| 434 |
router.post('/evaluate', checkAIAccess, async (req, res) => {
|
| 435 |
const { question, audio, image, images } = req.body;
|
| 436 |
res.setHeader('Content-Type', 'text/event-stream');
|
|
@@ -440,42 +338,22 @@ router.post('/evaluate', checkAIAccess, async (req, res) => {
|
|
| 440 |
|
| 441 |
try {
|
| 442 |
res.write(`data: ${JSON.stringify({ status: 'analyzing' })}\n\n`);
|
|
|
|
|
|
|
|
|
|
|
|
|
| 443 |
|
| 444 |
-
|
| 445 |
-
if (audio) {
|
| 446 |
-
evalParts.push({ text: "学生的回答在音频中。" });
|
| 447 |
-
evalParts.push({ inlineData: { mimeType: 'audio/webm', data: audio } });
|
| 448 |
-
}
|
| 449 |
-
|
| 450 |
-
if (images && Array.isArray(images) && images.length > 0) {
|
| 451 |
-
evalParts.push({ text: "学生的回答写在以下图片中,请识别所有图片中的文字内容并进行批改:" });
|
| 452 |
-
images.forEach(img => {
|
| 453 |
-
if(img) evalParts.push({ inlineData: { mimeType: 'image/jpeg', data: img } });
|
| 454 |
-
});
|
| 455 |
-
} else if (image) {
|
| 456 |
-
evalParts.push({ text: "学生的回答写在图片中,请识别图片中的文字内容并进行批改。" });
|
| 457 |
-
evalParts.push({ inlineData: { mimeType: 'image/jpeg', data: image } });
|
| 458 |
-
}
|
| 459 |
-
|
| 460 |
-
evalParts.push({ text: `请分析:1. 内容准确性 2. 表达/书写规范。
|
| 461 |
-
必须严格按照以下格式输出(不要使用Markdown代码块包裹):
|
| 462 |
-
|
| 463 |
## Transcription
|
| 464 |
-
(
|
| 465 |
-
|
| 466 |
## Feedback
|
| 467 |
-
(
|
| 468 |
-
|
| 469 |
## Score
|
| 470 |
-
(
|
| 471 |
-
|
| 472 |
-
const fullText = await streamContentWithSmartFallback({
|
| 473 |
-
contents: [{ role: 'user', parts: evalParts }],
|
| 474 |
-
}, res);
|
| 475 |
|
|
|
|
| 476 |
const feedbackMatch = fullText.match(/## Feedback\s+([\s\S]*?)(?=## Score|$)/i);
|
| 477 |
const feedbackText = feedbackMatch ? feedbackMatch[1].trim() : "";
|
| 478 |
-
|
| 479 |
if (feedbackText) {
|
| 480 |
res.write(`data: ${JSON.stringify({ status: 'tts' })}\n\n`);
|
| 481 |
try {
|
|
@@ -498,14 +376,9 @@ router.post('/evaluate', checkAIAccess, async (req, res) => {
|
|
| 498 |
else res.write(`data: ${JSON.stringify({ ttsSkipped: true })}\n\n`);
|
| 499 |
} catch (ttsErr) { res.write(`data: ${JSON.stringify({ ttsSkipped: true })}\n\n`); }
|
| 500 |
}
|
| 501 |
-
|
| 502 |
-
res.write('data: [DONE]\n\n');
|
| 503 |
-
res.end();
|
| 504 |
-
|
| 505 |
} catch (e) {
|
| 506 |
-
|
| 507 |
-
res.write(`data: ${JSON.stringify({ error: true, message: e.message || "Evaluation failed" })}\n\n`);
|
| 508 |
-
res.end();
|
| 509 |
}
|
| 510 |
});
|
| 511 |
|
|
|
|
| 29 |
} catch (e) { console.error("Failed to record AI usage stats:", e); }
|
| 30 |
}
|
| 31 |
|
| 32 |
+
// Fallback Speech-to-Text using Hugging Face (Fixed 410 Error)
|
| 33 |
async function transcribeAudioWithHF(audioBase64) {
|
| 34 |
const token = await getHFToken();
|
| 35 |
if (!token) {
|
|
|
|
| 38 |
}
|
| 39 |
|
| 40 |
try {
|
| 41 |
+
console.log("[AI] 🎤 Using Hugging Face ASR (Whisper v3)...");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
const buffer = Buffer.from(audioBase64, 'base64');
|
| 43 |
+
|
| 44 |
+
// Using the standard inference endpoint which is more stable
|
| 45 |
+
const response = await fetch(
|
| 46 |
"https://api-inference.huggingface.co/models/openai/whisper-large-v3",
|
| 47 |
{
|
| 48 |
headers: {
|
| 49 |
Authorization: `Bearer ${token}`,
|
|
|
|
| 50 |
},
|
| 51 |
method: "POST",
|
| 52 |
body: buffer,
|
| 53 |
}
|
| 54 |
);
|
| 55 |
|
| 56 |
+
if (!response.ok) {
|
| 57 |
+
const errText = await response.text();
|
| 58 |
+
if (response.status === 503) {
|
| 59 |
+
console.log("[AI] HF Model loading, retrying...");
|
| 60 |
+
await new Promise(r => setTimeout(r, 3000));
|
| 61 |
+
return transcribeAudioWithHF(audioBase64);
|
| 62 |
+
}
|
| 63 |
+
throw new Error(`HF API Error: ${response.status} ${errText}`);
|
| 64 |
}
|
| 65 |
|
| 66 |
+
const result = await response.json();
|
| 67 |
+
console.log("[AI] HF Transcribed:", result.text);
|
| 68 |
return result.text;
|
| 69 |
} catch (e) {
|
| 70 |
console.error("[AI] HF STT Failed:", e.message);
|
|
|
|
| 75 |
function convertGeminiToOpenAI(baseParams) {
|
| 76 |
const messages = [];
|
| 77 |
if (baseParams.config?.systemInstruction) messages.push({ role: 'system', content: baseParams.config.systemInstruction });
|
|
|
|
| 78 |
let contents = baseParams.contents;
|
| 79 |
+
if (contents && !Array.isArray(contents)) contents = [contents];
|
|
|
|
|
|
|
| 80 |
|
| 81 |
if (contents && Array.isArray(contents)) {
|
| 82 |
contents.forEach(content => {
|
| 83 |
let role = (content.role === 'model' || content.role === 'assistant') ? 'assistant' : 'user';
|
|
|
|
| 84 |
const messageContent = [];
|
| 85 |
if (content.parts) {
|
| 86 |
content.parts.forEach(p => {
|
|
|
|
| 89 |
if (p.inlineData.mimeType.startsWith('image/')) {
|
| 90 |
messageContent.push({ type: 'image_url', image_url: { url: `data:${p.inlineData.mimeType};base64,${p.inlineData.data}` } });
|
| 91 |
} else if (p.inlineData.mimeType.startsWith('audio/')) {
|
|
|
|
| 92 |
messageContent.push({ type: 'audio_base64', data: p.inlineData.data });
|
| 93 |
}
|
| 94 |
}
|
| 95 |
});
|
| 96 |
}
|
|
|
|
| 97 |
if (messageContent.length > 0) {
|
| 98 |
if (messageContent.length === 1 && messageContent[0].type === 'text') {
|
| 99 |
messages.push({ role: role, content: messageContent[0].text });
|
|
|
|
| 109 |
const PROVIDERS = { GEMINI: 'GEMINI', OPENROUTER: 'OPENROUTER', GEMMA: 'GEMMA' };
|
| 110 |
const DEFAULT_OPENROUTER_MODELS = ['qwen/qwen3-coder:free', 'openai/gpt-oss-120b:free', 'qwen/qwen3-235b-a22b:free', 'tngtech/deepseek-r1t-chimera:free'];
|
| 111 |
|
|
|
|
| 112 |
let runtimeProviderOrder = [];
|
| 113 |
|
| 114 |
function deprioritizeProvider(providerName) {
|
| 115 |
if (runtimeProviderOrder.length > 0 && runtimeProviderOrder[runtimeProviderOrder.length - 1] === providerName) return;
|
|
|
|
| 116 |
runtimeProviderOrder = runtimeProviderOrder.filter(p => p !== providerName).concat(providerName);
|
|
|
|
| 117 |
}
|
| 118 |
|
| 119 |
function isQuotaError(e) {
|
|
|
|
| 121 |
return e.status === 429 || e.status === 503 || msg.includes('quota') || msg.includes('overloaded') || msg.includes('resource_exhausted') || msg.includes('rate limit') || msg.includes('credits');
|
| 122 |
}
|
| 123 |
|
|
|
|
| 124 |
async function streamGemini(baseParams, res) {
|
| 125 |
const { GoogleGenAI } = await import("@google/genai");
|
| 126 |
const models = ['gemini-2.5-flash', 'gemini-2.5-flash-lite'];
|
|
|
|
| 131 |
const client = new GoogleGenAI({ apiKey });
|
| 132 |
for (const modelName of models) {
|
| 133 |
try {
|
|
|
|
| 134 |
const result = await client.models.generateContentStream({ ...baseParams, model: modelName });
|
|
|
|
| 135 |
let hasStarted = false;
|
| 136 |
let fullText = "";
|
|
|
|
| 137 |
for await (const chunk of result) {
|
| 138 |
+
if (!hasStarted) { recordUsage(modelName, PROVIDERS.GEMINI); hasStarted = true; }
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
if (chunk.text) {
|
| 140 |
fullText += chunk.text;
|
| 141 |
res.write(`data: ${JSON.stringify({ text: chunk.text })}\n\n`);
|
|
|
|
| 142 |
}
|
| 143 |
}
|
| 144 |
return fullText;
|
| 145 |
+
} catch (e) { if (isQuotaError(e)) continue; throw e; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
}
|
| 147 |
}
|
| 148 |
+
throw new Error("Gemini exhausted");
|
| 149 |
}
|
| 150 |
|
| 151 |
async function streamOpenRouter(baseParams, res) {
|
|
|
|
| 155 |
const keys = await getKeyPool('openrouter');
|
| 156 |
if (keys.length === 0) throw new Error("No OpenRouter API keys");
|
| 157 |
|
|
|
|
|
|
|
|
|
|
| 158 |
for (let msg of messages) {
|
| 159 |
if (Array.isArray(msg.content)) {
|
| 160 |
for (let part of msg.content) {
|
| 161 |
if (part.type === 'audio_base64') {
|
|
|
|
|
|
|
| 162 |
const text = await transcribeAudioWithHF(part.data);
|
| 163 |
if (text) {
|
| 164 |
part.type = 'text';
|
| 165 |
+
part.text = `[语音转文字]: ${text}`;
|
| 166 |
+
delete part.data;
|
| 167 |
} else {
|
| 168 |
+
throw new Error("语音转文字失败 (请检查 HF Token)");
|
| 169 |
}
|
| 170 |
}
|
| 171 |
}
|
|
|
|
| 176 |
for (const modelName of models) {
|
| 177 |
const modelConfig = config?.openRouterModels?.find(m => m.id === modelName);
|
| 178 |
const baseURL = modelConfig?.apiUrl ? modelConfig.apiUrl : "https://openrouter.ai/api/v1";
|
| 179 |
+
const client = new OpenAI({ baseURL, apiKey });
|
|
|
|
|
|
|
|
|
|
| 180 |
try {
|
|
|
|
|
|
|
| 181 |
const stream = await client.chat.completions.create({ model: modelName, messages, stream: true });
|
|
|
|
|
|
|
| 182 |
recordUsage(modelName, PROVIDERS.OPENROUTER);
|
|
|
|
| 183 |
let fullText = '';
|
| 184 |
for await (const chunk of stream) {
|
| 185 |
const text = chunk.choices[0]?.delta?.content || '';
|
| 186 |
if (text) {
|
| 187 |
fullText += text;
|
| 188 |
res.write(`data: ${JSON.stringify({ text: text })}\n\n`);
|
|
|
|
| 189 |
}
|
| 190 |
}
|
| 191 |
return fullText;
|
| 192 |
+
} catch (e) { if (isQuotaError(e)) break; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
}
|
| 194 |
}
|
| 195 |
+
throw new Error("OpenRouter exhausted");
|
| 196 |
}
|
| 197 |
|
| 198 |
async function streamGemma(baseParams, res) {
|
| 199 |
const { GoogleGenAI } = await import("@google/genai");
|
| 200 |
const models = ['gemma-3-27b-it', 'gemma-3-12b-it'];
|
| 201 |
const keys = await getKeyPool('gemini');
|
|
|
|
|
|
|
| 202 |
for (const apiKey of keys) {
|
| 203 |
const client = new GoogleGenAI({ apiKey });
|
| 204 |
for (const modelName of models) {
|
| 205 |
try {
|
|
|
|
| 206 |
const result = await client.models.generateContentStream({ ...baseParams, model: modelName });
|
|
|
|
| 207 |
let hasStarted = false;
|
| 208 |
let fullText = "";
|
| 209 |
for await (const chunk of result) {
|
| 210 |
+
if (!hasStarted) { recordUsage(modelName, PROVIDERS.GEMMA); hasStarted = true; }
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
if (chunk.text) {
|
| 212 |
fullText += chunk.text;
|
| 213 |
res.write(`data: ${JSON.stringify({ text: chunk.text })}\n\n`);
|
|
|
|
| 214 |
}
|
| 215 |
}
|
| 216 |
return fullText;
|
| 217 |
+
} catch (e) { if (isQuotaError(e)) continue; }
|
|
|
|
|
|
|
|
|
|
| 218 |
}
|
| 219 |
}
|
| 220 |
+
throw new Error("Gemma failed");
|
| 221 |
}
|
| 222 |
|
| 223 |
async function streamContentWithSmartFallback(baseParams, res) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
const config = await ConfigModel.findOne({ key: 'main' });
|
| 225 |
+
const configuredOrder = config?.aiProviderOrder && config.aiProviderOrder.length > 0 ? config.aiProviderOrder : [PROVIDERS.GEMINI, PROVIDERS.OPENROUTER, PROVIDERS.GEMMA];
|
| 226 |
+
if (runtimeProviderOrder.length === 0) runtimeProviderOrder = [...configuredOrder];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 227 |
|
| 228 |
let finalError = null;
|
|
|
|
| 229 |
for (const provider of runtimeProviderOrder) {
|
| 230 |
try {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
if (provider === PROVIDERS.GEMINI) return await streamGemini(baseParams, res);
|
| 232 |
else if (provider === PROVIDERS.OPENROUTER) return await streamOpenRouter(baseParams, res);
|
| 233 |
else if (provider === PROVIDERS.GEMMA) return await streamGemma(baseParams, res);
|
|
|
|
| 234 |
} catch (e) {
|
|
|
|
| 235 |
finalError = e;
|
| 236 |
+
if (isQuotaError(e)) { deprioritizeProvider(provider); continue; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
continue;
|
| 238 |
}
|
| 239 |
}
|
| 240 |
+
throw finalError || new Error('All failed');
|
|
|
|
|
|
|
| 241 |
}
|
| 242 |
|
| 243 |
const checkAIAccess = async (req, res, next) => {
|
|
|
|
| 252 |
next();
|
| 253 |
};
|
| 254 |
|
|
|
|
| 255 |
router.get('/live-access', checkAIAccess, async (req, res) => {
|
| 256 |
try {
|
| 257 |
const keys = await getKeyPool('gemini');
|
|
|
|
| 278 |
} catch (e) { res.status(500).json({ error: e.message }); }
|
| 279 |
});
|
| 280 |
|
| 281 |
+
router.post('/reset-pool', checkAIAccess, (req, res) => { runtimeProviderOrder = []; res.json({ success: true }); });
|
|
|
|
|
|
|
|
|
|
|
|
|
| 282 |
|
| 283 |
router.post('/chat', checkAIAccess, async (req, res) => {
|
| 284 |
const { text, audio, history } = req.body;
|
|
|
|
| 299 |
|
| 300 |
const answerText = await streamContentWithSmartFallback({
|
| 301 |
contents: fullContents,
|
| 302 |
+
config: { systemInstruction: "你是一位友善且知识渊博的中小学AI助教。请用简洁、鼓励性的语言回答。支持 Markdown。" }
|
| 303 |
}, res);
|
| 304 |
|
| 305 |
if (answerText) {
|
|
|
|
| 325 |
}
|
| 326 |
res.write('data: [DONE]\n\n'); res.end();
|
| 327 |
} catch (e) {
|
|
|
|
| 328 |
res.write(`data: ${JSON.stringify({ error: true, message: e.message })}\n\n`); res.end();
|
| 329 |
}
|
| 330 |
});
|
| 331 |
|
|
|
|
| 332 |
router.post('/evaluate', checkAIAccess, async (req, res) => {
|
| 333 |
const { question, audio, image, images } = req.body;
|
| 334 |
res.setHeader('Content-Type', 'text/event-stream');
|
|
|
|
| 338 |
|
| 339 |
try {
|
| 340 |
res.write(`data: ${JSON.stringify({ status: 'analyzing' })}\n\n`);
|
| 341 |
+
const evalParts = [{ text: `请对学生的回答评分。题目:${question}。` }];
|
| 342 |
+
if (audio) evalParts.push({ inlineData: { mimeType: 'audio/webm', data: audio } });
|
| 343 |
+
if (images && Array.isArray(images)) images.forEach(img => { if(img) evalParts.push({ inlineData: { mimeType: 'image/jpeg', data: img } }); });
|
| 344 |
+
else if (image) evalParts.push({ inlineData: { mimeType: 'image/jpeg', data: image } });
|
| 345 |
|
| 346 |
+
evalParts.push({ text: `格式:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 347 |
## Transcription
|
| 348 |
+
(内容)
|
|
|
|
| 349 |
## Feedback
|
| 350 |
+
(评语)
|
|
|
|
| 351 |
## Score
|
| 352 |
+
(0-100数字)` });
|
|
|
|
|
|
|
|
|
|
|
|
|
| 353 |
|
| 354 |
+
const fullText = await streamContentWithSmartFallback({ contents: [{ role: 'user', parts: evalParts }] }, res);
|
| 355 |
const feedbackMatch = fullText.match(/## Feedback\s+([\s\S]*?)(?=## Score|$)/i);
|
| 356 |
const feedbackText = feedbackMatch ? feedbackMatch[1].trim() : "";
|
|
|
|
| 357 |
if (feedbackText) {
|
| 358 |
res.write(`data: ${JSON.stringify({ status: 'tts' })}\n\n`);
|
| 359 |
try {
|
|
|
|
| 376 |
else res.write(`data: ${JSON.stringify({ ttsSkipped: true })}\n\n`);
|
| 377 |
} catch (ttsErr) { res.write(`data: ${JSON.stringify({ ttsSkipped: true })}\n\n`); }
|
| 378 |
}
|
| 379 |
+
res.write('data: [DONE]\n\n'); res.end();
|
|
|
|
|
|
|
|
|
|
| 380 |
} catch (e) {
|
| 381 |
+
res.write(`data: ${JSON.stringify({ error: true, message: e.message })}\n\n`); res.end();
|
|
|
|
|
|
|
| 382 |
}
|
| 383 |
});
|
| 384 |
|
components/ai/AssessmentPanel.tsx
CHANGED
|
@@ -16,17 +16,8 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 16 |
const [isAssessmentRecording, setIsAssessmentRecording] = useState(false);
|
| 17 |
const [isWebSpeechListening, setIsWebSpeechListening] = useState(false);
|
| 18 |
const [assessmentStatus, setAssessmentStatus] = useState<'IDLE' | 'UPLOADING' | 'ANALYZING' | 'TTS'>('IDLE');
|
| 19 |
-
|
| 20 |
-
// For Web Speech Text Accumulation
|
| 21 |
const [recognizedText, setRecognizedText] = useState('');
|
| 22 |
-
|
| 23 |
-
const [streamedAssessment, setStreamedAssessment] = useState<{
|
| 24 |
-
transcription: string;
|
| 25 |
-
feedback: string;
|
| 26 |
-
score: number | null;
|
| 27 |
-
audio?: string;
|
| 28 |
-
}>({ transcription: '', feedback: '', score: null });
|
| 29 |
-
|
| 30 |
const [toast, setToast] = useState<ToastState>({ show: false, message: '', type: 'success' });
|
| 31 |
|
| 32 |
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
|
@@ -35,14 +26,12 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 35 |
const currentSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
| 36 |
const recognitionRef = useRef<any>(null);
|
| 37 |
|
| 38 |
-
// Initialize AudioContext
|
| 39 |
useEffect(() => {
|
| 40 |
// @ts-ignore
|
| 41 |
const AudioCtor = window.AudioContext || window.webkitAudioContext;
|
| 42 |
audioContextRef.current = new AudioCtor();
|
| 43 |
return () => {
|
| 44 |
stopPlayback();
|
| 45 |
-
window.speechSynthesis.cancel();
|
| 46 |
if (recognitionRef.current) recognitionRef.current.abort();
|
| 47 |
};
|
| 48 |
}, []);
|
|
@@ -58,13 +47,8 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 58 |
const speakWithBrowser = (text: string) => {
|
| 59 |
if (!text) return;
|
| 60 |
stopPlayback();
|
| 61 |
-
const
|
| 62 |
-
const utterance = new SpeechSynthesisUtterance(cleanText);
|
| 63 |
utterance.lang = 'zh-CN';
|
| 64 |
-
utterance.rate = 1.0;
|
| 65 |
-
const voices = window.speechSynthesis.getVoices();
|
| 66 |
-
const zhVoice = voices.find(v => v.lang === 'zh-CN' && !v.name.includes('Hong Kong') && !v.name.includes('Taiwan'));
|
| 67 |
-
if (zhVoice) utterance.voice = zhVoice;
|
| 68 |
window.speechSynthesis.speak(utterance);
|
| 69 |
};
|
| 70 |
|
|
@@ -76,9 +60,6 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 76 |
const AudioCtor = window.AudioContext || window.webkitAudioContext;
|
| 77 |
audioContextRef.current = new AudioCtor();
|
| 78 |
}
|
| 79 |
-
if (audioContextRef.current?.state === 'suspended') {
|
| 80 |
-
await audioContextRef.current.resume();
|
| 81 |
-
}
|
| 82 |
const bytes = base64ToUint8Array(base64Audio);
|
| 83 |
const audioBuffer = decodePCM(bytes, audioContextRef.current!);
|
| 84 |
const source = audioContextRef.current!.createBufferSource();
|
|
@@ -86,13 +67,11 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 86 |
source.connect(audioContextRef.current!.destination);
|
| 87 |
source.start(0);
|
| 88 |
currentSourceRef.current = source;
|
| 89 |
-
} catch (e) {
|
| 90 |
-
console.error("Audio playback error", e);
|
| 91 |
-
setToast({ show: true, message: '语音播放失败', type: 'error' });
|
| 92 |
-
}
|
| 93 |
};
|
| 94 |
|
| 95 |
const startRecording = async () => {
|
|
|
|
| 96 |
// @ts-ignore
|
| 97 |
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
|
| 98 |
if (SpeechRecognition) {
|
|
@@ -101,6 +80,7 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 101 |
const recognition = new SpeechRecognition();
|
| 102 |
recognition.lang = 'zh-CN';
|
| 103 |
recognition.interimResults = true;
|
|
|
|
| 104 |
|
| 105 |
recognition.onstart = () => {
|
| 106 |
setIsWebSpeechListening(true);
|
|
@@ -113,36 +93,21 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 113 |
for (let i = event.resultIndex; i < event.results.length; ++i) {
|
| 114 |
if (event.results[i].isFinal) final += event.results[i][0].transcript;
|
| 115 |
}
|
| 116 |
-
if (final)
|
| 117 |
-
setRecognizedText(prev => prev + final);
|
| 118 |
-
// For assessment, we usually want full context, so keep appending
|
| 119 |
-
}
|
| 120 |
-
};
|
| 121 |
-
|
| 122 |
-
recognition.onend = () => {
|
| 123 |
-
// Logic to handle stop: if it stopped but user is still holding button?
|
| 124 |
-
// Typically Web Speech stops on silence. We rely on stopRecording() to finalize.
|
| 125 |
-
// If it stopped automatically, we might just restart or consider it done.
|
| 126 |
-
// For now, let's update state but not trigger submit yet unless triggered by user.
|
| 127 |
-
setIsWebSpeechListening(false);
|
| 128 |
};
|
| 129 |
|
| 130 |
recognition.onerror = (e: any) => {
|
| 131 |
-
console.warn("Web Speech
|
| 132 |
-
|
| 133 |
-
if (e.error !== 'aborted')
|
| 134 |
-
startAudioRecordingFallback();
|
| 135 |
-
} else {
|
| 136 |
-
setIsWebSpeechListening(false);
|
| 137 |
-
setIsAssessmentRecording(false);
|
| 138 |
-
}
|
| 139 |
};
|
| 140 |
|
|
|
|
| 141 |
recognitionRef.current = recognition;
|
| 142 |
recognition.start();
|
| 143 |
return;
|
| 144 |
} catch (e) {
|
| 145 |
-
console.
|
| 146 |
startAudioRecordingFallback();
|
| 147 |
}
|
| 148 |
} else {
|
|
@@ -156,13 +121,15 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 156 |
const mediaRecorder = new MediaRecorder(stream);
|
| 157 |
mediaRecorderRef.current = mediaRecorder;
|
| 158 |
audioChunksRef.current = [];
|
| 159 |
-
|
| 160 |
mediaRecorder.ondataavailable = (event) => {
|
| 161 |
-
if (event.data.size > 0)
|
| 162 |
-
|
| 163 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
};
|
| 165 |
-
|
| 166 |
mediaRecorder.start();
|
| 167 |
setIsAssessmentRecording(true);
|
| 168 |
setIsWebSpeechListening(false);
|
|
@@ -173,35 +140,11 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 173 |
|
| 174 |
const stopRecording = () => {
|
| 175 |
setIsAssessmentRecording(false);
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
if (
|
| 179 |
-
// Wait slightly for final results then submit
|
| 180 |
-
setTimeout(() => {
|
| 181 |
-
if (recognizedText) {
|
| 182 |
-
// Send text as prompt supplement since we don't have audio
|
| 183 |
-
// Construct a virtual "audio" payload which is just text instructions for the backend to handle?
|
| 184 |
-
// No, backend expects audio or image. We should modify backend to accept text for assessment too.
|
| 185 |
-
// Actually, simpler: backend takes 'audio' or 'image'.
|
| 186 |
-
// Let's modify frontend to send `audio` as undefined but include text in prompt.
|
| 187 |
-
|
| 188 |
-
// Hack: Prepend "学生回答内容:..." to the question if we have text but no audio.
|
| 189 |
-
// Or, add a `studentAnswerText` field to the API.
|
| 190 |
-
// Let's modify the handleAssessmentStreamingSubmit to take text.
|
| 191 |
-
handleAssessmentStreamingSubmit({ text: recognizedText });
|
| 192 |
-
} else {
|
| 193 |
-
setToast({ show: true, message: '未检测到语音输入', type: 'error' });
|
| 194 |
-
}
|
| 195 |
-
}, 500);
|
| 196 |
} else if (mediaRecorderRef.current) {
|
| 197 |
mediaRecorderRef.current.stop();
|
| 198 |
-
|
| 199 |
-
mediaRecorderRef.current.onstop = async () => {
|
| 200 |
-
const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' });
|
| 201 |
-
const base64 = await blobToBase64(audioBlob);
|
| 202 |
-
handleAssessmentStreamingSubmit({ audio: base64 });
|
| 203 |
-
mediaRecorderRef.current?.stream.getTracks().forEach(track => track.stop());
|
| 204 |
-
};
|
| 205 |
}
|
| 206 |
};
|
| 207 |
|
|
@@ -211,11 +154,8 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 211 |
stopPlayback();
|
| 212 |
|
| 213 |
try {
|
| 214 |
-
// If we have text from Web Speech, augment the question
|
| 215 |
let finalQuestion = assessmentTopic;
|
| 216 |
-
if (text) {
|
| 217 |
-
finalQuestion += `\n\n学生口述回答内容:${text}\n(请基于此文本进行评分,忽略音频缺失)`;
|
| 218 |
-
}
|
| 219 |
|
| 220 |
const response = await fetch('/api/ai/evaluate', {
|
| 221 |
method: 'POST',
|
|
@@ -229,11 +169,9 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 229 |
});
|
| 230 |
|
| 231 |
if (!response.ok) throw new Error(response.statusText);
|
| 232 |
-
if (!response.body) throw new Error('No response body');
|
| 233 |
-
|
| 234 |
setAssessmentStatus('ANALYZING');
|
| 235 |
|
| 236 |
-
const reader = response.body
|
| 237 |
const decoder = new TextDecoder();
|
| 238 |
let accumulatedRaw = '';
|
| 239 |
let buffer = '';
|
|
@@ -241,7 +179,6 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 241 |
while (true) {
|
| 242 |
const { done, value } = await reader.read();
|
| 243 |
if (done) break;
|
| 244 |
-
|
| 245 |
buffer += decoder.decode(value, { stream: true });
|
| 246 |
const parts = buffer.split('\n\n');
|
| 247 |
buffer = parts.pop() || '';
|
|
@@ -250,72 +187,38 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 250 |
if (line.startsWith('data: ')) {
|
| 251 |
const jsonStr = line.replace('data: ', '').trim();
|
| 252 |
if (jsonStr === '[DONE]') break;
|
| 253 |
-
|
| 254 |
try {
|
| 255 |
const data = JSON.parse(jsonStr);
|
| 256 |
-
|
| 257 |
-
if (data.status) {
|
| 258 |
-
if (data.status === 'analyzing') setAssessmentStatus('ANALYZING');
|
| 259 |
-
if (data.status === 'tts') setAssessmentStatus('TTS');
|
| 260 |
-
}
|
| 261 |
-
|
| 262 |
if (data.text) {
|
| 263 |
accumulatedRaw += data.text;
|
| 264 |
-
const
|
| 265 |
-
const
|
| 266 |
-
const
|
| 267 |
-
|
| 268 |
setStreamedAssessment(prev => ({
|
| 269 |
...prev,
|
| 270 |
-
transcription:
|
| 271 |
-
feedback:
|
| 272 |
-
score:
|
| 273 |
}));
|
| 274 |
}
|
| 275 |
-
|
| 276 |
if (data.audio) {
|
| 277 |
setStreamedAssessment(prev => ({ ...prev, audio: data.audio }));
|
| 278 |
playPCMAudio(data.audio);
|
| 279 |
}
|
| 280 |
-
|
| 281 |
if (data.ttsSkipped) {
|
| 282 |
const fb = streamedAssessment.feedback || accumulatedRaw.match(/## Feedback\s+([\s\S]*?)(?=## Score|$)/i)?.[1] || '';
|
| 283 |
if (fb) speakWithBrowser(fb);
|
| 284 |
}
|
| 285 |
-
|
| 286 |
-
if (data.error) {
|
| 287 |
-
setToast({ show: true, message: data.message || '评分出错', type: 'error' });
|
| 288 |
-
}
|
| 289 |
-
|
| 290 |
} catch (e) {}
|
| 291 |
}
|
| 292 |
}
|
| 293 |
}
|
| 294 |
setAssessmentStatus('IDLE');
|
| 295 |
-
|
| 296 |
} catch (error: any) {
|
| 297 |
-
|
| 298 |
-
setToast({ show: true, message: '评分失败: ' + error.message, type: 'error' });
|
| 299 |
-
setAssessmentStatus('IDLE');
|
| 300 |
-
}
|
| 301 |
-
};
|
| 302 |
-
|
| 303 |
-
const handleImageUpload = (e: React.ChangeEvent<HTMLInputElement>) => {
|
| 304 |
-
if (e.target.files && e.target.files.length > 0) {
|
| 305 |
-
setSelectedImages(prev => [...prev, ...Array.from(e.target.files!)]);
|
| 306 |
-
}
|
| 307 |
-
};
|
| 308 |
-
|
| 309 |
-
const confirmImageSubmission = async () => {
|
| 310 |
-
if (selectedImages.length === 0) return;
|
| 311 |
-
setAssessmentStatus('UPLOADING');
|
| 312 |
-
try {
|
| 313 |
-
const base64Promises = selectedImages.map(file => compressImage(file));
|
| 314 |
-
const base64Images = await Promise.all(base64Promises);
|
| 315 |
-
handleAssessmentStreamingSubmit({ images: base64Images });
|
| 316 |
-
} catch(e) {
|
| 317 |
setAssessmentStatus('IDLE');
|
| 318 |
-
setToast({ show: true, message: '图片压缩上传失败', type: 'error' });
|
| 319 |
}
|
| 320 |
};
|
| 321 |
|
|
@@ -325,7 +228,6 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 325 |
<button onClick={stopPlayback} className="absolute top-4 right-4 z-50 bg-white/80 backdrop-blur p-2 rounded-full shadow-md text-red-500 hover:bg-white border border-gray-200" title="停止播放"><StopIcon size={20}/></button>
|
| 326 |
|
| 327 |
<div className="max-w-3xl mx-auto space-y-6">
|
| 328 |
-
{/* Topic Card */}
|
| 329 |
<div className="bg-white p-6 rounded-2xl border border-purple-100 shadow-sm">
|
| 330 |
<h3 className="text-lg font-bold text-gray-800 mb-2 flex items-center justify-between">
|
| 331 |
<span className="flex items-center"><Brain className="mr-2 text-purple-600"/> 今日测评题目</span>
|
|
@@ -335,111 +237,48 @@ export const AssessmentPanel: React.FC<AssessmentPanelProps> = ({ currentUser })
|
|
| 335 |
</div>
|
| 336 |
</h3>
|
| 337 |
<textarea className="w-full bg-purple-50/50 border border-purple-100 rounded-xl p-4 text-gray-700 font-medium text-lg resize-none focus:ring-2 focus:ring-purple-200 outline-none" value={assessmentTopic} onChange={e => setAssessmentTopic(e.target.value)} rows={3}/>
|
| 338 |
-
|
| 339 |
<div className="mt-6 flex justify-center">
|
| 340 |
{assessmentMode === 'audio' ? (
|
| 341 |
-
<button
|
| 342 |
-
onMouseDown={startRecording} onMouseUp={stopRecording} onTouchStart={startRecording} onTouchEnd={stopRecording}
|
| 343 |
-
disabled={assessmentStatus !== 'IDLE'}
|
| 344 |
-
className={`px-8 py-4 rounded-full font-bold text-white flex items-center gap-3 shadow-lg transition-all ${isAssessmentRecording ? 'bg-red-500 scale-105' : 'bg-gradient-to-r from-purple-600 to-indigo-600 hover:shadow-purple-200 hover:scale-105 disabled:opacity-50'}`}
|
| 345 |
-
>
|
| 346 |
{assessmentStatus !== 'IDLE' ? <Loader2 className="animate-spin"/> : (isAssessmentRecording ? <StopCircle/> : <Mic/>)}
|
| 347 |
-
{assessmentStatus === 'UPLOADING' ? '上传中...' : assessmentStatus === 'ANALYZING' ? 'AI
|
| 348 |
</button>
|
| 349 |
) : (
|
| 350 |
<div className="w-full">
|
| 351 |
<div className="relative border-2 border-dashed border-purple-200 rounded-xl p-4 text-center hover:bg-purple-50 transition-colors cursor-pointer min-h-[160px] flex flex-col items-center justify-center">
|
| 352 |
-
<input
|
| 353 |
-
|
| 354 |
-
accept="image/*"
|
| 355 |
-
multiple
|
| 356 |
-
className="absolute inset-0 opacity-0 cursor-pointer w-full h-full z-10"
|
| 357 |
-
onChange={handleImageUpload}
|
| 358 |
-
onClick={(e) => (e.currentTarget.value = '')}
|
| 359 |
-
/>
|
| 360 |
-
{selectedImages.length === 0 ? (
|
| 361 |
-
<>
|
| 362 |
-
<ImageIcon className="mx-auto text-purple-300 mb-2" size={40}/>
|
| 363 |
-
<p className="text-purple-600 font-bold">点击上传作业图片</p>
|
| 364 |
-
<p className="text-xs text-gray-400">支持批量上传 • 自动压缩处理</p>
|
| 365 |
-
</>
|
| 366 |
-
) : (
|
| 367 |
-
<div className="z-0 w-full pointer-events-none opacity-50 flex items-center justify-center">
|
| 368 |
-
<Plus className="text-purple-300" size={40}/>
|
| 369 |
-
<span className="text-purple-400 font-bold ml-2">继续添加图片</span>
|
| 370 |
-
</div>
|
| 371 |
-
)}
|
| 372 |
</div>
|
| 373 |
-
|
| 374 |
{selectedImages.length > 0 && (
|
| 375 |
<div className="mt-4 grid grid-cols-3 sm:grid-cols-4 gap-3 animate-in fade-in">
|
| 376 |
-
{selectedImages.map((file, idx) => (
|
| 377 |
-
|
| 378 |
-
<img src={URL.createObjectURL(file)} className="w-full h-full object-cover rounded-lg shadow-sm border border-gray-200" />
|
| 379 |
-
<button
|
| 380 |
-
onClick={() => setSelectedImages(prev => prev.filter((_, i) => i !== idx))}
|
| 381 |
-
className="absolute -top-2 -right-2 bg-red-500 text-white rounded-full p-1 shadow-md hover:bg-red-600 transition-colors z-20 scale-90 hover:scale-100"
|
| 382 |
-
>
|
| 383 |
-
<X size={14}/>
|
| 384 |
-
</button>
|
| 385 |
-
</div>
|
| 386 |
-
))}
|
| 387 |
</div>
|
| 388 |
)}
|
| 389 |
-
|
| 390 |
-
{selectedImages.length > 0 && (
|
| 391 |
-
<button
|
| 392 |
-
onClick={confirmImageSubmission}
|
| 393 |
-
disabled={assessmentStatus !== 'IDLE'}
|
| 394 |
-
className="mt-6 w-full px-8 py-3 bg-purple-600 text-white rounded-lg font-bold hover:bg-purple-700 flex items-center justify-center gap-2 shadow-md transition-all"
|
| 395 |
-
>
|
| 396 |
-
{assessmentStatus !== 'IDLE' ? <Loader2 className="animate-spin" size={18}/> : <CheckCircle size={18}/>}
|
| 397 |
-
{assessmentStatus === 'UPLOADING' ? '压缩上传中...' : assessmentStatus === 'ANALYZING' ? 'AI 正在分析...' : assessmentStatus === 'TTS' ? '生成语音...' : `开始批改 (${selectedImages.length}张)`}
|
| 398 |
-
</button>
|
| 399 |
-
)}
|
| 400 |
</div>
|
| 401 |
)}
|
| 402 |
</div>
|
| 403 |
</div>
|
| 404 |
-
|
| 405 |
-
{/* Streamed Result Card */}
|
| 406 |
{(streamedAssessment.transcription || streamedAssessment.feedback || streamedAssessment.score !== null) && (
|
| 407 |
<div className="bg-white p-6 rounded-2xl border border-gray-200 shadow-lg animate-in slide-in-from-bottom-4">
|
| 408 |
<div className="flex items-center justify-between border-b border-gray-100 pb-4 mb-4">
|
| 409 |
<div className="flex items-center gap-2">
|
| 410 |
<h3 className="font-bold text-xl text-gray-800">测评报告</h3>
|
| 411 |
-
{assessmentStatus !== 'IDLE' &&
|
| 412 |
-
<div className="flex items-center gap-1 text-xs px-2 py-1 bg-purple-50 text-purple-600 rounded-full animate-pulse">
|
| 413 |
-
<Zap size={12}/>
|
| 414 |
-
{assessmentStatus === 'ANALYZING' ? '正在智能分析内容...' : assessmentStatus === 'TTS' ? '正在生成语音点评...' : '处理中...'}
|
| 415 |
-
</div>
|
| 416 |
-
)}
|
| 417 |
</div>
|
| 418 |
<div className="flex items-center gap-4">
|
| 419 |
-
{streamedAssessment.audio && (
|
| 420 |
-
|
| 421 |
-
<Volume2 size={16}/> 听AI点评
|
| 422 |
-
</button>
|
| 423 |
-
)}
|
| 424 |
-
{streamedAssessment.score !== null ? (
|
| 425 |
-
<div className={`text-3xl font-black ${streamedAssessment.score >= 80 ? 'text-green-500' : streamedAssessment.score >= 60 ? 'text-yellow-500' : 'text-red-500'}`}>
|
| 426 |
-
{streamedAssessment.score}<span className="text-sm text-gray-400 ml-1">分</span>
|
| 427 |
-
</div>
|
| 428 |
-
) : (
|
| 429 |
-
<div className="text-sm text-gray-400 italic">评分中...</div>
|
| 430 |
-
)}
|
| 431 |
</div>
|
| 432 |
</div>
|
| 433 |
<div className="space-y-4">
|
| 434 |
<div className="bg-gray-50 p-4 rounded-xl">
|
| 435 |
<p className="text-xs font-bold text-gray-500 uppercase mb-1">AI 识别内容</p>
|
| 436 |
-
<p className="text-gray-700 leading-relaxed text-sm whitespace-pre-wrap">{streamedAssessment.transcription ||
|
| 437 |
</div>
|
| 438 |
<div>
|
| 439 |
<p className="text-xs font-bold text-gray-500 uppercase mb-2">AI 点评建议</p>
|
| 440 |
-
<div className="p-4 bg-purple-50 text-purple-900 rounded-xl border border-purple-100 text-sm leading-relaxed whitespace-pre-wrap">
|
| 441 |
-
{streamedAssessment.feedback || <span className="text-purple-300">AI 正在思考评语...</span>}
|
| 442 |
-
</div>
|
| 443 |
</div>
|
| 444 |
</div>
|
| 445 |
</div>
|
|
|
|
| 16 |
const [isAssessmentRecording, setIsAssessmentRecording] = useState(false);
|
| 17 |
const [isWebSpeechListening, setIsWebSpeechListening] = useState(false);
|
| 18 |
const [assessmentStatus, setAssessmentStatus] = useState<'IDLE' | 'UPLOADING' | 'ANALYZING' | 'TTS'>('IDLE');
|
|
|
|
|
|
|
| 19 |
const [recognizedText, setRecognizedText] = useState('');
|
| 20 |
+
const [streamedAssessment, setStreamedAssessment] = useState<{ transcription: string; feedback: string; score: number | null; audio?: string; }>({ transcription: '', feedback: '', score: null });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
const [toast, setToast] = useState<ToastState>({ show: false, message: '', type: 'success' });
|
| 22 |
|
| 23 |
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
|
|
|
| 26 |
const currentSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
| 27 |
const recognitionRef = useRef<any>(null);
|
| 28 |
|
|
|
|
| 29 |
useEffect(() => {
|
| 30 |
// @ts-ignore
|
| 31 |
const AudioCtor = window.AudioContext || window.webkitAudioContext;
|
| 32 |
audioContextRef.current = new AudioCtor();
|
| 33 |
return () => {
|
| 34 |
stopPlayback();
|
|
|
|
| 35 |
if (recognitionRef.current) recognitionRef.current.abort();
|
| 36 |
};
|
| 37 |
}, []);
|
|
|
|
| 47 |
const speakWithBrowser = (text: string) => {
|
| 48 |
if (!text) return;
|
| 49 |
stopPlayback();
|
| 50 |
+
const utterance = new SpeechSynthesisUtterance(cleanTextForTTS(text));
|
|
|
|
| 51 |
utterance.lang = 'zh-CN';
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
window.speechSynthesis.speak(utterance);
|
| 53 |
};
|
| 54 |
|
|
|
|
| 60 |
const AudioCtor = window.AudioContext || window.webkitAudioContext;
|
| 61 |
audioContextRef.current = new AudioCtor();
|
| 62 |
}
|
|
|
|
|
|
|
|
|
|
| 63 |
const bytes = base64ToUint8Array(base64Audio);
|
| 64 |
const audioBuffer = decodePCM(bytes, audioContextRef.current!);
|
| 65 |
const source = audioContextRef.current!.createBufferSource();
|
|
|
|
| 67 |
source.connect(audioContextRef.current!.destination);
|
| 68 |
source.start(0);
|
| 69 |
currentSourceRef.current = source;
|
| 70 |
+
} catch (e) { console.error("Audio playback error", e); }
|
|
|
|
|
|
|
|
|
|
| 71 |
};
|
| 72 |
|
| 73 |
const startRecording = async () => {
|
| 74 |
+
console.log("[Assessment] Starting Recording...");
|
| 75 |
// @ts-ignore
|
| 76 |
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
|
| 77 |
if (SpeechRecognition) {
|
|
|
|
| 80 |
const recognition = new SpeechRecognition();
|
| 81 |
recognition.lang = 'zh-CN';
|
| 82 |
recognition.interimResults = true;
|
| 83 |
+
recognition.continuous = true;
|
| 84 |
|
| 85 |
recognition.onstart = () => {
|
| 86 |
setIsWebSpeechListening(true);
|
|
|
|
| 93 |
for (let i = event.resultIndex; i < event.results.length; ++i) {
|
| 94 |
if (event.results[i].isFinal) final += event.results[i][0].transcript;
|
| 95 |
}
|
| 96 |
+
if (final) setRecognizedText(prev => prev + final);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
};
|
| 98 |
|
| 99 |
recognition.onerror = (e: any) => {
|
| 100 |
+
console.warn("[Assessment] Web Speech Error:", e.error);
|
| 101 |
+
stopRecording();
|
| 102 |
+
if (e.error !== 'aborted') startAudioRecordingFallback();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
};
|
| 104 |
|
| 105 |
+
recognition.onend = () => setIsWebSpeechListening(false);
|
| 106 |
recognitionRef.current = recognition;
|
| 107 |
recognition.start();
|
| 108 |
return;
|
| 109 |
} catch (e) {
|
| 110 |
+
console.error("[Assessment] Web Speech Failed", e);
|
| 111 |
startAudioRecordingFallback();
|
| 112 |
}
|
| 113 |
} else {
|
|
|
|
| 121 |
const mediaRecorder = new MediaRecorder(stream);
|
| 122 |
mediaRecorderRef.current = mediaRecorder;
|
| 123 |
audioChunksRef.current = [];
|
|
|
|
| 124 |
mediaRecorder.ondataavailable = (event) => {
|
| 125 |
+
if (event.data.size > 0) audioChunksRef.current.push(event.data);
|
| 126 |
+
};
|
| 127 |
+
mediaRecorder.onstop = async () => {
|
| 128 |
+
const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' });
|
| 129 |
+
const base64 = await blobToBase64(audioBlob);
|
| 130 |
+
handleAssessmentStreamingSubmit({ audio: base64 });
|
| 131 |
+
stream.getTracks().forEach(track => track.stop());
|
| 132 |
};
|
|
|
|
| 133 |
mediaRecorder.start();
|
| 134 |
setIsAssessmentRecording(true);
|
| 135 |
setIsWebSpeechListening(false);
|
|
|
|
| 140 |
|
| 141 |
const stopRecording = () => {
|
| 142 |
setIsAssessmentRecording(false);
|
| 143 |
+
if (isWebSpeechListening && recognitionRef.current) {
|
| 144 |
+
recognitionRef.current.stop();
|
| 145 |
+
setTimeout(() => { if (recognizedText) handleAssessmentStreamingSubmit({ text: recognizedText }); }, 500);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
} else if (mediaRecorderRef.current) {
|
| 147 |
mediaRecorderRef.current.stop();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
}
|
| 149 |
};
|
| 150 |
|
|
|
|
| 154 |
stopPlayback();
|
| 155 |
|
| 156 |
try {
|
|
|
|
| 157 |
let finalQuestion = assessmentTopic;
|
| 158 |
+
if (text) finalQuestion += `\n\n学生���述回答内容:${text}\n(请基于此文本进行评分)`;
|
|
|
|
|
|
|
| 159 |
|
| 160 |
const response = await fetch('/api/ai/evaluate', {
|
| 161 |
method: 'POST',
|
|
|
|
| 169 |
});
|
| 170 |
|
| 171 |
if (!response.ok) throw new Error(response.statusText);
|
|
|
|
|
|
|
| 172 |
setAssessmentStatus('ANALYZING');
|
| 173 |
|
| 174 |
+
const reader = response.body!.getReader();
|
| 175 |
const decoder = new TextDecoder();
|
| 176 |
let accumulatedRaw = '';
|
| 177 |
let buffer = '';
|
|
|
|
| 179 |
while (true) {
|
| 180 |
const { done, value } = await reader.read();
|
| 181 |
if (done) break;
|
|
|
|
| 182 |
buffer += decoder.decode(value, { stream: true });
|
| 183 |
const parts = buffer.split('\n\n');
|
| 184 |
buffer = parts.pop() || '';
|
|
|
|
| 187 |
if (line.startsWith('data: ')) {
|
| 188 |
const jsonStr = line.replace('data: ', '').trim();
|
| 189 |
if (jsonStr === '[DONE]') break;
|
|
|
|
| 190 |
try {
|
| 191 |
const data = JSON.parse(jsonStr);
|
| 192 |
+
if (data.status === 'tts') setAssessmentStatus('TTS');
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
if (data.text) {
|
| 194 |
accumulatedRaw += data.text;
|
| 195 |
+
const trMatch = accumulatedRaw.match(/## Transcription\s+([\s\S]*?)(?=## Feedback|$)/i);
|
| 196 |
+
const fbMatch = accumulatedRaw.match(/## Feedback\s+([\s\S]*?)(?=## Score|$)/i);
|
| 197 |
+
const scMatch = accumulatedRaw.match(/## Score\s+(\d+)/i);
|
|
|
|
| 198 |
setStreamedAssessment(prev => ({
|
| 199 |
...prev,
|
| 200 |
+
transcription: trMatch ? trMatch[1].trim() : (text || prev.transcription),
|
| 201 |
+
feedback: fbMatch ? fbMatch[1].trim() : prev.feedback,
|
| 202 |
+
score: scMatch ? parseInt(scMatch[1]) : prev.score
|
| 203 |
}));
|
| 204 |
}
|
|
|
|
| 205 |
if (data.audio) {
|
| 206 |
setStreamedAssessment(prev => ({ ...prev, audio: data.audio }));
|
| 207 |
playPCMAudio(data.audio);
|
| 208 |
}
|
|
|
|
| 209 |
if (data.ttsSkipped) {
|
| 210 |
const fb = streamedAssessment.feedback || accumulatedRaw.match(/## Feedback\s+([\s\S]*?)(?=## Score|$)/i)?.[1] || '';
|
| 211 |
if (fb) speakWithBrowser(fb);
|
| 212 |
}
|
| 213 |
+
if (data.error) setToast({ show: true, message: data.message || '评分出错', type: 'error' });
|
|
|
|
|
|
|
|
|
|
|
|
|
| 214 |
} catch (e) {}
|
| 215 |
}
|
| 216 |
}
|
| 217 |
}
|
| 218 |
setAssessmentStatus('IDLE');
|
|
|
|
| 219 |
} catch (error: any) {
|
| 220 |
+
setToast({ show: true, message: '评分失败', type: 'error' });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
setAssessmentStatus('IDLE');
|
|
|
|
| 222 |
}
|
| 223 |
};
|
| 224 |
|
|
|
|
| 228 |
<button onClick={stopPlayback} className="absolute top-4 right-4 z-50 bg-white/80 backdrop-blur p-2 rounded-full shadow-md text-red-500 hover:bg-white border border-gray-200" title="停止播放"><StopIcon size={20}/></button>
|
| 229 |
|
| 230 |
<div className="max-w-3xl mx-auto space-y-6">
|
|
|
|
| 231 |
<div className="bg-white p-6 rounded-2xl border border-purple-100 shadow-sm">
|
| 232 |
<h3 className="text-lg font-bold text-gray-800 mb-2 flex items-center justify-between">
|
| 233 |
<span className="flex items-center"><Brain className="mr-2 text-purple-600"/> 今日测评题目</span>
|
|
|
|
| 237 |
</div>
|
| 238 |
</h3>
|
| 239 |
<textarea className="w-full bg-purple-50/50 border border-purple-100 rounded-xl p-4 text-gray-700 font-medium text-lg resize-none focus:ring-2 focus:ring-purple-200 outline-none" value={assessmentTopic} onChange={e => setAssessmentTopic(e.target.value)} rows={3}/>
|
|
|
|
| 240 |
<div className="mt-6 flex justify-center">
|
| 241 |
{assessmentMode === 'audio' ? (
|
| 242 |
+
<button onMouseDown={startRecording} onMouseUp={stopRecording} onTouchStart={startRecording} onTouchEnd={stopRecording} disabled={assessmentStatus !== 'IDLE'} className={`px-8 py-4 rounded-full font-bold text-white flex items-center gap-3 shadow-lg transition-all ${isAssessmentRecording ? 'bg-red-500 scale-105' : 'bg-gradient-to-r from-purple-600 to-indigo-600 hover:shadow-purple-200 hover:scale-105 disabled:opacity-50'}`}>
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
{assessmentStatus !== 'IDLE' ? <Loader2 className="animate-spin"/> : (isAssessmentRecording ? <StopCircle/> : <Mic/>)}
|
| 244 |
+
{assessmentStatus === 'UPLOADING' ? '上传中...' : assessmentStatus === 'ANALYZING' ? 'AI 分析中...' : assessmentStatus === 'TTS' ? '生成语音...' : isAssessmentRecording ? '正在识别...' : '长按开始回答'}
|
| 245 |
</button>
|
| 246 |
) : (
|
| 247 |
<div className="w-full">
|
| 248 |
<div className="relative border-2 border-dashed border-purple-200 rounded-xl p-4 text-center hover:bg-purple-50 transition-colors cursor-pointer min-h-[160px] flex flex-col items-center justify-center">
|
| 249 |
+
<input type="file" accept="image/*" multiple className="absolute inset-0 opacity-0 cursor-pointer w-full h-full z-10" onChange={(e) => { if (e.target.files) setSelectedImages(prev => [...prev, ...Array.from(e.target.files!)]); }} onClick={(e) => (e.currentTarget.value = '')} />
|
| 250 |
+
{selectedImages.length === 0 ? (<><ImageIcon className="mx-auto text-purple-300 mb-2" size={40}/><p className="text-purple-600 font-bold">点击上传作业图片</p></>) : (<div className="z-0 w-full pointer-events-none opacity-50 flex items-center justify-center"><Plus className="text-purple-300" size={40}/><span className="text-purple-400 font-bold ml-2">继续添加</span></div>)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
</div>
|
|
|
|
| 252 |
{selectedImages.length > 0 && (
|
| 253 |
<div className="mt-4 grid grid-cols-3 sm:grid-cols-4 gap-3 animate-in fade-in">
|
| 254 |
+
{selectedImages.map((file, idx) => (<div key={idx} className="relative group aspect-square"><img src={URL.createObjectURL(file)} className="w-full h-full object-cover rounded-lg shadow-sm border border-gray-200" /><button onClick={() => setSelectedImages(prev => prev.filter((_, i) => i !== idx))} className="absolute -top-2 -right-2 bg-red-500 text-white rounded-full p-1 shadow-md hover:bg-red-600 transition-colors z-20 scale-90 hover:scale-100"><X size={14}/></button></div>))}
|
| 255 |
+
<button onClick={async () => { setAssessmentStatus('UPLOADING'); try { const images = await Promise.all(selectedImages.map(f => compressImage(f))); handleAssessmentStreamingSubmit({ images }); } catch(e) { setAssessmentStatus('IDLE'); } }} disabled={assessmentStatus !== 'IDLE'} className="mt-6 w-full px-8 py-3 bg-purple-600 text-white rounded-lg font-bold hover:bg-purple-700 flex items-center justify-center gap-2 shadow-md transition-all">开始批改</button>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 256 |
</div>
|
| 257 |
)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 258 |
</div>
|
| 259 |
)}
|
| 260 |
</div>
|
| 261 |
</div>
|
|
|
|
|
|
|
| 262 |
{(streamedAssessment.transcription || streamedAssessment.feedback || streamedAssessment.score !== null) && (
|
| 263 |
<div className="bg-white p-6 rounded-2xl border border-gray-200 shadow-lg animate-in slide-in-from-bottom-4">
|
| 264 |
<div className="flex items-center justify-between border-b border-gray-100 pb-4 mb-4">
|
| 265 |
<div className="flex items-center gap-2">
|
| 266 |
<h3 className="font-bold text-xl text-gray-800">测评报告</h3>
|
| 267 |
+
{assessmentStatus !== 'IDLE' && <div className="flex items-center gap-1 text-xs px-2 py-1 bg-purple-50 text-purple-600 rounded-full animate-pulse"><Zap size={12}/>正在处理...</div>}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
</div>
|
| 269 |
<div className="flex items-center gap-4">
|
| 270 |
+
{streamedAssessment.audio && <button onClick={() => playPCMAudio(streamedAssessment.audio!)} className="flex items-center gap-1 text-sm bg-purple-100 text-purple-700 px-3 py-1 rounded-full hover:bg-purple-200"><Volume2 size={16}/> 听AI点评</button>}
|
| 271 |
+
{streamedAssessment.score !== null && <div className={`text-3xl font-black ${streamedAssessment.score >= 80 ? 'text-green-500' : streamedAssessment.score >= 60 ? 'text-yellow-500' : 'text-red-500'}`}>{streamedAssessment.score}<span className="text-sm text-gray-400 ml-1">分</span></div>}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 272 |
</div>
|
| 273 |
</div>
|
| 274 |
<div className="space-y-4">
|
| 275 |
<div className="bg-gray-50 p-4 rounded-xl">
|
| 276 |
<p className="text-xs font-bold text-gray-500 uppercase mb-1">AI 识别内容</p>
|
| 277 |
+
<p className="text-gray-700 leading-relaxed text-sm whitespace-pre-wrap">{streamedAssessment.transcription || '正在识别...'}</p>
|
| 278 |
</div>
|
| 279 |
<div>
|
| 280 |
<p className="text-xs font-bold text-gray-500 uppercase mb-2">AI 点评建议</p>
|
| 281 |
+
<div className="p-4 bg-purple-50 text-purple-900 rounded-xl border border-purple-100 text-sm leading-relaxed whitespace-pre-wrap">{streamedAssessment.feedback || 'AI 思考中...'}</div>
|
|
|
|
|
|
|
| 282 |
</div>
|
| 283 |
</div>
|
| 284 |
</div>
|
components/ai/ChatPanel.tsx
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
|
| 2 |
import React, { useState, useRef, useEffect } from 'react';
|
| 3 |
import { AIChatMessage, User } from '../../types';
|
| 4 |
-
import { Bot, Mic,
|
| 5 |
import ReactMarkdown from 'react-markdown';
|
| 6 |
import remarkGfm from 'remark-gfm';
|
| 7 |
import { blobToBase64, base64ToUint8Array, decodePCM, cleanTextForTTS } from '../../utils/mediaHelpers';
|
|
@@ -22,16 +22,11 @@ export const ChatPanel: React.FC<ChatPanelProps> = ({ currentUser }) => {
|
|
| 22 |
timestamp: Date.now()
|
| 23 |
}];
|
| 24 |
} catch (e) {
|
| 25 |
-
return [{
|
| 26 |
-
id: 'welcome',
|
| 27 |
-
role: 'model',
|
| 28 |
-
text: '你好!我是你的 AI 智能助教。有什么可以帮你的吗?',
|
| 29 |
-
timestamp: Date.now()
|
| 30 |
-
}];
|
| 31 |
}
|
| 32 |
});
|
|
|
|
| 33 |
const [textInput, setTextInput] = useState('');
|
| 34 |
-
const [inputMode, setInputMode] = useState<'text' | 'audio'>('text');
|
| 35 |
const [isChatProcessing, setIsChatProcessing] = useState(false);
|
| 36 |
const [isChatRecording, setIsChatRecording] = useState(false);
|
| 37 |
const [isWebSpeechListening, setIsWebSpeechListening] = useState(false);
|
|
@@ -43,32 +38,18 @@ export const ChatPanel: React.FC<ChatPanelProps> = ({ currentUser }) => {
|
|
| 43 |
const currentSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
| 44 |
const messagesEndRef = useRef<HTMLDivElement>(null);
|
| 45 |
const recognitionRef = useRef<any>(null);
|
|
|
|
| 46 |
|
| 47 |
-
// Initialize AudioContext
|
| 48 |
useEffect(() => {
|
| 49 |
// @ts-ignore
|
| 50 |
const AudioCtor = window.AudioContext || window.webkitAudioContext;
|
| 51 |
audioContextRef.current = new AudioCtor();
|
| 52 |
return () => {
|
| 53 |
stopPlayback();
|
| 54 |
-
window.speechSynthesis.cancel();
|
| 55 |
if (recognitionRef.current) recognitionRef.current.abort();
|
| 56 |
};
|
| 57 |
}, []);
|
| 58 |
|
| 59 |
-
// Persist messages
|
| 60 |
-
useEffect(() => {
|
| 61 |
-
try {
|
| 62 |
-
const MAX_COUNT = 50;
|
| 63 |
-
const welcome = messages.find(m => m.id === 'welcome');
|
| 64 |
-
const others = messages.filter(m => m.id !== 'welcome');
|
| 65 |
-
const recent = others.slice(-MAX_COUNT);
|
| 66 |
-
const messagesToSave = (welcome ? [welcome] : []).concat(recent);
|
| 67 |
-
localStorage.setItem('ai_chat_history', JSON.stringify(messagesToSave));
|
| 68 |
-
} catch (e) {}
|
| 69 |
-
}, [messages]);
|
| 70 |
-
|
| 71 |
-
// Scroll to bottom
|
| 72 |
useEffect(() => {
|
| 73 |
messagesEndRef.current?.scrollIntoView({ behavior: isChatProcessing ? 'auto' : 'smooth', block: 'end' });
|
| 74 |
}, [messages, isChatProcessing]);
|
|
@@ -84,13 +65,8 @@ export const ChatPanel: React.FC<ChatPanelProps> = ({ currentUser }) => {
|
|
| 84 |
const speakWithBrowser = (text: string) => {
|
| 85 |
if (!text) return;
|
| 86 |
stopPlayback();
|
| 87 |
-
const
|
| 88 |
-
const utterance = new SpeechSynthesisUtterance(cleanText);
|
| 89 |
utterance.lang = 'zh-CN';
|
| 90 |
-
utterance.rate = 1.0;
|
| 91 |
-
const voices = window.speechSynthesis.getVoices();
|
| 92 |
-
const zhVoice = voices.find(v => v.lang === 'zh-CN' && !v.name.includes('Hong Kong') && !v.name.includes('Taiwan'));
|
| 93 |
-
if (zhVoice) utterance.voice = zhVoice;
|
| 94 |
window.speechSynthesis.speak(utterance);
|
| 95 |
};
|
| 96 |
|
|
@@ -114,61 +90,63 @@ export const ChatPanel: React.FC<ChatPanelProps> = ({ currentUser }) => {
|
|
| 114 |
currentSourceRef.current = source;
|
| 115 |
} catch (e) {
|
| 116 |
console.error("Audio playback error", e);
|
| 117 |
-
setToast({ show: true, message: '语音播放失败', type: 'error' });
|
| 118 |
}
|
| 119 |
};
|
| 120 |
|
| 121 |
-
const startRecording = async () => {
|
| 122 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
// @ts-ignore
|
| 124 |
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
|
|
|
|
| 125 |
if (SpeechRecognition) {
|
| 126 |
try {
|
| 127 |
if (recognitionRef.current) recognitionRef.current.abort();
|
|
|
|
| 128 |
const recognition = new SpeechRecognition();
|
| 129 |
recognition.lang = 'zh-CN';
|
| 130 |
recognition.interimResults = true;
|
|
|
|
| 131 |
|
| 132 |
recognition.onstart = () => {
|
|
|
|
| 133 |
setIsWebSpeechListening(true);
|
| 134 |
-
setIsChatRecording(true);
|
| 135 |
};
|
| 136 |
|
| 137 |
recognition.onresult = (event: any) => {
|
| 138 |
-
let
|
| 139 |
-
let interim = '';
|
| 140 |
for (let i = event.resultIndex; i < event.results.length; ++i) {
|
| 141 |
-
|
| 142 |
-
else interim += event.results[i][0].transcript;
|
| 143 |
}
|
| 144 |
-
if (
|
| 145 |
-
setTextInput(
|
| 146 |
-
setInputMode('text'); // Switch UI to text to show result
|
| 147 |
}
|
| 148 |
};
|
| 149 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
recognition.onend = () => {
|
|
|
|
| 151 |
setIsWebSpeechListening(false);
|
| 152 |
setIsChatRecording(false);
|
| 153 |
};
|
| 154 |
|
| 155 |
-
recognition.onerror = (e: any) => {
|
| 156 |
-
console.warn("Web Speech API Error:", e.error);
|
| 157 |
-
recognition.stop();
|
| 158 |
-
// Fallback to audio recording if it was a permission/network issue not aborted by user
|
| 159 |
-
if (e.error !== 'aborted') {
|
| 160 |
-
startAudioRecordingFallback();
|
| 161 |
-
} else {
|
| 162 |
-
setIsWebSpeechListening(false);
|
| 163 |
-
setIsChatRecording(false);
|
| 164 |
-
}
|
| 165 |
-
};
|
| 166 |
-
|
| 167 |
recognitionRef.current = recognition;
|
| 168 |
recognition.start();
|
| 169 |
return;
|
| 170 |
} catch (e) {
|
| 171 |
-
console.
|
| 172 |
startAudioRecordingFallback();
|
| 173 |
}
|
| 174 |
} else {
|
|
@@ -177,6 +155,7 @@ export const ChatPanel: React.FC<ChatPanelProps> = ({ currentUser }) => {
|
|
| 177 |
};
|
| 178 |
|
| 179 |
const startAudioRecordingFallback = async () => {
|
|
|
|
| 180 |
try {
|
| 181 |
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
| 182 |
const mediaRecorder = new MediaRecorder(stream);
|
|
@@ -184,55 +163,51 @@ export const ChatPanel: React.FC<ChatPanelProps> = ({ currentUser }) => {
|
|
| 184 |
audioChunksRef.current = [];
|
| 185 |
|
| 186 |
mediaRecorder.ondataavailable = (event) => {
|
| 187 |
-
if (event.data.size > 0)
|
| 188 |
-
|
| 189 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
};
|
| 191 |
|
| 192 |
mediaRecorder.start();
|
| 193 |
setIsChatRecording(true);
|
| 194 |
setIsWebSpeechListening(false);
|
| 195 |
} catch (e) {
|
| 196 |
-
setToast({ show: true, message: '
|
| 197 |
}
|
| 198 |
};
|
| 199 |
|
| 200 |
const stopRecording = () => {
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
} else if (mediaRecorderRef.current && isChatRecording) {
|
| 205 |
mediaRecorderRef.current.stop();
|
| 206 |
-
setIsChatRecording(false);
|
| 207 |
-
|
| 208 |
-
mediaRecorderRef.current.onstop = async () => {
|
| 209 |
-
const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' });
|
| 210 |
-
const base64 = await blobToBase64(audioBlob);
|
| 211 |
-
handleChatSubmit(undefined, base64);
|
| 212 |
-
mediaRecorderRef.current?.stream.getTracks().forEach(track => track.stop());
|
| 213 |
-
};
|
| 214 |
}
|
|
|
|
|
|
|
| 215 |
};
|
| 216 |
|
| 217 |
const handleChatSubmit = async (text?: string, audioBase64?: string) => {
|
| 218 |
-
|
|
|
|
|
|
|
| 219 |
stopPlayback();
|
| 220 |
const historyPayload = messages.filter(m => m.id !== 'welcome').map(m => ({ role: m.role, text: m.text }));
|
| 221 |
|
| 222 |
const newUserMsg: AIChatMessage = {
|
| 223 |
id: Date.now().toString(),
|
| 224 |
role: 'user',
|
| 225 |
-
text:
|
| 226 |
isAudioMessage: !!audioBase64,
|
| 227 |
timestamp: Date.now()
|
| 228 |
};
|
| 229 |
const newAiMsgId = (Date.now() + 1).toString();
|
| 230 |
-
const newAiMsg: AIChatMessage = {
|
| 231 |
-
id: newAiMsgId,
|
| 232 |
-
role: 'model',
|
| 233 |
-
text: '',
|
| 234 |
-
timestamp: Date.now()
|
| 235 |
-
};
|
| 236 |
|
| 237 |
setMessages(prev => [...prev, newUserMsg, newAiMsg]);
|
| 238 |
setTextInput('');
|
|
@@ -247,13 +222,11 @@ export const ChatPanel: React.FC<ChatPanelProps> = ({ currentUser }) => {
|
|
| 247 |
'x-user-role': currentUser?.role || '',
|
| 248 |
'x-school-id': currentUser?.schoolId || ''
|
| 249 |
},
|
| 250 |
-
body: JSON.stringify({ text, audio: audioBase64, history: historyPayload })
|
| 251 |
});
|
| 252 |
|
| 253 |
if (!response.ok) throw new Error(response.statusText);
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
const reader = response.body.getReader();
|
| 257 |
const decoder = new TextDecoder();
|
| 258 |
let aiTextAccumulated = '';
|
| 259 |
let buffer = '';
|
|
@@ -280,36 +253,23 @@ export const ChatPanel: React.FC<ChatPanelProps> = ({ currentUser }) => {
|
|
| 280 |
playPCMAudio(data.audio);
|
| 281 |
}
|
| 282 |
if (data.ttsSkipped) {
|
| 283 |
-
setToast({ show: true, message: 'AI 语音额度已用尽,已切换至本地语音播报', type: 'error' });
|
| 284 |
speakWithBrowser(aiTextAccumulated);
|
| 285 |
}
|
| 286 |
-
if (data.error) {
|
| 287 |
-
setMessages(prev => prev.map(m => m.id === newAiMsgId ? { ...m, text: `⚠️ 错误: ${data.message || '未知错误'}` } : m));
|
| 288 |
-
}
|
| 289 |
} catch (e) {}
|
| 290 |
}
|
| 291 |
}
|
| 292 |
}
|
| 293 |
} catch (error: any) {
|
| 294 |
-
setMessages(prev => prev.map(m => m.id === newAiMsgId ? { ...m, text: '
|
| 295 |
} finally { setIsChatProcessing(false); }
|
| 296 |
};
|
| 297 |
|
| 298 |
-
const clearHistory = () => {
|
| 299 |
-
setMessages([{
|
| 300 |
-
id: 'welcome',
|
| 301 |
-
role: 'model',
|
| 302 |
-
text: '你好!我是你的 AI 智能助教。有什么可以帮你的吗?',
|
| 303 |
-
timestamp: Date.now()
|
| 304 |
-
}]);
|
| 305 |
-
};
|
| 306 |
-
|
| 307 |
return (
|
| 308 |
<div className="flex-1 flex flex-col max-w-4xl mx-auto w-full min-h-0 relative overflow-hidden h-full">
|
| 309 |
{toast.show && <Toast message={toast.message} type={toast.type} onClose={()=>setToast({...toast, show: false})}/>}
|
| 310 |
|
| 311 |
<div className="absolute top-2 right-4 z-10">
|
| 312 |
-
<button onClick={
|
| 313 |
<Trash2 size={14}/> 清除记录
|
| 314 |
</button>
|
| 315 |
</div>
|
|
@@ -322,20 +282,68 @@ export const ChatPanel: React.FC<ChatPanelProps> = ({ currentUser }) => {
|
|
| 322 |
</div>
|
| 323 |
<div className={`max-w-[80%] p-3 rounded-2xl text-sm overflow-hidden ${msg.role === 'user' ? 'bg-blue-600 text-white rounded-tr-none' : 'bg-white border border-gray-200 text-gray-800 rounded-tl-none shadow-sm'}`}>
|
| 324 |
<div className="markdown-body"><ReactMarkdown remarkPlugins={[remarkGfm]}>{msg.text || ''}</ReactMarkdown></div>
|
| 325 |
-
{msg.
|
| 326 |
-
{msg.audio ? (<button onClick={() => playPCMAudio(msg.audio!)} className="mt-2 flex items-center gap-2 text-xs bg-blue-50 text-blue-600 px-3 py-1.5 rounded-full hover:bg-blue-100 border border-blue-100 transition-colors w-fit"><Volume2 size={14}/> 播放语音 (AI)</button>) : (msg.role === 'model' && msg.text && !isChatProcessing) && (<button onClick={() => speakWithBrowser(msg.text!)} className="mt-2 flex items-center gap-2 text-xs bg-gray-50 text-gray-600 px-3 py-1.5 rounded-full hover:bg-gray-100 border border-gray-200 transition-colors w-fit"><Volume2 size={14}/> 朗读 (本地)</button>)}
|
| 327 |
</div>
|
| 328 |
</div>
|
| 329 |
))}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
<div ref={messagesEndRef} />
|
| 331 |
</div>
|
| 332 |
|
|
|
|
| 333 |
<div className="p-4 bg-white border-t border-gray-200 shrink-0 z-20">
|
| 334 |
-
<div className="flex items-center gap-
|
| 335 |
-
|
| 336 |
-
{
|
| 337 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
</div>
|
|
|
|
| 339 |
</div>
|
| 340 |
</div>
|
| 341 |
);
|
|
|
|
| 1 |
|
| 2 |
import React, { useState, useRef, useEffect } from 'react';
|
| 3 |
import { AIChatMessage, User } from '../../types';
|
| 4 |
+
import { Bot, Mic, Volume2, Send, Sparkles, Loader2, StopCircle, Trash2 } from 'lucide-react';
|
| 5 |
import ReactMarkdown from 'react-markdown';
|
| 6 |
import remarkGfm from 'remark-gfm';
|
| 7 |
import { blobToBase64, base64ToUint8Array, decodePCM, cleanTextForTTS } from '../../utils/mediaHelpers';
|
|
|
|
| 22 |
timestamp: Date.now()
|
| 23 |
}];
|
| 24 |
} catch (e) {
|
| 25 |
+
return [{ id: 'welcome', role: 'model', text: '你好!我是你的 AI 智能助教。', timestamp: Date.now() }];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
}
|
| 27 |
});
|
| 28 |
+
|
| 29 |
const [textInput, setTextInput] = useState('');
|
|
|
|
| 30 |
const [isChatProcessing, setIsChatProcessing] = useState(false);
|
| 31 |
const [isChatRecording, setIsChatRecording] = useState(false);
|
| 32 |
const [isWebSpeechListening, setIsWebSpeechListening] = useState(false);
|
|
|
|
| 38 |
const currentSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
| 39 |
const messagesEndRef = useRef<HTMLDivElement>(null);
|
| 40 |
const recognitionRef = useRef<any>(null);
|
| 41 |
+
const inputRef = useRef<HTMLInputElement>(null);
|
| 42 |
|
|
|
|
| 43 |
useEffect(() => {
|
| 44 |
// @ts-ignore
|
| 45 |
const AudioCtor = window.AudioContext || window.webkitAudioContext;
|
| 46 |
audioContextRef.current = new AudioCtor();
|
| 47 |
return () => {
|
| 48 |
stopPlayback();
|
|
|
|
| 49 |
if (recognitionRef.current) recognitionRef.current.abort();
|
| 50 |
};
|
| 51 |
}, []);
|
| 52 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
useEffect(() => {
|
| 54 |
messagesEndRef.current?.scrollIntoView({ behavior: isChatProcessing ? 'auto' : 'smooth', block: 'end' });
|
| 55 |
}, [messages, isChatProcessing]);
|
|
|
|
| 65 |
const speakWithBrowser = (text: string) => {
|
| 66 |
if (!text) return;
|
| 67 |
stopPlayback();
|
| 68 |
+
const utterance = new SpeechSynthesisUtterance(cleanTextForTTS(text));
|
|
|
|
| 69 |
utterance.lang = 'zh-CN';
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
window.speechSynthesis.speak(utterance);
|
| 71 |
};
|
| 72 |
|
|
|
|
| 90 |
currentSourceRef.current = source;
|
| 91 |
} catch (e) {
|
| 92 |
console.error("Audio playback error", e);
|
|
|
|
| 93 |
}
|
| 94 |
};
|
| 95 |
|
| 96 |
+
const startRecording = async (e?: React.MouseEvent | React.TouchEvent) => {
|
| 97 |
+
if (e) {
|
| 98 |
+
e.preventDefault();
|
| 99 |
+
e.stopPropagation();
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
console.log("[Voice] Starting...");
|
| 103 |
// @ts-ignore
|
| 104 |
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
|
| 105 |
+
|
| 106 |
if (SpeechRecognition) {
|
| 107 |
try {
|
| 108 |
if (recognitionRef.current) recognitionRef.current.abort();
|
| 109 |
+
|
| 110 |
const recognition = new SpeechRecognition();
|
| 111 |
recognition.lang = 'zh-CN';
|
| 112 |
recognition.interimResults = true;
|
| 113 |
+
recognition.continuous = true;
|
| 114 |
|
| 115 |
recognition.onstart = () => {
|
| 116 |
+
console.log("[Voice] Web Speech Active");
|
| 117 |
setIsWebSpeechListening(true);
|
| 118 |
+
setIsChatRecording(true);
|
| 119 |
};
|
| 120 |
|
| 121 |
recognition.onresult = (event: any) => {
|
| 122 |
+
let transcript = '';
|
|
|
|
| 123 |
for (let i = event.resultIndex; i < event.results.length; ++i) {
|
| 124 |
+
transcript += event.results[i][0].transcript;
|
|
|
|
| 125 |
}
|
| 126 |
+
if (transcript) {
|
| 127 |
+
setTextInput(transcript);
|
|
|
|
| 128 |
}
|
| 129 |
};
|
| 130 |
|
| 131 |
+
recognition.onerror = (e: any) => {
|
| 132 |
+
console.warn("[Voice] Web Speech Error:", e.error);
|
| 133 |
+
if (e.error === 'not-allowed') {
|
| 134 |
+
setToast({ show: true, message: '请允许麦克风访问', type: 'error' });
|
| 135 |
+
}
|
| 136 |
+
stopRecording();
|
| 137 |
+
};
|
| 138 |
+
|
| 139 |
recognition.onend = () => {
|
| 140 |
+
console.log("[Voice] Web Speech Stopped");
|
| 141 |
setIsWebSpeechListening(false);
|
| 142 |
setIsChatRecording(false);
|
| 143 |
};
|
| 144 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
recognitionRef.current = recognition;
|
| 146 |
recognition.start();
|
| 147 |
return;
|
| 148 |
} catch (e) {
|
| 149 |
+
console.error("[Voice] Web Speech Failed, falling back", e);
|
| 150 |
startAudioRecordingFallback();
|
| 151 |
}
|
| 152 |
} else {
|
|
|
|
| 155 |
};
|
| 156 |
|
| 157 |
const startAudioRecordingFallback = async () => {
|
| 158 |
+
console.log("[Voice] Using MediaRecorder Fallback");
|
| 159 |
try {
|
| 160 |
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
| 161 |
const mediaRecorder = new MediaRecorder(stream);
|
|
|
|
| 163 |
audioChunksRef.current = [];
|
| 164 |
|
| 165 |
mediaRecorder.ondataavailable = (event) => {
|
| 166 |
+
if (event.data.size > 0) audioChunksRef.current.push(event.data);
|
| 167 |
+
};
|
| 168 |
+
|
| 169 |
+
mediaRecorder.onstop = async () => {
|
| 170 |
+
const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' });
|
| 171 |
+
const base64 = await blobToBase64(audioBlob);
|
| 172 |
+
handleChatSubmit(undefined, base64);
|
| 173 |
+
stream.getTracks().forEach(track => track.stop());
|
| 174 |
};
|
| 175 |
|
| 176 |
mediaRecorder.start();
|
| 177 |
setIsChatRecording(true);
|
| 178 |
setIsWebSpeechListening(false);
|
| 179 |
} catch (e) {
|
| 180 |
+
setToast({ show: true, message: '麦克风不可用', type: 'error' });
|
| 181 |
}
|
| 182 |
};
|
| 183 |
|
| 184 |
const stopRecording = () => {
|
| 185 |
+
console.log("[Voice] Stopping...");
|
| 186 |
+
if (isWebSpeechListening && recognitionRef.current) {
|
| 187 |
+
recognitionRef.current.stop();
|
| 188 |
} else if (mediaRecorderRef.current && isChatRecording) {
|
| 189 |
mediaRecorderRef.current.stop();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
}
|
| 191 |
+
setIsChatRecording(false);
|
| 192 |
+
setIsWebSpeechListening(false);
|
| 193 |
};
|
| 194 |
|
| 195 |
const handleChatSubmit = async (text?: string, audioBase64?: string) => {
|
| 196 |
+
const finalContent = text || textInput;
|
| 197 |
+
if (!finalContent && !audioBase64) return;
|
| 198 |
+
|
| 199 |
stopPlayback();
|
| 200 |
const historyPayload = messages.filter(m => m.id !== 'welcome').map(m => ({ role: m.role, text: m.text }));
|
| 201 |
|
| 202 |
const newUserMsg: AIChatMessage = {
|
| 203 |
id: Date.now().toString(),
|
| 204 |
role: 'user',
|
| 205 |
+
text: finalContent || '(语音消息)',
|
| 206 |
isAudioMessage: !!audioBase64,
|
| 207 |
timestamp: Date.now()
|
| 208 |
};
|
| 209 |
const newAiMsgId = (Date.now() + 1).toString();
|
| 210 |
+
const newAiMsg: AIChatMessage = { id: newAiMsgId, role: 'model', text: '', timestamp: Date.now() };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
|
| 212 |
setMessages(prev => [...prev, newUserMsg, newAiMsg]);
|
| 213 |
setTextInput('');
|
|
|
|
| 222 |
'x-user-role': currentUser?.role || '',
|
| 223 |
'x-school-id': currentUser?.schoolId || ''
|
| 224 |
},
|
| 225 |
+
body: JSON.stringify({ text: finalContent, audio: audioBase64, history: historyPayload })
|
| 226 |
});
|
| 227 |
|
| 228 |
if (!response.ok) throw new Error(response.statusText);
|
| 229 |
+
const reader = response.body!.getReader();
|
|
|
|
|
|
|
| 230 |
const decoder = new TextDecoder();
|
| 231 |
let aiTextAccumulated = '';
|
| 232 |
let buffer = '';
|
|
|
|
| 253 |
playPCMAudio(data.audio);
|
| 254 |
}
|
| 255 |
if (data.ttsSkipped) {
|
|
|
|
| 256 |
speakWithBrowser(aiTextAccumulated);
|
| 257 |
}
|
|
|
|
|
|
|
|
|
|
| 258 |
} catch (e) {}
|
| 259 |
}
|
| 260 |
}
|
| 261 |
}
|
| 262 |
} catch (error: any) {
|
| 263 |
+
setMessages(prev => prev.map(m => m.id === newAiMsgId ? { ...m, text: '抱歉,连接断开,请重试。' } : m));
|
| 264 |
} finally { setIsChatProcessing(false); }
|
| 265 |
};
|
| 266 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 267 |
return (
|
| 268 |
<div className="flex-1 flex flex-col max-w-4xl mx-auto w-full min-h-0 relative overflow-hidden h-full">
|
| 269 |
{toast.show && <Toast message={toast.message} type={toast.type} onClose={()=>setToast({...toast, show: false})}/>}
|
| 270 |
|
| 271 |
<div className="absolute top-2 right-4 z-10">
|
| 272 |
+
<button onClick={() => setMessages([{ id: 'welcome', role: 'model', text: '你好!我是你的 AI 智能助教。有什么可以帮你的吗?', timestamp: Date.now() }])} className="text-xs text-gray-400 hover:text-red-500 flex items-center gap-1 bg-white/80 p-1.5 rounded-lg border border-transparent hover:border-red-100 transition-all shadow-sm backdrop-blur">
|
| 273 |
<Trash2 size={14}/> 清除记录
|
| 274 |
</button>
|
| 275 |
</div>
|
|
|
|
| 282 |
</div>
|
| 283 |
<div className={`max-w-[80%] p-3 rounded-2xl text-sm overflow-hidden ${msg.role === 'user' ? 'bg-blue-600 text-white rounded-tr-none' : 'bg-white border border-gray-200 text-gray-800 rounded-tl-none shadow-sm'}`}>
|
| 284 |
<div className="markdown-body"><ReactMarkdown remarkPlugins={[remarkGfm]}>{msg.text || ''}</ReactMarkdown></div>
|
| 285 |
+
{msg.audio && <button onClick={() => playPCMAudio(msg.audio!)} className="mt-2 flex items-center gap-2 text-xs bg-blue-50 text-blue-600 px-3 py-1.5 rounded-full hover:bg-blue-100 border border-blue-100 transition-colors w-fit"><Volume2 size={14}/> 播放语音 (AI)</button>}
|
|
|
|
| 286 |
</div>
|
| 287 |
</div>
|
| 288 |
))}
|
| 289 |
+
{isChatProcessing && (
|
| 290 |
+
<div className="flex gap-3">
|
| 291 |
+
<div className="w-10 h-10 rounded-full bg-blue-100 text-blue-600 flex items-center justify-center shrink-0">
|
| 292 |
+
<Loader2 className="animate-spin" size={20}/>
|
| 293 |
+
</div>
|
| 294 |
+
<div className="bg-white border border-gray-100 p-3 rounded-2xl rounded-tl-none shadow-sm flex items-center gap-2 text-gray-400 text-xs">
|
| 295 |
+
思考中...
|
| 296 |
+
</div>
|
| 297 |
+
</div>
|
| 298 |
+
)}
|
| 299 |
<div ref={messagesEndRef} />
|
| 300 |
</div>
|
| 301 |
|
| 302 |
+
{/* Unified Input Bar (No switching, Mic and Text coexist) */}
|
| 303 |
<div className="p-4 bg-white border-t border-gray-200 shrink-0 z-20">
|
| 304 |
+
<div className="flex items-center gap-3 max-w-4xl mx-auto bg-gray-50 p-2 rounded-2xl border border-gray-200">
|
| 305 |
+
{/* Recording Status Dot */}
|
| 306 |
+
{isChatRecording && (
|
| 307 |
+
<div className="flex items-center gap-1.5 px-3 py-1 bg-red-100 text-red-600 rounded-full animate-pulse text-[10px] font-bold shrink-0">
|
| 308 |
+
<div className="w-2 h-2 bg-red-600 rounded-full"></div>
|
| 309 |
+
{isWebSpeechListening ? '正在识别' : '正在录制'}
|
| 310 |
+
</div>
|
| 311 |
+
)}
|
| 312 |
+
|
| 313 |
+
<input
|
| 314 |
+
ref={inputRef}
|
| 315 |
+
className="flex-1 bg-transparent border-none outline-none px-3 text-sm py-2"
|
| 316 |
+
placeholder={isChatRecording ? "正在倾听..." : "输入问题..."}
|
| 317 |
+
value={textInput}
|
| 318 |
+
onChange={e => setTextInput(e.target.value)}
|
| 319 |
+
onKeyDown={e => e.key === 'Enter' && !isChatProcessing && handleChatSubmit(textInput)}
|
| 320 |
+
disabled={isChatProcessing}
|
| 321 |
+
/>
|
| 322 |
+
|
| 323 |
+
<div className="flex items-center gap-2 shrink-0">
|
| 324 |
+
{/* Mic Button - Coexists with input */}
|
| 325 |
+
<button
|
| 326 |
+
onMouseDown={startRecording}
|
| 327 |
+
onMouseUp={stopRecording}
|
| 328 |
+
onTouchStart={startRecording}
|
| 329 |
+
onTouchEnd={stopRecording}
|
| 330 |
+
className={`p-3 rounded-xl transition-all ${isChatRecording ? 'bg-red-500 scale-110 shadow-lg text-white ring-4 ring-red-100' : 'bg-gray-100 text-gray-500 hover:bg-gray-200'}`}
|
| 331 |
+
title="按住说话"
|
| 332 |
+
>
|
| 333 |
+
{isChatRecording ? <StopCircle size={22}/> : <Mic size={22}/>}
|
| 334 |
+
</button>
|
| 335 |
+
|
| 336 |
+
{/* Send Button */}
|
| 337 |
+
<button
|
| 338 |
+
onClick={() => handleChatSubmit(textInput)}
|
| 339 |
+
className={`p-3 rounded-xl transition-all ${!textInput.trim() || isChatProcessing ? 'bg-gray-100 text-gray-300' : 'bg-blue-600 text-white hover:bg-blue-700 shadow-md'}`}
|
| 340 |
+
disabled={!textInput.trim() || isChatProcessing}
|
| 341 |
+
>
|
| 342 |
+
<Send size={22}/>
|
| 343 |
+
</button>
|
| 344 |
+
</div>
|
| 345 |
</div>
|
| 346 |
+
<div className="text-[10px] text-gray-400 text-center mt-2">支持文字输入或按住麦克风图标进行语音提问</div>
|
| 347 |
</div>
|
| 348 |
</div>
|
| 349 |
);
|