Upload 54 files

components/LiveAssistant.tsx

@@ -1,13 +1,12 @@
 
 import React, { useState, useRef, useEffect } from 'react';
-import { GoogleGenAI, LiveServerMessage, Modality } from "@google/genai";
 import { Mic, X, Power, Loader2, Bot, Volume2, Radio, Activity, RefreshCw, ChevronDown } from 'lucide-react';
 import { api } from '../services/api';
 
 // --- Audio Types & Helpers ---
 // 16kHz for Gemini Input
 const INPUT_SAMPLE_RATE = 16000;
-// 24kHz for Gemini Output
+// 24kHz for Gemini Output (Typically)
 const OUTPUT_SAMPLE_RATE = 24000;
 
 function base64ToUint8Array(base64: string) {
@@ -23,7 +22,6 @@ function base64ToUint8Array(base64: string) {
 export const LiveAssistant: React.FC = () => {
   const [isOpen, setIsOpen] = useState(false);
   const [status, setStatus] = useState<'DISCONNECTED' | 'CONNECTING' | 'CONNECTED' | 'LISTENING' | 'THINKING' | 'SPEAKING'>('DISCONNECTED');
-  const [apiKey, setApiKey] = useState('');
   const [transcript, setTranscript] = useState(''); // Current subtitle
   const [volumeLevel, setVolumeLevel] = useState(0);
 
@@ -34,30 +32,12 @@ export const LiveAssistant: React.FC = () => {
   const sourceNodeRef = useRef<MediaStreamAudioSourceNode | null>(null);
   const gainNodeRef = useRef<GainNode | null>(null);
   
-  const sessionRef = useRef<any>(null); // The GenAI Session
+  const wsRef = useRef<WebSocket | null>(null); 
   const nextPlayTimeRef = useRef<number>(0);
   const analyserRef = useRef<AnalyserNode | null>(null);
   const volumeIntervalRef = useRef<any>(null);
 
-  // 1. Fetch Key on Open
-  useEffect(() => {
-      if (isOpen && !apiKey) {
-          api.ai.getStats().catch(() => {}); // Warm up
-          fetch('/api/ai/live-access', {
-              headers: { 
-                  'x-user-username': api.auth.getCurrentUser()?.username || '',
-                  'x-user-role': api.auth.getCurrentUser()?.role || ''
-              }
-          })
-          .then(res => res.json())
-          .then(data => {
-              if (data.key) setApiKey(data.key);
-          })
-          .catch(err => console.error("Failed to get live key", err));
-      }
-  }, [isOpen]);
-
-  // 2. Clean up on unmount or close
+  // 1. Clean up on unmount or close
   useEffect(() => {
       if (!isOpen) {
           handleDisconnect();
@@ -106,44 +86,45 @@ export const LiveAssistant: React.FC = () => {
   };
 
   const handleConnect = async () => {
-      if (!apiKey) return;
       setStatus('CONNECTING');
-      setTranscript('正在建立连接...');
+      setTranscript('正在连接服务器...');
 
       try {
           initAudioContext();
-          const client = new GoogleGenAI({ apiKey });
           
-          const session = await client.live.connect({
-              model: 'gemini-2.5-flash-native-audio-preview-09-2025',
-              callbacks: {
-                  onopen: () => {
-                      console.log('Session Open');
-                  },
-                  onmessage: (msg: LiveServerMessage) => {
-                      handleServerMessage(msg);
-                  },
-                  onclose: () => {
-                      console.log('Session Close');
-                      handleDisconnect();
-                  },
-                  onerror: (e) => {
-                      console.error('Session Error', e);
-                      setTranscript('连接发生错误，请重试');
-                      handleDisconnect();
-                  }
-              },
-              config: {
-                  responseModalities: [Modality.AUDIO],
-                  speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } } },
-                  // Strong instruction to force Chinese
-                  systemInstruction: "You are a helpful school assistant. You MUST reply in spoken Chinese (Mandarin). Keep answers concise and friendly. Do not use markdown.",
+          // Connect to Backend Proxy via WebSocket
+          const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
+          const wsUrl = `${protocol}//${window.location.host}/ws/live`;
+          
+          console.log("Connecting to", wsUrl);
+          const ws = new WebSocket(wsUrl);
+          wsRef.current = ws;
+
+          ws.onopen = () => {
+              console.log('WS Open');
+              setStatus('CONNECTED');
+              setTranscript('连接成功，请按住麦克风说话');
+          };
+
+          ws.onmessage = async (event) => {
+              try {
+                  const msg = JSON.parse(event.data);
+                  handleServerMessage(msg);
+              } catch (e) {
+                  console.error("Parse error", e);
               }
-          });
+          };
+
+          ws.onclose = () => {
+              console.log('WS Close');
+              handleDisconnect();
+          };
 
-          sessionRef.current = session;
-          setStatus('CONNECTED');
-          setTranscript('连接成功，请按住麦克风说话');
+          ws.onerror = (e) => {
+              console.error('WS Error', e);
+              setTranscript('连接服务器失败');
+              handleDisconnect();
+          };
 
       } catch (e) {
           console.error("Connect failed", e);
@@ -152,17 +133,14 @@ export const LiveAssistant: React.FC = () => {
       }
   };
 
-  const handleServerMessage = async (msg: LiveServerMessage) => {
-      const serverContent = msg.serverContent;
-      
+  const handleServerMessage = async (msg: any) => {
       // 1. Audio Output
-      const audioData = serverContent?.modelTurn?.parts?.[0]?.inlineData?.data;
-      if (audioData && audioContextRef.current) {
-          setStatus('SPEAKING'); // Receiving audio means speaking
+      if (msg.type === 'audio' && msg.data && audioContextRef.current) {
+          setStatus('SPEAKING'); 
           const ctx = audioContextRef.current;
-          const bytes = base64ToUint8Array(audioData);
+          const bytes = base64ToUint8Array(msg.data);
           
-          // Decode Raw PCM (16-bit, 24kHz, Mono)
+          // Decode Raw PCM (16-bit, 24kHz, Mono) from Gemini
           const int16 = new Int16Array(bytes.buffer);
           const float32 = new Float32Array(int16.length);
           for(let i=0; i<int16.length; i++) float32[i] = int16[i] / 32768.0;
@@ -195,20 +173,18 @@ export const LiveAssistant: React.FC = () => {
           };
       }
 
-      // 2. Text Transcription (Subtitle)
-      // Note: The model sometimes returns 'thought' or 'search' logs here. 
-      // We rely on audio mostly, but show text if it looks like a response.
-      const text = serverContent?.modelTurn?.parts?.[0]?.text;
-      if (text) {
-          if (!text.startsWith('**') && !text.includes('Finding')) {
-             setTranscript(text);
-          }
+      // 2. Text Transcription
+      if (msg.type === 'text' && msg.content) {
+          setTranscript(msg.content);
       }
       
-      // 3. User Turn Finished (Model starts thinking)
-      if (serverContent?.turnComplete) {
+      // 3. Status/Error
+      if (msg.type === 'turnComplete') {
           setStatus('THINKING');
       }
+      if (msg.type === 'error') {
+          setTranscript(`错误: ${msg.message}`);
+      }
   };
 
   const startRecording = async () => {
@@ -217,13 +193,17 @@ export const LiveAssistant: React.FC = () => {
       try {
           // Interrupt model if speaking
           if (status === 'SPEAKING') {
-              // We can send an interruption message or just stop playing, but API handles new input as interrupt usually
               setStatus('CONNECTED'); 
           }
 
-          const stream = await navigator.mediaDevices.getUserMedia({ audio: { sampleRate: INPUT_SAMPLE_RATE } });
+          const stream = await navigator.mediaDevices.getUserMedia({ audio: { 
+              sampleRate: INPUT_SAMPLE_RATE,
+              channelCount: 1,
+              echoCancellation: true
+          }});
           mediaStreamRef.current = stream;
           
+          // Use a new context for input to ensure 16k rate if browser supports specific ctx rate
           const ctx = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: INPUT_SAMPLE_RATE });
           const source = ctx.createMediaStreamSource(stream);
           const processor = ctx.createScriptProcessor(4096, 1, 1);
@@ -231,14 +211,13 @@ export const LiveAssistant: React.FC = () => {
           processor.onaudioprocess = (e) => {
               const inputData = e.inputBuffer.getChannelData(0);
               
-              // Downconvert Float32 to Int16 for Gemini
+              // Float32 -> Int16 -> Base64
               const l = inputData.length;
               const int16Data = new Int16Array(l);
               for (let i = 0; i < l; i++) {
                   int16Data[i] = inputData[i] * 32768;
               }
               
-              // Convert to Base64 manually to avoid large lib dependency
               let binary = '';
               const bytes = new Uint8Array(int16Data.buffer);
               const len = bytes.byteLength;
@@ -247,15 +226,16 @@ export const LiveAssistant: React.FC = () => {
               }
               const b64 = btoa(binary);
 
-              if (sessionRef.current) {
-                  sessionRef.current.sendRealtimeInput({
-                      media: { mimeType: `audio/pcm;rate=${INPUT_SAMPLE_RATE}`, data: b64 }
-                  });
+              if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
+                  wsRef.current.send(JSON.stringify({
+                      type: 'audio',
+                      data: b64
+                  }));
               }
           };
 
           source.connect(processor);
-          processor.connect(ctx.destination);
+          processor.connect(ctx.destination); // Destination is mute, just for processing pipeline
           
           sourceNodeRef.current = source;
           processorRef.current = processor;
@@ -290,19 +270,17 @@ export const LiveAssistant: React.FC = () => {
   };
 
   const handleDisconnect = () => {
-      if (sessionRef.current) {
-          // sessionRef.current.close(); // SDK might not have close method exposed directly depending on version, but usually does
-          sessionRef.current = null;
+      if (wsRef.current) {
+          wsRef.current.close();
+          wsRef.current = null;
       }
       // Cleanup Audio
       if (audioContextRef.current) {
-          audioContextRef.current.suspend(); // Suspend instead of close to reuse? Or close.
-          // For robustness, let's just close and nullify.
           audioContextRef.current.close().catch(()=>{});
           audioContextRef.current = null;
       }
       
-      stopRecording(); // Ensure mic is off
+      stopRecording(); 
       
       setStatus('DISCONNECTED');
       setTranscript('');
@@ -330,7 +308,7 @@ export const LiveAssistant: React.FC = () => {
                 <div className="bg-slate-800/50 p-4 flex justify-between items-center text-white shrink-0 backdrop-blur-md">
                     <div className="flex items-center gap-2">
                         <div className={`w-2 h-2 rounded-full ${status === 'DISCONNECTED' ? 'bg-red-500' : 'bg-green-500 animate-pulse'}`}></div>
-                        <span className="font-bold text-sm">AI 实时通话</span>
+                        <span className="font-bold text-sm">AI 实时通话 (代理模式)</span>
                     </div>
                     <div className="flex gap-2">
                         <button onClick={handleDisconnect} title="重置" className="hover:bg-white/10 p-1.5 rounded-full text-gray-400 hover:text-white transition-colors"><RefreshCw size={16}/></button>
@@ -400,8 +378,7 @@ export const LiveAssistant: React.FC = () => {
                     {status === 'DISCONNECTED' ? (
                         <button 
                             onClick={handleConnect} 
-                            disabled={!apiKey}
-                            className="w-full py-4 bg-blue-600 hover:bg-blue-500 text-white rounded-2xl font-bold flex items-center justify-center gap-2 transition-all hover:scale-[1.02] active:scale-95 disabled:opacity-50 disabled:cursor-not-allowed"
+                            className="w-full py-4 bg-blue-600 hover:bg-blue-500 text-white rounded-2xl font-bold flex items-center justify-center gap-2 transition-all hover:scale-[1.02] active:scale-95"
                         >
                             <Power size={20}/> 开启 AI 语音
                         </button>

package.json

@@ -23,7 +23,8 @@
     "openai": "^4.28.0",
     "@google/genai": "*",
     "react-markdown": "^9.0.1",
-    "remark-gfm": "^4.0.0"
+    "remark-gfm": "^4.0.0",
+    "ws": "^8.16.0"
   },
   "devDependencies": {
     "@types/react": "^18.2.43",

server.js

@@ -15,11 +15,14 @@ const cors = require('cors');
 const bodyParser = require('body-parser');
 const path = require('path');
 const compression = require('compression'); 
+const WebSocket = require('ws'); // Import WS
+const http = require('http'); // Import HTTP for server integration
 
 const PORT = 7860;
 const MONGO_URI = 'mongodb+srv://dv890a:db8822723@chatpro.gw3v0v7.mongodb.net/chatpro?retryWrites=true&w=majority&appName=chatpro&authSource=admin';
 
 const app = express();
+const server = http.createServer(app); // Wrap express app in HTTP server
 
 app.use(compression({
   filter: (req, res) => {
@@ -47,15 +50,10 @@ const connectDB = async () => {
   try {
     await mongoose.connect(MONGO_URI, { serverSelectionTimeoutMS: 30000 });
     console.log('✅ MongoDB 连接成功 (Real Data)');
-    
-    // FIX: Drop the restrictive index that prevents multiple schedules per slot
     try {
         await ScheduleModel.collection.dropIndex('schoolId_1_className_1_dayOfWeek_1_period_1');
         console.log('✅ Dropped restrictive schedule index');
-    } catch (e) {
-        // Ignore error if index doesn't exist
-    }
-
+    } catch (e) { }
   } catch (err) {
     console.error('❌ MongoDB 连接失败:', err.message);
     InMemoryDB.isFallback = true;
@@ -63,6 +61,155 @@ const connectDB = async () => {
 };
 connectDB();
 
+// --- WebSocket Proxy for Gemini Live API ---
+const wss = new WebSocket.Server({ noServer: true });
+
+server.on('upgrade', (request, socket, head) => {
+    // Basic path check
+    if (request.url.startsWith('/ws/live')) {
+        wss.handleUpgrade(request, socket, head, (ws) => {
+            wss.emit('connection', ws, request);
+        });
+    } else {
+        socket.destroy();
+    }
+});
+
+wss.on('connection', async (ws, req) => {
+    console.log('🔌 Client connected to Live Proxy');
+    let geminiSession = null;
+    let isGeminiConnected = false;
+
+    try {
+        // 1. Get API Key
+        const config = await ConfigModel.findOne({ key: 'main' });
+        let apiKey = process.env.API_KEY;
+        if (config && config.apiKeys && config.apiKeys.gemini && config.apiKeys.gemini.length > 0) {
+            apiKey = config.apiKeys.gemini[0]; // Use first available key
+        }
+
+        if (!apiKey) {
+            ws.send(JSON.stringify({ type: 'error', message: 'No Server API Key Configured' }));
+            ws.close();
+            return;
+        }
+
+        // 2. Initialize Gemini SDK (Dynamic Import)
+        const { GoogleGenAI, Modality } = await import("@google/genai");
+        const client = new GoogleGenAI({ apiKey });
+
+        geminiSession = await client.live.connect({
+            model: 'gemini-2.5-flash-native-audio-preview-09-2025',
+            config: {
+                responseModalities: [Modality.AUDIO],
+                speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } } },
+                systemInstruction: { parts: [{ text: "你是一位乐于助人的校园AI助手。请必须使用中文（普通话）回答。回答要简短、自然、口语化。不要使用Markdown格式。" }] },
+                outputAudioTranscription: { model: true }
+            }
+        });
+
+        isGeminiConnected = true;
+        console.log('🔗 Backend connected to Gemini Live');
+        ws.send(JSON.stringify({ type: 'status', content: 'CONNECTED' }));
+
+        // 3. Forward Gemini -> Client
+        // Note: The SDK's session object relies on callbacks/listeners in newer versions, 
+        // or uses a stream iterator. Adjusting based on standard SDK usage for proxying.
+        // We need to attach listeners to the session manually if the SDK supports it, 
+        // or loop through the incoming stream. 
+        
+        // Since `live.connect` returns a session that is also an event emitter (typically) or we passed callbacks.
+        // Let's re-instantiate using callbacks approach for easier piping which matches the frontend logic I wrote earlier,
+        // but now applied to backend.
+        
+        // RE-DO Connect with Callbacks to handle stream piping properly
+        geminiSession = await client.live.connect({
+            model: 'gemini-2.5-flash-native-audio-preview-09-2025',
+            config: {
+                responseModalities: [Modality.AUDIO],
+                speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } } },
+                systemInstruction: { parts: [{ text: "你是一位乐于助人的校园AI助手。请必须使用中文（普通话）回答。回答要简短、自然、口语化。不要使用Markdown格式。" }] },
+                outputAudioTranscription: { model: true }
+            },
+            callbacks: {
+                onopen: () => {
+                    console.log('Gemini Stream Open');
+                },
+                onmessage: (msg) => {
+                    // Extract relevant data and send to frontend
+                    const serverContent = msg.serverContent;
+                    
+                    // Audio
+                    const audioData = serverContent?.modelTurn?.parts?.[0]?.inlineData?.data;
+                    if (audioData) {
+                        if (ws.readyState === WebSocket.OPEN) {
+                            ws.send(JSON.stringify({ type: 'audio', data: audioData }));
+                        }
+                    }
+
+                    // Text
+                    const text = serverContent?.modelTurn?.parts?.[0]?.text;
+                    if (text) {
+                        if (ws.readyState === WebSocket.OPEN) {
+                            ws.send(JSON.stringify({ type: 'text', content: text }));
+                        }
+                    }
+                    
+                    // Turn Complete
+                    if (serverContent?.turnComplete) {
+                        if (ws.readyState === WebSocket.OPEN) {
+                            ws.send(JSON.stringify({ type: 'turnComplete' }));
+                        }
+                    }
+                },
+                onclose: () => {
+                    console.log('Gemini Stream Closed');
+                    if (ws.readyState === WebSocket.OPEN) ws.close();
+                },
+                onerror: (err) => {
+                    console.error('Gemini Stream Error', err);
+                    if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: 'error', message: 'Upstream Error' }));
+                }
+            }
+        });
+
+        // 4. Forward Client -> Gemini
+        ws.on('message', (message) => {
+            if (!isGeminiConnected) return;
+            try {
+                const parsed = JSON.parse(message);
+                if (parsed.type === 'audio') {
+                    // Frontend sends base64, SDK needs base64 string directly in media object
+                    // data: { mimeType: ..., data: ... }
+                    geminiSession.sendRealtimeInput({
+                        media: { 
+                            mimeType: 'audio/pcm;rate=16000', 
+                            data: parsed.data 
+                        }
+                    });
+                }
+            } catch (e) {
+                console.error('Error parsing client message', e);
+            }
+        });
+
+        ws.on('close', () => {
+            console.log('Client disconnected');
+            if (geminiSession) {
+                // Try to close session if method exists, otherwise it might close automatically
+                try { geminiSession.close(); } catch(e){}
+            }
+        });
+
+    } catch (e) {
+        console.error('WebSocket Handshake Error:', e);
+        ws.send(JSON.stringify({ type: 'error', message: e.message }));
+        ws.close();
+    }
+});
+
+// ... (Rest of Express logic) ...
+
 const getQueryFilter = (req) => { 
   const s = req.headers['x-school-id']; 
   const role = req.headers['x-user-role'];
@@ -113,8 +260,6 @@ const generateStudentNo = async () => {
 // MOUNT AI ROUTES
 app.use('/api/ai', aiRoutes);
 
-// ... (Rest of Existing Routes) ...
-
 // --- TODO LIST ENDPOINTS ---
 app.get('/api/todos', async (req, res) => {
     const username = req.headers['x-user-username'];
@@ -162,7 +307,6 @@ app.put('/api/schedules/:id', async (req, res) => {
 // Create or Update by Logic (Upsert)
 app.post('/api/schedules', async (req, res) => { 
     try {
-        // Updated Filter: Include weekType to allow separate ODD/EVEN records for same slot
         const filter = { 
             className: req.body.className, 
             dayOfWeek: req.body.dayOfWeek, 
@@ -175,14 +319,12 @@ app.post('/api/schedules', async (req, res) => {
         await ScheduleModel.findOneAndUpdate(filter, injectSchoolId(req, req.body), {upsert:true}); 
         res.json({}); 
     } catch (e) {
-        console.error("Save schedule error:", e);
         res.status(500).json({ error: e.message });
     }
 });
 
 app.delete('/api/schedules', async (req, res) => { 
     try {
-        // Support deleting by ID if provided
         if (req.query.id) {
             await ScheduleModel.findByIdAndDelete(req.query.id);
         } else {
@@ -194,14 +336,12 @@ app.delete('/api/schedules', async (req, res) => {
     }
 });
 
-// --- USER MENU ORDER ---
 app.put('/api/users/:id/menu-order', async (req, res) => {
     const { menuOrder } = req.body;
     await User.findByIdAndUpdate(req.params.id, { menuOrder });
     res.json({ success: true });
 });
 
-// ... (Rest of existing routes unchanged) ...
 app.get('/api/classes/:className/teachers', async (req, res) => {
     const { className } = req.params;
     const schoolId = req.headers['x-school-id'];
@@ -545,4 +685,4 @@ app.delete('/api/attendance/calendar/:id', async (req, res) => { await SchoolCal
 app.post('/api/batch-delete', async (req, res) => { if(req.body.type==='student') await Student.deleteMany({_id:{$in:req.body.ids}}); if(req.body.type==='score') await Score.deleteMany({_id:{$in:req.body.ids}}); res.json({}); });
 
 app.get('*', (req, res) => { res.sendFile(path.join(__dirname, 'dist', 'index.html')); });
-app.listen(PORT, () => console.log(`🚀 Server running on port ${PORT}`));
+server.listen(PORT, () => console.log(`🚀 Server running on port ${PORT}`));