import { Component, OnDestroy, OnInit, signal, ViewChild, ElementRef, ChangeDetectorRef } from '@angular/core'; import { CommonModule } from '@angular/common'; import { Router, NavigationStart } from '@angular/router'; import { Subscription } from 'rxjs'; declare global { interface Window { webkitSpeechRecognition?: any; SpeechRecognition?: any; } } type QAResult = { question: string; transcript: string; language: string; avgPitchHz: number | null; avgVolume: number | null; // 0..1 (rough RMS) audioUrl: string; startedAt: number; endedAt: number; }; @Component({ standalone: true, selector: 'app-py-detect', imports: [CommonModule], templateUrl: './py-detect.component.html', styleUrls: ['./py-detect.component.css'] }) export class PyDetectComponent implements OnInit, OnDestroy { // ---- UI state ---- status = signal<'idle' | 'asking' | 'idle-wait' | 'recording' | 'processing'>('idle'); autoMode = signal(true); // Auto Next is always enabled micOn = signal(false); ttsEnabled = signal(true); // Speak Questions is always enabled recognizerReady = signal(false); // ---- TTS active flag ---- private isActive = false; // ---- Q/A data ---- currentQuestion = signal(''); questionIndex = signal(0); log: QAResult[] = []; // ---- Constructor with Router Injection ---- private routerSubscription?: Subscription; constructor(private router: Router, private cdr: ChangeDetectorRef) { // Cancel TTS on any navigation away this.routerSubscription = this.router.events.subscribe(event => { if (event instanceof NavigationStart) { if (window.speechSynthesis) { window.speechSynthesis.cancel(); } } }); } // ---- Recording/analysis handles ---- private mediaStream?: MediaStream; private mediaRecorder?: MediaRecorder; private audioChunks: Blob[] = []; private audioCtx?: AudioContext; private analyser?: AnalyserNode; private sourceNode?: MediaStreamAudioSourceNode; private pitchSamples: number[] = []; private volumeSamples: number[] = []; private analyserBuffer?: Float32Array; private analyserTimer?: any; // ---- Speech Recognition ---- private recognition?: any; // webkitSpeechRecognition private transcriptSoFar = ''; private detectedLang = 'auto'; // ---- Settings ---- private maxAnswerMs = 10_000; // per answer recording window private silenceTimeout?: any; // Declare silenceTimeout here private analyserWindowMs = 100; // Declare analyserWindowMs property // Example question source (replace with API call when ready) private seedQuestions = [ 'Please introduce yourself in two sentences.', 'What motivates you to take on challenging tasks?', 'Describe a situation where you solved a tough problem.', 'How do you handle disagreements in a team?', 'What is a recent technology you learned and why?' ]; // Button state signals startDisabled = signal(false); // Always enabled stopDisabled = signal(true); resumeDisabled = signal(true); submitDisabled = signal(true); // Add missing public methods and properties for template binding public videoStatus: string = ''; public videoStream?: MediaStream; public videoRecorder?: MediaRecorder; @ViewChild('videoElement', { static: false }) videoElement?: ElementRef; public videoChunks: Blob[] = []; public videoAnswers: Blob[] = []; // UI properties for template progress: number = 0; caseId: string = 'CASE-007'; officer: string = 'Ganesh'; currentQuestionText: string = ''; isRecording: boolean = false; isProcessing: boolean = false; transcriptLines: string[] = []; showSummary: boolean = false; summaryData: { question: string; answer: string; duration: number }[] = []; // Navigate back to the homepage navigateHome() { if (window.speechSynthesis) { window.speechSynthesis.cancel(); // Stop any TTS audio immediately on navigation } this.router.navigate(['/']); // Navigates to the root (homepage) } // Navigate to the info page goToInfoPage() { if (window.speechSynthesis) { window.speechSynthesis.cancel(); // Stop any TTS audio immediately on navigation } this.router.navigate(['/infopage']); } // ======== Lifecycle ======== ngOnInit(): void { this.isActive = true; } ngOnDestroy(): void { this.isActive = false; if (this.routerSubscription) { this.routerSubscription.unsubscribe(); } this.cleanupAll(); this.stopVideoRecording(); this.micOn.set(false); // Stop mic indicator this.recognizerReady.set(false); // Stop recognizer indicator this.videoStatus = ''; if (this.videoStream) { this.videoStream.getTracks().forEach(t => t.stop()); this.videoStream = undefined; } this.videoRecorder = undefined; if (window.speechSynthesis) { window.speechSynthesis.cancel(); // Stop any TTS audio } } // ======== Main flow ======== async start(): Promise { if (this.status() !== 'idle') return; this.status.set('asking'); this.setupRecognition(); this.recognizerReady.set(!!this.recognition); await this.startCamera(); this.startDisabled.set(false); // Always enabled this.stopDisabled.set(false); this.resumeDisabled.set(true); this.submitDisabled.set(true); this.autoMode.set(true); // Enable auto mode for continuous questions this.nextQuestionLoopRunning = true; this.nextQuestionLoop(); } stopAll(): void { this.autoMode.set(false); // Stop auto mode to halt question loop this.nextQuestionLoopRunning = false; if (this.videoRecorder && this.videoRecorder.state === 'recording') { this.videoRecorder.stop(); } if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') { this.cleanupRecording(); } this.stopRecognition(); if (window.speechSynthesis) { window.speechSynthesis.cancel(); // Stop any TTS audio } // Stop and clear video stream if (this.videoStream) { this.videoStream.getTracks().forEach(t => t.stop()); this.videoStream = undefined; if (this.videoElement?.nativeElement) { this.videoElement.nativeElement.srcObject = null; } } this.videoRecorder = undefined; this.videoStatus = ''; this.micOn.set(false); this.recognizerReady.set(false); this.status.set('idle'); this.startDisabled.set(false); // Always enabled this.stopDisabled.set(false); // Ensure Stop button is enabled after stopping this.resumeDisabled.set(false); this.submitDisabled.set(false); } public resume() { // Start camera if not already started this.startCamera(); // Resume video recording if paused if (this.videoRecorder && this.videoRecorder.state === 'paused') { this.videoRecorder.resume(); this.videoStatus = 'Recording...'; this.stopDisabled.set(false); this.resumeDisabled.set(true); this.submitDisabled.set(true); } // Resume question loop if stopped if (this.status() === 'idle') { this.autoMode.set(true); this.nextQuestionLoopRunning = true; this.startDisabled.set(false); this.stopDisabled.set(false); this.resumeDisabled.set(true); this.submitDisabled.set(true); this.nextQuestionLoop(); } } public submitAll() { this.stopAll(); this.stopVideoRecording(); this.videoStatus = 'Submitted!'; this.startDisabled.set(true); this.stopDisabled.set(true); this.resumeDisabled.set(true); this.submitDisabled.set(true); // TODO: Upload all videoAnswers to backend // Example: Calculate dummy percentages (replace with real logic) const truePercentage = 70; const falsePercentage = 30; this.router.navigate(['/validationpage'], { state: { truePercentage, falsePercentage } }); } private nextQuestionLoopRunning = false; private async nextQuestionLoop(): Promise { while (this.autoMode() && this.nextQuestionLoopRunning) { // 1) Get next question const q = await this.fetchNextQuestion(); this.currentQuestion.set(q); this.status.set('asking'); // Show asking popup if (this.ttsEnabled()) { await this.speak(q); } // After question finishes playing, show idle popup for 5 seconds this.status.set('idle-wait'); // Show idle popup await this.sleep(5000); // Wait for suspect reply // Now start speech recognition only this.status.set('recording'); let userSpoke = false; this.transcriptSoFar = ''; this.startRecognition('en-IN'); // Wait for user to start speaking await new Promise((resolve) => { const checkSpeech = () => { if (this.transcriptSoFar.trim().length > 0 && !userSpoke) { userSpoke = true; // Start video recording and analysis when user speaks this.startVideoRecording(); resolve(); } else { setTimeout(checkSpeech, 200); } }; checkSpeech(); }); // Stop recognition and video recording after answer window await this.sleep(this.maxAnswerMs); this.stopRecognition(); this.stopVideoRecording(); const startedAt = Date.now(); const { audioUrl, avgPitchHz, avgVolume, transcript, language } = await this.captureAnswerWithAnalysis(this.maxAnswerMs); const endedAt = Date.now(); this.log.push({ question: q, transcript, language, avgPitchHz, avgVolume, audioUrl, startedAt, endedAt }); this.questionIndex.set(this.questionIndex() + 1); this.status.set('processing'); await this.sleep(700); } this.status.set('idle'); this.stopVideoRecording(); this.startDisabled.set(false); this.stopDisabled.set(true); this.resumeDisabled.set(false); this.submitDisabled.set(false); } // ======== Question source ======== private async fetchNextQuestion(): Promise { // Replace this with HTTP call to your backend if needed. // Example: const { question } = await this.http.get<{question:string}>('/api/next-question').toPromise(); const i = this.questionIndex() % this.seedQuestions.length; return this.seedQuestions[i]; } // ======== TTS (question playback) ======== private speak(text: string): Promise { return new Promise((resolve) => { if (!this.isActive) return resolve(); // Only play TTS if component is active const synth = window.speechSynthesis; if (!synth) return resolve(); // gracefully continue without TTS const utter = new SpeechSynthesisUtterance(text); // Optional voice selection: pick an Indian English voice if available const prefer = ['en-IN', 'en-GB', 'en-US']; const voices = synth.getVoices(); const v = voices.find(v => prefer.includes(v.lang) || v.lang.toLowerCase().startsWith('en')); if (v) utter.voice = v; utter.rate = 1.0; utter.pitch = 1.0; utter.onend = () => resolve(); utter.onerror = () => resolve(); // do not block flow synth.cancel(); // ensure clean queue synth.speak(utter); }); } // ======== Recording + Recognition + Analysis ======== private async captureAnswerWithAnalysis(ms: number): Promise<{ audioUrl: string; avgPitchHz: number | null; avgVolume: number | null; transcript: string; language: string; }> { // reset buffers this.audioChunks = []; this.pitchSamples = []; this.volumeSamples = []; this.transcriptSoFar = ''; this.detectedLang = 'auto'; // 1) mic stream this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true }, video: false }); this.micOn.set(true); // 2) prepare MediaRecorder const mime = this.chooseMimeType(); this.mediaRecorder = new MediaRecorder(this.mediaStream, { mimeType: mime }); this.mediaRecorder.ondataavailable = (e) => { if (e.data && e.data.size > 0) this.audioChunks.push(e.data); }; // 3) start recognition (if available) this.startRecognition('en-IN'); // 4) start analysis await this.startAnalyser(this.mediaStream); // 5) record for fixed window const recordPromise = new Promise((resolve) => { this.mediaRecorder!.onstop = () => resolve(); this.mediaRecorder!.start(200); // gather chunks setTimeout(() => { if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') { this.mediaRecorder.stop(); } }, ms); }); await recordPromise; // 6) stop analysis + recognition + mic this.stopAnalyser(); this.stopRecognition(); this.cleanupMediaStream(); this.micOn.set(false); // 7) build audio URL const blob = new Blob(this.audioChunks, { type: mime }); const audioUrl = URL.createObjectURL(blob); // 8) aggregate metrics const avgPitchHz = this.averageNonZero(this.pitchSamples) ?? null; const avgVolume = this.averageNonZero(this.volumeSamples) ?? null; const transcript = this.transcriptSoFar.trim(); const language = this.detectedLang; return { audioUrl, avgPitchHz, avgVolume, transcript, language }; } private waitForSilenceOrContinue() { if (this.silenceTimeout) clearTimeout(this.silenceTimeout); this.silenceTimeout = setTimeout(() => { this.stopAll(); }, 5000); // Timeout after 5 seconds of silence } private chooseMimeType(): string { const candidates = [ 'audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/mpeg' ]; for (const c of candidates) { if (MediaRecorder.isTypeSupported(c)) return c; } return ''; } // ======== Web Audio analysis (pitch + volume) ======== private async startAnalyser(stream: MediaStream) { this.audioCtx = new (window.AudioContext || (window as any).webkitAudioContext)(); this.sourceNode = this.audioCtx.createMediaStreamSource(stream); this.analyser = this.audioCtx.createAnalyser(); this.analyser.fftSize = 2048; this.sourceNode.connect(this.analyser); this.analyserBuffer = new Float32Array(this.analyser.fftSize); const tick = () => { if (!this.analyser || !this.analyserBuffer) return; this.analyser.getFloatTimeDomainData(this.analyserBuffer); const pitch = this.estimatePitchFromAutocorrelation( this.analyserBuffer, this.audioCtx!.sampleRate ); const vol = this.rootMeanSquare(this.analyserBuffer); if (pitch) this.pitchSamples.push(pitch); this.volumeSamples.push(vol); this.analyserTimer = setTimeout(tick, this.analyserWindowMs); }; tick(); } private stopAnalyser() { if (this.analyserTimer) clearTimeout(this.analyserTimer); this.analyserTimer = null; if (this.sourceNode) { try { this.sourceNode.disconnect(); } catch { } } if (this.analyser) { try { this.analyser.disconnect(); } catch { } } if (this.audioCtx) { try { this.audioCtx.close(); } catch { } } this.sourceNode = undefined; this.analyser = undefined; this.audioCtx = undefined; } // Simple autocorrelation-based pitch estimator private estimatePitchFromAutocorrelation(buf: Float32Array, sampleRate: number): number | null { // 1) normalize let size = buf.length; let rms = 0; for (let i = 0; i < size; i++) rms += buf[i] * buf[i]; rms = Math.sqrt(rms / size); if (rms < 0.01) return null; // too quiet // 2) autocorrelation const MAX_SAMPLES = Math.floor(size / 2); let bestOffset = -1; let bestCorr = 0; let lastCorr = 1; for (let offset = 1; offset < MAX_SAMPLES; offset++) { let corr = 0; for (let i = 0; i < MAX_SAMPLES; i++) { corr += Math.abs(buf[i] - buf[i + offset]); } corr = 1 - (corr / MAX_SAMPLES); if (corr > 0.9 && corr > lastCorr) { bestCorr = corr; bestOffset = offset; } lastCorr = corr; } if (bestOffset > 0) { const freq = sampleRate / bestOffset; if (freq >= 50 && freq <= 400) return Math.round(freq); // human speech band (rough) } return null; } private rootMeanSquare(buf: Float32Array): number { let sum = 0; for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i]; return Math.sqrt(sum / buf.length); // 0..~0.5 typical speech } private averageNonZero(arr: number[]): number | undefined { const f = arr.filter(x => x && isFinite(x)); if (!f.length) return undefined; return Math.round((f.reduce((a, b) => a + b, 0) / f.length) * 100) / 100; } // ======== Speech Recognition ======== private setupRecognition() { const Ctor = window.webkitSpeechRecognition || window.SpeechRecognition; if (!Ctor) return; this.recognition = new Ctor(); this.recognition.continuous = true; this.recognition.interimResults = false; // Disable interim results (only final results) this.recognition.onresult = (event: any) => { let finalText = ''; // Loop over all result sets, but only process final results for (let i = event.resultIndex; i < event.results.length; i++) { const result = event.results[i]; // Get the transcript of the final recognized word if (result.isFinal) { finalText += result[0].transcript.trim(); // Append final result and clean up } } // Clean up the result by removing filler words (optional) this.transcriptSoFar = this.removeFillerWords(finalText.trim()); }; this.recognition.onerror = (error: any) => { console.error('Speech recognition error', error); }; this.recognition.onend = () => { console.log('Speech recognition has ended'); }; } private removeFillerWords(text: string): string { const fillerWords = ['um', 'ah', 'like', 'you know', 'so', 'actually', 'basically']; const regex = new RegExp(`\\b(${fillerWords.join('|')})\\b`, 'gi'); return text.replace(regex, '').replace(/\s+/g, ' ').trim(); // Clean extra spaces after removal } private startRecognition(lang: string) { if (!this.recognition) return; try { this.recognition.lang = lang; // set your target language; change to 'ta-IN' for Tamil, etc. this.recognition.start(); } catch { /* ignore double-start */ } } private stopRecognition() { if (!this.recognition) return; try { this.recognition.stop(); } catch { /* ignore */ } } // ======== Clean-up ======== private cleanupMediaStream() { if (this.mediaStream) { this.mediaStream.getTracks().forEach(t => t.stop()); } this.mediaStream = undefined; } private cleanupRecording() { try { if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') this.mediaRecorder.stop(); } catch { } this.mediaRecorder = undefined; this.audioChunks = []; this.stopAnalyser(); this.cleanupMediaStream(); } private cleanupAll() { this.stopRecognition(); this.cleanupRecording(); } // ======== Helpers ======== private sleep(ms: number) { return new Promise(res => setTimeout(res, ms)); } public async startCamera() { if (!this.videoStream) { this.videoStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false }); if (this.videoElement?.nativeElement) { this.videoElement.nativeElement.srcObject = this.videoStream; } } } public async startVideoRecording() { if (!this.videoStream) return; this.videoChunks = []; this.videoRecorder = new MediaRecorder(this.videoStream, { mimeType: 'video/webm' }); this.videoRecorder.ondataavailable = (e) => { if (e.data && e.data.size > 0) this.videoChunks.push(e.data); }; this.videoRecorder.onstart = () => { this.videoStatus = 'Recording...'; }; this.videoRecorder.onstop = () => { this.videoStatus = 'Stopped'; const videoBlob = new Blob(this.videoChunks, { type: 'video/webm' }); this.videoAnswers.push(videoBlob); }; this.videoRecorder.start(); } public stopVideoRecording() { if (this.videoRecorder && this.videoRecorder.state !== 'inactive') { this.videoRecorder.stop(); } } // UI methods for template onStartInterview() { this.currentQuestionText = this.seedQuestions[0]; this.progress = 0; this.isRecording = false; this.isProcessing = false; } pauseRecording() { this.isRecording = false; // Add logic to pause recording } skipQuestion() { const idx = this.seedQuestions.indexOf(this.currentQuestionText); if (idx >= 0 && idx < this.seedQuestions.length - 1) { this.currentQuestionText = this.seedQuestions[idx + 1]; this.progress = Math.round(((idx + 2) / this.seedQuestions.length) * 100); } else { this.currentQuestionText = ''; this.progress = 100; this.showSummary = true; } } closeSummary() { this.showSummary = false; } onStartRecording() { this.progress = 0; this.transcriptLines = []; this.showSummary = false; this.startQuestionFlow(0); // Play the question using TTS, then wait 2s, then start recognition this.speakQuestion(this.seedQuestions[0], () => { setTimeout(() => { this.startRecognitionWithRecording(0); }, 2000); }); } speakQuestion(text: string, onEnd?: () => void) { const synth = window.speechSynthesis; if (!synth) { if (onEnd) onEnd(); return; } const utter = new SpeechSynthesisUtterance(text); utter.lang = 'en-IN'; synth.cancel(); // Stop any previous speech utter.onend = () => { if (onEnd) onEnd(); }; synth.speak(utter); } startRecognitionWithRecording(idx: number) { const Ctor = window.webkitSpeechRecognition || window.SpeechRecognition; if (!Ctor) return; this.recognition = new Ctor(); this.recognition.lang = 'en-IN'; this.recognition.continuous = false; this.recognition.interimResults = false; let recordingStarted = false; const startTime = Date.now(); // Timeout logic for 5 seconds if no answer const silenceTimeout = setTimeout(() => { if (!recordingStarted) { this.recognition.stop(); // Do NOT set isRecording = false here this.status.set('idle'); setTimeout(() => this.playNextQuestion(idx + 1), 5000); } }, 5000); this.recognition.onresult = (event: any) => { let finalText = ''; for (let i = event.resultIndex; i < event.results.length; i++) { const result = event.results[i]; if (result.isFinal) { finalText += result[0].transcript.trim(); } } if (!recordingStarted) { recordingStarted = true; clearTimeout(silenceTimeout); // Cancel timeout if answer detected } this.transcriptLines.push(finalText); }; this.recognition.onend = () => { this.isRecording = false; this.status.set('processing'); setTimeout(() => this.playNextQuestion(idx + 1), 5000); }; this.recognition.onerror = () => { this.isRecording = false; this.status.set('idle'); }; this.recognition.start(); } playNextQuestion(idx: number) { if (idx >= this.seedQuestions.length) { this.currentQuestionText = ''; this.progress = 100; this.showSummary = true; this.status.set('idle'); this.cdr.detectChanges(); this.isRecording = false; return; } this.currentQuestionText = this.seedQuestions[idx]; this.cdr.detectChanges(); this.status.set('asking'); // Set status to asking this.isRecording = false; // Not recording yet this.speakQuestion(this.seedQuestions[idx], () => { // After TTS, wait 5 seconds, then start recording setTimeout(() => { this.status.set('recording'); this.isRecording = true; this.startRecognitionWithRecording(idx); }, 5000); }); } startQuestionFlow(idx: number) { if (idx >= this.seedQuestions.length) { this.currentQuestionText = ''; this.progress = 100; this.showSummary = true; this.status.set('idle'); return; } this.currentQuestionText = this.seedQuestions[idx]; this.status.set('asking'); setTimeout(() => { this.status.set('recording'); this.isRecording = true; this.isProcessing = false; this.transcriptLines = []; this.startRecognition('en-IN'); }, 1200); // 1.2s delay for "Asking..." } get currentQuestionIndex(): number { return this.seedQuestions.indexOf(this.currentQuestionText) + 1; } get totalQuestions(): number { return this.seedQuestions.length; } startVoice() { // Add logic to start voice recording or recognition here this.status.set('recording'); } }