Py-detect / src /app /py-detect /py-detect.component.ts
Oviya
casedetailspage-update
6f093ab
raw
history blame
25.1 kB
import { Component, OnDestroy, OnInit, signal, ViewChild, ElementRef, ChangeDetectorRef } from '@angular/core';
import { CommonModule } from '@angular/common';
import { Router, NavigationStart } from '@angular/router';
import { Subscription } from 'rxjs';
declare global {
interface Window {
webkitSpeechRecognition?: any;
SpeechRecognition?: any;
}
}
type QAResult = {
question: string;
transcript: string;
language: string;
avgPitchHz: number | null;
avgVolume: number | null; // 0..1 (rough RMS)
audioUrl: string;
startedAt: number;
endedAt: number;
};
@Component({
standalone: true,
selector: 'app-py-detect',
imports: [CommonModule],
templateUrl: './py-detect.component.html',
styleUrls: ['./py-detect.component.css']
})
export class PyDetectComponent implements OnInit, OnDestroy {
// ---- UI state ----
status = signal<'idle' | 'asking' | 'idle-wait' | 'recording' | 'processing'>('idle');
autoMode = signal(true); // Auto Next is always enabled
micOn = signal(false);
ttsEnabled = signal(true); // Speak Questions is always enabled
recognizerReady = signal(false);
// ---- TTS active flag ----
private isActive = false;
// ---- Q/A data ----
currentQuestion = signal<string>('');
questionIndex = signal<number>(0);
log: QAResult[] = [];
// ---- Constructor with Router Injection ----
private routerSubscription?: Subscription;
constructor(private router: Router, private cdr: ChangeDetectorRef) {
// Cancel TTS on any navigation away
this.routerSubscription = this.router.events.subscribe(event => {
if (event instanceof NavigationStart) {
if (window.speechSynthesis) {
window.speechSynthesis.cancel();
}
}
});
}
// ---- Recording/analysis handles ----
private mediaStream?: MediaStream;
private mediaRecorder?: MediaRecorder;
private audioChunks: Blob[] = [];
private audioCtx?: AudioContext;
private analyser?: AnalyserNode;
private sourceNode?: MediaStreamAudioSourceNode;
private pitchSamples: number[] = [];
private volumeSamples: number[] = [];
private analyserBuffer?: Float32Array;
private analyserTimer?: any;
// ---- Speech Recognition ----
private recognition?: any; // webkitSpeechRecognition
private transcriptSoFar = '';
private detectedLang = 'auto';
// ---- Settings ----
private maxAnswerMs = 10_000; // per answer recording window
private silenceTimeout?: any; // Declare silenceTimeout here
private analyserWindowMs = 100; // Declare analyserWindowMs property
// Example question source (replace with API call when ready)
private seedQuestions = [
'Please introduce yourself in two sentences.',
'What motivates you to take on challenging tasks?',
'Describe a situation where you solved a tough problem.',
'How do you handle disagreements in a team?',
'What is a recent technology you learned and why?'
];
// Button state signals
startDisabled = signal(false); // Always enabled
stopDisabled = signal(true);
resumeDisabled = signal(true);
submitDisabled = signal(true);
// Add missing public methods and properties for template binding
public videoStatus: string = '';
public videoStream?: MediaStream;
public videoRecorder?: MediaRecorder;
@ViewChild('videoElement', { static: false }) videoElement?: ElementRef<HTMLVideoElement>;
public videoChunks: Blob[] = [];
public videoAnswers: Blob[] = [];
// UI properties for template
progress: number = 0;
caseId: string = 'CASE-007';
officer: string = 'Ganesh';
currentQuestionText: string = '';
isRecording: boolean = false;
isProcessing: boolean = false;
transcriptLines: string[] = [];
showSummary: boolean = false;
summaryData: { question: string; answer: string; duration: number }[] = [];
// Navigate back to the homepage
navigateHome() {
if (window.speechSynthesis) {
window.speechSynthesis.cancel(); // Stop any TTS audio immediately on navigation
}
this.router.navigate(['/']); // Navigates to the root (homepage)
}
// Navigate to the info page
goToInfoPage() {
if (window.speechSynthesis) {
window.speechSynthesis.cancel(); // Stop any TTS audio immediately on navigation
}
this.router.navigate(['/infopage']);
}
// ======== Lifecycle ========
ngOnInit(): void {
this.isActive = true;
}
ngOnDestroy(): void {
this.isActive = false;
if (this.routerSubscription) {
this.routerSubscription.unsubscribe();
}
this.cleanupAll();
this.stopVideoRecording();
this.micOn.set(false); // Stop mic indicator
this.recognizerReady.set(false); // Stop recognizer indicator
this.videoStatus = '';
if (this.videoStream) {
this.videoStream.getTracks().forEach(t => t.stop());
this.videoStream = undefined;
}
this.videoRecorder = undefined;
if (window.speechSynthesis) {
window.speechSynthesis.cancel(); // Stop any TTS audio
}
}
// ======== Main flow ========
async start(): Promise<void> {
if (this.status() !== 'idle') return;
this.status.set('asking');
this.setupRecognition();
this.recognizerReady.set(!!this.recognition);
await this.startCamera();
this.startDisabled.set(false); // Always enabled
this.stopDisabled.set(false);
this.resumeDisabled.set(true);
this.submitDisabled.set(true);
this.autoMode.set(true); // Enable auto mode for continuous questions
this.nextQuestionLoopRunning = true;
this.nextQuestionLoop();
}
stopAll(): void {
this.autoMode.set(false); // Stop auto mode to halt question loop
this.nextQuestionLoopRunning = false;
if (this.videoRecorder && this.videoRecorder.state === 'recording') {
this.videoRecorder.stop();
}
if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') {
this.cleanupRecording();
}
this.stopRecognition();
if (window.speechSynthesis) {
window.speechSynthesis.cancel(); // Stop any TTS audio
}
// Stop and clear video stream
if (this.videoStream) {
this.videoStream.getTracks().forEach(t => t.stop());
this.videoStream = undefined;
if (this.videoElement?.nativeElement) {
this.videoElement.nativeElement.srcObject = null;
}
}
this.videoRecorder = undefined;
this.videoStatus = '';
this.micOn.set(false);
this.recognizerReady.set(false);
this.status.set('idle');
this.startDisabled.set(false); // Always enabled
this.stopDisabled.set(false); // Ensure Stop button is enabled after stopping
this.resumeDisabled.set(false);
this.submitDisabled.set(false);
}
public resume() {
// Start camera if not already started
this.startCamera();
// Resume video recording if paused
if (this.videoRecorder && this.videoRecorder.state === 'paused') {
this.videoRecorder.resume();
this.videoStatus = 'Recording...';
this.stopDisabled.set(false);
this.resumeDisabled.set(true);
this.submitDisabled.set(true);
}
// Resume question loop if stopped
if (this.status() === 'idle') {
this.autoMode.set(true);
this.nextQuestionLoopRunning = true;
this.startDisabled.set(false);
this.stopDisabled.set(false);
this.resumeDisabled.set(true);
this.submitDisabled.set(true);
this.nextQuestionLoop();
}
}
public submitAll() {
this.stopAll();
this.stopVideoRecording();
this.videoStatus = 'Submitted!';
this.startDisabled.set(true);
this.stopDisabled.set(true);
this.resumeDisabled.set(true);
this.submitDisabled.set(true);
// TODO: Upload all videoAnswers to backend
// Example: Calculate dummy percentages (replace with real logic)
const truePercentage = 70;
const falsePercentage = 30;
this.router.navigate(['/validationpage'], { state: { truePercentage, falsePercentage } });
}
private nextQuestionLoopRunning = false;
private async nextQuestionLoop(): Promise<void> {
while (this.autoMode() && this.nextQuestionLoopRunning) {
// 1) Get next question
const q = await this.fetchNextQuestion();
this.currentQuestion.set(q);
this.status.set('asking'); // Show asking popup
if (this.ttsEnabled()) {
await this.speak(q);
}
// After question finishes playing, show idle popup for 5 seconds
this.status.set('idle-wait'); // Show idle popup
await this.sleep(5000); // Wait for suspect reply
// Now start speech recognition only
this.status.set('recording');
let userSpoke = false;
this.transcriptSoFar = '';
this.startRecognition('en-IN');
// Wait for user to start speaking
await new Promise<void>((resolve) => {
const checkSpeech = () => {
if (this.transcriptSoFar.trim().length > 0 && !userSpoke) {
userSpoke = true;
// Start video recording and analysis when user speaks
this.startVideoRecording();
resolve();
} else {
setTimeout(checkSpeech, 200);
}
};
checkSpeech();
});
// Stop recognition and video recording after answer window
await this.sleep(this.maxAnswerMs);
this.stopRecognition();
this.stopVideoRecording();
const startedAt = Date.now();
const { audioUrl, avgPitchHz, avgVolume, transcript, language } =
await this.captureAnswerWithAnalysis(this.maxAnswerMs);
const endedAt = Date.now();
this.log.push({
question: q,
transcript,
language,
avgPitchHz,
avgVolume,
audioUrl,
startedAt,
endedAt
});
this.questionIndex.set(this.questionIndex() + 1);
this.status.set('processing');
await this.sleep(700);
}
this.status.set('idle');
this.stopVideoRecording();
this.startDisabled.set(false);
this.stopDisabled.set(true);
this.resumeDisabled.set(false);
this.submitDisabled.set(false);
}
// ======== Question source ========
private async fetchNextQuestion(): Promise<string> {
// Replace this with HTTP call to your backend if needed.
// Example: const { question } = await this.http.get<{question:string}>('/api/next-question').toPromise();
const i = this.questionIndex() % this.seedQuestions.length;
return this.seedQuestions[i];
}
// ======== TTS (question playback) ========
private speak(text: string): Promise<void> {
return new Promise<void>((resolve) => {
if (!this.isActive) return resolve(); // Only play TTS if component is active
const synth = window.speechSynthesis;
if (!synth) return resolve(); // gracefully continue without TTS
const utter = new SpeechSynthesisUtterance(text);
// Optional voice selection: pick an Indian English voice if available
const prefer = ['en-IN', 'en-GB', 'en-US'];
const voices = synth.getVoices();
const v = voices.find(v =>
prefer.includes(v.lang) || v.lang.toLowerCase().startsWith('en'));
if (v) utter.voice = v;
utter.rate = 1.0;
utter.pitch = 1.0;
utter.onend = () => resolve();
utter.onerror = () => resolve(); // do not block flow
synth.cancel(); // ensure clean queue
synth.speak(utter);
});
}
// ======== Recording + Recognition + Analysis ========
private async captureAnswerWithAnalysis(ms: number): Promise<{
audioUrl: string; avgPitchHz: number | null; avgVolume: number | null; transcript: string; language: string;
}> {
// reset buffers
this.audioChunks = [];
this.pitchSamples = [];
this.volumeSamples = [];
this.transcriptSoFar = '';
this.detectedLang = 'auto';
// 1) mic stream
this.mediaStream = await navigator.mediaDevices.getUserMedia({
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
video: false
});
this.micOn.set(true);
// 2) prepare MediaRecorder
const mime = this.chooseMimeType();
this.mediaRecorder = new MediaRecorder(this.mediaStream, { mimeType: mime });
this.mediaRecorder.ondataavailable = (e) => {
if (e.data && e.data.size > 0) this.audioChunks.push(e.data);
};
// 3) start recognition (if available)
this.startRecognition('en-IN');
// 4) start analysis
await this.startAnalyser(this.mediaStream);
// 5) record for fixed window
const recordPromise = new Promise<void>((resolve) => {
this.mediaRecorder!.onstop = () => resolve();
this.mediaRecorder!.start(200); // gather chunks
setTimeout(() => {
if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') {
this.mediaRecorder.stop();
}
}, ms);
});
await recordPromise;
// 6) stop analysis + recognition + mic
this.stopAnalyser();
this.stopRecognition();
this.cleanupMediaStream();
this.micOn.set(false);
// 7) build audio URL
const blob = new Blob(this.audioChunks, { type: mime });
const audioUrl = URL.createObjectURL(blob);
// 8) aggregate metrics
const avgPitchHz = this.averageNonZero(this.pitchSamples) ?? null;
const avgVolume = this.averageNonZero(this.volumeSamples) ?? null;
const transcript = this.transcriptSoFar.trim();
const language = this.detectedLang;
return { audioUrl, avgPitchHz, avgVolume, transcript, language };
}
private waitForSilenceOrContinue() {
if (this.silenceTimeout) clearTimeout(this.silenceTimeout);
this.silenceTimeout = setTimeout(() => {
this.stopAll();
}, 5000); // Timeout after 5 seconds of silence
}
private chooseMimeType(): string {
const candidates = [
'audio/webm;codecs=opus',
'audio/webm',
'audio/mp4',
'audio/mpeg'
];
for (const c of candidates) {
if (MediaRecorder.isTypeSupported(c)) return c;
}
return '';
}
// ======== Web Audio analysis (pitch + volume) ========
private async startAnalyser(stream: MediaStream) {
this.audioCtx = new (window.AudioContext || (window as any).webkitAudioContext)();
this.sourceNode = this.audioCtx.createMediaStreamSource(stream);
this.analyser = this.audioCtx.createAnalyser();
this.analyser.fftSize = 2048;
this.sourceNode.connect(this.analyser);
this.analyserBuffer = new Float32Array(this.analyser.fftSize);
const tick = () => {
if (!this.analyser || !this.analyserBuffer) return;
this.analyser.getFloatTimeDomainData(this.analyserBuffer);
const pitch = this.estimatePitchFromAutocorrelation(
this.analyserBuffer, this.audioCtx!.sampleRate
);
const vol = this.rootMeanSquare(this.analyserBuffer);
if (pitch) this.pitchSamples.push(pitch);
this.volumeSamples.push(vol);
this.analyserTimer = setTimeout(tick, this.analyserWindowMs);
};
tick();
}
private stopAnalyser() {
if (this.analyserTimer) clearTimeout(this.analyserTimer);
this.analyserTimer = null;
if (this.sourceNode) { try { this.sourceNode.disconnect(); } catch { } }
if (this.analyser) { try { this.analyser.disconnect(); } catch { } }
if (this.audioCtx) { try { this.audioCtx.close(); } catch { } }
this.sourceNode = undefined;
this.analyser = undefined;
this.audioCtx = undefined;
}
// Simple autocorrelation-based pitch estimator
private estimatePitchFromAutocorrelation(buf: Float32Array, sampleRate: number): number | null {
// 1) normalize
let size = buf.length;
let rms = 0;
for (let i = 0; i < size; i++) rms += buf[i] * buf[i];
rms = Math.sqrt(rms / size);
if (rms < 0.01) return null; // too quiet
// 2) autocorrelation
const MAX_SAMPLES = Math.floor(size / 2);
let bestOffset = -1;
let bestCorr = 0;
let lastCorr = 1;
for (let offset = 1; offset < MAX_SAMPLES; offset++) {
let corr = 0;
for (let i = 0; i < MAX_SAMPLES; i++) {
corr += Math.abs(buf[i] - buf[i + offset]);
}
corr = 1 - (corr / MAX_SAMPLES);
if (corr > 0.9 && corr > lastCorr) {
bestCorr = corr;
bestOffset = offset;
}
lastCorr = corr;
}
if (bestOffset > 0) {
const freq = sampleRate / bestOffset;
if (freq >= 50 && freq <= 400) return Math.round(freq); // human speech band (rough)
}
return null;
}
private rootMeanSquare(buf: Float32Array): number {
let sum = 0;
for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i];
return Math.sqrt(sum / buf.length); // 0..~0.5 typical speech
}
private averageNonZero(arr: number[]): number | undefined {
const f = arr.filter(x => x && isFinite(x));
if (!f.length) return undefined;
return Math.round((f.reduce((a, b) => a + b, 0) / f.length) * 100) / 100;
}
// ======== Speech Recognition ========
private setupRecognition() {
const Ctor = window.webkitSpeechRecognition || window.SpeechRecognition;
if (!Ctor) return;
this.recognition = new Ctor();
this.recognition.continuous = true;
this.recognition.interimResults = false; // Disable interim results (only final results)
this.recognition.onresult = (event: any) => {
let finalText = '';
// Loop over all result sets, but only process final results
for (let i = event.resultIndex; i < event.results.length; i++) {
const result = event.results[i];
// Get the transcript of the final recognized word
if (result.isFinal) {
finalText += result[0].transcript.trim(); // Append final result and clean up
}
}
// Clean up the result by removing filler words (optional)
this.transcriptSoFar = this.removeFillerWords(finalText.trim());
};
this.recognition.onerror = (error: any) => {
console.error('Speech recognition error', error);
};
this.recognition.onend = () => {
console.log('Speech recognition has ended');
};
}
private removeFillerWords(text: string): string {
const fillerWords = ['um', 'ah', 'like', 'you know', 'so', 'actually', 'basically'];
const regex = new RegExp(`\\b(${fillerWords.join('|')})\\b`, 'gi');
return text.replace(regex, '').replace(/\s+/g, ' ').trim(); // Clean extra spaces after removal
}
private startRecognition(lang: string) {
if (!this.recognition) return;
try {
this.recognition.lang = lang; // set your target language; change to 'ta-IN' for Tamil, etc.
this.recognition.start();
} catch { /* ignore double-start */ }
}
private stopRecognition() {
if (!this.recognition) return;
try { this.recognition.stop(); } catch { /* ignore */ }
}
// ======== Clean-up ========
private cleanupMediaStream() {
if (this.mediaStream) {
this.mediaStream.getTracks().forEach(t => t.stop());
}
this.mediaStream = undefined;
}
private cleanupRecording() {
try { if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') this.mediaRecorder.stop(); } catch { }
this.mediaRecorder = undefined;
this.audioChunks = [];
this.stopAnalyser();
this.cleanupMediaStream();
}
private cleanupAll() {
this.stopRecognition();
this.cleanupRecording();
}
// ======== Helpers ========
private sleep(ms: number) { return new Promise(res => setTimeout(res, ms)); }
public async startCamera() {
if (!this.videoStream) {
this.videoStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false });
if (this.videoElement?.nativeElement) {
this.videoElement.nativeElement.srcObject = this.videoStream;
}
}
}
public async startVideoRecording() {
if (!this.videoStream) return;
this.videoChunks = [];
this.videoRecorder = new MediaRecorder(this.videoStream, { mimeType: 'video/webm' });
this.videoRecorder.ondataavailable = (e) => {
if (e.data && e.data.size > 0) this.videoChunks.push(e.data);
};
this.videoRecorder.onstart = () => {
this.videoStatus = 'Recording...';
};
this.videoRecorder.onstop = () => {
this.videoStatus = 'Stopped';
const videoBlob = new Blob(this.videoChunks, { type: 'video/webm' });
this.videoAnswers.push(videoBlob);
};
this.videoRecorder.start();
}
public stopVideoRecording() {
if (this.videoRecorder && this.videoRecorder.state !== 'inactive') {
this.videoRecorder.stop();
}
}
// UI methods for template
onStartInterview() {
this.currentQuestionText = this.seedQuestions[0];
this.progress = 0;
this.isRecording = false;
this.isProcessing = false;
}
pauseRecording() {
this.isRecording = false;
// Add logic to pause recording
}
skipQuestion() {
const idx = this.seedQuestions.indexOf(this.currentQuestionText);
if (idx >= 0 && idx < this.seedQuestions.length - 1) {
this.currentQuestionText = this.seedQuestions[idx + 1];
this.progress = Math.round(((idx + 2) / this.seedQuestions.length) * 100);
} else {
this.currentQuestionText = '';
this.progress = 100;
this.showSummary = true;
}
}
closeSummary() {
this.showSummary = false;
}
onStartRecording() {
this.progress = 0;
this.transcriptLines = [];
this.showSummary = false;
this.startQuestionFlow(0);
// Play the question using TTS, then wait 2s, then start recognition
this.speakQuestion(this.seedQuestions[0], () => {
setTimeout(() => {
this.startRecognitionWithRecording(0);
}, 2000);
});
}
speakQuestion(text: string, onEnd?: () => void) {
const synth = window.speechSynthesis;
if (!synth) {
if (onEnd) onEnd();
return;
}
const utter = new SpeechSynthesisUtterance(text);
utter.lang = 'en-IN';
synth.cancel(); // Stop any previous speech
utter.onend = () => { if (onEnd) onEnd(); };
synth.speak(utter);
}
startRecognitionWithRecording(idx: number) {
const Ctor = window.webkitSpeechRecognition || window.SpeechRecognition;
if (!Ctor) return;
this.recognition = new Ctor();
this.recognition.lang = 'en-IN';
this.recognition.continuous = false;
this.recognition.interimResults = false;
let recordingStarted = false;
const startTime = Date.now();
// Timeout logic for 5 seconds if no answer
const silenceTimeout = setTimeout(() => {
if (!recordingStarted) {
this.recognition.stop();
// Do NOT set isRecording = false here
this.status.set('idle');
setTimeout(() => this.playNextQuestion(idx + 1), 5000);
}
}, 5000);
this.recognition.onresult = (event: any) => {
let finalText = '';
for (let i = event.resultIndex; i < event.results.length; i++) {
const result = event.results[i];
if (result.isFinal) {
finalText += result[0].transcript.trim();
}
}
if (!recordingStarted) {
recordingStarted = true;
clearTimeout(silenceTimeout); // Cancel timeout if answer detected
}
this.transcriptLines.push(finalText);
};
this.recognition.onend = () => {
this.isRecording = false;
this.status.set('processing');
setTimeout(() => this.playNextQuestion(idx + 1), 5000);
};
this.recognition.onerror = () => {
this.isRecording = false;
this.status.set('idle');
};
this.recognition.start();
}
playNextQuestion(idx: number) {
if (idx >= this.seedQuestions.length) {
this.currentQuestionText = '';
this.progress = 100;
this.showSummary = true;
this.status.set('idle');
this.cdr.detectChanges();
this.isRecording = false;
return;
}
this.currentQuestionText = this.seedQuestions[idx];
this.cdr.detectChanges();
this.status.set('asking'); // Set status to asking
this.isRecording = false; // Not recording yet
this.speakQuestion(this.seedQuestions[idx], () => {
// After TTS, wait 5 seconds, then start recording
setTimeout(() => {
this.status.set('recording');
this.isRecording = true;
this.startRecognitionWithRecording(idx);
}, 5000);
});
}
startQuestionFlow(idx: number) {
if (idx >= this.seedQuestions.length) {
this.currentQuestionText = '';
this.progress = 100;
this.showSummary = true;
this.status.set('idle');
return;
}
this.currentQuestionText = this.seedQuestions[idx];
this.status.set('asking');
setTimeout(() => {
this.status.set('recording');
this.isRecording = true;
this.isProcessing = false;
this.transcriptLines = [];
this.startRecognition('en-IN');
}, 1200); // 1.2s delay for "Asking..."
}
get currentQuestionIndex(): number {
return this.seedQuestions.indexOf(this.currentQuestionText) + 1;
}
get totalQuestions(): number {
return this.seedQuestions.length;
}
startVoice() {
// Add logic to start voice recording or recognition here
this.status.set('recording');
}
}