Update process_interview.py

process_interview.py

@@ -66,60 +66,58 @@ def download_audio_from_url(url: str) -> str:
         logger.error(f"Failed to download audio from URL {url}: {e}")
         raise
 
+
+
+
+# Initialize services
 def initialize_services():
-    try:
-        pc = Pinecone(api_key=PINECONE_KEY)
-        index_name = "interview-speaker-embeddings"
-        if index_name not in pc.list_indexes().names():
-            pc.create_index(
-                name=index_name,
-                dimension=192,
-                metric="cosine",
-                spec=ServerlessSpec(cloud="aws", region="us-east-1")
-            )
-        index = pc.Index(index_name)
-        genai.configure(api_key=GEMINI_API_KEY)
-        gemini_model = genai.GenerativeModel('gemini-1.5-flash')
-        return index, gemini_model
-    except Exception as e:
-        logger.error(f"Error initializing services: {str(e)}")
-        raise
+    # Pinecone
+    pc = Pinecone(api_key=PINECONE_KEY)
+    index_name = "interview-speaker-embeddings"
+    if index_name not in pc.list_indexes().names():
+        pc.create_index(
+            name=index_name,
+            dimension=192,
+            metric="cosine",
+            spec=ServerlessSpec(cloud="aws", region="us-east-1")
+        )
+    index = pc.Index(index_name)
+    
+    # Gemini
+    genai.configure(api_key=GEMINI_API_KEY)
+    gemini_model = genai.GenerativeModel('gemini-1.5-flash')
+    
+    return index, gemini_model
 
 index, gemini_model = initialize_services()
 
+# Device setup
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 logger.info(f"Using device: {device}")
 
-def load_speaker_model():
-    try:
-        import torch
-        torch.set_num_threads(5)
-        model = EncDecSpeakerLabelModel.from_pretrained(
-            "nvidia/speakerverification_en_titanet_large",
-            map_location=torch.device('cpu')
-        )
-        model.eval()
-        return model
-    except Exception as e:
-        logger.error(f"Model loading failed: {str(e)}")
-        raise RuntimeError("Could not load speaker verification model")
-
+# Load ML models
 def load_models():
-    speaker_model = load_speaker_model()
+    speaker_model = EncDecSpeakerLabelModel.from_pretrained("nvidia/speakerverification_en_titanet_large").to(device)
+    speaker_model.eval()
+    
     nlp = spacy.load("en_core_web_sm")
+    
     tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
     llm_model = AutoModel.from_pretrained("distilbert-base-uncased").to(device)
     llm_model.eval()
+    
     return speaker_model, nlp, tokenizer, llm_model
 
 speaker_model, nlp, tokenizer, llm_model = load_models()
 
+# Audio processing functions
 def convert_to_wav(audio_path: str, output_dir: str = OUTPUT_DIR) -> str:
     try:
         audio = AudioSegment.from_file(audio_path)
         if audio.channels > 1:
             audio = audio.set_channels(1)
         audio = audio.set_frame_rate(16000)
+        
         wav_file = os.path.join(output_dir, f"{uuid.uuid4()}.wav")
         audio.export(wav_file, format="wav")
         return wav_file
@@ -133,11 +131,11 @@ def extract_prosodic_features(audio_path: str, start_ms: int, end_ms: int) -> Di
         segment = audio[start_ms:end_ms]
         temp_path = os.path.join(OUTPUT_DIR, f"temp_{uuid.uuid4()}.wav")
         segment.export(temp_path, format="wav")
-
+        
         y, sr = librosa.load(temp_path, sr=16000)
         pitches = librosa.piptrack(y=y, sr=sr)[0]
         pitches = pitches[pitches > 0]
-
+        
         features = {
             'duration': (end_ms - start_ms) / 1000,
             'mean_pitch': float(np.mean(pitches)) if len(pitches) > 0 else 0.0,
@@ -149,7 +147,7 @@ def extract_prosodic_features(audio_path: str, start_ms: int, end_ms: int) -> Di
             'intensityMax': float(np.max(librosa.feature.rms(y=y)[0])),
             'intensitySD': float(np.std(librosa.feature.rms(y=y)[0])),
         }
-
+        
         os.remove(temp_path)
         return features
     except Exception as e:
@@ -166,9 +164,10 @@ def extract_prosodic_features(audio_path: str, start_ms: int, end_ms: int) -> Di
             'intensitySD': 0.0,
         }
 
-
+# Transcription
 def transcribe(audio_path: str) -> Dict:
     try:
+        # Upload audio
         with open(audio_path, 'rb') as f:
             upload_response = requests.post(
                 "https://api.assemblyai.com/v2/upload",
@@ -176,62 +175,78 @@ def transcribe(audio_path: str) -> Dict:
                 data=f
             )
         audio_url = upload_response.json()['upload_url']
+        
+        # Start transcription
         transcript_response = requests.post(
             "https://api.assemblyai.com/v2/transcript",
             headers={"authorization": ASSEMBLYAI_KEY},
             json={
                 "audio_url": audio_url,
                 "speaker_labels": True,
-                "filter_profanity": True,
-                "speakers_expected": 2
+                "filter_profanity": True
             }
         )
         transcript_id = transcript_response.json()['id']
+        
+        # Poll for results
         while True:
             result = requests.get(
                 f"https://api.assemblyai.com/v2/transcript/{transcript_id}",
                 headers={"authorization": ASSEMBLYAI_KEY}
             ).json()
+            
             if result['status'] == 'completed':
                 return result
             elif result['status'] == 'error':
                 raise Exception(result['error'])
+            
             time.sleep(5)
     except Exception as e:
         logger.error(f"Transcription failed: {str(e)}")
         raise
 
+
+# Speaker identification
 def process_utterance(utterance, full_audio, wav_file):
     try:
+        # Extract audio segment
         start = utterance['start']
         end = utterance['end']
         segment = full_audio[start:end]
         temp_path = os.path.join(OUTPUT_DIR, f"temp_{uuid.uuid4()}.wav")
         segment.export(temp_path, format="wav")
+        
+        # Get speaker embedding
         with torch.no_grad():
-            embedding = speaker_model.get_embedding(temp_path).cpu().numpy()
-        embedding_list = embedding.flatten().tolist()
+            embedding = speaker_model.get_embedding(temp_path).to(device)
+        
+        # Query speaker database
         query_result = index.query(
-            vector=embedding_list,
+            vector=embedding.cpu().numpy().tolist(),
             top_k=1,
             include_metadata=True
         )
+        
+        # Identify speaker
         if query_result['matches'] and query_result['matches'][0]['score'] > 0.7:
             speaker_id = query_result['matches'][0]['id']
             speaker_name = query_result['matches'][0]['metadata']['speaker_name']
         else:
             speaker_id = f"unknown_{uuid.uuid4().hex[:6]}"
             speaker_name = f"Speaker_{speaker_id[-4:]}"
-            index.upsert([(speaker_id, embedding_list, {"speaker_name": speaker_name})])
+            index.upsert([(speaker_id, embedding.tolist(), {"speaker_name": speaker_name})])
+        
+        # Cleanup
         os.remove(temp_path)
+        
         return {
             **utterance,
             'speaker': speaker_name,
             'speaker_id': speaker_id,
-            'embedding': embedding_list
+            'embedding': embedding.cpu().numpy().tolist()
         }
     except Exception as e:
-        logger.error(f"Utterance processing failed: {str(e)}", exc_info=True)
+        logger.error(f"Utterance processing failed: {str(e)}")
         return {
             **utterance,
             'speaker': 'Unknown',
@@ -239,36 +254,54 @@ def process_utterance(utterance, full_audio, wav_file):
             'embedding': None
         }
 
+
 def identify_speakers(transcript: Dict, wav_file: str) -> List[Dict]:
     try:
         full_audio = AudioSegment.from_wav(wav_file)
         utterances = transcript['utterances']
-        with ThreadPoolExecutor(max_workers=5) as executor:
+        
+        # Process utterances in parallel
+        with ThreadPoolExecutor(max_workers=4) as executor:
             futures = [
                 executor.submit(process_utterance, utterance, full_audio, wav_file)
                 for utterance in utterances
             ]
             results = [f.result() for f in futures]
+        
         return results
     except Exception as e:
         logger.error(f"Speaker identification failed: {str(e)}")
         raise
-
+# Role classification
 def train_role_classifier(utterances: List[Dict]):
     try:
+        # Prepare data
         texts = [u['text'] for u in utterances]
         vectorizer = TfidfVectorizer(max_features=500, ngram_range=(1, 2))
         X_text = vectorizer.fit_transform(texts)
+        
         features = []
         labels = []
+        
         for i, utterance in enumerate(utterances):
+            # Prosodic features
             prosodic = utterance['prosodic_features']
             feat = [
-                prosodic['duration'], prosodic['mean_pitch'], prosodic['min_pitch'],
-                prosodic['max_pitch'], prosodic['pitch_sd'], prosodic['intensityMean'],
-                prosodic['intensityMin'], prosodic['intensityMax'], prosodic['intensitySD'],
+                prosodic['duration'],
+                prosodic['mean_pitch'],
+                prosodic['min_pitch'],
+                prosodic['max_pitch'],
+                prosodic['pitch_sd'],
+                prosodic['intensityMean'],
+                prosodic['intensityMin'],
+                prosodic['intensityMax'],
+                prosodic['intensitySD'],
             ]
+            
+            # Text features
             feat.extend(X_text[i].toarray()[0].tolist())
+            
+            # Linguistic features
             doc = nlp(utterance['text'])
             feat.extend([
                 int(utterance['text'].endswith('?')),
@@ -277,35 +310,59 @@ def train_role_classifier(utterances: List[Dict]):
                 sum(1 for token in doc if token.pos_ == 'VERB'),
                 sum(1 for token in doc if token.pos_ == 'NOUN')
             ])
+            
             features.append(feat)
-            labels.append(0 if i % 2 == 0 else 1)
+            labels.append(0 if i % 2 == 0 else 1)  # Temporary labeling
+        
+        # Train classifier
         scaler = StandardScaler()
         X = scaler.fit_transform(features)
+        
         clf = RandomForestClassifier(
-            n_estimators=150, max_depth=10, random_state=42, class_weight='balanced'
+            n_estimators=150,
+            max_depth=10,
+            random_state=42,
+            class_weight='balanced'
         )
         clf.fit(X, labels)
+        
+        # Save models
         joblib.dump(clf, os.path.join(OUTPUT_DIR, 'role_classifier.pkl'))
         joblib.dump(vectorizer, os.path.join(OUTPUT_DIR, 'text_vectorizer.pkl'))
         joblib.dump(scaler, os.path.join(OUTPUT_DIR, 'feature_scaler.pkl'))
+        
         return clf, vectorizer, scaler
     except Exception as e:
         logger.error(f"Classifier training failed: {str(e)}")
         raise
 
+
 def classify_roles(utterances: List[Dict], clf, vectorizer, scaler):
     try:
+        # Prepare features for classification
         texts = [u['text'] for u in utterances]
         X_text = vectorizer.transform(texts)
+        
         results = []
         for i, utterance in enumerate(utterances):
+            # Prosodic features
             prosodic = utterance['prosodic_features']
             feat = [
-                prosodic['duration'], prosodic['mean_pitch'], prosodic['min_pitch'],
-                prosodic['max_pitch'], prosodic['pitch_sd'], prosodic['intensityMean'],
-                prosodic['intensityMin'], prosodic['intensityMax'], prosodic['intensitySD'],
+                prosodic['duration'],
+                prosodic['mean_pitch'],
+                prosodic['min_pitch'],
+                prosodic['max_pitch'],
+                prosodic['pitch_sd'],
+                prosodic['intensityMean'],
+                prosodic['intensityMin'],
+                prosodic['intensityMax'],
+                prosodic['intensitySD'],
             ]
+            
+            # Text features
             feat.extend(X_text[i].toarray()[0].tolist())
+            
+            # Linguistic features
             doc = nlp(utterance['text'])
             feat.extend([
                 int(utterance['text'].endswith('?')),
@@ -314,70 +371,120 @@ def classify_roles(utterances: List[Dict], clf, vectorizer, scaler):
                 sum(1 for token in doc if token.pos_ == 'VERB'),
                 sum(1 for token in doc if token.pos_ == 'NOUN')
             ])
+            
+            # Predict
             X = scaler.transform([feat])
             role = 'Interviewer' if clf.predict(X)[0] == 0 else 'Interviewee'
+            
             results.append({**utterance, 'role': role})
+        
         return results
     except Exception as e:
         logger.error(f"Role classification failed: {str(e)}")
         raise
 
+# Voice analysis for interviewee
 def analyze_interviewee_voice(audio_path: str, utterances: List[Dict]) -> Dict:
     try:
+        # Load full audio
         y, sr = librosa.load(audio_path, sr=16000)
+        
+        # Filter interviewee utterances
         interviewee_utterances = [u for u in utterances if u['role'] == 'Interviewee']
         if not interviewee_utterances:
             return {'error': 'No interviewee utterances found'}
+        
+        # Extract all interviewee segments
         segments = []
         for u in interviewee_utterances:
             start = int(u['start'] * sr / 1000)
             end = int(u['end'] * sr / 1000)
             segments.append(y[start:end])
+        
+        # Combine all segments
+        combined_audio = np.concatenate(segments)
+        
+        # Speaking rate analysis
         total_duration = sum(u['prosodic_features']['duration'] for u in interviewee_utterances)
         total_words = sum(len(u['text'].split()) for u in interviewee_utterances)
         speaking_rate = total_words / total_duration if total_duration > 0 else 0
+        
+        # Filler words analysis
         filler_words = ['um', 'uh', 'like', 'you know', 'so', 'i mean']
-        filler_count = sum(sum(u['text'].lower().count(fw) for fw in filler_words) for u in interviewee_utterances)
+        filler_count = sum(
+            sum(u['text'].lower().count(fw) for fw in filler_words)
+            for u in interviewee_utterances
+        )
         filler_ratio = filler_count / total_words if total_words > 0 else 0
+        
+        # Repetition analysis
         all_words = ' '.join(u['text'].lower() for u in interviewee_utterances).split()
         word_counts = {}
         for i in range(len(all_words) - 1):
-            bigram = (all_words[i], all_words[i + 1])
+            bigram = (all_words[i], all_words[i+1])
             word_counts[bigram] = word_counts.get(bigram, 0) + 1
         repetition_score = sum(1 for count in word_counts.values() if count > 1) / len(word_counts) if word_counts else 0
+        
+        # Pitch analysis (anxiety)
         pitches = []
         for segment in segments:
             f0, voiced_flag, _ = librosa.pyin(segment, fmin=80, fmax=300, sr=sr)
             pitches.extend(f0[voiced_flag])
+        
         pitch_mean = np.mean(pitches) if len(pitches) > 0 else 0
         pitch_std = np.std(pitches) if len(pitches) > 0 else 0
         jitter = np.mean(np.abs(np.diff(pitches))) / pitch_mean if len(pitches) > 1 and pitch_mean > 0 else 0
+        
+        # Intensity analysis (confidence)
         intensities = []
         for segment in segments:
             rms = librosa.feature.rms(y=segment)[0]
             intensities.extend(rms)
+        
         intensity_mean = np.mean(intensities) if intensities else 0
         intensity_std = np.std(intensities) if intensities else 0
         shimmer = np.mean(np.abs(np.diff(intensities))) / intensity_mean if len(intensities) > 1 and intensity_mean > 0 else 0
+        
+        # Composite scores
         anxiety_score = 0.6 * (pitch_std / pitch_mean) + 0.4 * (jitter + shimmer) if pitch_mean > 0 else 0
         confidence_score = 0.7 * (1 / (1 + intensity_std)) + 0.3 * (1 / (1 + filler_ratio))
         hesitation_score = filler_ratio + repetition_score
-        anxiety_level = 'High' if anxiety_score > 0.15 else 'Moderate' if anxiety_score > 0.07 else 'Low'
-        confidence_level = 'High' if confidence_score > 0.7 else 'Moderate' if confidence_score > 0.5 else 'Low'
-        fluency_level = 'Fluent' if (filler_ratio < 0.05 and repetition_score < 0.1) else 'Moderate' if (filler_ratio < 0.1 and repetition_score < 0.2) else 'Disfluent'
+        
+        # Interpretation
+        anxiety_level = 'high' if anxiety_score > 0.15 else 'moderate' if anxiety_score > 0.07 else 'low'
+        confidence_level = 'high' if confidence_score > 0.7 else 'moderate' if confidence_score > 0.5 else 'low'
+        fluency_level = 'fluent' if (filler_ratio < 0.05 and repetition_score < 0.1) else 'moderate' if (filler_ratio < 0.1 and repetition_score < 0.2) else 'disfluent'
+        
         return {
-            'speaking_rate': float(round(speaking_rate, 2)),
+            'speaking_rate':float (round(speaking_rate, 2)),
             'filler_ratio': float(round(filler_ratio, 4)),
             'repetition_score': float(round(repetition_score, 4)),
-            'pitch_analysis': {'mean': float(round(pitch_mean, 2)), 'std_dev': float(round(pitch_std, 2)), 'jitter': float(round(jitter, 4))},
-            'intensity_analysis': {'mean': float(round(intensity_mean, 2)), 'std_dev': float(round(intensity_std, 2)), 'shimmer': float(round(shimmer, 4))},
-            'composite_scores': {'anxiety': float(round(anxiety_score, 4)), 'confidence': float(round(confidence_score, 4)), 'hesitation': float(round(hesitation_score, 4))},
-            'interpretation': {'anxiety_level': anxiety_level, 'confidence_level': confidence_level, 'fluency_level': fluency_level}
+            'pitch_analysis': {
+                'mean': float(round(pitch_mean, 2)),
+                'std_dev':float(round(pitch_std, 2)),
+                'jitter': float(round(jitter, 4))
+            },
+            'intensity_analysis': {
+                'mean': float(round(intensity_mean, 2)),
+                'std_dev': float(round(intensity_std, 2)),
+                'shimmer': float(round(shimmer, 4))
+            },
+            'composite_scores': {
+                'anxiety': float(round(anxiety_score, 4)),
+                'confidence': float(round(confidence_score, 4)),
+                'hesitation': float(round(hesitation_score, 4))
+            },
+            'interpretation': {
+                'anxiety_level': anxiety_level,
+                'confidence_level': confidence_level,
+                'fluency_level': fluency_level
+            }
         }
     except Exception as e:
         logger.error(f"Voice analysis failed: {str(e)}")
         return {'error': str(e)}
 
+
 def generate_voice_interpretation(analysis: Dict) -> str:
     if 'error' in analysis:
         return "Voice analysis unavailable due to processing limitations."
@@ -698,10 +805,15 @@ def create_pdf_report(analysis_data: Dict, output_path: str, gemini_report_text:
         return False
 
 def convert_to_serializable(obj):
-    if isinstance(obj, np.generic): return obj.item()
-    if isinstance(obj, dict): return {k: convert_to_serializable(v) for k, v in obj.items()}
-    if isinstance(obj, list): return [convert_to_serializable(i) for i in obj]
-    if isinstance(obj, np.ndarray): return obj.tolist()
+    """Convert numpy data types to Python native types for JSON serialization"""
+    if isinstance(obj, np.generic):
+        return obj.item()
+    elif isinstance(obj, dict):
+        return {key: convert_to_serializable(value) for key, value in obj.items()}
+    elif isinstance(obj, list):
+        return [convert_to_serializable(item) for item in obj]
+    elif isinstance(obj, np.ndarray):
+        return obj.tolist()
     return obj
 
 def process_interview(audio_path_or_url: str):
@@ -720,7 +832,7 @@ def process_interview(audio_path_or_url: str):
         for utterance in transcript['utterances']:
             utterance['prosodic_features'] = extract_prosodic_features(wav_file, utterance['start'], utterance['end'])
         utterances_with_speakers = identify_speakers(transcript, wav_file)
-        clf, vectorizer, scaler = None, None, None
+        
         if os.path.exists(os.path.join(OUTPUT_DIR, 'role_classifier.pkl')):
             clf = joblib.load(os.path.join(OUTPUT_DIR, 'role_classifier.pkl'))
             vectorizer = joblib.load(os.path.join(OUTPUT_DIR, 'text_vectorizer.pkl'))