Update evaluate.py

evaluate.py

@@ -32,7 +32,7 @@ def calculate_similarity(text1, text2):
     return matcher.ratio() * 100
 
 def setup_reference_patterns(reference_dir, sample_rate=16000):
-    """Create standard reference pattern directories and dummy files if needed"""
+    """Create standard reference pattern directories without dummy files"""
     reference_patterns = [
         "mayap_a_abak", "mayap_a_ugtu", "mayap_a_gatpanapun", "mayap_a_bengi", 
         "komusta_ka", "malaus_ko_pu", "malaus_kayu", "agaganaka_da_ka", 
@@ -47,7 +47,6 @@ def setup_reference_patterns(reference_dir, sample_rate=16000):
     ]
     
     created_dirs = 0
-    created_files = 0
     
     for pattern in reference_patterns:
         pattern_dir = os.path.join(reference_dir, pattern)
@@ -59,26 +58,8 @@ def setup_reference_patterns(reference_dir, sample_rate=16000):
             except Exception as e:
                 logger.error(f"❌ Failed to create reference pattern directory {pattern_dir}: {str(e)}")
                 continue
-        
-        # Check if directory has any WAV files, add a dummy if not
-        wav_files = glob.glob(os.path.join(pattern_dir, "*.wav"))
-        if not wav_files:
-            try:
-                dummy_path = os.path.join(pattern_dir, "dummy_reference.wav")
-                # Create a 1-second silent audio file - not completely silent to avoid transcription issues
-                # Adding a small amount of noise helps ASR models detect something
-                silent = AudioSegment.silent(duration=1000, frame_rate=sample_rate)
-                # Add a tiny bit of noise
-                for i in range(50, 950, 300):
-                    silent = silent.overlay(AudioSegment.silent(duration=50, frame_rate=sample_rate) + 3, position=i)
-                silent.export(dummy_path, format="wav")
-                logger.info(f"📄 Created dummy reference file: {dummy_path}")
-                created_files += 1
-            except Exception as e:
-                logger.error(f"❌ Failed to create dummy file in {pattern_dir}: {str(e)}")
     
-    return created_dirs, created_files
-
+    return created_dirs, 0  # Return 0 created files since we're not creating dummy files anymore
 def search_reference_directories():
     """Search for possible reference directories in various locations"""
     possible_locations = [
@@ -152,12 +133,12 @@ def init_reference_audio(reference_dir, output_dir):
         os.makedirs(working_dir, exist_ok=True)
         logger.info(f"📁 Using reference directory: {working_dir}")
         
-        # Set up reference pattern directories with dummy files if needed
-        dirs_created, files_created = setup_reference_patterns(working_dir)
-        logger.info(f"📊 Created {dirs_created} directories and {files_created} dummy files")
+        # Set up reference pattern directories WITHOUT dummy files
+        dirs_created, _ = setup_reference_patterns(working_dir)
+        logger.info(f"📊 Created {dirs_created} directories")
         
         # Try to copy reference files from other found directories to working directory if needed
-        if files_created > 0 and len(found_dirs) > 1:
+        if len(found_dirs) > 1:
             # Try to find a directory with existing WAV files
             for directory in found_dirs:
                 if directory['path'] != working_dir and directory['wav_files'] > 0:
@@ -166,24 +147,32 @@ def init_reference_audio(reference_dir, output_dir):
                         logger.info(f"🔄 Copying reference files from {source_dir} to {working_dir}")
                         
                         # Copy pattern directories that have WAV files
+                        # But skip any dummy reference files
                         for item in os.listdir(source_dir):
                             src_path = os.path.join(source_dir, item)
-                            if os.path.isdir(src_path) and glob.glob(os.path.join(src_path, "*.wav")):
-                                dst_path = os.path.join(working_dir, item)
+                            if os.path.isdir(src_path):
+                                wav_files = glob.glob(os.path.join(src_path, "*.wav"))
+                                # Filter out dummy references
+                                wav_files = [f for f in wav_files if "dummy_reference" not in f]
                                 
-                                # Copy each WAV file individually
-                                for wav_file in glob.glob(os.path.join(src_path, "*.wav")):
-                                    wav_name = os.path.basename(wav_file)
-                                    dst_file = os.path.join(dst_path, wav_name)
-                                    if not os.path.exists(dst_file):
-                                        shutil.copy2(wav_file, dst_file)
-                                        logger.info(f"📄 Copied {wav_name} to {dst_path}")
+                                if wav_files:  # Only proceed if there are valid files
+                                    dst_path = os.path.join(working_dir, item)
+                                    os.makedirs(dst_path, exist_ok=True)
+                                    
+                                    # Copy each valid WAV file individually
+                                    for wav_file in wav_files:
+                                        wav_name = os.path.basename(wav_file)
+                                        if "dummy_reference" not in wav_name:  # Extra check
+                                            dst_file = os.path.join(dst_path, wav_name)
+                                            if not os.path.exists(dst_file):
+                                                shutil.copy2(wav_file, dst_file)
+                                                logger.info(f"📄 Copied {wav_name} to {dst_path}")
                         
                         break
                     except Exception as e:
                         logger.warning(f"⚠️ Failed to copy reference files: {str(e)}")
         
-        # Log the final contents
+        # Log the final contents, excluding dummy files
         pattern_dirs = [d for d in os.listdir(working_dir)
                        if os.path.isdir(os.path.join(working_dir, d))]
         logger.info(f"📊 Final reference directory has {len(pattern_dirs)} pattern directories")
@@ -192,22 +181,35 @@ def init_reference_audio(reference_dir, output_dir):
         for pattern in pattern_dirs:
             pattern_path = os.path.join(working_dir, pattern)
             wav_files = glob.glob(os.path.join(pattern_path, "*.wav"))
-            total_wav_files += len(wav_files)
-            logger.info(f"  - {pattern}: {len(wav_files)} WAV files")
+            # Count only non-dummy files
+            valid_files = [f for f in wav_files if "dummy_reference" not in f]
+            total_wav_files += len(valid_files)
+            logger.info(f"  - {pattern}: {len(valid_files)} valid WAV files")
         
         logger.info(f"📊 Total reference WAV files: {total_wav_files}")
         
+        # Check for and remove any dummy files
+        for pattern in pattern_dirs:
+            pattern_path = os.path.join(working_dir, pattern)
+            dummy_files = glob.glob(os.path.join(pattern_path, "dummy_reference.wav"))
+            for dummy in dummy_files:
+                try:
+                    os.remove(dummy)
+                    logger.info(f"🗑️ Removed dummy file: {dummy}")
+                except Exception as e:
+                    logger.warning(f"⚠️ Failed to remove dummy file {dummy}: {str(e)}")
+        
         return working_dir
             
     except Exception as e:
         logger.error(f"❌ Failed to set up reference audio directory: {str(e)}")
         logger.debug(f"Stack trace: {traceback.format_exc()}")
         
-        # As a last resort, try to use /tmp
+        # As a last resort, try to use /tmp but without dummy files
         fallback_dir = os.path.join('/tmp', 'reference_audios')
         try:
             os.makedirs(fallback_dir, exist_ok=True)
-            setup_reference_patterns(fallback_dir)
+            setup_reference_patterns(fallback_dir)  # This now doesn't create dummy files
             logger.warning(f"⚠️ Using emergency fallback directory: {fallback_dir}")
             return fallback_dir
         except:
@@ -344,30 +346,18 @@ def handle_evaluation_request(request, reference_dir, output_dir, sample_rate):
 
         # Check for reference files
         reference_files = glob.glob(os.path.join(reference_dir_path, "*.wav"))
-        logger.info(f"[{request_id}] 📁 Found {len(reference_files)} reference files")
+        # Filter out any dummy reference files
+        reference_files = [f for f in reference_files if "dummy_reference" not in f]
+        logger.info(f"[{request_id}] 📁 Found {len(reference_files)} valid reference files")
 
-        # If no reference files exist, create a dummy reference file
+        # If no reference files exist, return a more detailed error message
         if not reference_files:
-            logger.warning(f"[{request_id}] ⚠️ No reference audio files found in {reference_dir_path}")
-    
-            # Create a dummy reference file
-            try:
-                dummy_file_path = os.path.join(reference_dir_path, "dummy_reference.wav")
-                logger.info(f"[{request_id}] 🔄 Creating dummy reference file: {dummy_file_path}")
-        
-                # Create a 1-second audio file with a slight sound
-                silent_audio = AudioSegment.silent(duration=1000, frame_rate=sample_rate)
-                # Add a tiny bit of noise to help ASR
-                for i in range(50, 950, 300):
-                    silent_audio = silent_audio.overlay(AudioSegment.silent(duration=50, frame_rate=sample_rate) + 3, position=i)
-                silent_audio.export(dummy_file_path, format="wav")
-        
-                # Add it to the list of reference files
-                reference_files = [dummy_file_path]
-                logger.info(f"[{request_id}] ✅ Created dummy reference file for testing")
-            except Exception as e:
-                logger.error(f"[{request_id}] ❌ Failed to create dummy reference: {str(e)}")
-                return jsonify({"error": f"No reference audio found for {reference_locator}"}), 404
+            logger.warning(f"[{request_id}] ⚠️ No valid reference audio files found in {reference_dir_path}")
+            return jsonify({
+                "error": f"No reference audio found for {reference_locator}",
+                "message": "Please upload a reference audio file before evaluation.",
+                "status": "MISSING_REFERENCE"
+            }), 404
 
         lang_code = LANGUAGE_CODES.get(language, language)
         logger.info(f"[{request_id}] 🔄 Evaluating pronunciation for reference: {reference_locator} with language code: {lang_code}")