debug_1

src/predict.py

@@ -260,63 +260,7 @@ class Predictor:
         return {"voice_clone_1":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-clone-1.mp3",
                 "voice_clone_2":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-clone-2.mp3"
                 }
-        if method_type == 'voice_clone_with_emotions':
-            try:
-                print("INSIDE emotions")        
-                base_speaker_tts,tone_color_converter = self.base_speaker_tts,self.tone_color_converter
-                reference_speaker = local_file_path
-                print("here 1")
-                target_se, audio_name = se_extractor.get_se(reference_speaker, tone_color_converter, target_dir=openvoice_dir, vad=False)
-                print("here 2")
-                src_path = os.path.join(results_dir,f"{gen_id}-tmp-emotions.wav")
-                openvoice_output = os.path.join(results_dir,f"{gen_id}-4.wav")
-                base_speaker_tts.tts(passage,src_path,speaker='default',language='English',speed=1.0,use_emotions=True)
-                source_se = torch.load(f'{self.ckpt_base}/en_style_se.pth').to(self.device)
-                tone_color_converter.convert(audio_src_path=src_path,src_se=source_se,tgt_se=target_se,output_path=openvoice_output,message='')
-                if process_audio:
-                    (new_sr, wav1) = self._fn(openvoice_output,"Midpoint",32,0.5)
-                    sf.write(openvoice_output,wav1,new_sr)
-                
-                mp3_final_output_1 = str(openvoice_output).replace('wav','mp3')
-                self.convert_wav_to_mp3(openvoice_output,mp3_final_output_1)
-                print(mp3_final_output_1)
-                self.upload_file_to_s3(mp3_final_output_1,'demovidelyusergenerations',f"{gen_id}-voice-with-emotions.mp3")
-                shutil.rmtree(os.path.join(output_dir,gen_id))
-                return {"voice_clone_with_emotions":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-with-emotions.mp3"
-                        }
-            except Exception as e:
-                return {"error":f"Unexpected error{e}"}
-        if method_type == 'voice_clone_with_multi_lang':
-            print("Inside multilang")
-            #voice clone with multi-lingugal
-            _,tone_color_converter = self.base_speaker_tts,self.tone_color_converter
-            reference_speaker = local_file_path
-            target_se, audio_name = se_extractor.get_se(reference_speaker, tone_color_converter, target_dir=openvoice_dir, vad=False)
-            src_path = 'openai_source_output.mp3'
-            source_se, audio_name = se_extractor.get_se(src_path, tone_color_converter, vad=True)
-
             
-            client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
-            response = client.audio.speech.create(
-                model="tts-1",
-                voice="fable",
-                input=passage
-            )
-
-            openai_multi_lang_path = os.path.join(results_dir,f"{gen_id}-openai-gen.wav")
-            response.stream_to_file(openai_multi_lang_path)
-            multi_lang_with_voice_clone_path = os.path.join(results_dir,f"{gen_id}-voice-clone-multi-lang.wav")
-
-            source_se, audio_name = se_extractor.get_se(src_path, tone_color_converter, vad=True)
-            self.tone_color_converter.convert(audio_src_path=openai_multi_lang_path, src_se=source_se, tgt_se=target_se, output_path=multi_lang_with_voice_clone_path,message='')
-
-            mp3_final_output_1 = str(multi_lang_with_voice_clone_path).replace('wav','mp3')
-            self.convert_wav_to_mp3(multi_lang_with_voice_clone_path,mp3_final_output_1)
-            print(mp3_final_output_1)
-            self.upload_file_to_s3(mp3_final_output_1,'demovidelyusergenerations',f"{gen_id}-voice-clone-multi-lang.mp3")
-            shutil.rmtree(os.path.join(output_dir,gen_id))
-            return {"voice_clone_with_emotions":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-clone-multi-lang.mp3"
-            }
     
 
 
@@ -336,32 +280,68 @@ class Predictor:
         s3_key = s3_url.split('/')[-1]
         bucket_name = 'demovidelyuseruploads'
         local_file_path = os.path.join(raw_dir,s3_key)
-        self.download_file_from_s3(self.s3_client,bucket_name,s3_key,local_file_path)       
-        try:
-            print("INSIDE new emotions method")        
-            base_speaker_tts,tone_color_converter = self.base_speaker_tts,self.tone_color_converter
-            reference_speaker = local_file_path
-            print("here 1")
-            target_se, audio_name = se_extractor.get_se(reference_speaker, tone_color_converter, target_dir=openvoice_dir, vad=False)
-            print("here 2")
-            src_path = os.path.join(results_dir,f"{gen_id}-tmp-emotions.wav")
-            openvoice_output = os.path.join(results_dir,f"{gen_id}-4.wav")
-            base_speaker_tts.tts(passage,src_path,speaker='default',language='English',speed=1.0,use_emotions=True)
-            source_se = torch.load(f'{self.ckpt_base}/en_style_se.pth').to(self.device)
-            tone_color_converter.convert(audio_src_path=src_path,src_se=source_se,tgt_se=target_se,output_path=openvoice_output,message='')
-            if process_audio:
-                (new_sr, wav1) = self._fn(openvoice_output,"Midpoint",32,0.5)
-                sf.write(openvoice_output,wav1,new_sr)
-            
-            mp3_final_output_1 = str(openvoice_output).replace('wav','mp3')
-            self.convert_wav_to_mp3(openvoice_output,mp3_final_output_1)
-            print(mp3_final_output_1)
-            self.upload_file_to_s3(mp3_final_output_1,'demovidelyusergenerations',f"{gen_id}-voice-with-emotions.mp3")
-            shutil.rmtree(os.path.join(output_dir,gen_id))
-            return {"voice_clone_with_emotions":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-with-emotions.mp3"
-                    }
-        except Exception as e:
-            return {"error":f"Unexpected error{e}"}
+        self.download_file_from_s3(self.s3_client,bucket_name,s3_key,local_file_path)
+        #voice_clone with styletts2
+        model,sampler = self.model,self.sampler
+        result = self.process_audio_file(local_file_path,passage,model,sampler)
+
+        base_speaker_tts,tone_color_converter = self.base_speaker_tts,self.tone_color_converter
+        reference_speaker = local_file_path
+        target_se, audio_name = se_extractor.get_se(reference_speaker, tone_color_converter, target_dir=openvoice_dir, vad=False)
+        src_path = os.path.join(results_dir,f"{gen_id}-tmp.wav")
+        openvoice_output = os.path.join(results_dir,f"{gen_id}-voice-clone-2.wav")
+        base_speaker_tts.tts(passage,src_path,speaker='default',language='English',speed=1.0)
+        return {"status":"Done"}
+        
+
+    def predict_with_multi_lang(self,s3_url,passage,process_audio):
+        print("In multi lang voice cloning")
+        output_dir = 'processed'
+        gen_id = str(uuid.uuid4())
+        os.makedirs(output_dir,exist_ok=True)
+        raw_dir = os.path.join(output_dir,gen_id,'raw')
+        segments_dir = os.path.join(output_dir,gen_id,'segments')
+        results_dir = os.path.join(output_dir,gen_id,'results')
+        openvoice_dir = os.path.join(output_dir,gen_id,'openvoice')
+        os.makedirs(raw_dir)
+        os.makedirs(segments_dir)
+        os.makedirs(results_dir)
+        
+        
+        s3_key = s3_url.split('/')[-1]
+        bucket_name = 'demovidelyuseruploads'
+        local_file_path = os.path.join(raw_dir,s3_key)
+        self.download_file_from_s3(self.s3_client,bucket_name,s3_key,local_file_path)
+
+        _,tone_color_converter = self.base_speaker_tts,self.tone_color_converter
+        reference_speaker = local_file_path
+        target_se, audio_name = se_extractor.get_se(reference_speaker, tone_color_converter, target_dir=openvoice_dir, vad=False)
+        src_path = 'openai_source_output.mp3'
+        source_se, audio_name = se_extractor.get_se(src_path, tone_color_converter, vad=True)
+
+        
+        client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+        response = client.audio.speech.create(
+            model="tts-1",
+            voice="fable",
+            input=passage
+        )
+
+        openai_multi_lang_path = os.path.join(results_dir,f"{gen_id}-openai-gen.wav")
+        response.stream_to_file(openai_multi_lang_path)
+        multi_lang_with_voice_clone_path = os.path.join(results_dir,f"{gen_id}-voice-clone-multi-lang.wav")
+
+        source_se, audio_name = se_extractor.get_se(src_path, tone_color_converter, vad=True)
+        self.tone_color_converter.convert(audio_src_path=openai_multi_lang_path, src_se=source_se, tgt_se=target_se, output_path=multi_lang_with_voice_clone_path,message='')
+
+        mp3_final_output_1 = str(multi_lang_with_voice_clone_path).replace('wav','mp3')
+        self.convert_wav_to_mp3(multi_lang_with_voice_clone_path,mp3_final_output_1)
+        print(mp3_final_output_1)
+        self.upload_file_to_s3(mp3_final_output_1,'demovidelyusergenerations',f"{gen_id}-voice-clone-multi-lang.mp3")
+        shutil.rmtree(os.path.join(output_dir,gen_id))
+        return {"voice_clone_with_emotions":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-clone-multi-lang.mp3"
+        }
+
 
     def _fn(self,path, solver, nfe, tau):
         if path is None:

src/rp_handler.py

@@ -53,6 +53,8 @@ def run_voice_clone_job(job):
             result = MODEL.predict(s3_url,passage,process_audio)
         if method_type == 'voice_clone_with_emotions':
             result = MODEL.predict_with_emotions(s3_url,passage,process_audio)
+        if method_type == 'voice_clone_with_multi_lang':
+            result = MODEL.predict_with_multi_lang(s3_url,passage,process_audio)
 
         return result