improve performance

app.py

@@ -75,6 +75,13 @@ cache_schedule = {
     'ff':   presets[default_preset]['ff'][:]
 }
 
+recent_input = {
+    "ref_audio": None,
+    "ref_text": None,
+    "gen_text": None,
+    "nfe_step": None
+}
+
 seed = np.random.randint(0, 2**31 - 1)
 torch.manual_seed(seed)
 
@@ -170,7 +177,6 @@ def load_default():
     return render_grid(cache_schedule), default_preset
 
 
-@lru_cache(maxsize=1000)  # NOTE. need to ensure params of infer() hashable
 @gpu_decorator
 def infer(
     ref_audio_orig,
@@ -178,32 +184,36 @@ def infer(
     gen_text,
     nfe_step=32,
 ):
-    global cache_schedule
+    global cache_schedule, recent_input
     show_info = gr.Info
     if not ref_audio_orig:
         gr.Warning("Please provide reference audio.")
-        return gr.update(), gr.update(), ref_text
+        return gr.update(), gr.update(), ref_text, gr.update(), gr.update()
 
     if not gen_text.strip():
-        gr.Warning("Please enter text to generate or upload a text file.")
-        return gr.update(), gr.update(), ref_text
+        gr.Warning("Please enter text to generate.")
+        return gr.update(), gr.update(), ref_text, gr.update(), gr.update()
 
     ref_audio, ref_text = preprocess_ref_audio_text(
         ref_audio_orig, ref_text, show_info=show_info)
-    start_time = time.time()
-    final_wave, final_sample_rate, _ = infer_process(
-        ref_audio,
-        ref_text,
-        gen_text,
-        ema_model,
-        vocoder,
-        cross_fade_duration=cross_fade_duration,
-        nfe_step=nfe_step,
-        speed=speed,
-        show_info=show_info,
-        progress=gr.Progress(),
-    )
-    process_time = time.time() - start_time
+    skip_no_cache = False
+    if recent_input["ref_audio"] == ref_audio_orig and recent_input["ref_text"] == ref_text and recent_input["gen_text"] == gen_text and recent_input["nfe_step"] == nfe_step:
+        skip_no_cache = True
+    if not skip_no_cache:
+        start_time = time.time()
+        final_wave, final_sample_rate, _ = infer_process(
+            ref_audio,
+            ref_text,
+            gen_text,
+            ema_model,
+            vocoder,
+            cross_fade_duration=cross_fade_duration,
+            nfe_step=nfe_step,
+            speed=speed,
+            show_info=show_info,
+            progress=gr.Progress(),
+        )
+        process_time = time.time() - start_time
     cache_helper = SmoothCacheHelper(
         model=ema_model.transformer,
         block_classes=get_class("f5_tts.model.modules.DiTBlock"),
@@ -227,13 +237,21 @@ def infer(
     process_time_cache = time.time() - start_time
     cache_helper.disable()
 
-    return (final_sample_rate, final_wave), (final_sample_rate_cache, final_wave_cache), process_time, process_time_cache
+    recent_input["ref_audio"] = ref_audio_orig
+    recent_input["ref_text"] = ref_text
+    recent_input["gen_text"] = gen_text
+    recent_input["nfe_step"] = nfe_step
+
+    if skip_no_cache:
+        print("skip")
+        return gr.update(), (final_sample_rate_cache, final_wave_cache), ref_text, gr.update(), process_time_cache
+    return (final_sample_rate, final_wave), (final_sample_rate_cache, final_wave_cache), ref_text, process_time, process_time_cache
 
 
 with gr.Blocks() as demo:
     gr.Markdown("## F5-TTS + SmoothCache")
     ref_audio_input = gr.Audio(label="Reference Audio", type="filepath")
-    ref_text_input = gr.Textbox(label="Reference Text")
+    ref_text_input = gr.Textbox(label="Reference Text (Optional)")
     gen_text_input = gr.Textbox(label="Text to Generate")
     with gr.Row():
         with gr.Column(scale=0):
@@ -260,12 +278,12 @@ with gr.Blocks() as demo:
     preset_dropdown.change(
         fn=apply_preset, inputs=preset_dropdown, outputs=[image, nfe_slider])
     image.select(fn=toggle_cell, outputs=[image, preset_dropdown])
-    nfe_slider.input(fn=update_nfe, inputs=nfe_slider,
+    nfe_slider.release(fn=update_nfe, inputs=nfe_slider,
                      outputs=[image, preset_dropdown])
     generate_btn.click(
         infer,
         inputs=[ref_audio_input, ref_text_input, gen_text_input, nfe_slider],
-        outputs=[audio_output, audio_output_cache,
+        outputs=[audio_output, audio_output_cache, ref_text_input,
                  process_time, process_time_cache],
     )
     demo.load(fn=load_default, outputs=[image, preset_dropdown])