Update app.py

app.py

@@ -63,6 +63,16 @@ def chat_fn(message, history, system_text, temperature, top_p, max_new, min_new)
     if eos:
         gen_cfg_kwargs["eos_token_id"] = eos
 
+    gen_cfg = GenerationConfig(**gen_cfg_kwargs)
+
+    with torch.no_grad():
+        out = model.generate(**inputs, generation_config=gen_cfg)
+
+    # slice off the prompt so we show only the assistant reply
+    new_tokens = out[:, inputs["input_ids"].shape[1]:]
+    reply = tokenizer.batch_decode(new_tokens, skip_special_tokens=True)[0].strip()
+    return reply
+
 def infer_text(history, system_text=""):
     """
     Reply in the user’s language with 2–3 concise points (200–400 words); cite Gita verses when relevant.
@@ -85,16 +95,6 @@ def infer_text(history, system_text=""):
         min_new=128,
     )
 
-    gen_cfg = GenerationConfig(**gen_cfg_kwargs)
-
-    with torch.no_grad():
-        out = model.generate(**inputs, generation_config=gen_cfg)
-
-    # slice off the prompt so we show only the assistant reply
-    new_tokens = out[:, inputs["input_ids"].shape[1]:]
-    reply = tokenizer.batch_decode(new_tokens, skip_special_tokens=True)[0].strip()
-    return reply
-
 @spaces.GPU()
 def gradio_fn(message, history):
     response = infer_text(history + [(message, None)])
@@ -155,7 +155,5 @@ with gr.Blocks(css="""
       </div>
     """)
 
-
 if __name__ == "__main__":
     demo.launch()
-