Update backend/query_llm.py

backend/query_llm.py

@@ -50,7 +50,7 @@ def format_prompt(message: str, api_kind: str):
         raise ValueError("API is not supported")
 
 
-def generate_hf(prompt: str, history: str, temperature: float = 0.9, max_new_tokens: int = 3000,
+def generate_hf(prompt: str, history: str, temperature: float = 0.9, max_new_tokens: int = 4000,
              top_p: float = 0.95, repetition_penalty: float = 1.0) -> Generator[str, None, str]:
     """
     Generate a sequence of tokens based on a given prompt and history using Mistral client.
@@ -70,57 +70,15 @@ def generate_hf(prompt: str, history: str, temperature: float = 0.9, max_new_tok
 
     temperature = max(float(temperature), 1e-2)  # Ensure temperature isn't too low
     top_p = float(top_p)
-    generate_kwargs=[
-    gr.Textbox(
-        label="System Prompt",
-        max_lines=1,
-        interactive=True,
-    ),
-    gr.Slider(
-        label="Temperature",
-        value=0.9,
-        minimum=0.0,
-        maximum=1.0,
-        step=0.05,
-        interactive=True,
-        info="Higher values produce more diverse outputs",
-    ),
-    gr.Slider(
-        label="Max new tokens",
-        value=256,
-        minimum=0,
-        maximum=4048,
-        step=64,
-        interactive=True,
-        info="The maximum numbers of new tokens",
-    ),
-    gr.Slider(
-        label="Top-p (nucleus sampling)",
-        value=0.90,
-        minimum=0.0,
-        maximum=1,
-        step=0.05,
-        interactive=True,
-        info="Higher values sample more low-probability tokens",
-    ),
-    gr.Slider(
-        label="Repetition penalty",
-        value=1.2,
-        minimum=1.0,
-        maximum=2.0,
-        step=0.05,
-        interactive=True,
-        info="Penalize repeated tokens",
-    )
-    ]
-    # generate_kwargs = {
-    #     'temperature': temperature,
-    #     'max_new_tokens': max_new_tokens,
-    #     'top_p': top_p,
-    #     'repetition_penalty': repetition_penalty,
-    #     'do_sample': True,
-    #     'seed': 42,
-    #     }
+    
+    generate_kwargs = {
+        'temperature': temperature,
+        'max_new_tokens': max_new_tokens,
+        'top_p': top_p,
+        'repetition_penalty': repetition_penalty,
+        'do_sample': True,
+        'seed': 42,
+        }
     
     formatted_prompt = format_prompt(prompt, "hf")