max_tokens

app.py

@@ -2,11 +2,15 @@ import gradio as gr
 from huggingface_hub import InferenceClient
 import torch
 from transformers import pipeline
-import random
 
 # Inference client setup
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-pipe = pipeline("text-generation", "microsoft/Phi-3-mini-4k-instruct", torch_dtype=torch.bfloat16, device_map="auto")
+client = InferenceClient(model="HuggingFaceH4/zephyr-7b-beta")
+pipe = pipeline(
+    "text-generation",
+    "microsoft/Phi-3-mini-4k-instruct",
+    torch_dtype=torch.bfloat16,
+    device_map="auto"
+)
 
 # Global flag to handle cancellation
 stop_inference = False
@@ -20,20 +24,23 @@ def respond(
     history: list[tuple[str, str]],
     system_message_val,
     temperature=0.7,
-    practicality=None,
-    use_local_model=False,
+    # practicality=None,  # Commented out
     max_tokens=256,
+    use_local_model=False,
 ):
     global stop_inference
     stop_inference = False  # Reset cancellation flag
 
-    if practicality is None:
-        practicality = round(random.uniform(0,1), 1)  # Initialize random practicality score
-    if practicality > 0.5:
-        append_message = "Provide actionable advice or direct instructions."
-    else:
-        append_message = "Provide theoretical concepts or abstract quotes."
-    system_message_val = f"{base_message} {append_message}"
+    # if practicality is None:
+    #     practicality = round(random.uniform(0,1), 1)  # Initialize random practicality score
+    # if practicality > 0.5:
+    #     append_message = "Provide actionable advice or direct instructions."
+    # else:
+    #     append_message = "Provide theoretical concepts or abstract quotes."
+    # system_message_val = f"{base_message} {append_message}"
+
+    # Keeping the base message as it is without modifications
+    system_message_val = base_message
 
     # Initialize history if it's None
     if history is None:
@@ -53,7 +60,7 @@ def respond(
         output = pipe(
             input_text,
             temperature=temperature,
-            max_new_tokens=max_tokens,
+            max_new_tokens=256,
             do_sample=True,
             num_return_sequences=1,
         )
@@ -79,7 +86,7 @@ def respond(
             messages=messages,
             stream=True,
             temperature=temperature,
-            max_tokens=max_tokens
+            max_tokens=256
         ):
             if stop_inference:
                 response = "Inference cancelled."
@@ -157,7 +164,7 @@ with gr.Blocks(css=custom_css) as demo:
         )
         use_local_model = gr.Checkbox(label="Use Local Model", value=False)
         temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
-        practicality = gr.Slider(minimum=0.1, maximum=1.0, value=0.5, step=0.05, label="Practicality")
+        # practicality = gr.Slider(minimum=0.1, maximum=1.0, value=0.5, step=0.05, label="Practicality")  # Commented out
 
     chat_history = gr.Chatbot(label="Chat")
 
@@ -167,11 +174,12 @@ with gr.Blocks(css=custom_css) as demo:
     # Adjusted to ensure history is maintained and passed correctly
     user_input.submit(
         respond, 
-        inputs=[user_input, chat_history, system_message_box, temperature, practicality, use_local_model],
+        # Removed practicality from inputs
+        inputs=[user_input, chat_history, system_message_box, temperature, use_local_model],
         outputs=[chat_history, system_message_box]
     )
 
     cancel_button.click(cancel_inference)
 
 if __name__ == "__main__":
-    demo.launch(share=True)
+    demo.launch(share=False)

tests.py

@@ -1,4 +1,4 @@
-from app import respond, base_message 
+from app import respond, base_message
 import time
 
 def test_api():
@@ -13,22 +13,20 @@ def test_api():
 
     start_time = time.time()
 
-    # Call the respond function
+    # Call the respond function (generator)
     result_generator = respond(
         message=message,
         history=history,
         system_message_val=system_message_val,
         temperature=temperature,
         practicality=practicality,
-        max_tokens=max_tokens,
         use_local_model=use_local_model
     )
 
-    # Since respond is a generator, we need to iterate over it to get the final output
+    # Iterate over the generator to get the final output
+    final_history, final_system_message = None, None
     for result in result_generator:
         final_history, final_system_message = result
-        # Optionally, print intermediate results
-        # print("Intermediate history:", final_history)
 
     end_time = time.time()
     runtime = end_time - start_time
@@ -40,31 +38,42 @@ def test_api():
 
 def test_local():
     # Set up input parameters
-    message = "What is the meaning of life"
+    message = "What is the meaning of life?"
     history = []
     system_message_val = base_message
     temperature = 0.7
     practicality = 0.8  # This should modify the system message to provide actionable advice
-    max_tokens = 256
-    use_local_model = True
+    use_local_model = True  # Set to True to use the local model
 
     start_time = time.time()
+    print("start time: ", start_time)
 
-    # Call the respond function
-    result = list(respond(
+    # Call the respond function (generator)
+    result_generator = respond(
         message=message,
         history=history,
         system_message_val=system_message_val,
         temperature=temperature,
         practicality=practicality,
-        max_tokens=max_tokens,#test
-        use_local_model=use_local_model
-    ))
+        use_local_model=use_local_model,
+        max_tokens=256
+    )
+
+    # Iterate over the generator to get the final output
+    final_history, final_system_message = None, None
+    print("Iterating over results...")
+    for result in result_generator:
+        print("Result: ", result)
+        final_history, final_system_message = result
 
     end_time = time.time()
     runtime = end_time - start_time
 
-    final_history, final_system_message = result[-1]
     print("Local Runtime: ", runtime)
     print("Local Final conversation history:", final_history)
     print("Local Final system message:", final_system_message)
+
+# Run the tests
+if __name__ == "__main__":
+    # test_api()
+    test_local()