Update app.py

app.py

@@ -1,6 +1,7 @@
 import streamlit
 from llama_cpp import Llama
 import os
+import time
 from huggingface_hub import hf_hub_download
 # Load the LLM from GGUF file
 
@@ -11,7 +12,7 @@ model_path = hf_hub_download(repo_id = repo_id, filename=model_file)
 # n_threads
 llm = Llama(model_path=model_path,n_gpu_layers=30,n_ctx=512,temperature=0.2,repeat_penalty=1.1,top_k_sampling=40,top_p_sampling=0.95,min_p_sampling=0.05)
 def generate_llm_response(prompt):
-    output = llm(prompt, max_tokens=1024)
+    output = llm(prompt, max_tokens=512)
     return output["choices"][0]["text"]
 
 import streamlit as st
@@ -35,11 +36,11 @@ user_input = st.chat_input("Type a message, ask a coding question")
 if user_input:
     st.chat_message("user").write(user_input)
     st.session_state["messages"].append({"role": "user", "content": user_input})
-
+    start_time = time.time()
     # Get response from GGUF LLM
     response = generate_llm_response(user_input)
-
+    end_time = time.time()
     # Display response
     st.chat_message("assistant").write(response)
     st.session_state["messages"].append({"role": "assistant", "content": response})
-
+    st.caption(f"⏱️ Inference time: {inference_time:.2f} seconds")