Update app.py

app.py

@@ -1,91 +1,104 @@
+# # import gradio as gr
+# # from huggingface_hub import InferenceClient
+
+# # """
+# # For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+# # """
+# # client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+
+
+# # def respond(
+# #     message,
+# #     history: list[tuple[str, str]],
+# #     system_message,
+# #     max_tokens,
+# #     temperature,
+# #     top_p,
+# # ):
+# #     messages = [{"role": "system", "content": system_message}]
+
+# #     for val in history:
+# #         if val[0]:
+# #             messages.append({"role": "user", "content": val[0]})
+# #         if val[1]:
+# #             messages.append({"role": "assistant", "content": val[1]})
+
+# #     messages.append({"role": "user", "content": message})
+
+# #     response = ""
+
+# #     for message in client.chat_completion(
+# #         messages,
+# #         max_tokens=max_tokens,
+# #         stream=True,
+# #         temperature=temperature,
+# #         top_p=top_p,
+# #     ):
+# #         token = message.choices[0].delta.content
+
+# #         response += token
+# #         yield response
+
+
+# # """
+# # For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
+# # """
+# # demo = gr.ChatInterface(
+# #     respond,
+# #     additional_inputs=[
+# #         gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
+# #         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+# #         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+# #         gr.Slider(
+# #             minimum=0.1,
+# #             maximum=1.0,
+# #             value=0.95,
+# #             step=0.05,
+# #             label="Top-p (nucleus sampling)",
+# #         ),
+# #     ],
+# # )
+
+
+# # if __name__ == "__main__":
+# #     demo.launch()
+
+# import torch
+# from transformers import AutoTokenizer, AutoModelForCausalLM
 # import gradio as gr
-# from huggingface_hub import InferenceClient
-
-# """
-# For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-# """
-# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-# def respond(
-#     message,
-#     history: list[tuple[str, str]],
-#     system_message,
-#     max_tokens,
-#     temperature,
-#     top_p,
-# ):
-#     messages = [{"role": "system", "content": system_message}]
-
-#     for val in history:
-#         if val[0]:
-#             messages.append({"role": "user", "content": val[0]})
-#         if val[1]:
-#             messages.append({"role": "assistant", "content": val[1]})
-
-#     messages.append({"role": "user", "content": message})
-
-#     response = ""
-
-#     for message in client.chat_completion(
-#         messages,
-#         max_tokens=max_tokens,
-#         stream=True,
-#         temperature=temperature,
-#         top_p=top_p,
-#     ):
-#         token = message.choices[0].delta.content
-
-#         response += token
-#         yield response
-
-
-# """
-# For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-# """
-# demo = gr.ChatInterface(
-#     respond,
-#     additional_inputs=[
-#         gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-#         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-#         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-#         gr.Slider(
-#             minimum=0.1,
-#             maximum=1.0,
-#             value=0.95,
-#             step=0.05,
-#             label="Top-p (nucleus sampling)",
-#         ),
-#     ],
-# )
 
+# model_id = "Asit03/AI_Agent_V2_Merged"
 
-# if __name__ == "__main__":
-#     demo.launch()
+# # Load tokenizer
+# tokenizer = AutoTokenizer.from_pretrained(model_id)
 
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import gradio as gr
+# # Load model with 4-bit quantization
+# model = AutoModelForCausalLM.from_pretrained(
+#     model_id,
+#     device_map="auto",
+#     load_in_4bit=True,
+#     torch_dtype=torch.bfloat16,  # fallback to torch.float16 if needed
+#     trust_remote_code=True
+# )
 
-model_id = "Asit03/AI_Agent_V2_Merged"
+# # Generation function
+# def chat(prompt):
+#     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+#     outputs = model.generate(**inputs, max_new_tokens=200, pad_token_id=tokenizer.eos_token_id)
+#     return tokenizer.decode(outputs[0], skip_special_tokens=True)
 
-# Load tokenizer
-tokenizer = AutoTokenizer.from_pretrained(model_id)
+# # Launch Gradio app
+# gr.Interface(fn=chat, inputs="text", outputs="text", title="💬 AI Agent V2").launch()
 
-# Load model with 4-bit quantization
-model = AutoModelForCausalLM.from_pretrained(
-    model_id,
-    device_map="auto",
-    load_in_4bit=True,
-    torch_dtype=torch.bfloat16,  # fallback to torch.float16 if needed
-    trust_remote_code=True
-)
+from huggingface_hub import InferenceClient
+import gradio as gr
+
+# Create inference client for your model
+client = InferenceClient("Asit03/AI_Agent_V2_Merged")  # or private repo with token
 
-# Generation function
 def chat(prompt):
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    outputs = model.generate(**inputs, max_new_tokens=200, pad_token_id=tokenizer.eos_token_id)
-    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+    response = client.text_generation(prompt, max_new_tokens=150)
+    return response.strip()
+
+gr.Interface(fn=chat, inputs="text", outputs="text", title="💬 AI Agent via Inference API").launch()
 
-# Launch Gradio app
-gr.Interface(fn=chat, inputs="text", outputs="text", title="💬 AI Agent V2").launch()