Update app.py

app.py

@@ -1,143 +1,134 @@
 import gradio as gr
-import torch
-from transformers import (
-    AutoModelForCausalLM,
-    AutoTokenizer,
-    TextIteratorStreamer,
-)
-import os
-from threading import Thread
-import spaces
-import time
-import subprocess
-
-subprocess.run(
-    "pip install flash-attn --no-build-isolation",
-    env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
-    shell=True,
-)
+from huggingface_hub import InferenceClient
 
-token = os.environ["HF_TOKEN"]
+"""
+For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+"""
+client = InferenceClient("meta-llama/Llama-3.2-1B")
 
-model = AutoModelForCausalLM.from_pretrained(
-    "microsoft/Phi-3-mini-128k-instruct",
-    token=token,
-    trust_remote_code=True,
-)
-tok = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-128k-instruct", token=token)
-terminators = [
-    tok.eos_token_id,
-]
-
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-    print(f"Using GPU: {torch.cuda.get_device_name(device)}")
-else:
-    device = torch.device("cpu")
-    print("Using CPU")
-
-model = model.to(device)
-
-# Dispatch Errors
-
-@spaces.GPU(duration=60)
-def chat(message, history, temperature, do_sample, max_tokens):
-    # 定義 PTZ 控制助手的 prompt
-    prompt = (
-        "You are an assistant for controlling PTZ cameras.\n"
-        "When the user gives you a clear command, please JUST respond in the following format:\n"
-        "Camera:<camera_id>. Tracking_Target:<target_name> placement:<position>.\n"
-        "If multiple cameras are specified, provide separate lines for each camera.\n"
-        "The available placements are: top_left, top_middle, top_right, "
-        "center_left, center_middle, center_right, bottom_left, bottom_middle, bottom_right.\n"
-        "Default Values:\n"
-        "- camera_id: default\n"
-        "- tracking_target: default\n"
-        "- placement: center_middle\n"
-        "Rules for Defaults:\n"
-        "1. If the camera_id is not specified, use the default value `default`.\n"
-        "2. If the tracking_target is not specified, use the default value `default`.\n"
-        "3. If the position information is incomplete or not specified, default the placement to the middle position.\n"
-        "For example, if the user specifies 'top', interpret it as 'top_middle'.\n\n"
-        "Examples:\n"
-        "User: Please set camera 1 to track target A at bottom_right.\n"
-        "Assistant: Camera:1. Tracking_Target:A placement:bottom_right.\n\n"
-        "User: Please set camera 2 to track target B at top.\n"
-        "Assistant: Camera:2. Tracking_Target:B placement:top_middle.\n\n"
-        "User: Please set camera 3 to track target C.\n"
-        "Assistant: Camera:3. Tracking_Target:C placement:center_middle.\n\n"
-        "User: Please track target D at left.\n"
-        "Assistant: Camera:default. Tracking_Target:D placement:center_left.\n\n"
-        "User: Please control camera 4.\n"
-        "Assistant: Camera:4. Tracking_Target:default placement:center_middle.\n\n"
-        "User: Please start recording.\n"
-        "Assistant: Camera:default. Tracking_Target:default placement:center_middle.\n\n"
-        "User: Please set camera 2 and camera 3 to track target Kyle at bottom_right.\n"
-        "Assistant:\n"
-        "Camera:2. Tracking_Target:Kyle placement:bottom_right.\n"
-        "Camera:3. Tracking_Target:Kyle placement:bottom_right.\n\n"
-        "Now, respond to the following command:\n"
-    )
-
-    chat = []
-    for item in history:
-        chat.append({"role": "user", "content": item[0]})
-        if item[1] is not None:
-            chat.append({"role": "assistant", "content": item[1]})
-    chat.append({"role": "user", "content": message})
-    
-    # 將 prompt 添加到消息的開頭
-    full_message = prompt + "\n" + tok.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
-    
-    model_inputs = tok([full_message], return_tensors="pt").to(device)
-    streamer = TextIteratorStreamer(
-        tok, timeout=20.0, skip_prompt=True, skip_special_tokens=True
-    )
-    generate_kwargs = dict(
-        model_inputs,
-        streamer=streamer,
-        max_new_tokens=max_tokens,
-        do_sample=True,
-        temperature=temperature,
-        eos_token_id=terminators,
-    )
 
-    if temperature == 0:
-        generate_kwargs["do_sample"] = False
 
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
+def respond(
+    message,
+    history: list[tuple[str, str]],
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+):
+    MAX_HISTORY_LENGTH = 5  # 保留最近 5 條歷史
+
+    # 限制歷史對話的長度
+    history = history[-MAX_HISTORY_LENGTH:]
+
+    # 構建消息列表，從系統消息開始
+    messages = [{"role": "system", "content": system_message}]
+
+    # 添加歷史對話
+    for val in history:
+        if val[0]:
+            messages.append({"role": "user", "content": val[0]})
+        if val[1]:
+            messages.append({"role": "assistant", "content": val[1]})
+
+    # 添加最新的用戶輸入
+    messages.append({"role": "user", "content": message})
+
+    # 初始化空回應
+    response = ""
 
-    partial_text = ""
-    for new_text in streamer:
-        partial_text += new_text
-        yield partial_text
+    # 呼叫模型並生成回應，使用 stream 模式逐步更新
+    for message in client.chat_completion(
+        messages,
+        max_tokens=max_tokens,
+        stream=True,
+        temperature=temperature,
+        top_p=top_p,
+    ):
+        token = message.choices[0].delta.content
+        response += token
+        yield response
 
-    yield partial_text
 
 
+"""
+For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
+"""
 demo = gr.ChatInterface(
-    fn=chat,
-    examples=[["Please set camera 2 to track target A at top."]],
-    additional_inputs_accordion=gr.Accordion(
-        label="⚙️ Parameters", open=False, render=False
-    ),
+    respond,
     additional_inputs=[
+        gr.Textbox(
+    value=(
+            "You are an assistant for controlling PTZ cameras.\n"
+            "When the user gives you a clear command, please JUST respond in the following format:\n"
+            "Camera:<camera_id>. Tracking_Target:<target_name> placement:<position> speed:<speed> only_ptz_action:<only_ptz_action> tracking_action:<tracking_action>.\n"
+            "If multiple cameras are specified, provide separate lines for each camera.\n"
+            "Only provide commands for the cameras specified by the user.\n"
+            "Do not include additional cameras that the user did not mention.\n"
+            "Ensure all field names are spelled correctly.\n\n"
+            "The available placements are ONLY: top_left, top_middle, top_right, center_left, center_middle, center_right, bottom_left, bottom_middle, bottom_right.\n"
+            "The available speed options are ONLY: slow, medium, fast.\n"
+            "The available only_ptz_actions are ONLY: turn_right, turn_left, tilt_up, tilt_down, zoom_in, zoom_out, stop.\n"
+            "The available tracking_actions are ONLY: tracking.\n\n"
+            "Default Values:\n"
+            "- camera_id: default\n"
+            "- tracking_target: default\n"
+            "- placement: center_middle\n"
+            "- speed: medium\n"
+            "- only_ptz_action: default\n"
+            "- tracking_action: default\n\n"
+            "Rules for Defaults:\n"
+            "1. If the camera_id is not specified, use the default value `default`.\n"
+            "2. If the tracking_target is not specified, use the default value `default`.\n"
+            "3. If the position information is incomplete or not specified, default the placement to `center_middle`.\n"
+            "4. If only a general direction is specified, interpret it as the middle of that direction.\n"
+            "   For example, 'top' is interpreted as 'top_middle' and 'left' as 'center_left'.\n"
+            "5. If the speed is not specified, default to `medium`.\n"
+            "6. If the only_ptz_action is not specified, default to `default`.\n"
+            "7. If the tracking_action is not specified, default to `default`.\n"
+            "8. Camera IDs are restricted to 1, 2, 3, and 4. If an invalid camera_id is provided, use `default`.\n"
+            "9. If the user specifies 'all camera' or 'all cameras', apply the command to all cameras (1-4).\n\n"
+            "**Special Action Handling**:\n"
+            "- If only `camera_id` and `only_ptz_action` are specified (all other fields are `default`), execute only the specified `only_ptz_action`.\n"
+            "- If only `camera_id` and `tracking_action` are specified (all other fields are `default`), execute only the specified `tracking_action`.\n"
+            "- When tracking is involved, set `tracking_action` to `tracking`.\n\n"
+            "Examples:\n"
+            "User: Please set camera 1 to track target Alice at bottom_right with speed fast and action turn_right.\n"
+            "Assistant: Camera:1. Tracking_Target:Alice placement:bottom_right speed:fast only_ptz_action:turn_right tracking_action:default.\n\n"
+            "User: Please set camera 3 to track target Bob at top with speed slow.\n"
+            "Assistant: Camera:3. Tracking_Target:Bob placement:top_middle speed:slow only_ptz_action:default tracking_action:tracking.\n\n"
+            "User: Please set camera 2 to track target Carol.\n"
+            "Assistant: Camera:2. Tracking_Target:Carol placement:center_middle speed:medium only_ptz_action:default tracking_action:tracking.\n\n"
+            "User: Please track target Dave at left.\n"
+            "Assistant: Camera:default. Tracking_Target:Dave placement:center_left speed:medium only_ptz_action:default tracking_action:tracking.\n\n"
+            "User: Please control camera 4.\n"
+            "Assistant: Camera:4. Tracking_Target:default placement:center_middle speed:medium only_ptz_action:default tracking_action:default.\n\n"
+            "User: Please start recording.\n"
+            "Assistant: Camera:default. Tracking_Target:default placement:center_middle speed:medium only_ptz_action:default tracking_action:default.\n\n"
+            "User: Camera 5 action turn_left.\n"
+            "Assistant: Camera:default. Tracking_Target:default placement:center_middle speed:medium only_ptz_action:turn_left tracking_action:default.\n\n"
+            "User: camera 1 and 2 turn right.\n"
+            "Assistant:\n"
+            "Camera:1. Tracking_Target:default placement:center_middle speed:medium only_ptz_action:turn_right tracking_action:default.\n"
+            "Camera:2. Tracking_Target:default placement:center_middle speed:medium only_ptz_action:turn_right tracking_action:default.\n\n"
+            "Respond with ONLY the Assistant's output. Do NOT add any extra text."
+
+    ),
+    label="System message"
+)
+,
+        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
         gr.Slider(
-            minimum=0, maximum=1, step=0.1, value=0.9, label="Temperature", render=False
-        ),
-        gr.Checkbox(label="Sampling", value=True),
-        gr.Slider(
-            minimum=128,
-            maximum=4096,
-            step=1,
-            value=512,
-            label="Max new tokens",
-            render=False,
+            minimum=0.1,
+            maximum=1.0,
+            value=0.95,
+            step=0.05,
+            label="Top-p (nucleus sampling)",
         ),
     ],
-    stop_btn="Stop Generation",
-    title="PTZ Camera Control Chat",
-    description="Now Running [microsoft/Phi-3-mini-128k-instruct](https://huggingface.co/microsoft/Phi-3-mini-128k-instruct) for PTZ camera control.",
 )
-demo.launch()
+
+
+if __name__ == "__main__":
+    demo.launch()