Upload 2 files

app.py

@@ -8,132 +8,144 @@ import spaces
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 
-MAX_MAX_NEW_TOKENS = 128000
-DEFAULT_MAX_NEW_TOKENS = 4096
-total_count=0
-MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
+MAX_MAX_NEW_TOKENS = 2048
+DEFAULT_MAX_NEW_TOKENS = 2048
+total_count = 0
+MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "128000"))
 
 DESCRIPTION = """\
 # DeepSeek-R1-Chat
 
-This space demonstrates model [DeepSeek-R1](https://huggingface.co/deepseek-ai/deepseek-r1) by DeepSeek, a reasoning model with 6xxB parameters fine-tuned for chat instructions.
+This space demonstrates model [DeepSeek-Coder](https://huggingface.co/deepseek-ai/deepseek-coder-r1) by DeepSeek, a code model with 6XXB parameters fine-tuned for chat instructions.
 
 **You can also try our R1 model in [official homepage](https://r1.deepseek.com/chat).**
 """
 
 if not torch.cuda.is_available():
-    DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
+DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
 
 
 if torch.cuda.is_available():
-    model_id = "deepseek-ai/deepseek-r1"
-    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map="auto")
-    tokenizer = AutoTokenizer.from_pretrained(model_id)
-    tokenizer.use_default_system_prompt = False
-    
+model_id = "deepseek-ai/deepseek-r1"
+model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype = torch.bfloat16, device_map = "auto")
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+tokenizer.use_default_system_prompt = False
+
 
 
 @spaces.GPU
 def generate(
-    message: str,
-    chat_history: list[tuple[str, str]],
-    system_prompt: str,
-    max_new_tokens: int = 4096,
-    temperature: float = 0,
-    top_p: float = 0,
-    top_k: int = 0,
-    repetition_penalty: float = 4,
+  message: str,
+  chat_history: list[tuple[str, str]],
+  system_prompt: str,
+  max_new_tokens: int = 2048,
+  temperature: float = 0,
+  top_p: float = 0,
+  top_k: int = 50,
+  repetition_penalty: float = 2,
 ) -> Iterator[str]:
-    global total_count
-    total_count += 1
-    print(total_count)
-    os.system("nvidia-smi")
-    conversation = []
-    if system_prompt:
-        conversation.append({"role": "system", "content": system_prompt})
-    for user, assistant in chat_history:
-        conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
-    conversation.append({"role": "user", "content": message})
-
-    input_ids = tokenizer.apply_chat_template(conversation, return_tensors="pt")
-    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
-        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
-        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
-    input_ids = input_ids.to(model.device)
-
-    streamer = TextIteratorStreamer(tokenizer, timeout=400.0, skip_prompt=True, skip_special_tokens=True)
-    generate_kwargs = dict(
-        {"input_ids": input_ids},
-        streamer=streamer,
-        max_new_tokens=max_new_tokens,
-        do_sample=False,
-        top_p=top_p,
-        top_k=top_k,
-        num_beams=1,
-        # temperature=temperature,
-        repetition_penalty=repetition_penalty,
-        eos_token_id=32021
-    )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
-
-    outputs = []
-    for text in streamer:
-        outputs.append(text)
-        yield "".join(outputs).replace("<|EOT|>","")
+global total_count
+total_count += 1
+print(total_count)
+os.system("nvidia-smi")
+conversation = []
+if system_prompt:
+conversation.append({
+  "role": "system", "content": system_prompt
+})
+for user, assistant in chat_history:
+conversation.extend([{
+  "role": "user", "content": user
+}, {
+  "role": "assistant", "content": assistant
+}])
+conversation.append({
+  "role": "user", "content": message
+})
+
+input_ids = tokenizer.apply_chat_template(conversation, return_tensors = "pt")
+if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
+input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
+gr.Warning(f"Trimmed input from conversation as it was longer than {
+  MAX_INPUT_TOKEN_LENGTH
+} tokens.")
+input_ids = input_ids.to(model.device)
+
+streamer = TextIteratorStreamer(tokenizer, timeout = 10.0, skip_prompt = True, skip_special_tokens = True)
+generate_kwargs = dict(
+  {
+    "input_ids": input_ids
+  },
+  streamer = streamer,
+  max_new_tokens = max_new_tokens,
+  do_sample = False,
+  top_p = top_p,
+  top_k = top_k,
+  num_beams = 1,
+  # temperature=temperature,
+  repetition_penalty = repetition_penalty,
+  eos_token_id = 32021
+)
+t = Thread(target = model.generate, kwargs = generate_kwargs)
+t.start()
+
+outputs = []
+for text in streamer:
+outputs.append(text)
+yield "".join(outputs).replace("<|EOT|>","")
 
 
 chat_interface = gr.ChatInterface(
-    fn=generate,
-    additional_inputs=[
-        gr.Textbox(label="System prompt", lines=6),
-        gr.Slider(
-            label="Max new tokens",
-            minimum=1,
-            maximum=MAX_MAX_NEW_TOKENS,
-            step=1,
-            value=DEFAULT_MAX_NEW_TOKENS,
-        ),
-        # gr.Slider(
-        #     label="Temperature",
-        #     minimum=0,
-        #     maximum=4.0,
-        #     step=0.1,
-        #     value=0,
-        # ),
-        gr.Slider(
-            label="Top-p (nucleus sampling)",
-            minimum=0,
-            maximum=1.0,
-            step=0.001,
-            value=0,
-        ),
-        gr.Slider(
-            label="Top-k",
-            minimum=0,
-            maximum=400,
-            step=1,
-            value=0,
-        ),
-        gr.Slider(
-            label="Repetition penalty",
-            minimum=1.0,
-            maximum=4.0,
-            step=0.001,
-            value=4,
-        ),
-    ],
-    stop_btn=gr.Button("Stop"),
-    examples=[
-        ["implement snake game using pygame"],
-        ["Can you explain briefly to me what is the Python programming language?"],
-        ["write a program to find the factorial of a number"],
-    ],
+  fn = generate,
+  additional_inputs = [
+    gr.Textbox(label = "System prompt", lines = 6),
+    gr.Slider(
+      label = "Max new tokens",
+      minimum = 1,
+      maximum = MAX_MAX_NEW_TOKENS,
+      step = 1,
+      value = DEFAULT_MAX_NEW_TOKENS,
+    ),
+    gr.Slider(
+        label="Temperature",
+        minimum=0,
+        maximum=4.0,
+        step=0.01,
+        value=0,
+    ),
+    gr.Slider(
+      label = "Top-p (nucleus sampling)",
+      minimum = 0,
+      maximum = 4.0,
+      step = 0.01,
+      value = 0,
+    ),
+    gr.Slider(
+      label = "Top-k",
+      minimum = 1,
+      maximum = 1000,
+      step = 0.01,
+      value = 50,
+    ),
+    gr.Slider(
+      label = "Repetition penalty",
+      minimum = 0,
+      maximum = 2.0,
+      step = 0.01,
+      value = 2,
+    ),
+  ],
+  stop_btn = gr.Button("Stop"),
+  examples = [
+    ["implement snake game using pygame"],
+    ["Can you explain briefly to me what is the Python programming language?"],
+    ["write a program to find the factorial of a number"],
+  ],
 )
 
-with gr.Blocks(css="style.css") as demo:
-    gr.Markdown(DESCRIPTION)
-    chat_interface.render()
+with gr.Blocks(css = "style.css") as demo:
+gr.Markdown(DESCRIPTION)
+chat_interface.render()
 
 if __name__ == "__main__":
-    demo.queue(max_size=16384).launch()
+demo.queue(max_size = 20).launch()

requirements.txt

@@ -1,9 +1,9 @@
 https://github.com/huggingface/accelerate/archive/main.tar.gz
 https://github.com/bitsandbytes-foundation/bitsandbytes/archive/main.tar.gz
 https://github.com/gradio-app/gradio/archive/main.tar.gz
-https://files.pythonhosted.org/packages/f7/d1/e0a911544ca9993e0f17ce6d3cc0932752356c1b0a834397f28e63479344/protobuf-5.29.3.tar.gz
-https://files.pythonhosted.org/packages/76/c6/8eb0654ba0c7d0bb1bf67bf8fbace101a8e4f250f7722371105e8b6f68fc/scipy-1.15.1.tar.gz
+protobuf
+https://github.com/scipy/scipy/archive/main.tar.gz
 git+https://github.com/google/sentencepiece@master#sub_directory=python
 https://github.com/huggingface/huggingface_hub/archive/main.tar.gz
-https://github.com/pytorch/pytorch/archive/main.tar.gz
+https://github.com/pytorch/pytorch
 https://github.com/huggingface/transformers/archive/main.tar.gz