a

app.py

@@ -4,34 +4,83 @@ from huggingface_hub import InferenceClient
 """
 Copied from inference in colab notebook
 """
-# import torch
 
-# # Monkey-patch to avoid CUDA initialization issues
-# torch.cuda.get_device_capability = lambda *args, **kwargs: (0, 0)
+from transformers import LlamaForCausalLM, LlamaTokenizer
+import torch
 
-# from unsloth.chat_templates import get_chat_template
-# from unsloth import FastLanguageModel
+# Load model and tokenizer globally to avoid reloading for every request
+model_path = "llama_lora_model_1"
 
-# # IMPORTING MODEL AND TOKENIZER ————————
+# Load tokenizer
+tokenizer = LlamaTokenizer.from_pretrained(model_path)
 
-# max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
-# dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
-# load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
+# Load model
+model = LlamaForCausalLM.from_pretrained(
+    model_path,
+    torch_dtype=torch.float32,  # Adjust based on your environment
+    device_map="cpu"  # Use CPU for inference
+)
 
-# model, tokenizer = FastLanguageModel.from_pretrained(
-#     model_name = "llama_lora_model_1",
-#     max_seq_length = max_seq_length,
-#     dtype = dtype,
-#     load_in_4bit = load_in_4bit,
-# )
+# Define the response function
+def respond(
+    message: str,
+    history: list[tuple[str, str]],
+    system_message: str,
+    max_tokens: int,
+    temperature: float,
+    top_p: float,
+):
+    # Combine system message and history into a single prompt
+    messages = [{"role": "system", "content": system_message}]
+    for val in history:
+        if val[0]:
+            messages.append({"role": "user", "content": val[0]})
+        if val[1]:
+            messages.append({"role": "assistant", "content": val[1]})
+    messages.append({"role": "user", "content": message})
+    
+    # Create a single text prompt from the messages
+    prompt = ""
+    for msg in messages:
+        if msg["role"] == "system":
+            prompt += f"[System]: {msg['content']}\n\n"
+        elif msg["role"] == "user":
+            prompt += f"[User]: {msg['content']}\n\n"
+        elif msg["role"] == "assistant":
+            prompt += f"[Assistant]: {msg['content']}\n\n"
+    
+    # Tokenize the prompt
+    inputs = tokenizer(prompt, return_tensors="pt", truncation=True)
+    input_ids = inputs.input_ids.to("cpu")  # Ensure input is on the CPU
+
+    # Generate response
+    output_ids = model.generate(
+        input_ids,
+        max_length=input_ids.shape[1] + max_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        do_sample=True,
+    )
+    
+    # Decode the generated text
+    generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+
+    # Extract the assistant's response from the generated text
+    assistant_response = generated_text[len(prompt):].strip()
+
+    # Yield responses incrementally (simulate streaming)
+    response = ""
+    for token in assistant_response.split():  # Split tokens by whitespace
+        response += token + " "
+        yield response.strip()
+
+
+"""
+For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+"""
+# client = InferenceClient(model="https://huggingface.co/Heit39/llama_lora_model_1")
 
-# tokenizer = get_chat_template(
-#     tokenizer,
-#     chat_template = "llama-3.1",
-# )
-# FastLanguageModel.for_inference(model) # Enable native 2x faster inference
 
-# # RUNNING INFERENCE ————————————————————————
 
 # def respond(
 #     message,
@@ -51,57 +100,19 @@ Copied from inference in colab notebook
 
 #     messages.append({"role": "user", "content": message})
 
-#     inputs = tokenizer.apply_chat_template(
-#         messages,
-#         tokenize = True,
-#         add_generation_prompt = True, # Must add for generation
-#         return_tensors = "pt",
-#     )
-
-#     outputs = model.generate(input_ids = inputs, max_new_tokens = max_tokens, use_cache = True,
-#                             temperature = 1.5, min_p = 0.1)
-#     response = tokenizer.batch_decode(outputs)
-
-#     yield response
+#     response = ""
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient(model="https://huggingface.co/Heit39/llama_lora_model_1")
-
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
+#     for message in client.chat_completion(
+#         messages,
+#         max_tokens=max_tokens,
+#         stream=True,
+#         temperature=temperature,
+#         top_p=top_p,
+#     ):
+#         token = message.choices[0].delta.content
+
+#         response += token
+#         yield response
 
 
 """

requirements.txt

@@ -1,3 +1,5 @@
 huggingface_hub==0.25.2
 
-unsloth
+unsloth
+transformers
+accelerate