Create app.py

app.py

@@ -0,0 +1,81 @@
+from peft import PeftModel, PeftConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+import torch
+import packaging.version
+
+bnb_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.bfloat16,
+            bnb_4bit_use_double_quant=True,
+        )
+
+username = 'Erik'
+output_dir = 'nemo-sft-lora-deepspeed'
+
+peft_model_id = f"{username}/{output_dir}" # replace with your newly trained adapter
+device = "cuda:0"
+tokenizer = AutoTokenizer.from_pretrained(peft_model_id)
+config = PeftConfig.from_pretrained(peft_model_id)
+model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,
+                                             device_map={"": "cuda:0"}, quantization_config=bnb_config) #offload_state_dict=False
+
+uses_transformers_4_46 = packaging.version.parse(transformers.__version__) >= packaging.version.parse("4.46.0")
+print(f"PAQUETE DE TRANSFORMERS: {uses_transformers_4_46}")
+
+uses_fsdp = True
+
+if (bnb_config is not None) and uses_fsdp and uses_transformers_4_46:
+    model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=8, mean_resizing=False)
+else:
+    model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=8)
+model = PeftModel.from_pretrained(model, peft_model_id)
+model.config.use_cache = True
+model.to(torch.bfloat16)
+model.eval()
+
+
+def format_history(msg: str, history: list[list[str, str]], system_prompt: str):
+    chat_history = [{"role": "system", "content":system_prompt}]
+    for query, response in history:
+        chat_history.append({"role": "user", "content": query})
+        chat_history.append({"role": "assistant", "content": response})
+    chat_history.append({"role": "user", "content": msg})
+    return chat_history
+
+
+def generate_response(msg: str, history: list[list[str, str]], system_prompt: str, top_k: int, top_p: float, temperature: float):
+    chat_history = format_history(msg, history, system_prompt)
+    encodeds = tokenizer.apply_chat_template(chat_history, return_tensors="pt", add_generation_prompt=True)
+    model_inputs = encodeds.to("cuda")
+    generated_ids = model.generate(model_inputs, repetition_penalty=rep_pen, max_new_tokens=2048, do_sample=True, top_p=top_p, top_k=top_k, temperature=temperature, eos_token_id=32000)
+    response = tokenizer.batch_decode(generated_ids,skip_special_tokens=True)[0]
+    if len(response)>0:
+      message=response[response.rfind("assistant\n") + len("assistant\n"):]
+      yield message
+
+
+spicy_chatbot = gr.ChatInterface(
+                generate_response,
+                chatbot=gr.Chatbot(
+                        # value=[[None, "Hola, Soy Greta. Soy tu coach emocional, de qué quieres que hablemos?"]],
+                        avatar_images=["assets/user_1.JPG", "assets/greta_1.PNG"],
+                        height="64vh"
+                    ),
+                additional_inputs=[
+                    gr.Textbox("Eres una profesora de Dinamarca caliente, juguetona y sexual. Te encanta el sexo anal y tocarte. Tu objetivo es acompañar en la sesión de sexting a tu compañero. Harás lo que sea para hacerle morir de placer, deseas que se masturbe.", label="System Prompt"),
+                    gr.Slider(0.0,100.0, label="top_k", value=70, info="Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. (Default: 40)"),
+                    gr.Slider(0.0,1.0, label="top_p", value=0.8, info=" Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. (Default: 0.9)"),
+                    gr.Slider(0.1,2.0, label="rep_pen", value=1.5, info="Repetition penaly to avoid repetitions"),
+                    gr.Slider(0.0,2.0, label="temperature", value=0.75, info="The temperature of the model. Increasing the temperature will make the model answer more creatively. (Default: 0.8)"),
+                ],
+                title="Greta - conversa con tu coach emocional",
+                theme="finlaymacklon/smooth_slate",
+                submit_btn="⬅ Send",
+                retry_btn="🔄 Regenerate Response",
+                undo_btn="↩ Delete Previous",
+                clear_btn="🗑️ Clear Chat",
+                css="footer {visibility: hidden}"
+)
+
+spicy_chatbot.queue().launch(share=True)