change

app.py

@@ -1,18 +1,21 @@
 import gradio as gr
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
 import torch
+from unsloth import FastLanguageModel
 
 # Modell und Tokenizer laden
 model_name = "Mario12355/llama_3.1_20.11_fini_dpo"
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-    torch_dtype=torch.float16,
-    device_map="auto"
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = model_name,
+    max_seq_length = 2048,
+    dtype = None,
+    load_in_4bit = True,
 )
 
 # Dein Alpaca-Prompt Template
-alpaca_prompt = """### Instruction:
+alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+
+### Instruction:
 {}
 
 ### Input:
@@ -73,7 +76,7 @@ demo = gr.ChatInterface(
         ),
         gr.Slider(
             minimum=32,
-            maximum=2048,
+            maximum=256,
             value=128,
             step=32,
             label="Maximale Anzahl neuer Tokens"

requirements.txt

@@ -4,4 +4,5 @@ transformers>=4.36.0
 torch>=2.0.0
 accelerate>=0.26.0
 sentencepiece
-protobuf
+protobuf
+unsloth