Fixed it

app.py

@@ -20,7 +20,7 @@ MAX_INPUT_TOKEN_LENGTH = 4096
 
 if torch.cuda.is_available():
     model_id = "codys12/MergeLlama-7b"
-    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16)
+    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, device_map=0,)
     model.cuda()
     tokenizer = AutoTokenizer.from_pretrained(model_id)
 
@@ -44,7 +44,7 @@ def generate(
     current_input += message
     
     device = "cuda:0"
-    inputs_ids = tokenizer(current_input, return_tensors="pt").to(device)
+    input_ids = tokenizer(current_input, return_tensors="pt").to(device)
 
     if len(input_ids) > MAX_INPUT_TOKEN_LENGTH:
         input_ids = input_ids[-MAX_INPUT_TOKEN_LENGTH:]