Update app.py

app.py

@@ -10,6 +10,10 @@ text_generator = None
 is_hugging_face = True
 model_id = "google/gemma-2-9b-it"
 huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
+device = "auto" # torch.device("cuda" if torch.cuda.is_available() else "cpu")
+device = "cuda"
+dtype = torch.bfloat16
+
 if not huggingface_token:
     pass
     print("no HUGGINGFACE_TOKEN if you need set secret ")
@@ -22,9 +26,7 @@ def init():
     
     #model_id = "microsoft/Phi-3-mini-128k-instruct"
     
-    device = "auto" # torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    device = "cuda"
-    dtype = torch.bfloat16
+    
     
     tokenizer = AutoTokenizer.from_pretrained(model_id, token=huggingface_token)