Upload app.py

app.py

@@ -367,18 +367,21 @@ def load_model():
         base_phi_model = "microsoft/Phi-4-mini-instruct"
 
         # T4 does fp16 (not bf16)
-        DTYPE = torch.float16
+        DTYPE = torch.float32
         quantization_config = BitsAndBytesConfig(
             load_in_4bit=True,
             bnb_4bit_quant_type="nf4",
             bnb_4bit_compute_dtype=DTYPE,
         )
+
+
         classifier_backbone_base = AutoModelForCausalLM.from_pretrained(
             base_phi_model,
             quantization_config=quantization_config,
-            device_map={"": 0},          # single-GPU
-            trust_remote_code=False,     # <-- avoid remote LossKwargs import
-            attn_implementation="sdpa",
+            device_map={"": 0},
+            trust_remote_code=False,      # keep this if you switched it earlier
+            # safest with eager attention when mixing kernels:
+            attn_implementation="eager",
         )
 
         classifier_tokenizer = AutoTokenizer.from_pretrained(
@@ -401,7 +404,7 @@ def load_model():
         classifier_model.classifier.load_state_dict(torch.load(classifier_head_path, map_location=device))
 
         classifier_model.to(device)
-        classifier_model = classifier_model.to(torch.bfloat16)
+        classifier_model = classifier_model.to(device=DEVICE, dtype=torch.float32)
 
         classifier_model.eval() # Set model to evaluation mode