try different

app.py

@@ -21,7 +21,6 @@ app = FastAPI()
 # // FOR RUNNING IN SPACES
 model_name = "numind/NuExtract-1.5-tiny"
 # Path inside your container
-# MODEL_PATH = "/app/model_cache/models--numind--NuExtract-1.5-tiny/snapshots/df52efb3109d324cd52b30728f9e3fdedf19f742"
 # If you used local_dir="model", snapshot_download will still create models--… subfolder.
 # You can also symlink or copy it to /app/model directly in Dockerfile.
 
@@ -42,7 +41,6 @@ dtype = torch.float16 if device in ("mps", "cuda") else torch.float32
 
 print("CUDA available:", torch.cuda.is_available())       # True
 print("Device name:", torch.cuda.get_device_name(0)) 
-# bnb_config = BitsAndBytesConfig(load_in_8bit=True)
 
 
 # If lower memory usage needed: 
@@ -50,7 +48,8 @@ print("Device name:", torch.cuda.get_device_name(0))
 bnb_config = BitsAndBytesConfig(
     load_in_4bit=True,
     bnb_4bit_use_double_quant=True,
-    bnb_4bit_quant_type="nf4"
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.float16
 )
 
 @app.on_event("startup")
@@ -123,7 +122,7 @@ def predict_NuExtract(texts, template, batch_size=1, max_length=5096, max_new_to
                 max_length=max_length
             ).to(device)
             print(f"Generating outputs with model for batch {i//batch_size+1}...", flush=True)
-            ids = model.generate(**enc, max_new_tokens=max_new_tokens, num_beams=1, use_cache=True)
+            ids = model.generate(**enc, max_new_tokens=max_new_tokens, num_beams=1, use_cache=False)
             outputs += tokenizer.batch_decode(ids, skip_special_tokens=True)
     print("Outputs generated.", flush=True)
     elapsed = time.perf_counter() - start_time