Update app.py

app.py

@@ -3,20 +3,25 @@ from fastapi import FastAPI
 from pydantic import BaseModel
 from unsloth import FastModel
 
-# Set cache directories
+# Fix permission errors for cache dirs
 os.environ["TRITON_CACHE_DIR"] = "/tmp/triton_cache"
+os.environ["TORCHINDUCTOR_CACHE_DIR"] = "/tmp/torchinductor_cache"
+
+# Optional: HF model cache
 HF_CACHE = "/tmp/hf_cache"
 os.environ["TRANSFORMERS_CACHE"] = HF_CACHE
 os.environ["HF_HOME"] = HF_CACHE
+
+# Create cache dirs
 os.makedirs(HF_CACHE, exist_ok=True)
+os.makedirs("/tmp/triton_cache", exist_ok=True)
+os.makedirs("/tmp/torchinductor_cache", exist_ok=True)
 
-# Create app
 app = FastAPI()
 
-# Use separate base and adapter model
 model, tokenizer = FastModel.from_pretrained(
-    model_name = "microsoft/phi-2",                      # Base model
-    adapter_name = "srikar-v05/phi3-Mini-Medical-Chat",  # LoRA adapter
+    model_name = "microsoft/phi-2",
+    adapter_name = "srikar-v05/phi3-Mini-Medical-Chat",
     load_in_4bit = True,
     max_seq_length = 2048,
 )