🚀 Auto-deploy from GitHub

app/core/config.py

@@ -16,7 +16,7 @@ class Settings(BaseSettings):
 
     MODEL_PATH: str = str(_CARDSERVER_DIR_CLS / "models" / "lora-checkpoint") 
     DEFAULT_MODEL_ID: str = "teknium/OpenHermes-2.5-Mistral-7B" 
-    MODEL_LOAD_IN_4BIT: bool = True  # Disabled for macOS compatibility
+    MODEL_LOAD_IN_4BIT: bool = False  # Disabled for macOS compatibility
 
     GENERATED_PATH: str = str(_APP_DEFAULT_STATIC_DIR_CLS / "images" / "generated") 
     BASE_PATH: str = str(_APP_DEFAULT_STATIC_DIR_CLS / "images" / "base") 

app/core/model_loader.py

@@ -40,6 +40,11 @@ def load_model_and_tokenizer():
             logger.warning(f"Failed to import bitsandbytes: {e}. Disabling 4-bit quantization.")
             bitsandbytes_available = False
             use_4bit = False
+        except Exception as e:
+            # Catch other bitsandbytes related errors (like missing .dylib files)
+            logger.warning(f"Bitsandbytes import failed with error: {e}. Disabling 4-bit quantization.")
+            bitsandbytes_available = False
+            use_4bit = False
     else:
         bitsandbytes_available = False
         if is_macos:

requirements.txt

@@ -19,7 +19,7 @@ transformers>=4.52.0
 peft>=0.15.0
 accelerate>=1.7.0
 numpy>=1.26.0
-bitsandbytes>=0.46.0 
+# bitsandbytes>=0.46.0  # Disabled on macOS ARM64 - not compatible 
 
 protobuf>=3.20.0
 tiktoken>=0.5.0