bits

Dockerfile

@@ -5,6 +5,8 @@ WORKDIR /code
 COPY ./requirements.txt /code/requirements.txt
 
 RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+RUN apt-get update && apt-get install -y libbitsandbytes-dev
+
 
 RUN useradd -m -u 1000 user
 

app/model_loader.py

@@ -20,7 +20,8 @@ def load_model():
         use_auth_token=hf_token,
         cache_dir="/tmp/hf_cache",
         torch_dtype="auto",
-        device_map="auto"
+        device_map="auto",
+        load_in_8bit=True  # <-- Try enabling 8-bit
     )
     model = PeftModel.from_pretrained(
         base_model,

requirements.txt

@@ -5,3 +5,4 @@ torch
 accelerate
 fastapi
 uvicorn
+bitsandbytes