Fix bitsandbytes issue - attempt 2

app.py

@@ -1,29 +1,39 @@
+import os
 from fastapi import FastAPI
 from pydantic import BaseModel
-from transformers import pipeline
+from transformers import (
+    AutoModelForCausalLM, 
+    AutoTokenizer, 
+    pipeline, 
+    AutoConfig
+)
+
+# Load the configuration and remove any quantization config if present
+config = AutoConfig.from_pretrained("devops-bda/Abap")
+if hasattr(config, "quantization_config"):
+    config.quantization_config = None
 
-# Load your Hugging Face model
-# model = pipeline("text-generation", model="devops-bda/Abap")
-model = pipeline(
-    "text-generation",
-    model="devops-bda/Abap",
-    model_kwargs={"load_in_4bit": False}  # Disable 4-bit quantization
+# Load the model and tokenizer without 4-bit quantization
+model = AutoModelForCausalLM.from_pretrained(
+    "devops-bda/Abap",
+    config=config,
+    load_in_4bit=False  # explicitly disable 4-bit quantization
 )
+tokenizer = AutoTokenizer.from_pretrained("devops-bda/Abap")
+
+# Create a text-generation pipeline with the loaded model and tokenizer
+text_gen_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer)
 
-# Initialize FastAPI app
 app = FastAPI()
 
-# Define input format
 class InputData(BaseModel):
     input_text: str
 
-# Health check endpoint
 @app.get("/health")
 async def health_check():
     return {"status": "ok", "message": "Model is ready"}
 
-# Define prediction endpoint
 @app.post("/predict")
 async def predict(data: InputData):
-    result = model(data.input_text, max_length=500)
-    return {"output": result}
+    output = text_gen_pipeline(data.input_text, max_length=500)
+    return {"output": output}