Added app.py and requirements.txt

app.py

@@ -1,7 +1,6 @@
-from fastapi import FastAPI
+from fastapi import FastAPI 
 from pydantic import BaseModel
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from transformers import BitsAndBytesConfig
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 from peft import PeftModel
 import torch
 import os
@@ -17,11 +16,12 @@ LORA_MODEL = "varshithkumar/gemma-finetuned-sql"
 device = "cuda" if torch.cuda.is_available() else "cpu"
 print("Using device:", device)
 
-print("Loading base model...")
+print("Loading base model with 4-bit quantization...")
 
 bnb_config = BitsAndBytesConfig(
-    load_in_8bit=True,
-    llm_int8_enable_fp32_cpu_offload=True
+    load_in_4bit=True,                  # Use 4-bit
+    bnb_4bit_compute_dtype=torch.float16,  # Compute in float16
+    bnb_4bit_use_double_quant=True      # Optional, better accuracy
 )
 
 base_model = AutoModelForCausalLM.from_pretrained(
@@ -31,25 +31,27 @@ base_model = AutoModelForCausalLM.from_pretrained(
     use_auth_token=HF_TOKEN
 )
 
-
 print("Loading tokenizer...")
 tokenizer = AutoTokenizer.from_pretrained(
     BASE_MODEL,
     use_fast=True,
-    token=HF_TOKEN
+    use_auth_token=HF_TOKEN
 )
 
 print("Applying LoRA adapter...")
 model = PeftModel.from_pretrained(
     base_model,
     LORA_MODEL,
-    token=HF_TOKEN
+    use_auth_token=HF_TOKEN,
+    device_map="auto"  # ensure LoRA is loaded on the right device
 )
 
+model.to(device)
 print("Model loaded successfully!")
 
 class InputData(BaseModel):
     prompt: str
+    max_length: int = 256  # default max length if not provided
 
 @app.post("/generate")
 def generate_text(data: InputData):