Upload 4 files

app.py

@@ -0,0 +1,49 @@
+
+
+import io
+from fastapi import FastAPI,HTTPException, Request
+from typing import List
+from pydantic import BaseModel
+from model import generate_response, eval_tokenizer, model
+
+
+
+app=FastAPI(title="RAIZZ-FAQ-Bot")
+
+class Query(BaseModel):
+    query_prompt:str
+
+class response(BaseModel):
+    response:str
+
+#api endpoints
+
+@app.get("/")
+
+def read_root():
+  return{"message: Welcome to the FAQ Bot!"}
+
+@app.post("/chat")
+
+def chat(message:Query):
+
+    model_input = eval_tokenizer(message , return_tensors="pt").to("cuda")
+    model.eval()
+    with torch.no_grad():
+        response = (eval_tokenizer.decode(model.generate(**model_input, max_new_tokens=500)[0], skip_special_tokens=True))
+        #out = output.split(":")[-1]
+    return{"response":response}
+
+
+@app.post("/chatbot", response_model=response,status_code=200)
+
+async def make_prediction(request:Query):
+    try:
+        prompt=request.query_prompt
+        model_input = eval_tokenizer(prompt, return_tensors="pt").to("cuda")
+        with torch.no_grad():
+           model_answer = (eval_tokenizer.decode(model.generate(**model_input, max_new_tokens=500)[0], skip_special_tokens=True))
+          #out = output.split(":")[-1]
+           return response(response=model_answer)
+    except Exception as e:
+        raise HTTPException(status_code=500,detail=str(e))

dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.11.0
+
+WORKDIR /faq-chatbot
+
+COPY ./requirements.txt /faq-chatbot/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /faq-chatbot/requirements.txt
+
+RUN useradd -m -u 1000 user
+
+USER user 
+
+ENV HOME=/home/user \ 
+    PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+
+CMD ["uvicorn", "app:app","--host","0.0.0.0","--port","7860"]

model.py

@@ -0,0 +1,47 @@
+from huggingface_hub import login
+
+
+#loading base model
+
+import torch
+from transformers import AutoModelForCausalLM,AutoTokenizer,BitsAndBytesConfig
+
+
+base_model_id = "mistralai/Mistral-7B-Instruct-v0.2"
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+
+base_model = AutoModelForCausalLM.from_pretrained(
+    base_model_id,  # Mistral, same as before
+    quantization_config=bnb_config,  # Same quantization config as before
+    device_map="auto",
+    trust_remote_code=True,
+)
+
+eval_tokenizer = AutoTokenizer.from_pretrained(
+    base_model_id,
+    add_bos_token=True,
+    trust_remote_code=True,
+)
+from peft import PeftModel, PeftConfig
+from transformers import AutoModelForCausalLM
+
+peft_model_id="AgamP/results"
+
+config=PeftConfig.from_pretrained(peft_model_id)
+model= PeftModel.from_pretrained(base_model,peft_model_id)
+
+prompt="How do i track my fitness levels?"
+
+model.eval()
+
+with torch.no_grad():
+        def generate_response(prompt):
+           model_input = eval_tokenizer(prompt , return_tensors="pt").to("cuda")
+           response = (eval_tokenizer.decode(model.generate(**model_input, max_new_tokens=500)[0], skip_special_tokens=True))
+           #out = output.split(":")[-1]
+           return response

requirements.txt

@@ -0,0 +1,11 @@
+bitsandbytes
+accelerate
+torch 
+transformers
+huggingface_hub
+bitsandbytes
+peft
+fastapi
+uvicorn
+pydantic
+typing