Update app.py

app.py

@@ -1,63 +1,84 @@
-import gradio as gr
-from huggingface_hub import InferenceClient
-
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
+# app.py
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from peft import AutoPeftModelForCausalLM
+from transformers import AutoTokenizer
+import torch
 
+app = FastAPI()
+
+model_name = "ahmedembedded/AskFAST"
+load_in_4bit = True
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+model = AutoPeftModelForCausalLM.from_pretrained(model_name, load_in_4bit=load_in_4bit).to(device)
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+FAST_prompt_context = """You are an admission officer at Fast University Pakistan. Your role is to answer queries related to the admission process at Fast University. You are expected to provide detailed and accurate responses to questions regarding:
+
+- Application deadlines
+- Required documents
+- Eligibility criteria for different programs
+- Admission process details
+- Any other admissions-related information specific to Fast University Pakistan
+
+Do not respond to any questions that are not related to admissions at Fast University Pakistan. Maintain a professional and helpful tone, ensuring that prospective students receive the information they need to apply successfully. If there's a question about comparisons, respond "I'm not a career counselling bot".
+
+**Example Questions:**
+
+1. What is the application deadline for the upcoming semester?
+2. What documents are required for the application?
+3. What are the eligibility criteria for the Computer Science program?
+4. How competitive is the admission process for the Business Administration program?
+
+Stay focused on admissions-related topics only.
+
+### Instruction:
+{}
+
+### Input:
+{}
+
+### Response:
+{}"""
+
+past_prompts = []
+
+class Query(BaseModel):
+    question: str
+
+def get_answer(question: str) -> str:
+    if len(past_prompts) >= 10:
+        past_prompts.pop(0)
+
+    past_prompts.append(f"User: {question}")
+
+    inputs = tokenizer(
+        [
+            FAST_prompt_context.format(
+                past_prompts,
+                question,
+                "",
+            )
+        ], return_tensors="pt").to(device)
+
+    response = model.generate(**inputs, max_new_tokens=128)
+
+    response_text = tokenizer.decode(response[0], skip_special_tokens=True)
+
+    past_prompts.append(question)
+    past_prompts.append(response_text.split('Response:')[1].split('### Input:')[0])
+
+    return response_text.split('Response:')[1].split('### Input:')[0]
+
+@app.post("/predict/")
+async def predict(query: Query):
+    try:
+        answer = get_answer(query.question)
+        return {"answer": answer}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
 
 if __name__ == "__main__":
-    demo.launch()
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)