Create handler.py

handler.py

@@ -0,0 +1,90 @@
+import json
+import torch
+from unsloth import FastLanguageModel
+
+# Global model and tokenizer variables
+model = None
+tokenizer = None
+
+# Define your InstructABSA instruction text (as used in your finetuning script)
+instructabsa_instruction = (
+    "Definition: The output will be the aspects (both implicit and explicit) which have an associated opinion that are extracted from the input text. In cases where there are no aspects the output should be noaspectterm.\n"
+    "Positive example 1-\n"
+    "input: With the great variety on the menu, I eat here often and never get bored.\n"
+    "output: menu\n"
+    "Positive example 2-\n"
+    "input: Great food, good size menu, great service and an unpretensious setting.\n"
+    "output: food, menu, service, setting\n"
+    "Negative example 1-\n"
+    "input: They did not have mayonnaise, forgot our toast, left out ingredients (ie cheese in an omelet), below hot temperatures and the bacon was so over cooked it crumbled on the plate when you touched it.\n"
+    "output: toast, mayonnaise, bacon, ingredients, plate\n"
+    "Negative example 2-\n"
+    "input: The seats are uncomfortable if you are sitting against the wall on wooden benches.\n"
+    "output: seats\n"
+    "Neutral example 1-\n"
+    "input: I asked for seltzer with lime, no ice.\n"
+    "output: seltzer with lime\n"
+    "Neutral example 2-\n"
+    "input: They wouldnt even let me finish my glass of wine before offering another.\n"
+    "output: glass of wine\n"
+    "Now complete the following example:"
+)
+
+# Define the Alpaca-style prompt template that wraps the instruction
+alpaca_prompt = (
+    "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n"
+    "### Instruction:\n{}\n"
+    "### Input:\n{}\n"
+    "### Response:\n{}"
+)
+
+def initialize():
+    """
+    This function loads the finetuned unsloth model and tokenizer.
+    Adjust the 'model_name' to point to your model repository.
+    """
+    global model, tokenizer
+    # Set your finetuned model repository name here
+    model_name = "your-model-repo"  # e.g., "unsloth/mistral-7b-instruct-v0.3-bnb-4bit-finetuned"
+    model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name=model_name,
+        max_seq_length=2048,
+        load_in_4bit=True  # Change if necessary
+    )
+    # Set the model in inference mode
+    FastLanguageModel.for_inference(model)
+    model.eval()
+
+def preprocess(request):
+    """
+    Expects a JSON payload with an "inputs" field containing the raw review text.
+    It prepends the InstructABSA prompt (using the alpaca template) to the review.
+    """
+    data = json.loads(request.data.decode("utf-8"))
+    raw_text = data.get("inputs", "")
+    # Construct the full prompt using the defined instruction and template.
+    full_prompt = alpaca_prompt.format(instructabsa_instruction, raw_text, "")
+    inputs = tokenizer(full_prompt, return_tensors="pt", truncation=True)
+    device = next(model.parameters()).device
+    inputs = {key: tensor.to(device) for key, tensor in inputs.items()}
+    return inputs
+
+def inference(inputs):
+    """
+    Runs model.generate with a fixed maximum token limit.
+    """
+    with torch.no_grad():
+        output_ids = model.generate(**inputs, max_new_tokens=128)
+    return output_ids
+
+def postprocess(output_ids):
+    """
+    Decodes the generated tokens to a string. If the generated text includes the response marker,
+    it extracts the text after "### Response:".
+    """
+    output_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+    if "### Response:" in output_text:
+        response = output_text.split("### Response:")[-1].strip()
+    else:
+        response = output_text.strip()
+    return json.dumps({"predicted": response})