Add gen_entities function

app.py

@@ -11,16 +11,20 @@ model = AutoModelForCausalLM.from_pretrained(
     load_in_8bit=True,
     device_map="auto",
     revision="half",
-    # low_cpu_mem_usage=True
 )
 tokenizer = AutoTokenizer.from_pretrained(peft_model_id)
 # Load the Lora model
 model = PeftModel.from_pretrained(model, peft_model_id)
 
 
-def greet(name):
-    return "Hello " + name + "!!"
+def gen_entities(text):
+    text = f"<SP> text: {text}\n\n entities: "
+    batch = tokenizer(text, return_tensors="pt")
+    with torch.cuda.amp.autocast():
+        output_tokens = model.generate(**batch, max_new_tokens=256, eos_token_id=50258)
 
+    return tokenizer.decode(output_tokens[0], skip_special_tokens=False)
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
+
+iface = gr.Interface(fn=gen_entities, inputs="text", outputs="text")
 iface.launch()