Create app.py

app.py

@@ -0,0 +1,57 @@
+import torch
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+MODEL_NAME = "ibm-granite/granite-3.3-2b-instruct"
+
+# Load tokenizer and model
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_NAME,
+    torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
+)
+model.to("cuda" if torch.cuda.is_available() else "cpu")
+model.eval()
+
+
+def chat(user_input, history):
+    messages = []
+
+    # Convert chat history to Granite chat format
+    for user, assistant in history:
+        messages.append({"role": "user", "content": user})
+        messages.append({"role": "assistant", "content": assistant})
+
+    messages.append({"role": "user", "content": user_input})
+
+    inputs = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        tokenize=True,
+        return_dict=True,
+        return_tensors="pt",
+    ).to(model.device)
+
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs,
+            max_new_tokens=100,
+            do_sample=True,
+            temperature=0.7,
+            top_p=0.9,
+        )
+
+    response = tokenizer.decode(
+        outputs[0][inputs["input_ids"].shape[-1]:],
+        skip_special_tokens=True,
+    )
+
+    history.append((user_input, response))
+    return history, history
+
+
+gr.ChatInterface(
+    fn=chat,
+    title="IBM Granite 3.3 2B Instruct",
+    description="Chat with IBM Granite using Hugging Face Transformers",
+).launch()