demo

app.py

@@ -0,0 +1,27 @@
+import gradio as gr
+import spaces
+from guidance import gen
+from guidance.models import Transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-3.2-1B-Instruct")
+tok = AutoTokenizer.from_pretrained("meta-llama/Llama-3.2-1B-Instruct")
+model = model.to("cuda").eval()
+
+
+@spaces.GPU
+def greet(name):
+    lm = Transformers(model=model, tokenizer=tok)
+    lm += "Let the user know what the temperature for today is, in farenheit.\n"
+    lm += gen(
+        "response",
+        n=1,
+        temperature=0,
+        max_tokens=1024,
+        regex=f"Hello {name}, it is \d\d F today.",
+    )
+    return lm["response"]
+
+
+demo = gr.Interface(fn=greet, inputs=gr.Text(), outputs=gr.Text())
+demo.launch()

requirements.txt

@@ -0,0 +1,6 @@
+torch==2.5.1
+transformers==4.50.0
+gradio==5.35.0
+guidance==0.2.4
+gradio_rangeslider
+spaces