Create app.py

gradio==4.22.0
transformers==4.39.3
torch==2.2.2
accelerate==0.28.0
setuptools>=68.0.0
wheel>=0.41.0

app.py

@@ -0,0 +1,86 @@
+import gradio as gr
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+import torch
+import threading
+
+MODEL_ID = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"  # small enough for CPU Basic
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    torch_dtype=torch.float32,      # stay on CPU
+    low_cpu_mem_usage=True
+)
+model.eval()
+
+SYSTEM_PROMPT = "You are a helpful assistant."
+
+def chat_fn(history, max_new_tokens=256, temperature=0.7, top_p=0.9):
+    # history: list of [user, assistant]
+    messages = []
+    messages.append({"role": "system", "content": SYSTEM_PROMPT})
+    for u, a in history:
+        messages.append({"role": "user", "content": u})
+        if a:
+            messages.append({"role": "assistant", "content": a})
+
+    # Simple prompt format for TinyLlama chat
+    prompt = ""
+    for m in messages:
+        if m["role"] == "system":
+            prompt += f"<|system|>\n{m['content']}\n"
+        elif m["role"] == "user":
+            prompt += f"<|user|>\n{m['content']}\n"
+        elif m["role"] == "assistant":
+            prompt += f"<|assistant|>\n{m['content']}\n"
+    prompt += "<|assistant|>\n"
+
+    inputs = tokenizer(prompt, return_tensors="pt")
+    input_ids = inputs["input_ids"]
+
+    streamer = TextIteratorStreamer(tokenizer, skip_special_tokens=True)
+    generation_kwargs = dict(
+        input_ids=input_ids,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        temperature=temperature,
+        top_p=top_p,
+        streamer=streamer
+    )
+    thread = threading.Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
+
+    partial = ""
+    for new_text in streamer:
+        partial += new_text
+        yield partial
+
+with gr.Blocks(title="TinyLlama Chat (CPU Free)") as demo:
+    gr.Markdown("## TinyLlama Chat — Free CPU Space")
+    chatbot = gr.Chatbot(height=400)
+    with gr.Row():
+        msg = gr.Textbox(placeholder="Type your question...")
+    with gr.Row():
+        temperature = gr.Slider(0.1, 1.5, value=0.7, label="Temperature")
+        top_p = gr.Slider(0.1, 1.0, value=0.9, label="top_p")
+        max_new_tokens = gr.Slider(32, 512, value=256, step=8, label="Max new tokens")
+    send = gr.Button("Send")
+
+    def user_submit(user_message, chat_history):
+        chat_history = chat_history + [[user_message, None]]
+        return "", chat_history
+
+    def bot_respond(chat_history, max_new_tokens, temperature, top_p):
+        user_message = chat_history[-1][0]
+        gen = chat_fn(chat_history[:-1] + [[user_message, ""]], max_new_tokens, temperature, top_p)
+        partial = ""
+        for chunk in gen:
+            partial = chunk
+            chat_history[-1][1] = partial
+            yield chat_history
+
+    send.click(user_submit, [msg, chatbot], [msg, chatbot], queue=False).then(
+        bot_respond, [chatbot, max_new_tokens, temperature, top_p], [chatbot]
+    )
+
+demo.queue().launch()