Transformers: TildeOpen-30b Space

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/TildeAI.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="C:\Users\User\miniconda3" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="C:\Users\User\miniconda3" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="C:\Users\User\miniconda3" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/TildeAI.iml" filepath="$PROJECT_DIR$/.idea/TildeAI.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

app.py

@@ -0,0 +1,57 @@
+import gradio as gr
+import torch
+from threading import Thread
+from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
+
+MODEL_ID = "TildeAI/TildeOpen-30b"
+
+# Tokenizer MUST be slow version per model card
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=False)
+
+# Load model on GPU with BF16
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+)
+
+SYSTEM = (
+    "You are a helpful multilingual assistant. "
+    "The model is base (not instruction-tuned), so follow the user's request precisely."
+)
+
+def format_history(history, user_msg):
+    prompt = SYSTEM + "\n\n"
+    for u, a in history:
+        prompt += f"<|user|>\n{u}\n<|assistant|>\n{a}\n"
+    prompt += f"<|user|>\n{user_msg}\n<|assistant|>\n"
+    return prompt
+
+def chat_fn(message, history):
+    prompt = format_history(history, message)
+    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    gen_kwargs = dict(
+        **inputs,
+        max_new_tokens=512,
+        do_sample=True,
+        temperature=0.7,
+        top_p=0.9,
+        repetition_penalty=1.1,
+        streamer=streamer,
+    )
+
+    thread = Thread(target=model.generate, kwargs=gen_kwargs)
+    thread.start()
+    partial = ""
+    for new_text in streamer:
+        partial += new_text
+        yield partial
+
+demo = gr.ChatInterface(
+    fn=chat_fn,
+    title="TildeOpen-30B (Transformers, BF16)",
+    description="Base model (not instruction-tuned). Multilingual; context length 8192.",
+)
+demo.queue().launch()

requirements.txt

@@ -0,0 +1,7 @@
+transformers>=4.44.0
+accelerate>=0.33.0
+safetensors>=0.4.5
+huggingface_hub>=0.24.0
+# Torch GPU: Spaces often has a suitable CUDA. If install fails, remove the next line or switch cu121→cu124.
+torch --index-url https://download.pytorch.org/whl/cu121
+gradio>=4.44.0