Ana

Paused

OrbitMC commited on 11 days ago

Commit

5191be0

verified ·

1 Parent(s): 6894d69

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -3,17 +3,16 @@ import io
 import base64
 from flask import Flask, request, jsonify
 from huggingface_hub import hf_hub_download
-from llama_cpp import Llama
 from kittentts import KittenTTS
 import soundfile as sf
 app = Flask(__name__)
-# Load models
 MODEL_REPO = "unsloth/gemma-3-270m-it-GGUF"
 MODEL_FILE = "gemma-3-270m-it-F16.gguf"
 model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE, local_dir="models")
-llm = Llama(model_path=model_path, n_ctx=2048, n_threads=4, n_gpu_layers=0, verbose=False)
 tts = KittenTTS("KittenML/kitten-tts-nano-0.8-int8")
 HTML = """<!DOCTYPE html>
@@ -56,7 +55,11 @@ def index():
 @app.route('/api/chat', methods=['POST'])
 def chat():
     user_msg = request.json['message']
-    response = llm.create_chat_completion(messages=[{"role": "user", "content": user_msg}], max_tokens=512, temperature=0.7)['choices'][0]['message']['content']
     audio = tts.generate(text=response, voice="Kiki")
     buf = io.BytesIO()
     sf.write(buf, audio, 24000, format='WAV')

 import base64
 from flask import Flask, request, jsonify
 from huggingface_hub import hf_hub_download
+from ctransformers import AutoModelForCausalLM
 from kittentts import KittenTTS
 import soundfile as sf
 app = Flask(__name__)
 MODEL_REPO = "unsloth/gemma-3-270m-it-GGUF"
 MODEL_FILE = "gemma-3-270m-it-F16.gguf"
 model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE, local_dir="models")
+llm = AutoModelForCausalLM.from_pretrained(model_path, model_type="gemma", context_length=2048)
 tts = KittenTTS("KittenML/kitten-tts-nano-0.8-int8")
 HTML = """<!DOCTYPE html>
 @app.route('/api/chat', methods=['POST'])
 def chat():
     user_msg = request.json['message']
+    prompt = f"""<bos><start_of_turn>user
+{user_msg}<end_of_turn>
+<start_of_turn>model
+"""
+    response = llm(prompt, max_new_tokens=512, temperature=0.7, stop=["<end_of_turn>"])
     audio = tts.generate(text=response, voice="Kiki")
     buf = io.BytesIO()
     sf.write(buf, audio, 24000, format='WAV')