File size: 2,043 Bytes
7c31071 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | import json
import requests
import time
API_URL = "http://localhost:8000/v1/chat/completions"
HEADERS = {
"Content-Type": "application/json",
"Authorization": "Bearer 0",
}
def run_test(prompt: str, max_tokens=500):
payload = {
"model": "custom-model",
"messages": [
{"role": "system", "content": "Answer the user question about Markie Voss."},
{"role": "user", "content": prompt},
],
"max_tokens": max_tokens,
"do_sample": True,
"temperature": 0.6,
"top_p": 0.8,
"eos_token_id": [
151645,
151643,
151668
],
"max_tokens": 1024,
"enable_thinking": True,
"stream": True,
}
print("=" * 80)
print("Prompt:", prompt)
print("Streaming response:\n")
with requests.post(
API_URL,
headers=HEADERS,
json=payload,
stream=True, # 🔴 stream HTTP response
timeout=60,
) as r:
print("HTTP status:", r.status_code)
r.raise_for_status()
full_text = ""
for line in r.iter_lines(decode_unicode=True):
if not line:
continue
# OpenAI-style streaming uses "data: {...}"
if line.startswith("data:"):
data = line[len("data:"):].strip()
if data == "[DONE]":
break
try:
chunk = json.loads(data)
except json.JSONDecodeError:
continue
delta = chunk["choices"][0]["delta"]
if "content" in delta:
token = delta["content"]
full_text += token
print(token, end="", flush=True)
print("\n\n--- END OF STREAM ---")
print("✅ Full content repr:", repr(full_text))
if __name__ == "__main__":
print("Warming up...")
time.sleep(1)
while True:
p = input("User: ")
run_test(p)
|