Download server.py from UnimeType/Transliteration-Playground: direct link, hf CLI and curl.
- Browser
- Download file 18.1 kB
-
https://huggingface.co/spaces/UnimeType/Transliteration-Playground/resolve/main/server.py
- Command line
-
hf download hf://spaces/UnimeType/Transliteration-Playground/server.py
-
curl -L -o server.py https://huggingface.co/spaces/UnimeType/Transliteration-Playground/resolve/main/server.py
18.1 kB
| #!/usr/bin/env python3 | |
| """ | |
| UnimeType Transliteration 2.0 (`v2.6`) Live Playground Server. | |
| Serves the Editorial Design System UI on `:7860` immediately while warming up | |
| all three official `v2.6` GGUF engines (`0.8B`, `4B`, `9B` `Q4_K_M`) on local | |
| `llama-server` daemons inside Hugging Face's 16 GB RAM Docker container. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import collections | |
| import json | |
| import math | |
| import os | |
| import shutil | |
| import subprocess | |
| import threading | |
| import time | |
| from contextlib import asynccontextmanager | |
| from pathlib import Path | |
| from typing import Any, AsyncIterator | |
| import httpx | |
| from fastapi import FastAPI, HTTPException, Request | |
| from fastapi.responses import FileResponse, JSONResponse, StreamingResponse | |
| from huggingface_hub import hf_hub_download | |
| APP_DIR = Path(__file__).resolve().parent | |
| MODEL_DIR = Path(os.environ.get("MODEL_DIR", str(APP_DIR / "models"))) | |
| LLAMA_BIN_DIR = Path(os.environ.get("LLAMA_BIN_DIR", "/opt/llama")) | |
| LLAMA_SERVER_BIN = os.environ.get("LLAMA_SERVER_BIN", str(LLAMA_BIN_DIR / "llama-server")) | |
| TIER_SPECS: dict[str, dict[str, Any]] = { | |
| "08b": { | |
| "tier": "08b", | |
| "label": "0.8B", | |
| "repo_id": "UnimeType/Transliteration-2.0-0.8B-GGUF", | |
| "filename": "Transliteration-2.0-0.8B-Convert-Q4_K_M.gguf", | |
| "port": 8081, | |
| "size_mb": 533, | |
| }, | |
| "4b": { | |
| "tier": "4b", | |
| "label": "4B", | |
| "repo_id": "UnimeType/Transliteration-2.0-4B-GGUF", | |
| "filename": "Transliteration-2.0-4B-Convert-Q4_K_M.gguf", | |
| "port": 8082, | |
| "size_mb": 2740, | |
| }, | |
| "9b": { | |
| "tier": "9b", | |
| "label": "9B", | |
| "repo_id": "UnimeType/Transliteration-2.0-9B-GGUF", | |
| "filename": "Transliteration-2.0-9B-Convert-Q4_K_M.gguf", | |
| "port": 8083, | |
| "size_mb": 5680, | |
| }, | |
| } | |
| ENABLED_TIERS = [ | |
| t.strip() | |
| for t in os.environ.get("PLAYGROUND_TIERS", "08b,4b,9b").split(",") | |
| if t.strip() in TIER_SPECS | |
| ] | |
| def detect_cgroup_threads() -> int: | |
| """Detect real cgroup CPU quota so llama-server never over-spawns threads on 2-vCPU pods.""" | |
| override = os.environ.get("LLAMA_THREADS") | |
| if override and override.isdigit(): | |
| return max(1, int(override)) | |
| # cgroup v2 | |
| try: | |
| raw = Path("/sys/fs/cgroup/cpu.max").read_text(encoding="utf-8").strip().split() | |
| if len(raw) == 2 and raw[0] != "max": | |
| quota, period = int(raw[0]), int(raw[1]) | |
| if quota > 0 and period > 0: | |
| return max(1, min(16, math.ceil(quota / period))) | |
| except Exception: | |
| pass | |
| # cgroup v1 | |
| try: | |
| quota = int(Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read_text(encoding="utf-8").strip()) | |
| period = int(Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read_text(encoding="utf-8").strip()) | |
| if quota > 0 and period > 0: | |
| return max(1, min(16, math.ceil(quota / period))) | |
| except Exception: | |
| pass | |
| return max(1, min(4, os.cpu_count() or 2)) | |
| CPU_THREADS = detect_cgroup_threads() | |
| TIER_STATE: dict[str, dict[str, Any]] = { | |
| t: { | |
| "tier": t, | |
| "label": TIER_SPECS[t]["label"], | |
| "repo_id": TIER_SPECS[t]["repo_id"], | |
| "quant": "Q4_K_M", | |
| "state": "queued" if t in ENABLED_TIERS else "disabled", | |
| "detail": "Queued for warmup" if t in ENABLED_TIERS else "Disabled", | |
| "ready": False, | |
| } | |
| for t in TIER_SPECS | |
| } | |
| PROCS: dict[str, subprocess.Popen[str]] = {} | |
| STOP_EVENT = threading.Event() | |
| INFER_SEMAPHORE = asyncio.Semaphore(1) | |
| # Simple sliding-window per-IP rate limiter (36 requests / 60s) | |
| RATE_BUCKETS: dict[str, collections.deque[float]] = {} | |
| RATE_LIMIT_MAX = 36 | |
| RATE_LIMIT_WINDOW = 60.0 | |
| def check_rate_limit(client_ip: str) -> None: | |
| now = time.monotonic() | |
| dq = RATE_BUCKETS.setdefault(client_ip, collections.deque()) | |
| while dq and now - dq[0] > RATE_LIMIT_WINDOW: | |
| dq.popleft() | |
| if len(dq) >= RATE_LIMIT_MAX: | |
| raise HTTPException( | |
| status_code=429, | |
| detail="Rate limit reached (36 conversions/min). Wait a few seconds or download UnimeType for unlimited local use.", | |
| ) | |
| dq.append(now) | |
| def cleanup_temp_caches() -> None: | |
| for env_key in ("HF_XET_CACHE",): | |
| val = os.environ.get(env_key) | |
| if val: | |
| shutil.rmtree(val, ignore_errors=True) | |
| shutil.rmtree(MODEL_DIR / ".cache", ignore_errors=True) | |
| def start_llama_daemon(tier: str, gguf_path: Path) -> subprocess.Popen[str]: | |
| spec = TIER_SPECS[tier] | |
| env = os.environ.copy() | |
| existing_ld = env.get("LD_LIBRARY_PATH", "") | |
| env["LD_LIBRARY_PATH"] = f"{LLAMA_BIN_DIR}:{existing_ld}" if existing_ld else str(LLAMA_BIN_DIR) | |
| cmd = [ | |
| LLAMA_SERVER_BIN, | |
| "-m", | |
| str(gguf_path), | |
| "--host", | |
| "127.0.0.1", | |
| "--port", | |
| str(spec["port"]), | |
| "-c", | |
| "512", | |
| "-np", | |
| "1", | |
| "-t", | |
| str(CPU_THREADS), | |
| "-tb", | |
| str(CPU_THREADS), | |
| "--no-webui", | |
| "--reasoning", | |
| "off", | |
| ] | |
| return subprocess.Popen( | |
| cmd, | |
| env=env, | |
| stdout=subprocess.DEVNULL, | |
| stderr=subprocess.STDOUT, | |
| text=True, | |
| ) | |
| def wait_daemon_healthy(port: int, proc: subprocess.Popen[str], timeout_s: float = 90.0) -> bool: | |
| deadline = time.monotonic() + timeout_s | |
| url = f"http://127.0.0.1:{port}/health" | |
| while time.monotonic() < deadline and not STOP_EVENT.is_set(): | |
| if proc.poll() is not None: | |
| return False | |
| try: | |
| r = httpx.get(url, timeout=1.5) | |
| if r.status_code == 200 and r.json().get("status") == "ok": | |
| return True | |
| except Exception: | |
| pass | |
| time.sleep(0.4) | |
| return False | |
| def warmup_worker() -> None: | |
| MODEL_DIR.mkdir(parents=True, exist_ok=True) | |
| for tier in ENABLED_TIERS: | |
| if STOP_EVENT.is_set(): | |
| return | |
| spec = TIER_SPECS[tier] | |
| target_file = MODEL_DIR / spec["filename"] | |
| try: | |
| if not target_file.exists(): | |
| TIER_STATE[tier]["state"] = "downloading" | |
| TIER_STATE[tier]["detail"] = f"Pulling {spec['label']} Q4_K_M ({spec['size_mb']} MB)..." | |
| downloaded = hf_hub_download( | |
| repo_id=spec["repo_id"], | |
| filename=spec["filename"], | |
| revision="v2.6", | |
| local_dir=str(MODEL_DIR), | |
| ) | |
| target_file = Path(downloaded) | |
| cleanup_temp_caches() | |
| TIER_STATE[tier]["state"] = "starting" | |
| TIER_STATE[tier]["detail"] = f"Loading {spec['label']} into memory..." | |
| proc = start_llama_daemon(tier, target_file) | |
| PROCS[tier] = proc | |
| if wait_daemon_healthy(spec["port"], proc): | |
| TIER_STATE[tier]["state"] = "ready" | |
| TIER_STATE[tier]["ready"] = True | |
| TIER_STATE[tier]["detail"] = f"Live · {spec['label']} Q4_K_M ({CPU_THREADS} vCPU)" | |
| else: | |
| TIER_STATE[tier]["state"] = "error" | |
| TIER_STATE[tier]["detail"] = f"{spec['label']} failed healthcheck" | |
| except Exception as exc: | |
| TIER_STATE[tier]["state"] = "error" | |
| TIER_STATE[tier]["detail"] = f"{spec['label']} warmup error: {exc}" | |
| # Keep watching daemons and auto-restart if any exits unexpectedly | |
| while not STOP_EVENT.is_set(): | |
| for tier in ENABLED_TIERS: | |
| proc = PROCS.get(tier) | |
| spec = TIER_SPECS[tier] | |
| target_file = MODEL_DIR / spec["filename"] | |
| if proc is not None and proc.poll() is not None and target_file.exists(): | |
| TIER_STATE[tier]["ready"] = False | |
| TIER_STATE[tier]["state"] = "starting" | |
| TIER_STATE[tier]["detail"] = f"Restarting {spec['label']}..." | |
| new_proc = start_llama_daemon(tier, target_file) | |
| PROCS[tier] = new_proc | |
| if wait_daemon_healthy(spec["port"], new_proc): | |
| TIER_STATE[tier]["ready"] = True | |
| TIER_STATE[tier]["state"] = "ready" | |
| TIER_STATE[tier]["detail"] = f"Live · {spec['label']} Q4_K_M ({CPU_THREADS} vCPU)" | |
| STOP_EVENT.wait(3.0) | |
| async def lifespan(_: FastAPI) -> AsyncIterator[None]: | |
| worker = threading.Thread(target=warmup_worker, daemon=True, name="v26-warmup") | |
| worker.start() | |
| try: | |
| yield | |
| finally: | |
| STOP_EVENT.set() | |
| for proc in PROCS.values(): | |
| if proc.poll() is None: | |
| proc.terminate() | |
| app = FastAPI(title="UnimeType Transliteration 2.0 (v2.6) Playground", lifespan=lifespan) | |
| def normalize_tier(raw: str) -> str: | |
| s = (raw or "").strip().lower() | |
| if s in ("9b", "unimetype/transliteration-2.0-9b", "unimetype/transliteration-2.0-9b-gguf"): | |
| return "9b" | |
| if s in ("4b", "unimetype/transliteration-2.0-4b", "unimetype/transliteration-2.0-4b-gguf"): | |
| return "4b" | |
| if s in ("08b", "0.8b", "unimetype/transliteration-2.0-0.8b", "unimetype/transliteration-2.0-0.8b-gguf"): | |
| return "08b" | |
| if "9b" in s: | |
| return "9b" | |
| if "4b" in s: | |
| return "4b" | |
| if "0.8b" in s or "08b" in s: | |
| return "08b" | |
| return "9b" | |
| def build_v26_prompt(payload_Or_str: dict[str, Any] | str) -> str: | |
| if isinstance(payload_Or_str, str): | |
| user_json = payload_Or_str.strip() | |
| else: | |
| allowed_keys = ( | |
| "operation", | |
| "input", | |
| "enabled_languages", | |
| "primary_language", | |
| "japanese_output_style", | |
| "rejected_outputs", | |
| "lexicon", | |
| ) | |
| clean: dict[str, Any] = {} | |
| for k in allowed_keys: | |
| if k in payload_Or_str and payload_Or_str[k] is not None: | |
| clean[k] = payload_Or_str[k] | |
| inp = str(clean.get("input", "")).strip() | |
| if not inp: | |
| raise HTTPException(status_code=400, detail="Input cannot be empty.") | |
| if len(inp) > 600: | |
| raise HTTPException(status_code=400, detail="Input exceeds 600 characters.") | |
| clean["input"] = inp | |
| user_json = json.dumps(clean, ensure_ascii=False, separators=(",", ":")) | |
| if len(user_json) > 1600: | |
| raise HTTPException(status_code=400, detail="Wire payload too large.") | |
| return f"<|im_start|>user\n{user_json}<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n" | |
| async def wait_until_tier_ready(tier: str, max_wait_s: float = 45.0) -> None: | |
| if tier not in ENABLED_TIERS: | |
| raise HTTPException(status_code=404, detail=f"Model tier '{tier}' is not enabled on this server.") | |
| deadline = time.monotonic() + max_wait_s | |
| while time.monotonic() < deadline: | |
| if TIER_STATE[tier]["ready"]: | |
| return | |
| if TIER_STATE[tier]["state"] == "error": | |
| raise HTTPException(status_code=503, detail=TIER_STATE[tier]["detail"]) | |
| await asyncio.sleep(0.35) | |
| raise HTTPException(status_code=503, detail=f"{TIER_SPECS[tier]['label']} is still warming up — try again in a few seconds.") | |
| def strip_stop_tokens(text: str) -> str: | |
| for tok in ("<|im_end|>", "<|endoftext|>"): | |
| idx = text.find(tok) | |
| if idx != -1: | |
| text = text[:idx] | |
| return text | |
| async def serve_index() -> FileResponse: | |
| return FileResponse(APP_DIR / "index.html", media_type="text/html; charset=utf-8") | |
| async def serve_css() -> FileResponse: | |
| return FileResponse(APP_DIR / "style.css", media_type="text/css; charset=utf-8") | |
| async def api_status() -> JSONResponse: | |
| return JSONResponse( | |
| { | |
| "ok": True, | |
| "release": "v2.6", | |
| "cpu_threads": CPU_THREADS, | |
| "tiers": TIER_STATE, | |
| } | |
| ) | |
| async def api_convert(request: Request) -> Any: | |
| forwarded = request.headers.get("x-forwarded-for", "") | |
| client_ip = forwarded.split(",")[0].strip() if forwarded else (request.client.host if request.client else "local") | |
| check_rate_limit(client_ip) | |
| body = await request.json() | |
| tier = normalize_tier(str(body.get("tier", "9b"))) | |
| payload = body.get("payload") | |
| if not isinstance(payload, dict): | |
| raise HTTPException(status_code=400, detail="'payload' must be a JSON object.") | |
| stream = bool(body.get("stream", True)) | |
| prompt = build_v26_prompt(payload) | |
| await wait_until_tier_ready(tier) | |
| port = TIER_SPECS[tier]["port"] | |
| llama_req = { | |
| "prompt": prompt, | |
| "n_predict": 224, | |
| "temperature": 0.0, | |
| "top_k": 1, | |
| "stop": ["<|im_end|>", "<|endoftext|>"], | |
| "cache_prompt": True, | |
| "stream": stream, | |
| } | |
| if not stream: | |
| async with INFER_SEMAPHORE: | |
| t0 = time.perf_counter() | |
| async with httpx.AsyncClient(timeout=60.0) as client: | |
| r = await client.post(f"http://127.0.0.1:{port}/completion", json=llama_req) | |
| r.raise_for_status() | |
| data = r.json() | |
| elapsed_ms = round((time.perf_counter() - t0) * 1000) | |
| out = strip_stop_tokens(data.get("content", "")).strip() | |
| return JSONResponse( | |
| { | |
| "tier": tier, | |
| "label": TIER_SPECS[tier]["label"], | |
| "output": out, | |
| "elapsed_ms": elapsed_ms, | |
| "tokens_predicted": data.get("tokens_predicted", 0), | |
| } | |
| ) | |
| async def event_gen() -> AsyncIterator[bytes]: | |
| # Emit immediate status event if waiting in semaphore queue | |
| yield f"data: {json.dumps({'queued': True, 'tier': tier})}\n\n".encode("utf-8") | |
| async with INFER_SEMAPHORE: | |
| t0 = time.perf_counter() | |
| acc: list[str] = [] | |
| tokens_pred = 0 | |
| try: | |
| async with httpx.AsyncClient(timeout=75.0) as client: | |
| async with client.stream("POST", f"http://127.0.0.1:{port}/completion", json=llama_req) as resp: | |
| resp.raise_for_status() | |
| async for line in resp.aiter_lines(): | |
| if not line.startswith("data: "): | |
| continue | |
| chunk = json.loads(line[6:]) | |
| piece = chunk.get("content", "") | |
| if piece: | |
| acc.append(piece) | |
| clean_so_far = strip_stop_tokens("".join(acc)) | |
| yield f"data: {json.dumps({'delta': piece, 'text': clean_so_far, 'tier': tier}, ensure_ascii=False)}\n\n".encode("utf-8") | |
| if chunk.get("tokens_predicted"): | |
| tokens_pred = int(chunk["tokens_predicted"]) | |
| elapsed_ms = round((time.perf_counter() - t0) * 1000) | |
| final_text = strip_stop_tokens("".join(acc)).strip() | |
| yield f"data: {json.dumps({'done': True, 'output': final_text, 'elapsed_ms': elapsed_ms, 'tokens_predicted': tokens_pred, 'tier': tier}, ensure_ascii=False)}\n\n".encode("utf-8") | |
| except Exception as exc: | |
| yield f"data: {json.dumps({'error': str(exc), 'tier': tier}, ensure_ascii=False)}\n\n".encode("utf-8") | |
| return StreamingResponse( | |
| event_gen(), | |
| media_type="text/event-stream", | |
| headers={ | |
| "Cache-Control": "no-cache", | |
| "Connection": "keep-alive", | |
| "X-Accel-Buffering": "no", | |
| }, | |
| ) | |
| async def openai_chat_completions(request: Request) -> JSONResponse: | |
| """OpenAI-compatible endpoint so developers can run the Playground cURL snippet live.""" | |
| forwarded = request.headers.get("x-forwarded-for", "") | |
| client_ip = forwarded.split(",")[0].strip() if forwarded else (request.client.host if request.client else "local") | |
| check_rate_limit(client_ip) | |
| body = await request.json() | |
| tier = normalize_tier(str(body.get("model", "9b"))) | |
| messages = body.get("messages") or [] | |
| user_msgs = [m.get("content", "") for m in messages if isinstance(m, dict) and m.get("role") == "user"] | |
| if not user_msgs: | |
| raise HTTPException(status_code=400, detail="At least one user message is required.") | |
| prompt = build_v26_prompt(str(user_msgs[-1])) | |
| await wait_until_tier_ready(tier) | |
| port = TIER_SPECS[tier]["port"] | |
| max_tokens = max(1, min(224, int(body.get("max_tokens", 224)))) | |
| async with INFER_SEMAPHORE: | |
| t0 = time.perf_counter() | |
| async with httpx.AsyncClient(timeout=60.0) as client: | |
| r = await client.post( | |
| f"http://127.0.0.1:{port}/completion", | |
| json={ | |
| "prompt": prompt, | |
| "n_predict": max_tokens, | |
| "temperature": 0.0, | |
| "top_k": 1, | |
| "stop": ["<|im_end|>", "<|endoftext|>"], | |
| "cache_prompt": True, | |
| "stream": False, | |
| }, | |
| ) | |
| r.raise_for_status() | |
| data = r.json() | |
| elapsed_ms = round((time.perf_counter() - t0) * 1000) | |
| out = strip_stop_tokens(data.get("content", "")).strip() | |
| return JSONResponse( | |
| { | |
| "id": f"chatcmpl-v26-{tier}-{int(time.time())}", | |
| "object": "chat.completion", | |
| "created": int(time.time()), | |
| "model": TIER_SPECS[tier]["repo_id"], | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "message": {"role": "assistant", "content": out}, | |
| "finish_reason": "stop", | |
| } | |
| ], | |
| "usage": { | |
| "prompt_tokens": data.get("tokens_evaluated", 0), | |
| "completion_tokens": data.get("tokens_predicted", 0), | |
| "total_tokens": data.get("tokens_evaluated", 0) + data.get("tokens_predicted", 0), | |
| }, | |
| "unimetype_elapsed_ms": elapsed_ms, | |
| } | |
| ) | |