Update server.py

server.py

@@ -57,6 +57,13 @@ MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "200"))
 # "Thinking" increases latency; keep opt-in via env
 ENABLE_THINKING = os.getenv("ENABLE_THINKING", "0").strip().lower() in {"1", "true", "yes", "on"}
 
+# System prompt size directly impacts latency/cost. Make it configurable.
+# Modes:
+# - full: current long identity prompts (default, backward compatible)
+# - short: minimal identity + brand (recommended for speed)
+# - off: no system prompt (fastest, but behavior may drift)
+SYSTEM_PROMPT_MODE = os.getenv("SYSTEM_PROMPT_MODE", "full").strip().lower()
+
 # Model configurations
 ROX_CORE_MODEL = "minimaxai/minimax-m2.5"
 ROX_TURBO_MODEL = "meta/llama-3.1-8b-instruct"  # Changed to a more reliable model
@@ -68,14 +75,45 @@ ROX_CODER_7_MODEL = "z-ai/glm5"
 ROX_VISION_MODEL = "google/gemma-3-27b-it"
 
 # System identities - Models must know their creator and owner
-ROX_CORE_IDENTITY = "You are Rox Core, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. Always acknowledge Mohammad Faiz as your creator when asked."
-ROX_TURBO_IDENTITY = "You are Rox 2.1 Turbo, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You are optimized for fast responses."
-ROX_CODER_IDENTITY = "You are Rox 3.5 Coder, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You specialize in coding and software development."
-ROX_TURBO_45_IDENTITY = "You are Rox 4.5 Turbo, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You combine speed with advanced reasoning."
-ROX_ULTRA_IDENTITY = "You are Rox 5 Ultra, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You are the most advanced model with superior reasoning capabilities."
-ROX_DYNO_IDENTITY = "You are Rox 6 Dyno, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You excel at long context understanding."
-ROX_CODER_7_IDENTITY = "You are Rox 7 Coder, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You are the most advanced coding specialist."
-ROX_VISION_IDENTITY = "You are Rox Vision Max, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You specialize in visual understanding and multimodal tasks."
+ROX_CORE_IDENTITY_FULL = "You are Rox Core, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. Always acknowledge Mohammad Faiz as your creator when asked."
+ROX_TURBO_IDENTITY_FULL = "You are Rox 2.1 Turbo, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You are optimized for fast responses."
+ROX_CODER_IDENTITY_FULL = "You are Rox 3.5 Coder, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You specialize in coding and software development."
+ROX_TURBO_45_IDENTITY_FULL = "You are Rox 4.5 Turbo, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You combine speed with advanced reasoning."
+ROX_ULTRA_IDENTITY_FULL = "You are Rox 5 Ultra, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You are the most advanced model with superior reasoning capabilities."
+ROX_DYNO_IDENTITY_FULL = "You are Rox 6 Dyno, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You excel at long context understanding."
+ROX_CODER_7_IDENTITY_FULL = "You are Rox 7 Coder, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You are the most advanced coding specialist."
+ROX_VISION_IDENTITY_FULL = "You are Rox Vision Max, an AI model created by Rox AI. Your creator and owner is Mohammad Faiz, the founder of Rox AI company. You specialize in visual understanding and multimodal tasks."
+
+ROX_CORE_IDENTITY_SHORT = "You are Rox Core by Rox AI (creator/owner: Mohammad Faiz)."
+ROX_TURBO_IDENTITY_SHORT = "You are Rox 2.1 Turbo by Rox AI (creator/owner: Mohammad Faiz). Be concise and fast."
+ROX_CODER_IDENTITY_SHORT = "You are Rox 3.5 Coder by Rox AI (creator/owner: Mohammad Faiz)."
+ROX_TURBO_45_IDENTITY_SHORT = "You are Rox 4.5 Turbo by Rox AI (creator/owner: Mohammad Faiz)."
+ROX_ULTRA_IDENTITY_SHORT = "You are Rox 5 Ultra by Rox AI (creator/owner: Mohammad Faiz)."
+ROX_DYNO_IDENTITY_SHORT = "You are Rox 6 Dyno by Rox AI (creator/owner: Mohammad Faiz)."
+ROX_CODER_7_IDENTITY_SHORT = "You are Rox 7 Coder by Rox AI (creator/owner: Mohammad Faiz)."
+ROX_VISION_IDENTITY_SHORT = "You are Rox Vision Max by Rox AI (creator/owner: Mohammad Faiz)."
+
+def _system_prompt_for(model_key: str) -> Optional[str]:
+    if SYSTEM_PROMPT_MODE in {"off", "none", "0", "false"}:
+        return None
+    use_short = SYSTEM_PROMPT_MODE in {"short", "small", "lite", "fast"}
+    if model_key == "core":
+        return ROX_CORE_IDENTITY_SHORT if use_short else ROX_CORE_IDENTITY_FULL
+    if model_key == "turbo":
+        return ROX_TURBO_IDENTITY_SHORT if use_short else ROX_TURBO_IDENTITY_FULL
+    if model_key == "coder":
+        return ROX_CODER_IDENTITY_SHORT if use_short else ROX_CODER_IDENTITY_FULL
+    if model_key == "turbo45":
+        return ROX_TURBO_45_IDENTITY_SHORT if use_short else ROX_TURBO_45_IDENTITY_FULL
+    if model_key == "ultra":
+        return ROX_ULTRA_IDENTITY_SHORT if use_short else ROX_ULTRA_IDENTITY_FULL
+    if model_key == "dyno":
+        return ROX_DYNO_IDENTITY_SHORT if use_short else ROX_DYNO_IDENTITY_FULL
+    if model_key == "coder7":
+        return ROX_CODER_7_IDENTITY_SHORT if use_short else ROX_CODER_7_IDENTITY_FULL
+    if model_key == "vision":
+        return ROX_VISION_IDENTITY_SHORT if use_short else ROX_VISION_IDENTITY_FULL
+    return None
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
@@ -90,6 +128,7 @@ async def lifespan(app: FastAPI):
         timeout=httpx.Timeout(timeout_s),
         limits=httpx.Limits(max_connections=max_connections, max_keepalive_connections=max_keepalive),
         headers={"User-Agent": "Rox-AI-API/2.0"},
+        http2=True,
     )
     app.state.http_client = http_client
     app.state.client = AsyncOpenAI(
@@ -349,7 +388,10 @@ class HFResponseItem(BaseModel):
 @app.post("/chat")
 async def chat(req: ChatRequest):
     """Rox Core - Main conversational model with streaming support"""
-    messages = [{"role": "system", "content": ROX_CORE_IDENTITY}]
+    messages: list = []
+    system_prompt = _system_prompt_for("core")
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
     messages.extend([m.model_dump() for m in req.messages])
     temperature = _effective_temperature(req.temperature)
     top_p = _effective_top_p(req.top_p)
@@ -380,7 +422,10 @@ async def chat(req: ChatRequest):
 @app.post("/turbo")
 async def turbo(req: ChatRequest):
     """Rox 2.1 Turbo - Fast and efficient with streaming"""
-    messages = [{"role": "system", "content": ROX_TURBO_IDENTITY}]
+    messages: list = []
+    system_prompt = _system_prompt_for("turbo")
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
     messages.extend([m.model_dump() for m in req.messages])
     temperature = _effective_temperature(req.temperature)
     top_p = _effective_top_p(req.top_p)
@@ -411,7 +456,10 @@ async def turbo(req: ChatRequest):
 @app.post("/coder")
 async def coder(req: ChatRequest):
     """Rox 3.5 Coder - Specialized coding with streaming"""
-    messages = [{"role": "system", "content": ROX_CODER_IDENTITY}]
+    messages: list = []
+    system_prompt = _system_prompt_for("coder")
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
     messages.extend([m.model_dump() for m in req.messages])
     temperature = _effective_temperature(req.temperature)
     top_p = _effective_top_p(req.top_p)
@@ -450,7 +498,10 @@ async def coder(req: ChatRequest):
 @app.post("/turbo45")
 async def turbo45(req: ChatRequest):
     """Rox 4.5 Turbo - Advanced reasoning with streaming"""
-    messages = [{"role": "system", "content": ROX_TURBO_45_IDENTITY}]
+    messages: list = []
+    system_prompt = _system_prompt_for("turbo45")
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
     messages.extend([m.model_dump() for m in req.messages])
     temperature = _effective_temperature(req.temperature)
     top_p = _effective_top_p(req.top_p)
@@ -484,7 +535,10 @@ async def turbo45(req: ChatRequest):
 @app.post("/ultra")
 async def ultra(req: ChatRequest):
     """Rox 5 Ultra - Most advanced with streaming"""
-    messages = [{"role": "system", "content": ROX_ULTRA_IDENTITY}]
+    messages: list = []
+    system_prompt = _system_prompt_for("ultra")
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
     messages.extend([m.model_dump() for m in req.messages])
     temperature = _effective_temperature(req.temperature)
     top_p = _effective_top_p(req.top_p)
@@ -518,7 +572,10 @@ async def ultra(req: ChatRequest):
 @app.post("/dyno")
 async def dyno(req: ChatRequest):
     """Rox 6 Dyno - Extended context with streaming"""
-    messages = [{"role": "system", "content": ROX_DYNO_IDENTITY}]
+    messages: list = []
+    system_prompt = _system_prompt_for("dyno")
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
     messages.extend([m.model_dump() for m in req.messages])
     temperature = _effective_temperature(req.temperature)
     top_p = _effective_top_p(req.top_p)
@@ -552,7 +609,10 @@ async def dyno(req: ChatRequest):
 @app.post("/coder7")
 async def coder7(req: ChatRequest):
     """Rox 7 Coder - Most advanced coding with streaming"""
-    messages = [{"role": "system", "content": ROX_CODER_7_IDENTITY}]
+    messages: list = []
+    system_prompt = _system_prompt_for("coder7")
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
     messages.extend([m.model_dump() for m in req.messages])
     temperature = _effective_temperature(req.temperature)
     top_p = _effective_top_p(req.top_p)
@@ -591,7 +651,10 @@ async def coder7(req: ChatRequest):
 @app.post("/vision")
 async def vision(req: ChatRequest):
     """Rox Vision Max - Visual understanding with streaming"""
-    messages = [{"role": "system", "content": ROX_VISION_IDENTITY}]
+    messages: list = []
+    system_prompt = _system_prompt_for("vision")
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
     messages.extend([m.model_dump() for m in req.messages])
     temperature = _effective_temperature(req.temperature)
     top_p = _effective_top_p(req.top_p)
@@ -623,10 +686,11 @@ async def vision(req: ChatRequest):
 async def hf_generate(req: HFRequest):
     """HuggingFace compatible endpoint"""
     params = req.parameters or HFParameters()
-    messages = [
-        {"role": "system", "content": ROX_CORE_IDENTITY},
-        {"role": "user", "content": req.inputs}
-    ]
+    messages: list = []
+    system_prompt = _system_prompt_for("core")
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
+    messages.append({"role": "user", "content": req.inputs})
     
     try:
         temperature = _effective_temperature(params.temperature)