fix max_token > n_ctr(512 for llama_cpp) issue

README.md

@@ -4,7 +4,7 @@ emoji: "🧠"
 colorFrom: "indigo"
 colorTo: "red"
 sdk: "gradio"
-sdk_version: "4.44.0"
+sdk_version: "5.49.1"
 app_file: "ggufv2.py"
 pinned: false
 Script path: "build.sh"

ggufv2.py

@@ -10,7 +10,6 @@ from llama_cpp import Llama
 
 # Multi-session helpers from utils.py
 from utils import mk_msg_dir, _as_dir, persist_messages
-
 # ===================== Model =====================
 # You can swap to another GGUF by changing repo_id/filename.
 model = Llama.from_pretrained(
@@ -20,21 +19,62 @@ model = Llama.from_pretrained(
 
 assistant_name = "Nova"
 user_name = "Marshall"
-persona = f"""Your name is {assistant_name}. Address the user as "{user_name}". Use Markdown; put code in fenced blocks with a language tag. Be concise but never give empty feedback.""".strip()
+persona = f"""Your name is {assistant_name}. Address the user as "{user_name}". Use Markdown; put code in fenced blocks with a language tag.""".strip()
 
 # Where each conversation (session) persists its messages
 BASE_MSG_DIR = Path("./msgs/msgs_QwenGGUF")
 BASE_MSG_DIR.mkdir(parents=True, exist_ok=True)
 
 # ---------- Qwen chat template (no tools) ----------
-def render_qwen(messages: List[Dict[str, str]], add_generation_prompt: bool = True) -> str:
+# def render_qwen(messages: List[Dict[str, str]], add_generation_prompt: bool = True) -> str:
+#     """
+#     Convert OpenAI-style messages to Qwen2.5 Instruct format:
+#       <|im_start|>system ... <|im_end|>
+#       <|im_start|>user ...   <|im_end|>
+#       <|im_start|>assistant  (generation continues here)
+#     """
+#     # System prompt
+#     if messages and messages[0].get("role") == "system":
+#         sys_txt = messages[0]["content"]
+#         rest = messages[1:]
+#     else:
+#         sys_txt = persona
+#         rest = messages
+
+#     parts = [f"<|im_start|>system\n{sys_txt}<|im_end|>\n"]
+#     for m in rest:
+#         role = m.get("role")
+#         if role not in ("user", "assistant"):
+#             continue
+#         parts.append(f"<|im_start|>{role}\n{m['content']}<|im_end|>\n")
+
+#     if add_generation_prompt:
+#         parts.append("<|im_start|>assistant\n")
+#     return "".join(parts)
+
+def render_qwen_trim(
+    messages: List[Dict[str, str]],
+    model,                             # llama_cpp.Llama 实例（用于 token 计数）
+    n_ctx: Optional[int] = None,       # 不传则用 model.n_ctx()
+    add_generation_prompt: bool = True,
+    persona: str = "",
+    reserve_new: int = 256,            # 希望生成的新 token 预算（上限）
+    pad: int = 8,                      # 保险余量，避免越界
+    hard_user_tail_chars: int = 2000,  # 还不够时，最后一条 user 文本的硬截断字符数
+) -> Tuple[str, int]:
     """
-    Convert OpenAI-style messages to Qwen2.5 Instruct format:
-      <|im_start|>system ... <|im_end|>
-      <|im_start|>user ...   <|im_end|>
-      <|im_start|>assistant  (generation continues here)
+    - 只保留 system + 最近的若干轮对话，使得 total_tokens + reserve_new + pad <= n_ctx
+    - 若仍不够，则截短最后一条 user。
+    - 返回 (prompt, safe_max_new)，safe_max_new 已确保不越界。
     """
-    # System prompt
+    def _tok_len(txt: str) -> int:
+        # 与 llama_cpp 的计数保持一致
+        return len(model.tokenize(txt.encode("utf-8"), add_bos=True))
+
+    if n_ctx is None:
+        n_ctx = getattr(model, "n_ctx")() if callable(getattr(model, "n_ctx", None)) else model.n_ctx
+
+    # 1) 拆出 system 与其余消息
     if messages and messages[0].get("role") == "system":
         sys_txt = messages[0]["content"]
         rest = messages[1:]
@@ -42,16 +82,72 @@ def render_qwen(messages: List[Dict[str, str]], add_generation_prompt: bool = Tr
         sys_txt = persona
         rest = messages
 
-    parts = [f"<|im_start|>system\n{sys_txt}<|im_end|>\n"]
-    for m in rest:
-        role = m.get("role")
-        if role not in ("user", "assistant"):
-            continue
-        parts.append(f"<|im_start|>{role}\n{m['content']}<|im_end|>\n")
+    # 仅保留 user / assistant
+    rest = [m for m in rest if m.get("role") in ("user", "assistant")]
+
+    # 2) 生成函数：把 system + 若干轮对话渲染为 Qwen prompt
+    def _render(sys_text: str, turns: List[Dict[str, str]], add_gen: bool) -> str:
+        parts = [f"<|im_start|>system\n{sys_text}<|im_end|>\n"]
+        for m in turns:
+            parts.append(f"<|im_start|>{m['role']}\n{m['content']}<|im_end|>\n")
+        if add_gen:
+            parts.append("<|im_start|>assistant\n")
+        return "".join(parts)
+
+    # 3) 先尝试保留全部轮次，从最老开始裁剪直到 fits
+    kept = rest[:]  # 深拷贝
+    while True:
+        prompt = _render(sys_txt, kept, add_generation_prompt)
+        used = _tok_len(prompt)
+
+        # 计算还能安全生成的 token 数
+        safe_max_new = max(1, n_ctx - used - pad)
+        # 希望生成 reserve_new，但不能超过 safe_max_new
+        if used + reserve_new + pad <= n_ctx:
+            # 有余量，按 reserve_new 返回可生成上限
+            return prompt, min(reserve_new, safe_max_new)
+
+        # 没有余量——需要裁剪历史。如果可裁剪的 turns < 1，则进入硬截断
+        if len(kept) <= 1:
+            break  # 只剩最后一条，跳出去做硬截断
+
+        # 从最早的一条开始丢；为避免打断成对语义，可一次丢两条（user+assistant）
+        # 但如果开头不是成对，就按 1 条丢弃。
+        drop_count = 2 if len(kept) >= 2 else 1
+        # 保证留下至少 1 条（最后一条 user）用于上下文
+        while drop_count > 0 and len(kept) > 1:
+            kept.pop(0)
+            drop_count -= 1
+
+    # 4) 仍然不够：硬截断“最后一条 user”文本尾部
+    #    目标：尽量保留最近语义，同时立刻释放 token 空间
+    if kept and kept[-1]["role"] == "user":
+        kept[-1] = {
+            "role": "user",
+            "content": kept[-1]["content"][-hard_user_tail_chars:]
+        }
+    elif kept:
+        # 最后一条不是 user，则尽量截短它（通常是 assistant）
+        kept[-1] = {
+            "role": kept[-1]["role"],
+            "content": kept[-1]["content"][-hard_user_tail_chars:]
+        }
+
+    # 重新渲染并最终给出安全 max_new
+    prompt = _render(sys_txt, kept, add_generation_prompt)
+    used = _tok_len(prompt)
+    safe_max_new = max(1, n_ctx - used - pad)
+
+    # 如果仍然超（极端长的 system），进一步把 system 也截短
+    if used + pad > n_ctx:
+        trimmed_sys = sys_txt[-hard_user_tail_chars:]
+        prompt = _render(trimmed_sys, kept, add_generation_prompt)
+        used = _tok_len(prompt)
+        safe_max_new = max(1, n_ctx - used - pad)
+
+    # 不允许返回负或 0
+    return prompt, max(1, safe_max_new)
 
-    if add_generation_prompt:
-        parts.append("<|im_start|>assistant\n")
-    return "".join(parts)
 
 STOP_TOKENS = ["<|im_end|>", "<|endoftext|>"]
 
@@ -185,14 +281,24 @@ def on_send(user_text: str,
 
     # 3) append user, render, generate
     messages = messages + [{"role": "user", "content": user_text}]
-    prompt = render_qwen(messages, add_generation_prompt=True)
+    # prompt = render_qwen(messages, add_generation_prompt=True)
+    prompt, max_new = render_qwen_trim(
+        messages=messages,
+        model=model,        # llama_cpp.Llama 实例
+        n_ctx=None,         # 不传用 model.n_ctx()
+        add_generation_prompt=True,
+        persona=persona,    # 你之前的 persona 变量
+        reserve_new=max_new_tokens,  # 你希望的生成长度
+        pad=16
+    )
+    
 
     try:
         result = model.create_completion(
             prompt=prompt,
             temperature=float(temperature),
             top_p=float(top_p),
-            max_tokens=int(max_new_tokens),
+            max_tokens=int(max_new),
             repeat_penalty=float(repetition_penalty),
             stop=STOP_TOKENS,
         )
@@ -202,7 +308,7 @@ def on_send(user_text: str,
             prompt,
             temperature=float(temperature),
             top_p=float(top_p),
-            max_tokens=int(max_new_tokens),
+            max_tokens=int(max_new),
             repeat_penalty=float(repetition_penalty),
             stop=STOP_TOKENS,
         )
@@ -236,7 +342,7 @@ with gr.Blocks(title="Qwen GGUF — multi-session") as demo:
             with gr.Accordion("Generation settings", open=False):
                 temperature = gr.Slider(0.0, 2.0, value=0.7, step=0.05, label="temperature")
                 top_p = gr.Slider(0.1, 1.0, value=0.95, step=0.01, label="top_p")
-                max_new_tokens = gr.Slider(16, 1024, value=256, step=16, label="max_new_tokens")
+                max_new_tokens = gr.Slider(16, 512, value=256, step=16, label="max_new_tokens")
                 repetition_penalty = gr.Slider(1.0, 2.0, value=1.07, step=0.01, label="repetition_penalty")
 
             session_list = gr.Radio(choices=[], value=None, label="Conversations", interactive=True)