app.py

"""
app.py — CHIP HuggingFace Space 入口(美观版)

部署: https://huggingface.co/spaces/<user>/CHIP
"""
from __future__ import annotations

import os
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent))

import gradio as gr
import tiktoken

from chip import Compressor

# ============================================================
# Tokenizer 计数器
# ============================================================
TOKENIZERS = {}
try:
    TOKENIZERS["GPT-4 (cl100k)"] = tiktoken.get_encoding("cl100k_base")
    TOKENIZERS["GPT-4o (o200k)"] = tiktoken.get_encoding("o200k_base")
except Exception as e:
    print(f"[warn] tiktoken load failed: {e}")


def _lazy_load_qwen():
    from transformers import AutoTokenizer
    return AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B", trust_remote_code=True)


def _lazy_load_deepseek():
    from transformers import AutoTokenizer
    return AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V2-Lite", trust_remote_code=True)


_LAZY = {
    "Qwen2.5": _lazy_load_qwen,
    "DeepSeek-V2/V3": _lazy_load_deepseek,
}


def count_tokens(text: str, tokenizer_name: str) -> int:
    if tokenizer_name in TOKENIZERS:
        enc = TOKENIZERS[tokenizer_name]
    elif tokenizer_name in _LAZY:
        try:
            tok = _LAZY[tokenizer_name]()
            TOKENIZERS[tokenizer_name] = tok
            enc = tok
        except Exception:
            return -1
    else:
        return -1

    if hasattr(enc, "encode") and not hasattr(enc, "tokenize"):
        return len(enc.encode(text))
    return len(enc.encode(text, add_special_tokens=False))


# ============================================================
# 主压缩函数
# ============================================================
def run(text: str, target: str, use_l1: bool, use_l2: bool, use_l3: bool,
        use_l4: bool, tokenizer_name: str, use_jieba: bool):
    if not text.strip():
        empty_card = "<div class='stat-card empty'>👈 请在左侧输入 prompt</div>"
        return "", empty_card, "", ""

    layers = []
    for flag, name in [(use_l1, "L1"), (use_l2, "L2"), (use_l3, "L3"), (use_l4, "L4")]:
        if flag:
            layers.append(name)
    if not layers:
        layers = ["L1"]

    if use_jieba:
        os.environ["CHIP_USE_JIEBA"] = "1"
    else:
        os.environ.pop("CHIP_USE_JIEBA", None)
    import chip.compressor as cc
    cc._default_compressor = None

    compressor = Compressor(target=target, layers=layers)
    result = compressor.compress(text)

    n_orig = count_tokens(text, tokenizer_name)
    n_comp = count_tokens(result.compressed, tokenizer_name)

    # ---- 漂亮的统计卡片 ----
    if n_orig > 0 and n_comp > 0:
        saving_pct = (1 - n_comp / n_orig) * 100
        n_diff = n_orig - n_comp
        if saving_pct > 0:
            badge_class = "saving-badge good"
            arrow = "↓"
            verb = "节省"
        elif saving_pct < 0:
            badge_class = "saving-badge bad"
            arrow = "↑"
            verb = "增加"
        else:
            badge_class = "saving-badge neutral"
            arrow = "→"
            verb = "持平"

        char_orig, char_comp = len(text), len(result.compressed)
        char_pct = (1 - char_comp / max(char_orig, 1)) * 100

        stats_html = f"""
<div class="stats-grid">
  <div class="stat-card primary">
    <div class="stat-label">Token 节省</div>
    <div class="stat-value {badge_class}">
      <span class="arrow">{arrow}</span> {abs(saving_pct):.1f}%
    </div>
    <div class="stat-sub">{verb} {abs(n_diff)} token · 在 {tokenizer_name}</div>
  </div>
  <div class="stat-card">
    <div class="stat-label">原文</div>
    <div class="stat-value muted">{n_orig}</div>
    <div class="stat-sub">token · {char_orig} 字符</div>
  </div>
  <div class="stat-card">
    <div class="stat-label">压缩后</div>
    <div class="stat-value emphasis">{n_comp}</div>
    <div class="stat-sub">token · {char_comp} 字符</div>
  </div>
  <div class="stat-card">
    <div class="stat-label">字符压缩</div>
    <div class="stat-value muted">{char_pct:+.0f}%</div>
    <div class="stat-sub">字符数变化(供参考,token 才重要)</div>
  </div>
</div>"""
    else:
        stats_html = (
            "<div class='stat-card empty'>"
            "⚠️ Token 计数不可用(可能是 tokenizer 加载失败)"
            "</div>"
        )

    # ---- 漂亮的规则面板 ----
    if result.applied_rules:
        rules_items = []
        for r in result.applied_rules:
            # 解析 rule id 和命中次数
            parts = r.split("×")
            rid = parts[0]
            count = parts[1] if len(parts) > 1 else "1"
            # 按层着色
            layer_color = {"L1": "#3B7DD8", "L2": "#E6883C", "L3": "#7C3AED", "L4": "#10B981"}
            color = "#888"
            for l, c in layer_color.items():
                if l in rid:
                    color = c
                    break
            badge = (
                f"<span class='rule-pill' style='--rule-color:{color}'>"
                f"<span class='rule-id'>{rid}</span>"
                f"<span class='rule-count'>×{count}</span>"
                f"</span>"
            )
            rules_items.append(badge)
        rules_html = (
            "<div class='rules-header'>🎯 命中规则 "
            f"<span class='rules-count'>{len(result.applied_rules)} 条</span></div>"
            "<div class='rules-pills'>" + " ".join(rules_items) + "</div>"
        )
    else:
        rules_html = (
            "<div class='rules-header empty-rules'>"
            "📭 没有规则匹配 — 这段 prompt 已经够紧凑了"
            "</div>"
        )

    return result.compressed, stats_html, rules_html, result.compressed


# ============================================================
# 示例
# ============================================================
EXAMPLES = [
    [
        "请你帮我对下面这段文字进行一个全面的分析,如果可以的话麻烦你给出一些改进的建议",
        "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", False,
    ],
    [
        "请你扮演一位资深 Python 工程师,对下面的代码进行 code review,并以 JSON 格式输出结果",
        "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", True,
    ],
    [
        "因为最近在下雨,所以路面变得很滑,因此开车的时候需要特别注意安全",
        "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", False,
    ],
    [
        "Role: 资深产品经理\n任务: 评估这个需求,大家都知道产品决策需要数据支持",
        "deepseek_v3", True, True, True, True, "GPT-4 (cl100k)", False,
    ],
]


# ============================================================
# 自定义 CSS — 让 demo 看起来不像默认的 gradio
# ============================================================
CUSTOM_CSS = """
/* ===== 全局 ===== */
.gradio-container {
  max-width: 1280px !important;
  margin: 0 auto !important;
}

/* ===== Hero 区 ===== */
.hero {
  text-align: center;
  padding: 36px 16px 12px;
  background: linear-gradient(135deg, rgba(59,125,216,0.08) 0%, rgba(230,136,60,0.08) 100%);
  border-radius: 16px;
  margin-bottom: 24px;
  border: 1px solid rgba(0,0,0,0.05);
}
.hero h1 {
  font-size: 2.4em !important;
  margin: 0 !important;
  background: linear-gradient(135deg, #3B7DD8, #E6883C);
  -webkit-background-clip: text;
  -webkit-text-fill-color: transparent;
  background-clip: text;
  font-weight: 700 !important;
  letter-spacing: -0.02em;
}
.hero .tagline {
  font-size: 1.1em;
  color: #555;
  margin-top: 8px;
}
.hero .subtitle {
  font-size: 0.95em;
  color: #777;
  max-width: 760px;
  margin: 12px auto 0;
  line-height: 1.6;
}
.hero .badges {
  margin-top: 16px;
  display: flex;
  gap: 8px;
  justify-content: center;
  flex-wrap: wrap;
}
.hero .badge {
  display: inline-flex;
  align-items: center;
  padding: 4px 10px;
  background: rgba(255,255,255,0.7);
  border: 1px solid rgba(0,0,0,0.08);
  border-radius: 6px;
  font-size: 0.85em;
  color: #444;
  text-decoration: none;
  transition: all 0.2s;
}
.hero .badge:hover {
  transform: translateY(-1px);
  box-shadow: 0 2px 8px rgba(0,0,0,0.08);
}

/* ===== Key facts ===== */
.key-facts {
  display: grid;
  grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
  gap: 12px;
  margin: 24px 0;
}
.fact {
  padding: 14px 18px;
  background: white;
  border-radius: 10px;
  border-left: 3px solid var(--fact-color, #3B7DD8);
  box-shadow: 0 1px 3px rgba(0,0,0,0.05);
}
.fact-label {
  font-size: 0.75em;
  text-transform: uppercase;
  letter-spacing: 0.06em;
  color: #888;
  font-weight: 600;
}
.fact-value {
  font-size: 1.5em;
  font-weight: 700;
  color: var(--fact-color, #3B7DD8);
  margin-top: 4px;
}
.fact-detail {
  font-size: 0.85em;
  color: #666;
  margin-top: 2px;
}

/* ===== 统计卡片 ===== */
.stats-grid {
  display: grid;
  grid-template-columns: 2fr 1fr 1fr 1fr;
  gap: 12px;
  margin: 8px 0;
}
.stat-card {
  background: white;
  border-radius: 10px;
  padding: 14px 16px;
  border: 1px solid rgba(0,0,0,0.06);
  transition: all 0.2s;
}
.stat-card:hover {
  box-shadow: 0 2px 12px rgba(0,0,0,0.08);
  transform: translateY(-1px);
}
.stat-card.primary {
  background: linear-gradient(135deg, #FAFBFF, #F5F7FB);
  border: 1px solid #DDE5F0;
}
.stat-card.empty {
  text-align: center;
  padding: 28px;
  background: #F8F9FB;
  color: #888;
  border: 1px dashed #DDD;
  font-size: 0.95em;
}
.stat-label {
  font-size: 0.75em;
  text-transform: uppercase;
  letter-spacing: 0.06em;
  color: #888;
  font-weight: 600;
}
.stat-value {
  font-size: 1.6em;
  font-weight: 700;
  margin-top: 4px;
  display: flex;
  align-items: baseline;
  gap: 6px;
}
.stat-value.muted { color: #999; font-size: 1.4em; }
.stat-value.emphasis { color: #10B981; font-size: 1.8em; }
.saving-badge.good { color: #10B981; }
.saving-badge.bad { color: #E6883C; }
.saving-badge.neutral { color: #888; }
.arrow { font-weight: 700; }
.stat-sub {
  font-size: 0.78em;
  color: #777;
  margin-top: 2px;
}

/* ===== 规则面板 ===== */
.rules-header {
  font-size: 0.95em;
  font-weight: 600;
  color: #444;
  margin: 16px 0 8px;
  display: flex;
  align-items: center;
  gap: 8px;
}
.rules-count {
  font-size: 0.78em;
  background: #EFF2F7;
  padding: 2px 10px;
  border-radius: 10px;
  color: #555;
  font-weight: 500;
}
.rules-pills {
  display: flex;
  gap: 6px;
  flex-wrap: wrap;
  padding: 4px 0;
}
.rule-pill {
  display: inline-flex;
  align-items: center;
  gap: 4px;
  padding: 4px 10px;
  background: white;
  border: 1px solid var(--rule-color);
  color: var(--rule-color);
  border-radius: 999px;
  font-size: 0.82em;
  font-family: 'JetBrains Mono', 'Fira Code', monospace;
  font-weight: 500;
  transition: all 0.2s;
}
.rule-pill:hover {
  background: var(--rule-color);
  color: white;
}
.rule-count {
  font-size: 0.85em;
  opacity: 0.8;
}
.empty-rules {
  font-style: italic;
  color: #888;
  background: #F8F9FB;
  padding: 14px;
  border-radius: 8px;
  border: 1px dashed #DDD;
  text-align: center;
}

/* ===== 主操作按钮 ===== */
button.primary {
  background: linear-gradient(135deg, #3B7DD8 0%, #5B9BE5 100%) !important;
  border: none !important;
  font-size: 1.05em !important;
  letter-spacing: 0.02em !important;
  transition: all 0.2s !important;
}
button.primary:hover {
  transform: translateY(-1px);
  box-shadow: 0 4px 12px rgba(59,125,216,0.3) !important;
}

/* ===== Footer ===== */
.footer {
  margin-top: 36px;
  padding: 20px 0;
  border-top: 1px solid rgba(0,0,0,0.06);
  color: #777;
  font-size: 0.88em;
  text-align: center;
}
.footer a { color: #3B7DD8; text-decoration: none; }
.footer a:hover { text-decoration: underline; }

/* ===== 暗色模式适配 ===== */
.dark .hero {
  background: linear-gradient(135deg, rgba(59,125,216,0.15), rgba(230,136,60,0.15));
  border-color: rgba(255,255,255,0.05);
}
.dark .hero .tagline { color: #BBB; }
.dark .hero .subtitle { color: #999; }
.dark .hero .badge {
  background: rgba(255,255,255,0.05);
  border-color: rgba(255,255,255,0.1);
  color: #CCC;
}
.dark .stat-card { background: rgba(255,255,255,0.03); border-color: rgba(255,255,255,0.08); }
.dark .stat-card.primary { background: rgba(59,125,216,0.08); }
.dark .stat-card.empty { background: rgba(255,255,255,0.03); color: #888; border-color: rgba(255,255,255,0.1); }
.dark .stat-sub { color: #888; }
.dark .fact { background: rgba(255,255,255,0.03); }
.dark .fact-detail { color: #888; }
.dark .rules-count { background: rgba(255,255,255,0.06); color: #BBB; }
.dark .rule-pill { background: rgba(255,255,255,0.03); }
.dark .empty-rules { background: rgba(255,255,255,0.03); border-color: rgba(255,255,255,0.1); }

/* ===== 响应式 ===== */
@media (max-width: 768px) {
  .stats-grid { grid-template-columns: 1fr 1fr; }
  .key-facts { grid-template-columns: 1fr; }
  .hero h1 { font-size: 1.8em !important; }
}
"""


# ============================================================
# UI
# ============================================================
HERO_HTML = """
<div class="hero">
  <h1>🀄 CHIP</h1>
  <div class="tagline">Chinese High-density Instruction Protocol · 中文高密度提示协议</div>
  <div class="subtitle">
    把啰嗦的中文 prompt 自动压成结构化高密度形式 — <strong>数据驱动,不是品味</strong>
  </div>
  <div class="badges">
    <a class="badge" href="https://github.com/luancy1208/CHIP" target="_blank">⭐ GitHub</a>
    <a class="badge" href="https://github.com/luancy1208/CHIP/blob/main/SPEC.md" target="_blank">📄 SPEC</a>
    <a class="badge" href="https://github.com/luancy1208/CHIP/tree/main/results" target="_blank">📊 实测数据</a>
    <span class="badge">Apache-2.0</span>
    <span class="badge">v0.2 · 23 tests passing</span>
  </div>
</div>

<div class="key-facts">
  <div class="fact" style="--fact-color:#3B7DD8">
    <div class="fact-label">实测数据</div>
    <div class="fact-value">1800+ 行</div>
    <div class="fact-detail">9 tokenizer × 200 句 FLORES-200</div>
  </div>
  <div class="fact" style="--fact-color:#10B981">
    <div class="fact-label">国产模型上中文</div>
    <div class="fact-value">省 12.5%</div>
    <div class="fact-detail">Baichuan2 token / 等价英文</div>
  </div>
  <div class="fact" style="--fact-color:#E6883C">
    <div class="fact-label">cl100k 上中文</div>
    <div class="fact-value">贵 73%</div>
    <div class="fact-detail">所以 CHIP 的主战场是国产模型</div>
  </div>
  <div class="fact" style="--fact-color:#7C3AED">
    <div class="fact-label">### 标签</div>
    <div class="fact-value">1 token</div>
    <div class="fact-detail">在所有 9 个 tokenizer 上 — 完爆方括号</div>
  </div>
</div>
"""


FOOTER_HTML = """
<div class="footer">
  <p>
    🀄 CHIP v0.2 · Built with care for Chinese prompt engineering ·
    <a href="https://github.com/luancy1208/CHIP" target="_blank">GitHub</a> ·
    <a href="https://github.com/luancy1208/CHIP/issues" target="_blank">反馈 Issue</a>
  </p>
  <p style="margin-top:8px; font-size:0.82em; opacity:0.7">
    数据来源: FLORES-200 dev (n=200) · 200 个 HSK 5/6 高频成语 · 45 个常见标记符号 · 实测于 9 个 tokenizer
  </p>
</div>
"""


with gr.Blocks(
    title="CHIP — 中文高密度提示协议",
    theme=gr.themes.Soft(
        primary_hue="blue",
        secondary_hue="orange",
        neutral_hue="slate",
        font=["system-ui", "-apple-system", "Segoe UI", "Helvetica Neue", "sans-serif"],
    ),
    css=CUSTOM_CSS,
) as demo:
    gr.HTML(HERO_HTML)

    with gr.Row(equal_height=True):
        # ------- 输入侧 -------
        with gr.Column(scale=1):
            gr.Markdown("### 📝 输入 prompt")
            inp = gr.Textbox(
                show_label=False,
                lines=10,
                placeholder="把啰嗦的中文 prompt 粘贴进来,看看 CHIP 能压到多紧...\n\n💡 试试:'请你帮我对下面这段文字进行一个全面的分析,如果可以的话麻烦你给出一些建议'",
                container=False,
            )

            with gr.Accordion("⚙️ 高级设置", open=False):
                with gr.Row():
                    target = gr.Dropdown(
                        ["qwen2.5", "deepseek_v3", "glm4", "cl100k", "o200k"],
                        value="qwen2.5",
                        label="🎯 目标模型",
                        info="影响 L3 成语压缩等 target-aware 决策",
                    )
                    tokenizer_name = gr.Dropdown(
                        list(TOKENIZERS.keys()) + list(_LAZY.keys()),
                        value="GPT-4 (cl100k)",
                        label="🔢 计数 tokenizer",
                        info="决定 token 数怎么算(国产 tokenizer 首次需下载)",
                    )

                gr.Markdown("**压缩层** — 默认 L1+L2+L4(保险);L3 仅在国产模型上有意义")
                with gr.Row():
                    use_l1 = gr.Checkbox(value=True, label="L1 词法",
                                          info="套话剪枝 (~1.3-1.5×)")
                    use_l2 = gr.Checkbox(value=True, label="L2 句法",
                                          info="模式重排 (~2-3×)")
                    use_l3 = gr.Checkbox(value=False, label="L3 成语",
                                          info="长描述→成语")
                    use_l4 = gr.Checkbox(value=True, label="L4 协议",
                                          info="### 标签归一化")

                use_jieba = gr.Checkbox(
                    value=False,
                    label="🚀 jieba NP 增强模式",
                    info="复杂角色提取场景效果更好(默认关闭)",
                )

            btn = gr.Button("🔥  压缩", variant="primary", size="lg",
                            elem_classes="primary")

        # ------- 输出侧 -------
        with gr.Column(scale=1):
            gr.Markdown("### ✨ 压缩结果")
            try:
                out = gr.Textbox(
                    show_label=False,
                    lines=10,
                    interactive=False,
                    show_copy_button=True,
                    container=False,
                )
            except TypeError:
                out = gr.Textbox(show_label=False, lines=10,
                                  interactive=False, container=False)
            stats_panel = gr.HTML(
                "<div class='stat-card empty'>👈 在左侧输入并点击压缩按钮</div>"
            )
            rules_panel = gr.HTML()

    # 隐藏的 textbox 用于触发示例(因为示例需要更新所有 input)
    hidden_dup = gr.Textbox(visible=False)

    gr.Examples(
        examples=EXAMPLES,
        inputs=[inp, target, use_l1, use_l2, use_l3, use_l4, tokenizer_name, use_jieba],
        label="📌 试试这些示例(点击自动填充并运行)",
        examples_per_page=4,
    )

    with gr.Accordion("📖 关于 CHIP / 协议设计要点", open=False):
        gr.Markdown("""
**核心发现(基于 1800 行实测数据)**

| Tokenizer | ZH/EN ratio | 中文相对英文 |
|---|---|---|
| **baichuan2** 🟦 | 0.875 | 中文省 12.5% |
| **deepseek_v3** 🟦 | 0.916 | 中文省 8.4% |
| **glm4** 🟦 | 0.924 | 中文省 7.6% |
| qwen2.5/3 🟦 | 0.988 | 持平 |
| **o200k** (GPT-4o) 🟧 | 1.163 | 中文贵 16.3% |
| **cl100k** (GPT-4) 🟧 | 1.731 | 中文贵 73.1% |

🟦 = 国产 tokenizer  ·  🟧 = OpenAI tokenizer

---

**4 层压缩架构**

- **L1 词法层** · 啰嗦套话 → 紧凑动宾,纯正则,~1.3-1.5×
- **L2 句法层** · 模式重排 + 列表化,~2-3×
- **L3 成语层** · 长描述 → 成语(实测白名单),仅国产模型默认关闭
- **L4 协议层** · 标签归一化为 `### 标题`(实测全 tokenizer 1 token)

**不主张的事**(诚实声明)

- ❌ 不主张"中文因为信息密度高所以更省 token" — Mythbuster 2026 在 SWE-bench 上证伪
- ❌ 不主张"中文 prompt 让模型更聪明" — 在英文中心模型上常常相反
- ❌ 不主张"全程使用文言文压缩" — LLM 对生僻文言虚词理解不稳定

CHIP 主张的是:**在符合中文训练分布的国产模型上,通过协议化压缩可在 token 经济性、可读性、可审计性三个维度同时提供工程价值。**
""")

    gr.HTML(FOOTER_HTML)

    # 事件
    btn.click(
        run,
        inputs=[inp, target, use_l1, use_l2, use_l3, use_l4, tokenizer_name, use_jieba],
        outputs=[out, stats_panel, rules_panel, hidden_dup],
    )


if __name__ == "__main__":
    demo.launch(
        server_name=os.getenv("HOST", "0.0.0.0"),
        server_port=int(os.getenv("PORT", 7860)),
        share=os.getenv("SHARE") == "1",
    )