Delete chip-space/app.py

chip-space/app.py

@@ -1,665 +0,0 @@
-"""
-app.py — CHIP HuggingFace Space 入口(美观版)
-
-部署: https://huggingface.co/spaces/<user>/CHIP
-"""
-from __future__ import annotations
-
-import os
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-
-import gradio as gr
-import tiktoken
-
-from chip import Compressor
-
-# ============================================================
-# Tokenizer 计数器
-# ============================================================
-TOKENIZERS = {}
-try:
-    TOKENIZERS["GPT-4 (cl100k)"] = tiktoken.get_encoding("cl100k_base")
-    TOKENIZERS["GPT-4o (o200k)"] = tiktoken.get_encoding("o200k_base")
-except Exception as e:
-    print(f"[warn] tiktoken load failed: {e}")
-
-
-def _lazy_load_qwen():
-    from transformers import AutoTokenizer
-    return AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B", trust_remote_code=True)
-
-
-def _lazy_load_deepseek():
-    from transformers import AutoTokenizer
-    return AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V2-Lite", trust_remote_code=True)
-
-
-_LAZY = {
-    "Qwen2.5": _lazy_load_qwen,
-    "DeepSeek-V2/V3": _lazy_load_deepseek,
-}
-
-
-def count_tokens(text: str, tokenizer_name: str) -> int:
-    if tokenizer_name in TOKENIZERS:
-        enc = TOKENIZERS[tokenizer_name]
-    elif tokenizer_name in _LAZY:
-        try:
-            tok = _LAZY[tokenizer_name]()
-            TOKENIZERS[tokenizer_name] = tok
-            enc = tok
-        except Exception:
-            return -1
-    else:
-        return -1
-
-    if hasattr(enc, "encode") and not hasattr(enc, "tokenize"):
-        return len(enc.encode(text))
-    return len(enc.encode(text, add_special_tokens=False))
-
-
-# ============================================================
-# 主压缩函数
-# ============================================================
-def run(text: str, target: str, use_l1: bool, use_l2: bool, use_l3: bool,
-        use_l4: bool, tokenizer_name: str, use_jieba: bool):
-    if not text.strip():
-        empty_card = "<div class='stat-card empty'>👈 请在左侧输入 prompt</div>"
-        return "", empty_card, "", ""
-
-    layers = []
-    for flag, name in [(use_l1, "L1"), (use_l2, "L2"), (use_l3, "L3"), (use_l4, "L4")]:
-        if flag:
-            layers.append(name)
-    if not layers:
-        layers = ["L1"]
-
-    if use_jieba:
-        os.environ["CHIP_USE_JIEBA"] = "1"
-    else:
-        os.environ.pop("CHIP_USE_JIEBA", None)
-    import chip.compressor as cc
-    cc._default_compressor = None
-
-    compressor = Compressor(target=target, layers=layers)
-    result = compressor.compress(text)
-
-    n_orig = count_tokens(text, tokenizer_name)
-    n_comp = count_tokens(result.compressed, tokenizer_name)
-
-    # ---- 漂亮的统计卡片 ----
-    if n_orig > 0 and n_comp > 0:
-        saving_pct = (1 - n_comp / n_orig) * 100
-        n_diff = n_orig - n_comp
-        if saving_pct > 0:
-            badge_class = "saving-badge good"
-            arrow = "↓"
-            verb = "节省"
-        elif saving_pct < 0:
-            badge_class = "saving-badge bad"
-            arrow = "↑"
-            verb = "增加"
-        else:
-            badge_class = "saving-badge neutral"
-            arrow = "→"
-            verb = "持平"
-
-        char_orig, char_comp = len(text), len(result.compressed)
-        char_pct = (1 - char_comp / max(char_orig, 1)) * 100
-
-        stats_html = f"""
-<div class="stats-grid">
-  <div class="stat-card primary">
-    <div class="stat-label">Token 节省</div>
-    <div class="stat-value {badge_class}">
-      <span class="arrow">{arrow}</span> {abs(saving_pct):.1f}%
-    </div>
-    <div class="stat-sub">{verb} {abs(n_diff)} token · 在 {tokenizer_name}</div>
-  </div>
-  <div class="stat-card">
-    <div class="stat-label">原文</div>
-    <div class="stat-value muted">{n_orig}</div>
-    <div class="stat-sub">token · {char_orig} 字符</div>
-  </div>
-  <div class="stat-card">
-    <div class="stat-label">压缩后</div>
-    <div class="stat-value emphasis">{n_comp}</div>
-    <div class="stat-sub">token · {char_comp} 字符</div>
-  </div>
-  <div class="stat-card">
-    <div class="stat-label">字符压缩</div>
-    <div class="stat-value muted">{char_pct:+.0f}%</div>
-    <div class="stat-sub">字符数变化(供参考,token 才重要)</div>
-  </div>
-</div>"""
-    else:
-        stats_html = (
-            "<div class='stat-card empty'>"
-            "⚠️ Token 计数不可用(可能是 tokenizer 加载失败)"
-            "</div>"
-        )
-
-    # ---- 漂亮的规则面板 ----
-    if result.applied_rules:
-        rules_items = []
-        for r in result.applied_rules:
-            # 解析 rule id 和命中次数
-            parts = r.split("×")
-            rid = parts[0]
-            count = parts[1] if len(parts) > 1 else "1"
-            # 按层着色
-            layer_color = {"L1": "#3B7DD8", "L2": "#E6883C", "L3": "#7C3AED", "L4": "#10B981"}
-            color = "#888"
-            for l, c in layer_color.items():
-                if l in rid:
-                    color = c
-                    break
-            badge = (
-                f"<span class='rule-pill' style='--rule-color:{color}'>"
-                f"<span class='rule-id'>{rid}</span>"
-                f"<span class='rule-count'>×{count}</span>"
-                f"</span>"
-            )
-            rules_items.append(badge)
-        rules_html = (
-            "<div class='rules-header'>🎯 命中规则 "
-            f"<span class='rules-count'>{len(result.applied_rules)} 条</span></div>"
-            "<div class='rules-pills'>" + " ".join(rules_items) + "</div>"
-        )
-    else:
-        rules_html = (
-            "<div class='rules-header empty-rules'>"
-            "📭 没有规则匹配 — 这段 prompt 已经够紧凑了"
-            "</div>"
-        )
-
-    return result.compressed, stats_html, rules_html, result.compressed
-
-
-# ============================================================
-# 示例
-# ============================================================
-EXAMPLES = [
-    [
-        "请你帮我对下面这段文字进行一个全面的分析,如果可以的话麻烦你给出一些改进的建议",
-        "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", False,
-    ],
-    [
-        "请你扮演一位资深 Python 工程师,对下面的代码进行 code review,并以 JSON 格式输出结果",
-        "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", True,
-    ],
-    [
-        "因为最近在下雨,所以路面变得很滑,因此开车的时候需要特别注意安全",
-        "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", False,
-    ],
-    [
-        "Role: 资深产品经理\n任务: 评估这个需求,大家都知道产品决策需要数据支持",
-        "deepseek_v3", True, True, True, True, "GPT-4 (cl100k)", False,
-    ],
-]
-
-
-# ============================================================
-# 自定义 CSS — 让 demo 看起来不像默认的 gradio
-# ============================================================
-CUSTOM_CSS = """
-/* ===== 全局 ===== */
-.gradio-container {
-  max-width: 1280px !important;
-  margin: 0 auto !important;
-}
-
-/* ===== Hero 区 ===== */
-.hero {
-  text-align: center;
-  padding: 36px 16px 12px;
-  background: linear-gradient(135deg, rgba(59,125,216,0.08) 0%, rgba(230,136,60,0.08) 100%);
-  border-radius: 16px;
-  margin-bottom: 24px;
-  border: 1px solid rgba(0,0,0,0.05);
-}
-.hero h1 {
-  font-size: 2.4em !important;
-  margin: 0 !important;
-  background: linear-gradient(135deg, #3B7DD8, #E6883C);
-  -webkit-background-clip: text;
-  -webkit-text-fill-color: transparent;
-  background-clip: text;
-  font-weight: 700 !important;
-  letter-spacing: -0.02em;
-}
-.hero .tagline {
-  font-size: 1.1em;
-  color: #555;
-  margin-top: 8px;
-}
-.hero .subtitle {
-  font-size: 0.95em;
-  color: #777;
-  max-width: 760px;
-  margin: 12px auto 0;
-  line-height: 1.6;
-}
-.hero .badges {
-  margin-top: 16px;
-  display: flex;
-  gap: 8px;
-  justify-content: center;
-  flex-wrap: wrap;
-}
-.hero .badge {
-  display: inline-flex;
-  align-items: center;
-  padding: 4px 10px;
-  background: rgba(255,255,255,0.7);
-  border: 1px solid rgba(0,0,0,0.08);
-  border-radius: 6px;
-  font-size: 0.85em;
-  color: #444;
-  text-decoration: none;
-  transition: all 0.2s;
-}
-.hero .badge:hover {
-  transform: translateY(-1px);
-  box-shadow: 0 2px 8px rgba(0,0,0,0.08);
-}
-
-/* ===== Key facts ===== */
-.key-facts {
-  display: grid;
-  grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
-  gap: 12px;
-  margin: 24px 0;
-}
-.fact {
-  padding: 14px 18px;
-  background: white;
-  border-radius: 10px;
-  border-left: 3px solid var(--fact-color, #3B7DD8);
-  box-shadow: 0 1px 3px rgba(0,0,0,0.05);
-}
-.fact-label {
-  font-size: 0.75em;
-  text-transform: uppercase;
-  letter-spacing: 0.06em;
-  color: #888;
-  font-weight: 600;
-}
-.fact-value {
-  font-size: 1.5em;
-  font-weight: 700;
-  color: var(--fact-color, #3B7DD8);
-  margin-top: 4px;
-}
-.fact-detail {
-  font-size: 0.85em;
-  color: #666;
-  margin-top: 2px;
-}
-
-/* ===== 统计卡片 ===== */
-.stats-grid {
-  display: grid;
-  grid-template-columns: 2fr 1fr 1fr 1fr;
-  gap: 12px;
-  margin: 8px 0;
-}
-.stat-card {
-  background: white;
-  border-radius: 10px;
-  padding: 14px 16px;
-  border: 1px solid rgba(0,0,0,0.06);
-  transition: all 0.2s;
-}
-.stat-card:hover {
-  box-shadow: 0 2px 12px rgba(0,0,0,0.08);
-  transform: translateY(-1px);
-}
-.stat-card.primary {
-  background: linear-gradient(135deg, #FAFBFF, #F5F7FB);
-  border: 1px solid #DDE5F0;
-}
-.stat-card.empty {
-  text-align: center;
-  padding: 28px;
-  background: #F8F9FB;
-  color: #888;
-  border: 1px dashed #DDD;
-  font-size: 0.95em;
-}
-.stat-label {
-  font-size: 0.75em;
-  text-transform: uppercase;
-  letter-spacing: 0.06em;
-  color: #888;
-  font-weight: 600;
-}
-.stat-value {
-  font-size: 1.6em;
-  font-weight: 700;
-  margin-top: 4px;
-  display: flex;
-  align-items: baseline;
-  gap: 6px;
-}
-.stat-value.muted { color: #999; font-size: 1.4em; }
-.stat-value.emphasis { color: #10B981; font-size: 1.8em; }
-.saving-badge.good { color: #10B981; }
-.saving-badge.bad { color: #E6883C; }
-.saving-badge.neutral { color: #888; }
-.arrow { font-weight: 700; }
-.stat-sub {
-  font-size: 0.78em;
-  color: #777;
-  margin-top: 2px;
-}
-
-/* ===== 规则面板 ===== */
-.rules-header {
-  font-size: 0.95em;
-  font-weight: 600;
-  color: #444;
-  margin: 16px 0 8px;
-  display: flex;
-  align-items: center;
-  gap: 8px;
-}
-.rules-count {
-  font-size: 0.78em;
-  background: #EFF2F7;
-  padding: 2px 10px;
-  border-radius: 10px;
-  color: #555;
-  font-weight: 500;
-}
-.rules-pills {
-  display: flex;
-  gap: 6px;
-  flex-wrap: wrap;
-  padding: 4px 0;
-}
-.rule-pill {
-  display: inline-flex;
-  align-items: center;
-  gap: 4px;
-  padding: 4px 10px;
-  background: white;
-  border: 1px solid var(--rule-color);
-  color: var(--rule-color);
-  border-radius: 999px;
-  font-size: 0.82em;
-  font-family: 'JetBrains Mono', 'Fira Code', monospace;
-  font-weight: 500;
-  transition: all 0.2s;
-}
-.rule-pill:hover {
-  background: var(--rule-color);
-  color: white;
-}
-.rule-count {
-  font-size: 0.85em;
-  opacity: 0.8;
-}
-.empty-rules {
-  font-style: italic;
-  color: #888;
-  background: #F8F9FB;
-  padding: 14px;
-  border-radius: 8px;
-  border: 1px dashed #DDD;
-  text-align: center;
-}
-
-/* ===== 主操作按钮 ===== */
-button.primary {
-  background: linear-gradient(135deg, #3B7DD8 0%, #5B9BE5 100%) !important;
-  border: none !important;
-  font-size: 1.05em !important;
-  letter-spacing: 0.02em !important;
-  transition: all 0.2s !important;
-}
-button.primary:hover {
-  transform: translateY(-1px);
-  box-shadow: 0 4px 12px rgba(59,125,216,0.3) !important;
-}
-
-/* ===== Footer ===== */
-.footer {
-  margin-top: 36px;
-  padding: 20px 0;
-  border-top: 1px solid rgba(0,0,0,0.06);
-  color: #777;
-  font-size: 0.88em;
-  text-align: center;
-}
-.footer a { color: #3B7DD8; text-decoration: none; }
-.footer a:hover { text-decoration: underline; }
-
-/* ===== 暗色模式适配 ===== */
-.dark .hero {
-  background: linear-gradient(135deg, rgba(59,125,216,0.15), rgba(230,136,60,0.15));
-  border-color: rgba(255,255,255,0.05);
-}
-.dark .hero .tagline { color: #BBB; }
-.dark .hero .subtitle { color: #999; }
-.dark .hero .badge {
-  background: rgba(255,255,255,0.05);
-  border-color: rgba(255,255,255,0.1);
-  color: #CCC;
-}
-.dark .stat-card { background: rgba(255,255,255,0.03); border-color: rgba(255,255,255,0.08); }
-.dark .stat-card.primary { background: rgba(59,125,216,0.08); }
-.dark .stat-card.empty { background: rgba(255,255,255,0.03); color: #888; border-color: rgba(255,255,255,0.1); }
-.dark .stat-sub { color: #888; }
-.dark .fact { background: rgba(255,255,255,0.03); }
-.dark .fact-detail { color: #888; }
-.dark .rules-count { background: rgba(255,255,255,0.06); color: #BBB; }
-.dark .rule-pill { background: rgba(255,255,255,0.03); }
-.dark .empty-rules { background: rgba(255,255,255,0.03); border-color: rgba(255,255,255,0.1); }
-
-/* ===== 响应式 ===== */
-@media (max-width: 768px) {
-  .stats-grid { grid-template-columns: 1fr 1fr; }
-  .key-facts { grid-template-columns: 1fr; }
-  .hero h1 { font-size: 1.8em !important; }
-}
-"""
-
-
-# ============================================================
-# UI
-# ============================================================
-HERO_HTML = """
-<div class="hero">
-  <h1>🀄 CHIP</h1>
-  <div class="tagline">Chinese High-density Instruction Protocol · 中文高密度提示协议</div>
-  <div class="subtitle">
-    把啰嗦的中文 prompt 自动压成结构化高密度形式 — <strong>数据驱动,不是品味</strong>
-  </div>
-  <div class="badges">
-    <a class="badge" href="https://github.com/luancy1208/CHIP" target="_blank">⭐ GitHub</a>
-    <a class="badge" href="https://github.com/luancy1208/CHIP/blob/main/SPEC.md" target="_blank">📄 SPEC</a>
-    <a class="badge" href="https://github.com/luancy1208/CHIP/tree/main/results" target="_blank">📊 实测数据</a>
-    <span class="badge">Apache-2.0</span>
-    <span class="badge">v0.2 · 23 tests passing</span>
-  </div>
-</div>
-
-<div class="key-facts">
-  <div class="fact" style="--fact-color:#3B7DD8">
-    <div class="fact-label">实测数据</div>
-    <div class="fact-value">1800+ 行</div>
-    <div class="fact-detail">9 tokenizer × 200 句 FLORES-200</div>
-  </div>
-  <div class="fact" style="--fact-color:#10B981">
-    <div class="fact-label">国产模型上中文</div>
-    <div class="fact-value">省 12.5%</div>
-    <div class="fact-detail">Baichuan2 token / 等价英文</div>
-  </div>
-  <div class="fact" style="--fact-color:#E6883C">
-    <div class="fact-label">cl100k 上中文</div>
-    <div class="fact-value">贵 73%</div>
-    <div class="fact-detail">所以 CHIP 的主战场是国产模型</div>
-  </div>
-  <div class="fact" style="--fact-color:#7C3AED">
-    <div class="fact-label">### 标签</div>
-    <div class="fact-value">1 token</div>
-    <div class="fact-detail">在所有 9 个 tokenizer 上 — 完爆方括号</div>
-  </div>
-</div>
-"""
-
-
-FOOTER_HTML = """
-<div class="footer">
-  <p>
-    🀄 CHIP v0.2 · Built with care for Chinese prompt engineering ·
-    <a href="https://github.com/luancy1208/CHIP" target="_blank">GitHub</a> ·
-    <a href="https://github.com/luancy1208/CHIP/issues" target="_blank">反馈 Issue</a>
-  </p>
-  <p style="margin-top:8px; font-size:0.82em; opacity:0.7">
-    数据来源: FLORES-200 dev (n=200) · 200 个 HSK 5/6 高频成语 · 45 个常见标记符号 · 实测于 9 个 tokenizer
-  </p>
-</div>
-"""
-
-
-with gr.Blocks(
-    title="CHIP — 中文高密度提示协议",
-    theme=gr.themes.Soft(
-        primary_hue="blue",
-        secondary_hue="orange",
-        neutral_hue="slate",
-        font=["system-ui", "-apple-system", "Segoe UI", "Helvetica Neue", "sans-serif"],
-    ),
-    css=CUSTOM_CSS,
-) as demo:
-    gr.HTML(HERO_HTML)
-
-    with gr.Row(equal_height=True):
-        # ------- 输入侧 -------
-        with gr.Column(scale=1):
-            gr.Markdown("### 📝 输入 prompt")
-            inp = gr.Textbox(
-                show_label=False,
-                lines=10,
-                placeholder="把啰嗦的中文 prompt 粘贴进来,看看 CHIP 能压到多紧...\n\n💡 试试:'请你帮我对下面这段文字进行一个全面的分析,如果可以的话麻烦你给出一些建议'",
-                container=False,
-            )
-
-            with gr.Accordion("⚙️ 高级设置", open=False):
-                with gr.Row():
-                    target = gr.Dropdown(
-                        ["qwen2.5", "deepseek_v3", "glm4", "cl100k", "o200k"],
-                        value="qwen2.5",
-                        label="🎯 目标模型",
-                        info="影响 L3 成语压缩等 target-aware 决策",
-                    )
-                    tokenizer_name = gr.Dropdown(
-                        list(TOKENIZERS.keys()) + list(_LAZY.keys()),
-                        value="GPT-4 (cl100k)",
-                        label="🔢 计数 tokenizer",
-                        info="决定 token 数怎么算(国产 tokenizer 首次需下载)",
-                    )
-
-                gr.Markdown("**压缩层** — 默认 L1+L2+L4(保险);L3 仅在国产模型上有意义")
-                with gr.Row():
-                    use_l1 = gr.Checkbox(value=True, label="L1 词法",
-                                          info="套话剪枝 (~1.3-1.5×)")
-                    use_l2 = gr.Checkbox(value=True, label="L2 句法",
-                                          info="模式重排 (~2-3×)")
-                    use_l3 = gr.Checkbox(value=False, label="L3 成语",
-                                          info="长描述→成语")
-                    use_l4 = gr.Checkbox(value=True, label="L4 协议",
-                                          info="### 标签归一化")
-
-                use_jieba = gr.Checkbox(
-                    value=False,
-                    label="🚀 jieba NP 增强模式",
-                    info="复杂角色提取场景效果更好(默认关闭)",
-                )
-
-            btn = gr.Button("🔥  压缩", variant="primary", size="lg",
-                            elem_classes="primary")
-
-        # ------- 输出侧 -------
-        with gr.Column(scale=1):
-            gr.Markdown("### ✨ 压缩结果")
-            try:
-                out = gr.Textbox(
-                    show_label=False,
-                    lines=10,
-                    interactive=False,
-                    show_copy_button=True,
-                    container=False,
-                )
-            except TypeError:
-                out = gr.Textbox(show_label=False, lines=10,
-                                  interactive=False, container=False)
-            stats_panel = gr.HTML(
-                "<div class='stat-card empty'>👈 在左侧输入并点击压缩按钮</div>"
-            )
-            rules_panel = gr.HTML()
-
-    # 隐藏的 textbox 用于触发示例(因为示例需要更新所有 input)
-    hidden_dup = gr.Textbox(visible=False)
-
-    gr.Examples(
-        examples=EXAMPLES,
-        inputs=[inp, target, use_l1, use_l2, use_l3, use_l4, tokenizer_name, use_jieba],
-        label="📌 试试这些示例(点击自动填充并运行)",
-        examples_per_page=4,
-    )
-
-    with gr.Accordion("📖 关于 CHIP / 协议设计要点", open=False):
-        gr.Markdown("""
-**核心发现(基于 1800 行实测数据)**
-
-| Tokenizer | ZH/EN ratio | 中文相对英文 |
-|---|---|---|
-| **baichuan2** 🟦 | 0.875 | 中文省 12.5% |
-| **deepseek_v3** 🟦 | 0.916 | 中文省 8.4% |
-| **glm4** 🟦 | 0.924 | 中文省 7.6% |
-| qwen2.5/3 🟦 | 0.988 | 持平 |
-| **o200k** (GPT-4o) 🟧 | 1.163 | 中文贵 16.3% |
-| **cl100k** (GPT-4) 🟧 | 1.731 | 中文贵 73.1% |
-
-🟦 = 国产 tokenizer  ·  🟧 = OpenAI tokenizer
-
----
-
-**4 层压缩架构**
-
-- **L1 词法层** · 啰嗦套话 → 紧凑动宾,纯正则,~1.3-1.5×
-- **L2 句法层** · 模式重排 + 列表化,~2-3×
-- **L3 成语层** · 长描��� → 成语(实测白名单),仅国产模型默认关闭
-- **L4 协议层** · 标签归一化为 `### 标题`(实测全 tokenizer 1 token)
-
-**不主张的事**(诚实声明)
-
-- ❌ 不主张"中文因为信息密度高所以更省 token" — Mythbuster 2026 在 SWE-bench 上证伪
-- ❌ 不主张"中文 prompt 让模型更聪明" — 在英文中心模型上常常相反
-- ❌ 不主张"全程使用文言文压缩" — LLM 对生僻文言虚词理解不稳定
-
-CHIP 主张的是:**在符合中文训练分布的国产模型上,通过协议化压缩可在 token 经济性、可读性、可审计性三个维度同时提供工程价值。**
-""")
-
-    gr.HTML(FOOTER_HTML)
-
-    # 事件
-    btn.click(
-        run,
-        inputs=[inp, target, use_l1, use_l2, use_l3, use_l4, tokenizer_name, use_jieba],
-        outputs=[out, stats_panel, rules_panel, hidden_dup],
-    )
-
-
-if __name__ == "__main__":
-    demo.launch(
-        server_name=os.getenv("HOST", "0.0.0.0"),
-        server_port=int(os.getenv("PORT", 7860)),
-        share=os.getenv("SHARE") == "1",
-    )