Spaces:
Runtime error
Runtime error
| """ | |
| app.py — CHIP HuggingFace Space 入口(美观版) | |
| 部署: https://huggingface.co/spaces/<user>/CHIP | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).parent)) | |
| import gradio as gr | |
| import tiktoken | |
| from chip import Compressor | |
| # ============================================================ | |
| # Tokenizer 计数器 | |
| # ============================================================ | |
| TOKENIZERS = {} | |
| try: | |
| TOKENIZERS["GPT-4 (cl100k)"] = tiktoken.get_encoding("cl100k_base") | |
| TOKENIZERS["GPT-4o (o200k)"] = tiktoken.get_encoding("o200k_base") | |
| except Exception as e: | |
| print(f"[warn] tiktoken load failed: {e}") | |
| def _lazy_load_qwen(): | |
| from transformers import AutoTokenizer | |
| return AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B", trust_remote_code=True) | |
| def _lazy_load_deepseek(): | |
| from transformers import AutoTokenizer | |
| return AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V2-Lite", trust_remote_code=True) | |
| _LAZY = { | |
| "Qwen2.5": _lazy_load_qwen, | |
| "DeepSeek-V2/V3": _lazy_load_deepseek, | |
| } | |
| def count_tokens(text: str, tokenizer_name: str) -> int: | |
| if tokenizer_name in TOKENIZERS: | |
| enc = TOKENIZERS[tokenizer_name] | |
| elif tokenizer_name in _LAZY: | |
| try: | |
| tok = _LAZY[tokenizer_name]() | |
| TOKENIZERS[tokenizer_name] = tok | |
| enc = tok | |
| except Exception: | |
| return -1 | |
| else: | |
| return -1 | |
| if hasattr(enc, "encode") and not hasattr(enc, "tokenize"): | |
| return len(enc.encode(text)) | |
| return len(enc.encode(text, add_special_tokens=False)) | |
| # ============================================================ | |
| # 主压缩函数 | |
| # ============================================================ | |
| def run(text: str, target: str, use_l1: bool, use_l2: bool, use_l3: bool, | |
| use_l4: bool, tokenizer_name: str, use_jieba: bool): | |
| if not text.strip(): | |
| empty_card = "<div class='stat-card empty'>👈 请在左侧输入 prompt</div>" | |
| return "", empty_card, "", "" | |
| layers = [] | |
| for flag, name in [(use_l1, "L1"), (use_l2, "L2"), (use_l3, "L3"), (use_l4, "L4")]: | |
| if flag: | |
| layers.append(name) | |
| if not layers: | |
| layers = ["L1"] | |
| if use_jieba: | |
| os.environ["CHIP_USE_JIEBA"] = "1" | |
| else: | |
| os.environ.pop("CHIP_USE_JIEBA", None) | |
| import chip.compressor as cc | |
| cc._default_compressor = None | |
| compressor = Compressor(target=target, layers=layers) | |
| result = compressor.compress(text) | |
| n_orig = count_tokens(text, tokenizer_name) | |
| n_comp = count_tokens(result.compressed, tokenizer_name) | |
| # ---- 漂亮的统计卡片 ---- | |
| if n_orig > 0 and n_comp > 0: | |
| saving_pct = (1 - n_comp / n_orig) * 100 | |
| n_diff = n_orig - n_comp | |
| if saving_pct > 0: | |
| badge_class = "saving-badge good" | |
| arrow = "↓" | |
| verb = "节省" | |
| elif saving_pct < 0: | |
| badge_class = "saving-badge bad" | |
| arrow = "↑" | |
| verb = "增加" | |
| else: | |
| badge_class = "saving-badge neutral" | |
| arrow = "→" | |
| verb = "持平" | |
| char_orig, char_comp = len(text), len(result.compressed) | |
| char_pct = (1 - char_comp / max(char_orig, 1)) * 100 | |
| stats_html = f""" | |
| <div class="stats-grid"> | |
| <div class="stat-card primary"> | |
| <div class="stat-label">Token 节省</div> | |
| <div class="stat-value {badge_class}"> | |
| <span class="arrow">{arrow}</span> {abs(saving_pct):.1f}% | |
| </div> | |
| <div class="stat-sub">{verb} {abs(n_diff)} token · 在 {tokenizer_name}</div> | |
| </div> | |
| <div class="stat-card"> | |
| <div class="stat-label">原文</div> | |
| <div class="stat-value muted">{n_orig}</div> | |
| <div class="stat-sub">token · {char_orig} 字符</div> | |
| </div> | |
| <div class="stat-card"> | |
| <div class="stat-label">压缩后</div> | |
| <div class="stat-value emphasis">{n_comp}</div> | |
| <div class="stat-sub">token · {char_comp} 字符</div> | |
| </div> | |
| <div class="stat-card"> | |
| <div class="stat-label">字符压缩</div> | |
| <div class="stat-value muted">{char_pct:+.0f}%</div> | |
| <div class="stat-sub">字符数变化(供参考,token 才重要)</div> | |
| </div> | |
| </div>""" | |
| else: | |
| stats_html = ( | |
| "<div class='stat-card empty'>" | |
| "⚠️ Token 计数不可用(可能是 tokenizer 加载失败)" | |
| "</div>" | |
| ) | |
| # ---- 漂亮的规则面板 ---- | |
| if result.applied_rules: | |
| rules_items = [] | |
| for r in result.applied_rules: | |
| # 解析 rule id 和命中次数 | |
| parts = r.split("×") | |
| rid = parts[0] | |
| count = parts[1] if len(parts) > 1 else "1" | |
| # 按层着色 | |
| layer_color = {"L1": "#3B7DD8", "L2": "#E6883C", "L3": "#7C3AED", "L4": "#10B981"} | |
| color = "#888" | |
| for l, c in layer_color.items(): | |
| if l in rid: | |
| color = c | |
| break | |
| badge = ( | |
| f"<span class='rule-pill' style='--rule-color:{color}'>" | |
| f"<span class='rule-id'>{rid}</span>" | |
| f"<span class='rule-count'>×{count}</span>" | |
| f"</span>" | |
| ) | |
| rules_items.append(badge) | |
| rules_html = ( | |
| "<div class='rules-header'>🎯 命中规则 " | |
| f"<span class='rules-count'>{len(result.applied_rules)} 条</span></div>" | |
| "<div class='rules-pills'>" + " ".join(rules_items) + "</div>" | |
| ) | |
| else: | |
| rules_html = ( | |
| "<div class='rules-header empty-rules'>" | |
| "📭 没有规则匹配 — 这段 prompt 已经够紧凑了" | |
| "</div>" | |
| ) | |
| return result.compressed, stats_html, rules_html, result.compressed | |
| # ============================================================ | |
| # 示例 | |
| # ============================================================ | |
| EXAMPLES = [ | |
| [ | |
| "请你帮我对下面这段文字进行一个全面的分析,如果可以的话麻烦你给出一些改进的建议", | |
| "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", False, | |
| ], | |
| [ | |
| "请你扮演一位资深 Python 工程师,对下面的代码进行 code review,并以 JSON 格式输出结果", | |
| "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", True, | |
| ], | |
| [ | |
| "因为最近在下雨,所以路面变得很滑,因此开车的时候需要特别注意安全", | |
| "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", False, | |
| ], | |
| [ | |
| "Role: 资深产品经理\n任务: 评估这个需求,大家都知道产品决策需要数据支持", | |
| "deepseek_v3", True, True, True, True, "GPT-4 (cl100k)", False, | |
| ], | |
| ] | |
| # ============================================================ | |
| # 自定义 CSS — 让 demo 看起来不像默认的 gradio | |
| # ============================================================ | |
| CUSTOM_CSS = """ | |
| /* ===== 全局 ===== */ | |
| .gradio-container { | |
| max-width: 1280px !important; | |
| margin: 0 auto !important; | |
| } | |
| /* ===== Hero 区 ===== */ | |
| .hero { | |
| text-align: center; | |
| padding: 36px 16px 12px; | |
| background: linear-gradient(135deg, rgba(59,125,216,0.08) 0%, rgba(230,136,60,0.08) 100%); | |
| border-radius: 16px; | |
| margin-bottom: 24px; | |
| border: 1px solid rgba(0,0,0,0.05); | |
| } | |
| .hero h1 { | |
| font-size: 2.4em !important; | |
| margin: 0 !important; | |
| background: linear-gradient(135deg, #3B7DD8, #E6883C); | |
| -webkit-background-clip: text; | |
| -webkit-text-fill-color: transparent; | |
| background-clip: text; | |
| font-weight: 700 !important; | |
| letter-spacing: -0.02em; | |
| } | |
| .hero .tagline { | |
| font-size: 1.1em; | |
| color: #555; | |
| margin-top: 8px; | |
| } | |
| .hero .subtitle { | |
| font-size: 0.95em; | |
| color: #777; | |
| max-width: 760px; | |
| margin: 12px auto 0; | |
| line-height: 1.6; | |
| } | |
| .hero .badges { | |
| margin-top: 16px; | |
| display: flex; | |
| gap: 8px; | |
| justify-content: center; | |
| flex-wrap: wrap; | |
| } | |
| .hero .badge { | |
| display: inline-flex; | |
| align-items: center; | |
| padding: 4px 10px; | |
| background: rgba(255,255,255,0.7); | |
| border: 1px solid rgba(0,0,0,0.08); | |
| border-radius: 6px; | |
| font-size: 0.85em; | |
| color: #444; | |
| text-decoration: none; | |
| transition: all 0.2s; | |
| } | |
| .hero .badge:hover { | |
| transform: translateY(-1px); | |
| box-shadow: 0 2px 8px rgba(0,0,0,0.08); | |
| } | |
| /* ===== Key facts ===== */ | |
| .key-facts { | |
| display: grid; | |
| grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); | |
| gap: 12px; | |
| margin: 24px 0; | |
| } | |
| .fact { | |
| padding: 14px 18px; | |
| background: white; | |
| border-radius: 10px; | |
| border-left: 3px solid var(--fact-color, #3B7DD8); | |
| box-shadow: 0 1px 3px rgba(0,0,0,0.05); | |
| } | |
| .fact-label { | |
| font-size: 0.75em; | |
| text-transform: uppercase; | |
| letter-spacing: 0.06em; | |
| color: #888; | |
| font-weight: 600; | |
| } | |
| .fact-value { | |
| font-size: 1.5em; | |
| font-weight: 700; | |
| color: var(--fact-color, #3B7DD8); | |
| margin-top: 4px; | |
| } | |
| .fact-detail { | |
| font-size: 0.85em; | |
| color: #666; | |
| margin-top: 2px; | |
| } | |
| /* ===== 统计卡片 ===== */ | |
| .stats-grid { | |
| display: grid; | |
| grid-template-columns: 2fr 1fr 1fr 1fr; | |
| gap: 12px; | |
| margin: 8px 0; | |
| } | |
| .stat-card { | |
| background: white; | |
| border-radius: 10px; | |
| padding: 14px 16px; | |
| border: 1px solid rgba(0,0,0,0.06); | |
| transition: all 0.2s; | |
| } | |
| .stat-card:hover { | |
| box-shadow: 0 2px 12px rgba(0,0,0,0.08); | |
| transform: translateY(-1px); | |
| } | |
| .stat-card.primary { | |
| background: linear-gradient(135deg, #FAFBFF, #F5F7FB); | |
| border: 1px solid #DDE5F0; | |
| } | |
| .stat-card.empty { | |
| text-align: center; | |
| padding: 28px; | |
| background: #F8F9FB; | |
| color: #888; | |
| border: 1px dashed #DDD; | |
| font-size: 0.95em; | |
| } | |
| .stat-label { | |
| font-size: 0.75em; | |
| text-transform: uppercase; | |
| letter-spacing: 0.06em; | |
| color: #888; | |
| font-weight: 600; | |
| } | |
| .stat-value { | |
| font-size: 1.6em; | |
| font-weight: 700; | |
| margin-top: 4px; | |
| display: flex; | |
| align-items: baseline; | |
| gap: 6px; | |
| } | |
| .stat-value.muted { color: #999; font-size: 1.4em; } | |
| .stat-value.emphasis { color: #10B981; font-size: 1.8em; } | |
| .saving-badge.good { color: #10B981; } | |
| .saving-badge.bad { color: #E6883C; } | |
| .saving-badge.neutral { color: #888; } | |
| .arrow { font-weight: 700; } | |
| .stat-sub { | |
| font-size: 0.78em; | |
| color: #777; | |
| margin-top: 2px; | |
| } | |
| /* ===== 规则面板 ===== */ | |
| .rules-header { | |
| font-size: 0.95em; | |
| font-weight: 600; | |
| color: #444; | |
| margin: 16px 0 8px; | |
| display: flex; | |
| align-items: center; | |
| gap: 8px; | |
| } | |
| .rules-count { | |
| font-size: 0.78em; | |
| background: #EFF2F7; | |
| padding: 2px 10px; | |
| border-radius: 10px; | |
| color: #555; | |
| font-weight: 500; | |
| } | |
| .rules-pills { | |
| display: flex; | |
| gap: 6px; | |
| flex-wrap: wrap; | |
| padding: 4px 0; | |
| } | |
| .rule-pill { | |
| display: inline-flex; | |
| align-items: center; | |
| gap: 4px; | |
| padding: 4px 10px; | |
| background: white; | |
| border: 1px solid var(--rule-color); | |
| color: var(--rule-color); | |
| border-radius: 999px; | |
| font-size: 0.82em; | |
| font-family: 'JetBrains Mono', 'Fira Code', monospace; | |
| font-weight: 500; | |
| transition: all 0.2s; | |
| } | |
| .rule-pill:hover { | |
| background: var(--rule-color); | |
| color: white; | |
| } | |
| .rule-count { | |
| font-size: 0.85em; | |
| opacity: 0.8; | |
| } | |
| .empty-rules { | |
| font-style: italic; | |
| color: #888; | |
| background: #F8F9FB; | |
| padding: 14px; | |
| border-radius: 8px; | |
| border: 1px dashed #DDD; | |
| text-align: center; | |
| } | |
| /* ===== 主操作按钮 ===== */ | |
| button.primary { | |
| background: linear-gradient(135deg, #3B7DD8 0%, #5B9BE5 100%) !important; | |
| border: none !important; | |
| font-size: 1.05em !important; | |
| letter-spacing: 0.02em !important; | |
| transition: all 0.2s !important; | |
| } | |
| button.primary:hover { | |
| transform: translateY(-1px); | |
| box-shadow: 0 4px 12px rgba(59,125,216,0.3) !important; | |
| } | |
| /* ===== Footer ===== */ | |
| .footer { | |
| margin-top: 36px; | |
| padding: 20px 0; | |
| border-top: 1px solid rgba(0,0,0,0.06); | |
| color: #777; | |
| font-size: 0.88em; | |
| text-align: center; | |
| } | |
| .footer a { color: #3B7DD8; text-decoration: none; } | |
| .footer a:hover { text-decoration: underline; } | |
| /* ===== 暗色模式适配 ===== */ | |
| .dark .hero { | |
| background: linear-gradient(135deg, rgba(59,125,216,0.15), rgba(230,136,60,0.15)); | |
| border-color: rgba(255,255,255,0.05); | |
| } | |
| .dark .hero .tagline { color: #BBB; } | |
| .dark .hero .subtitle { color: #999; } | |
| .dark .hero .badge { | |
| background: rgba(255,255,255,0.05); | |
| border-color: rgba(255,255,255,0.1); | |
| color: #CCC; | |
| } | |
| .dark .stat-card { background: rgba(255,255,255,0.03); border-color: rgba(255,255,255,0.08); } | |
| .dark .stat-card.primary { background: rgba(59,125,216,0.08); } | |
| .dark .stat-card.empty { background: rgba(255,255,255,0.03); color: #888; border-color: rgba(255,255,255,0.1); } | |
| .dark .stat-sub { color: #888; } | |
| .dark .fact { background: rgba(255,255,255,0.03); } | |
| .dark .fact-detail { color: #888; } | |
| .dark .rules-count { background: rgba(255,255,255,0.06); color: #BBB; } | |
| .dark .rule-pill { background: rgba(255,255,255,0.03); } | |
| .dark .empty-rules { background: rgba(255,255,255,0.03); border-color: rgba(255,255,255,0.1); } | |
| /* ===== 响应式 ===== */ | |
| @media (max-width: 768px) { | |
| .stats-grid { grid-template-columns: 1fr 1fr; } | |
| .key-facts { grid-template-columns: 1fr; } | |
| .hero h1 { font-size: 1.8em !important; } | |
| } | |
| """ | |
| # ============================================================ | |
| # UI | |
| # ============================================================ | |
| HERO_HTML = """ | |
| <div class="hero"> | |
| <h1>🀄 CHIP</h1> | |
| <div class="tagline">Chinese High-density Instruction Protocol · 中文高密度提示协议</div> | |
| <div class="subtitle"> | |
| 把啰嗦的中文 prompt 自动压成结构化高密度形式 — <strong>数据驱动,不是品味</strong> | |
| </div> | |
| <div class="badges"> | |
| <a class="badge" href="https://github.com/luancy1208/CHIP" target="_blank">⭐ GitHub</a> | |
| <a class="badge" href="https://github.com/luancy1208/CHIP/blob/main/SPEC.md" target="_blank">📄 SPEC</a> | |
| <a class="badge" href="https://github.com/luancy1208/CHIP/tree/main/results" target="_blank">📊 实测数据</a> | |
| <span class="badge">Apache-2.0</span> | |
| <span class="badge">v0.2 · 23 tests passing</span> | |
| </div> | |
| </div> | |
| <div class="key-facts"> | |
| <div class="fact" style="--fact-color:#3B7DD8"> | |
| <div class="fact-label">实测数据</div> | |
| <div class="fact-value">1800+ 行</div> | |
| <div class="fact-detail">9 tokenizer × 200 句 FLORES-200</div> | |
| </div> | |
| <div class="fact" style="--fact-color:#10B981"> | |
| <div class="fact-label">国产模型上中文</div> | |
| <div class="fact-value">省 12.5%</div> | |
| <div class="fact-detail">Baichuan2 token / 等价英文</div> | |
| </div> | |
| <div class="fact" style="--fact-color:#E6883C"> | |
| <div class="fact-label">cl100k 上中文</div> | |
| <div class="fact-value">贵 73%</div> | |
| <div class="fact-detail">所以 CHIP 的主战场是国产模型</div> | |
| </div> | |
| <div class="fact" style="--fact-color:#7C3AED"> | |
| <div class="fact-label">### 标签</div> | |
| <div class="fact-value">1 token</div> | |
| <div class="fact-detail">在所有 9 个 tokenizer 上 — 完爆方括号</div> | |
| </div> | |
| </div> | |
| """ | |
| FOOTER_HTML = """ | |
| <div class="footer"> | |
| <p> | |
| 🀄 CHIP v0.2 · Built with care for Chinese prompt engineering · | |
| <a href="https://github.com/luancy1208/CHIP" target="_blank">GitHub</a> · | |
| <a href="https://github.com/luancy1208/CHIP/issues" target="_blank">反馈 Issue</a> | |
| </p> | |
| <p style="margin-top:8px; font-size:0.82em; opacity:0.7"> | |
| 数据来源: FLORES-200 dev (n=200) · 200 个 HSK 5/6 高频成语 · 45 个常见标记符号 · 实测于 9 个 tokenizer | |
| </p> | |
| </div> | |
| """ | |
| with gr.Blocks( | |
| title="CHIP — 中文高密度提示协议", | |
| theme=gr.themes.Soft( | |
| primary_hue="blue", | |
| secondary_hue="orange", | |
| neutral_hue="slate", | |
| font=["system-ui", "-apple-system", "Segoe UI", "Helvetica Neue", "sans-serif"], | |
| ), | |
| css=CUSTOM_CSS, | |
| ) as demo: | |
| gr.HTML(HERO_HTML) | |
| with gr.Row(equal_height=True): | |
| # ------- 输入侧 ------- | |
| with gr.Column(scale=1): | |
| gr.Markdown("### 📝 输入 prompt") | |
| inp = gr.Textbox( | |
| show_label=False, | |
| lines=10, | |
| placeholder="把啰嗦的中文 prompt 粘贴进来,看看 CHIP 能压到多紧...\n\n💡 试试:'请你帮我对下面这段文字进行一个全面的分析,如果可以的话麻烦你给出一些建议'", | |
| container=False, | |
| ) | |
| with gr.Accordion("⚙️ 高级设置", open=False): | |
| with gr.Row(): | |
| target = gr.Dropdown( | |
| ["qwen2.5", "deepseek_v3", "glm4", "cl100k", "o200k"], | |
| value="qwen2.5", | |
| label="🎯 目标模型", | |
| info="影响 L3 成语压缩等 target-aware 决策", | |
| ) | |
| tokenizer_name = gr.Dropdown( | |
| list(TOKENIZERS.keys()) + list(_LAZY.keys()), | |
| value="GPT-4 (cl100k)", | |
| label="🔢 计数 tokenizer", | |
| info="决定 token 数怎么算(国产 tokenizer 首次需下载)", | |
| ) | |
| gr.Markdown("**压缩层** — 默认 L1+L2+L4(保险);L3 仅在国产模型上有意义") | |
| with gr.Row(): | |
| use_l1 = gr.Checkbox(value=True, label="L1 词法", | |
| info="套话剪枝 (~1.3-1.5×)") | |
| use_l2 = gr.Checkbox(value=True, label="L2 句法", | |
| info="模式重排 (~2-3×)") | |
| use_l3 = gr.Checkbox(value=False, label="L3 成语", | |
| info="长描述→成语") | |
| use_l4 = gr.Checkbox(value=True, label="L4 协议", | |
| info="### 标签归一化") | |
| use_jieba = gr.Checkbox( | |
| value=False, | |
| label="🚀 jieba NP 增强模式", | |
| info="复杂角色提取场景效果更好(默认关闭)", | |
| ) | |
| btn = gr.Button("🔥 压缩", variant="primary", size="lg", | |
| elem_classes="primary") | |
| # ------- 输出侧 ------- | |
| with gr.Column(scale=1): | |
| gr.Markdown("### ✨ 压缩结果") | |
| try: | |
| out = gr.Textbox( | |
| show_label=False, | |
| lines=10, | |
| interactive=False, | |
| show_copy_button=True, | |
| container=False, | |
| ) | |
| except TypeError: | |
| out = gr.Textbox(show_label=False, lines=10, | |
| interactive=False, container=False) | |
| stats_panel = gr.HTML( | |
| "<div class='stat-card empty'>👈 在左侧输入并点击压缩按钮</div>" | |
| ) | |
| rules_panel = gr.HTML() | |
| # 隐藏的 textbox 用于触发示例(因为示例需要更新所有 input) | |
| hidden_dup = gr.Textbox(visible=False) | |
| gr.Examples( | |
| examples=EXAMPLES, | |
| inputs=[inp, target, use_l1, use_l2, use_l3, use_l4, tokenizer_name, use_jieba], | |
| label="📌 试试这些示例(点击自动填充并运行)", | |
| examples_per_page=4, | |
| ) | |
| with gr.Accordion("📖 关于 CHIP / 协议设计要点", open=False): | |
| gr.Markdown(""" | |
| **核心发现(基于 1800 行实测数据)** | |
| | Tokenizer | ZH/EN ratio | 中文相对英文 | | |
| |---|---|---| | |
| | **baichuan2** 🟦 | 0.875 | 中文省 12.5% | | |
| | **deepseek_v3** 🟦 | 0.916 | 中文省 8.4% | | |
| | **glm4** 🟦 | 0.924 | 中文省 7.6% | | |
| | qwen2.5/3 🟦 | 0.988 | 持平 | | |
| | **o200k** (GPT-4o) 🟧 | 1.163 | 中文贵 16.3% | | |
| | **cl100k** (GPT-4) 🟧 | 1.731 | 中文贵 73.1% | | |
| 🟦 = 国产 tokenizer · 🟧 = OpenAI tokenizer | |
| --- | |
| **4 层压缩架构** | |
| - **L1 词法层** · 啰嗦套话 → 紧凑动宾,纯正则,~1.3-1.5× | |
| - **L2 句法层** · 模式重排 + 列表化,~2-3× | |
| - **L3 成语层** · 长描述 → 成语(实测白名单),仅国产模型默认关闭 | |
| - **L4 协议层** · 标签归一化为 `### 标题`(实测全 tokenizer 1 token) | |
| **不主张的事**(诚实声明) | |
| - ❌ 不主张"中文因为信息密度高所以更省 token" — Mythbuster 2026 在 SWE-bench 上证伪 | |
| - ❌ 不主张"中文 prompt 让模型更聪明" — 在英文中心模型上常常相反 | |
| - ❌ 不主张"全程使用文言文压缩" — LLM 对生僻文言虚词理解不稳定 | |
| CHIP 主张的是:**在符合中文训练分布的国产模型上,通过协议化压缩可在 token 经济性、可读性、可审计性三个维度同时提供工程价值。** | |
| """) | |
| gr.HTML(FOOTER_HTML) | |
| # 事件 | |
| btn.click( | |
| run, | |
| inputs=[inp, target, use_l1, use_l2, use_l3, use_l4, tokenizer_name, use_jieba], | |
| outputs=[out, stats_panel, rules_panel, hidden_dup], | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch( | |
| server_name=os.getenv("HOST", "0.0.0.0"), | |
| server_port=int(os.getenv("PORT", 7860)), | |
| share=os.getenv("SHARE") == "1", | |
| ) | |