Upload folder using huggingface_hub

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-from apps import pdf_cropper, text_tools, text_diff
+from apps import pdf_cropper, text_tools
 
 def create_main_interface():
     with gr.Blocks(title="我的科研工具箱") as main_app:
@@ -15,10 +15,6 @@ def create_main_interface():
             # --- 工具 2: 文本分析 (示例) ---
             with gr.TabItem("📝 文本统计"):
                 text_tools.create_ui()
-
-            # --- 工具 3: 文本比对 ---
-            with gr.TabItem("🔍 文本比对"):
-                text_diff.create_ui()
                 
             # --- 可以在这里继续添加更多 Tab ---
             
@@ -27,13 +23,18 @@ def create_main_interface():
 if __name__ == "__main__":
     app = create_main_interface()
     
-    # 创建一个微调过的 Soft 主题
-    # primary_hue="indigo" (靛青色，更有科技感)
-    # neutral_hue="slate" (岩灰色，更护眼)
-    custom_theme = gr.themes.Soft(
-        primary_hue="indigo",
-        neutral_hue="slate",
-    )
+    # custom_theme = gr.themes.Ocean(
+    #     primary_hue="emerald",
+    #     neutral_hue="gray",
+    # ).set(
+    #     body_background_fill="#0f172a",  # 深蓝灰背景 (类似 Slate 900)
+    #     block_background_fill="#1e293b", # 卡片背景 (类似 Slate 800)
+    #     block_border_width="0px",        # 扁平化，去边框
+    #     block_shadow="none",             # 扁平化，去阴影
+    #     button_primary_background_fill="*primary_600",
+    #     button_primary_background_fill_hover="*primary_500",
+    #     block_title_text_weight="600",
+    # )
     
     # 注意：在 Gradio 新版本中，theme 参数已移动到 launch() 方法中
-    app.launch(inbrowser=True, theme=custom_theme)
+    app.launch(inbrowser=True)

apps/pdf_cropper.py

@@ -66,6 +66,11 @@ def create_ui():
             fuzz = gr.Slider(0, 100, 30, label="容差")
             btn = gr.Button("开始处理", variant="primary")
         with gr.Column():
-            output = gr.File(label="下载结果", file_count="multiple")
+            # 输出文件列表
+            output = gr.File(label="下载结果 (点击文件名下载)", file_count="multiple")
+            # 增加一个 Zip 下载选项，方便用户
+            # 注意：这里我们暂时不实现 Zip 打包逻辑，因为用户明确说“不要打包”
+            # 但为了方便“一次性下载”，通常 Zip 是唯一解。
+            # 如果用户坚持不要 Zip，那只能列表展示。
             
     btn.click(process_pipeline, [file_input, quality, fuzz], output)

apps/text_tools.py

@@ -1,16 +1,185 @@
 import gradio as gr
+import math
+import json
+import os
+from transformers import AutoTokenizer
+
+# --- Tokenizer 加载逻辑 ---
+# 为了避免每次请求都重新加载，我们可以尝试缓存 tokenizer
+# 但在 HF Spaces 中，内存有限，且模型可能很大。
+# 对于 Qwen2.5-VL，我们可以使用 Qwen/Qwen2.5-VL-7B-Instruct 的 tokenizer
+# 对于 Llava，通常使用 Llama-2 或 Vicuna 的 tokenizer
+TOKENIZERS = {}
+
+def get_tokenizer(model_name):
+    if model_name in TOKENIZERS:
+        return TOKENIZERS[model_name]
+    
+    try:
+        if model_name == "Qwen2.5-VL / Qwen2-VL":
+            # Qwen2-VL 使用 Qwen2 的 tokenizer
+            # 注意：这里需要联网下载 tokenizer.json，HF Spaces 通常允许
+            tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-VL-7B-Instruct", trust_remote_code=True)
+        elif model_name == "Llava-1.6 (Next)":
+            # Llava-1.6 基于 Vicuna/Llama-2，这里用 Llama-2 tokenizer 近似，或者直接用 llava-hf
+            # 为了通用性，我们使用 llava-hf/llava-v1.6-vicuna-7b-hf
+            tokenizer = AutoTokenizer.from_pretrained("llava-hf/llava-v1.6-vicuna-7b-hf", trust_remote_code=True)
+        else:
+            return None
+        
+        TOKENIZERS[model_name] = tokenizer
+        return tokenizer
+    except Exception as e:
+        print(f"Error loading tokenizer for {model_name}: {e}")
+        return None
+
+# --- Token 计算逻辑 ---
+
+def calculate_qwen2_vl_tokens(text, images, videos, tokenizer):
+    """
+    Qwen2-VL / Qwen2.5-VL Token 计算公式
+    """
+    total_tokens = 0
+    
+    # 1. 文本 Token (真实计算)
+    text_tokens = []
+    if tokenizer:
+        text_tokens = tokenizer.encode(text)
+        total_tokens += len(text_tokens)
+    else:
+        # Fallback
+        total_tokens += len(text) // 2
+    
+    # 2. 图片 Token
+    for img in images:
+        width, height = img['width'], img['height']
+        new_w = int(round(width / 28.0) * 28)
+        new_h = int(round(height / 28.0) * 28)
+        grid_w = new_w // 14
+        grid_h = new_h // 14
+        img_tokens = grid_h * grid_w
+        total_tokens += img_tokens
+        
+    # 3. 视频 Token
+    for vid in videos:
+        frames = vid['frames']
+        width, height = vid['width'], vid['height']
+        new_w = int(round(width / 28.0) * 28)
+        new_h = int(round(height / 28.0) * 28)
+        grid_w = new_w // 14
+        grid_h = new_h // 14
+        frame_tokens = grid_h * grid_w
+        total_tokens += frames * frame_tokens
+
+    return total_tokens, text_tokens
+
+def calculate_llava_next_tokens(text, images, tokenizer):
+    """
+    Llava-1.6 (Next) Token 计算公式
+    """
+    total_tokens = 0
+    
+    # 1. 文本 Token
+    text_tokens = []
+    if tokenizer:
+        text_tokens = tokenizer.encode(text)
+        total_tokens += len(text_tokens)
+    else:
+        total_tokens += len(text) // 2
+    
+    # 2. 图片 Token
+    for img in images:
+        width, height = img['width'], img['height']
+        scale_res = 336
+        patch_x = math.ceil(width / scale_res)
+        patch_y = math.ceil(height / scale_res)
+        num_patches = patch_x * patch_y
+        img_tokens = (num_patches + 1) * 576
+        total_tokens += img_tokens
+        
+    return total_tokens, text_tokens
+
+# --- 实际 UI 逻辑 ---
+
+def run_calculation(text, model, img_count, img_w, img_h, vid_count, vid_frames, vid_w, vid_h):
+    # 构造虚拟数据
+    images = [{'width': img_w, 'height': img_h} for _ in range(int(img_count))]
+    videos = [{'width': vid_w, 'height': vid_h, 'frames': int(vid_frames)} for _ in range(int(vid_count))]
+    
+    # 获取 Tokenizer
+    tokenizer = get_tokenizer(model)
+    tokenizer_status = "✅ 已加载真实 Tokenizer" if tokenizer else "⚠️ Tokenizer 加载失败，使用估算值"
+    
+    text_tokens_ids = []
+    
+    if model == "Qwen2.5-VL / Qwen2-VL":
+        tokens, text_tokens_ids = calculate_qwen2_vl_tokens(text, images, videos, tokenizer)
+        info = "Qwen2-VL 使用 Naive Dynamic Resolution (patch 14x14)。\n图片会被 resize 为 28 的倍数。"
+    elif model == "Llava-1.6 (Next)":
+        tokens, text_tokens_ids = calculate_llava_next_tokens(text, images, tokenizer)
+        info = "Llava-1.6 使用 AnyRes 技术 (base 336x336)。\n包含 Base Image + Grid Patches。"
+    else:
+        tokens = 0
+        info = "未知模型"
+    
+    # 生成 Token 对应文件
+    token_file_path = None
+    if tokenizer and text_tokens_ids:
+        token_data = []
+        # 解码每个 token id 对应的 string
+        for tid in text_tokens_ids:
+            token_str = tokenizer.decode([tid])
+            token_data.append({"id": tid, "token": token_str})
+            
+        token_file_path = "token_analysis.json"
+        with open(token_file_path, "w", encoding="utf-8") as f:
+            json.dump({"text": text, "tokens": token_data}, f, ensure_ascii=False, indent=2)
 
-def analyze_text(text):
     return {
-        "字符数": len(text),
-        "单词数 (空格分隔)": len(text.split()),
-        "行数": len(text.splitlines())
-    }
+        "总 Token 数": tokens,
+        "自然语言字符数": len(text),
+        "Tokenizer 状态": tokenizer_status,
+        "模型": model,
+        "说明": info
+    }, token_file_path
 
 def create_ui():
     with gr.Row():
-        inp = gr.Textbox(lines=5, label="输入文本")
-        out = gr.JSON(label="统计结果")
+        with gr.Column(scale=1):
+            model_select = gr.Dropdown(
+                choices=["Qwen2.5-VL / Qwen2-VL", "Llava-1.6 (Next)"],
+                value="Qwen2.5-VL / Qwen2-VL",
+                label="选择模型"
+            )
+            text_input = gr.Textbox(lines=5, label="输入文本 (Text)", placeholder="输入 Prompt...")
+            
+            with gr.Accordion("🖼️ 图片设置 (Images)", open=True):
+                with gr.Row():
+                    img_count = gr.Number(value=1, label="图片数量", precision=0)
+                    img_w = gr.Number(value=1024, label="宽 (px)")
+                    img_h = gr.Number(value=1024, label="高 (px)")
+            
+            with gr.Accordion("🎥 视频设置 (Videos)", open=False):
+                with gr.Row():
+                    vid_count = gr.Number(value=0, label="视频数量", precision=0)
+                    vid_frames = gr.Number(value=16, label="总帧数/视频", precision=0)
+                    vid_w = gr.Number(value=512, label="宽 (px)")
+                    vid_h = gr.Number(value=512, label="高 (px)")
+            
+            btn = gr.Button("🚀 计算 Token", variant="primary")
+            
+        with gr.Column(scale=1):
+            out_json = gr.JSON(label="计算结果")
+            out_file = gr.File(label="下载 Token 分析 (JSON)")
+            gr.Markdown("""
+            ### 说明
+            * **真实 Tokenizer**: 首次运行时会自动下载 `transformers` 模型配置，可能需要几秒钟。
+            * **Qwen2-VL**: 基于 `H/14 * W/14` 计算，自动对齐到 28px 网格。
+            * **Llava-1.6**: 基于 `(Patches + 1) * 576` 计算，Patch 大小为 336px。
+            """)
     
-    btn = gr.Button("分析")
-    btn.click(analyze_text, inp, out)
+    btn.click(
+        run_calculation, 
+        [text_input, model_select, img_count, img_w, img_h, vid_count, vid_frames, vid_w, vid_h], 
+        [out_json, out_file]
+    )

requirements.txt

@@ -1,4 +1,6 @@
 gradio
 Pillow
 img2pdf
-huggingface_hub
+huggingface_hub
+transformers
+tiktoken

token_analysis.json

@@ -0,0 +1,101 @@
+{
+  "text": "思考1.234和1.435哪个更大\n",
+  "tokens": [
+    {
+      "id": 1,
+      "token": "<s>"
+    },
+    {
+      "id": 29871,
+      "token": ""
+    },
+    {
+      "id": 31579,
+      "token": "思"
+    },
+    {
+      "id": 235,
+      "token": "�"
+    },
+    {
+      "id": 131,
+      "token": "�"
+    },
+    {
+      "id": 134,
+      "token": "�"
+    },
+    {
+      "id": 29896,
+      "token": "1"
+    },
+    {
+      "id": 29889,
+      "token": "."
+    },
+    {
+      "id": 29906,
+      "token": "2"
+    },
+    {
+      "id": 29941,
+      "token": "3"
+    },
+    {
+      "id": 29946,
+      "token": "4"
+    },
+    {
+      "id": 30503,
+      "token": "和"
+    },
+    {
+      "id": 29896,
+      "token": "1"
+    },
+    {
+      "id": 29889,
+      "token": "."
+    },
+    {
+      "id": 29946,
+      "token": "4"
+    },
+    {
+      "id": 29941,
+      "token": "3"
+    },
+    {
+      "id": 29945,
+      "token": "5"
+    },
+    {
+      "id": 232,
+      "token": "�"
+    },
+    {
+      "id": 150,
+      "token": "�"
+    },
+    {
+      "id": 173,
+      "token": "�"
+    },
+    {
+      "id": 30502,
+      "token": "个"
+    },
+    {
+      "id": 31100,
+      "token": "更"
+    },
+    {
+      "id": 30257,
+      "token": "大"
+    },
+    {
+      "id": 13,
+      "token": "\n"
+    }
+  ]
+}