support latex formula

app.py

@@ -4,19 +4,16 @@ import json
 
 def count_tokens(text):
     """
-    Calculate the number of tokens in the input text using tiktoken.
-
+    计算输入文本中的 token 数量，并根据用户选择格式化文本。
+    
     Args:
-        text (str): The input text to be tokenized.
-
+        text (str): 输入文本。
+        use_markdown (bool): 是否使用 Markdown/LaTeX 格式输出。
+    
     Returns:
-        int: The number of tokens in the input text.
+        tuple: 返回 token 数量和格式化后的文本。
     """
-    # Choose the encoding based on the model you are targeting.
-    # Here, we use 'gpt-3.5-turbo' as an example.
     encoding = tiktoken.encoding_for_model("gpt-4")
-    
-    # Encode the input text to get the list of token IDs
     tokens = encoding.encode(text)
     
     try:
@@ -27,26 +24,36 @@ def count_tokens(text):
    
     text = text.replace("\\n", "\n")
     
-    return len(tokens), text
+    formatted_text = text
+    
+    return len(tokens), gr.update(value=formatted_text)
 
-# Define the Gradio interface
+# 定义 Gradio 接口
 iface = gr.Interface(
-    fn=count_tokens,                   # The function to call
-    inputs=gr.Textbox(lines=1, max_lines=1000000, placeholder="Enter your text here..."),  # Input component
+    fn=count_tokens,                   
+    inputs=[
+        gr.Textbox(
+            lines=10, 
+            max_lines=1000000, 
+            placeholder="Enter your text here..."
+        ),
+        # gr.Checkbox(label="使用 Markdown/LaTeX 格式输出", value=True)  # 格式选择开关
+    ],  
     outputs=[
         "number",
-        gr.Textbox(label="Beautified Text", lines=30)
+        gr.Markdown(label="Beautified Text")
     ],
     title="Token Counter with tiktoken",
-    description="Enter text below to calculate the number of tokens using the tiktoken library.",
+    description="Enter text below to calculate the number of tokens using the tiktoken library. Supports LaTeX formulas using $ for inline and $$ for block formulas.",
     examples=[
-        ["Hello, how are you doing today?"],
-        ["Gradio makes it easy to create web apps for machine learning models."],
-        ["OpenAI's GPT models are powerful tools for natural language processing tasks."]
+        ["这是一个行内公式示例：$E=mc^2$"],
+        ["这是一个块级公式示例：$$\\sum_{i=1}^n i = \\frac{n(n+1)}{2}$$"],
+        ["这是混合示例：\n行内公式：$\\alpha + \\beta$\n块级公式：$$\\int_0^\\infty e^{-x^2} dx = \\frac{\\sqrt{\\pi}}{2}$$"],
+        ["普通文本示例：Hello, how are you doing today?"],
     ],
     theme="default"
 )
 
-# Launch the app
+# 启动应用
 if __name__ == "__main__":
     iface.launch()