init commit

app.py

@@ -0,0 +1,52 @@
+from vertexai.preview import tokenization
+
+color_map = {
+    '0': '#ccbfee',
+    '1': '#beedc6',
+    '2': '#f4d7ab',
+    '3': '#f4aeb1',
+    '4': '#a4dcf3',
+}
+
+def tokens_len(prompt, model):
+    model_name = "gemini-1.5-pro"
+    tokenizer = tokenization.get_tokenizer_for_model(model)
+
+    result = tokenizer.compute_tokens(prompt)
+    tokens = result.tokens_info[0].tokens
+
+    return (
+        f'{len(tokens)} tokens, {len(prompt)} chracters',
+        [
+            (
+                token.decode('utf-8').replace("\n", "\\n"),
+                str(i % len(color_map))
+            ) for i, token in enumerate(tokens)
+        ]
+    )
+
+web_tokenizer = gr.Interface(
+    fn = tokens_len,
+    inputs = [
+        gr.Textbox(
+            label='輸入提示文字',
+            lines=6),
+        gr.Dropdown(
+            label='模型名稱',
+            choices = [
+                'gemini-1.5-pro',
+                'gemini-1.5-flash',
+            ],
+            value='gemini-1.5-pro')
+    ],
+    outputs = [
+        gr.Textbox(
+            label='統計量'),
+        gr.Highlightedtext(
+            label='切割結果：',
+            color_map = color_map,
+            show_inline_category=False)
+    ]
+)
+
+web_tokenizer.launch(share=True, debug=True)