Spaces:
Sleeping
Sleeping
show number of characters.
Browse files
app.py
CHANGED
|
@@ -18,8 +18,10 @@ def gen_num():
|
|
| 18 |
def tokens_len(prompt, model):
|
| 19 |
encoder = tiktoken.encoding_for_model(model)
|
| 20 |
tokens = encoder.encode(prompt)
|
| 21 |
-
return (
|
| 22 |
-
|
|
|
|
|
|
|
| 23 |
token, category in zip(tokens, gen_num())])
|
| 24 |
|
| 25 |
web_tokenizer = gr.Interface(
|
|
@@ -39,7 +41,8 @@ web_tokenizer = gr.Interface(
|
|
| 39 |
value='gpt-4o-mini')
|
| 40 |
],
|
| 41 |
outputs = [
|
| 42 |
-
gr.Number(label='
|
|
|
|
| 43 |
gr.Highlightedtext(
|
| 44 |
label='切割結果:',
|
| 45 |
color_map = color_map,
|
|
|
|
| 18 |
def tokens_len(prompt, model):
|
| 19 |
encoder = tiktoken.encoding_for_model(model)
|
| 20 |
tokens = encoder.encode(prompt)
|
| 21 |
+
return (
|
| 22 |
+
len(tokens),
|
| 23 |
+
len(prompt),
|
| 24 |
+
[(encoder.decode([token]), category) for
|
| 25 |
token, category in zip(tokens, gen_num())])
|
| 26 |
|
| 27 |
web_tokenizer = gr.Interface(
|
|
|
|
| 41 |
value='gpt-4o-mini')
|
| 42 |
],
|
| 43 |
outputs = [
|
| 44 |
+
gr.Number(label='Tokens'),
|
| 45 |
+
gr.Number(label='Characters'),
|
| 46 |
gr.Highlightedtext(
|
| 47 |
label='切割結果:',
|
| 48 |
color_map = color_map,
|