meebox commited on
Commit
bb86d37
·
1 Parent(s): 89c1cb0

show number of characters.

Browse files
Files changed (1) hide show
  1. app.py +6 -3
app.py CHANGED
@@ -18,8 +18,10 @@ def gen_num():
18
  def tokens_len(prompt, model):
19
  encoder = tiktoken.encoding_for_model(model)
20
  tokens = encoder.encode(prompt)
21
- return (len(tokens),
22
- [(encoder.decode([token]), category) for
 
 
23
  token, category in zip(tokens, gen_num())])
24
 
25
  web_tokenizer = gr.Interface(
@@ -39,7 +41,8 @@ web_tokenizer = gr.Interface(
39
  value='gpt-4o-mini')
40
  ],
41
  outputs = [
42
- gr.Number(label='Token 數量:'),
 
43
  gr.Highlightedtext(
44
  label='切割結果:',
45
  color_map = color_map,
 
18
  def tokens_len(prompt, model):
19
  encoder = tiktoken.encoding_for_model(model)
20
  tokens = encoder.encode(prompt)
21
+ return (
22
+ len(tokens),
23
+ len(prompt),
24
+ [(encoder.decode([token]), category) for
25
  token, category in zip(tokens, gen_num())])
26
 
27
  web_tokenizer = gr.Interface(
 
41
  value='gpt-4o-mini')
42
  ],
43
  outputs = [
44
+ gr.Number(label='Tokens'),
45
+ gr.Number(label='Characters'),
46
  gr.Highlightedtext(
47
  label='切割結果:',
48
  color_map = color_map,