sid-0313 commited on
Commit
d91a750
·
verified ·
1 Parent(s): 68db3f0

Create utilities.py

Browse files
Files changed (1) hide show
  1. src/utils/utilities.py +5 -0
src/utils/utilities.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ import tiktoken
2
+
3
+ def count_num_tokens(text: str, model: str) -> int:
4
+ encoding = tiktoken.encoding_for_model(model)
5
+ return len(encoding.encode(text))