Backedman commited on
Commit
c8dfeef
verified
1 Parent(s): e84ce90

Create custom_tokenizer.py

Browse files
Files changed (1) hide show
  1. custom_tokenizer.py +7 -0
custom_tokenizer.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ class CustomTokenizer:
2
+ def __init__(self):
3
+ pass
4
+
5
+ def __call__(self, text):
6
+ # No tokenization, return input text as-is
7
+ return text