Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -4,10 +4,10 @@ import tiktoken
|
|
| 4 |
|
| 5 |
|
| 6 |
CHARACTER_LENGTH = "length_function=lambda x: len(x)"
|
| 7 |
-
TOKEN_LENGTH =
|
| 8 |
length_function = lambda text: len(enc.encode(text))
|
| 9 |
|
| 10 |
-
RECURSIVE_CHARACTER =
|
| 11 |
chunk_size={chunk_size},
|
| 12 |
chunk_overlap={chunk_overlap},
|
| 13 |
length_function={length_function}
|
|
|
|
| 4 |
|
| 5 |
|
| 6 |
CHARACTER_LENGTH = "length_function=lambda x: len(x)"
|
| 7 |
+
TOKEN_LENGTH = enc = tiktoken.get_encoding("cl100k_base")
|
| 8 |
length_function = lambda text: len(enc.encode(text))
|
| 9 |
|
| 10 |
+
RECURSIVE_CHARACTER = RecursiveCharacterTextSplitter(
|
| 11 |
chunk_size={chunk_size},
|
| 12 |
chunk_overlap={chunk_overlap},
|
| 13 |
length_function={length_function}
|