Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -7,13 +7,6 @@ CHARACTER_LENGTH = "length_function=lambda x: len(x)"
|
|
| 7 |
TOKEN_LENGTH = enc = tiktoken.get_encoding("cl100k_base")
|
| 8 |
length_function = lambda text: len(enc.encode(text))
|
| 9 |
|
| 10 |
-
RECURSIVE_CHARACTER = RecursiveCharacterTextSplitter(
|
| 11 |
-
chunk_size={chunk_size},
|
| 12 |
-
chunk_overlap={chunk_overlap},
|
| 13 |
-
length_function={length_function}
|
| 14 |
-
)
|
| 15 |
-
|
| 16 |
-
|
| 17 |
# Streamlit UI
|
| 18 |
st.title("Understand Chunk and Token")
|
| 19 |
|
|
|
|
| 7 |
TOKEN_LENGTH = enc = tiktoken.get_encoding("cl100k_base")
|
| 8 |
length_function = lambda text: len(enc.encode(text))
|
| 9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
# Streamlit UI
|
| 11 |
st.title("Understand Chunk and Token")
|
| 12 |
|