Update README.md
Browse files
README.md
CHANGED
|
@@ -92,7 +92,7 @@ def split_text(text, max_tokens_per_chunk):
|
|
| 92 |
|
| 93 |
return splits
|
| 94 |
|
| 95 |
-
|
| 96 |
token_split_points = find_split_points(prob_list, max_tokens_per_chunk)
|
| 97 |
|
| 98 |
str_split_points = [tokens.token_to_chars(pos + 1).start for pos in token_split_points]
|
|
|
|
| 92 |
|
| 93 |
return splits
|
| 94 |
|
| 95 |
+
|
| 96 |
token_split_points = find_split_points(prob_list, max_tokens_per_chunk)
|
| 97 |
|
| 98 |
str_split_points = [tokens.token_to_chars(pos + 1).start for pos in token_split_points]
|