Update README.md
Browse files
README.md
CHANGED
|
@@ -248,8 +248,8 @@ def chunk_text_with_max_chunk_size(model, text, tokenizer, prob_threshold=0.5,ma
|
|
| 248 |
best_logits = torch.finfo(torch.float32).min
|
| 249 |
STEP = round(((MAX_TOKENS - 2)//2)*1.75 )
|
| 250 |
print(f"Processing {input_ids.shape[1]} tokens...")
|
| 251 |
-
# while windows_end <= input_ids.shape[1]
|
| 252 |
-
while windows_start < input_ids.shape[1]
|
| 253 |
windows_end = windows_start + MAX_TOKENS - 2
|
| 254 |
ids = torch.cat((CLS, input_ids[:, windows_start:windows_end], SEP), 1)
|
| 255 |
ids = ids.to(model.device)
|
|
|
|
| 248 |
best_logits = torch.finfo(torch.float32).min
|
| 249 |
STEP = round(((MAX_TOKENS - 2)//2)*1.75 )
|
| 250 |
print(f"Processing {input_ids.shape[1]} tokens...")
|
| 251 |
+
# while windows_end <= input_ids.shape[1]:
|
| 252 |
+
while windows_start < input_ids.shape[1]:
|
| 253 |
windows_end = windows_start + MAX_TOKENS - 2
|
| 254 |
ids = torch.cat((CLS, input_ids[:, windows_start:windows_end], SEP), 1)
|
| 255 |
ids = ids.to(model.device)
|