tim1900 commited on
Commit
563c4f6
verified
1 Parent(s): e36c2ab

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +2 -2
README.md CHANGED
@@ -248,8 +248,8 @@ def chunk_text_with_max_chunk_size(model, text, tokenizer, prob_threshold=0.5,ma
248
  best_logits = torch.finfo(torch.float32).min
249
  STEP = round(((MAX_TOKENS - 2)//2)*1.75 )
250
  print(f"Processing {input_ids.shape[1]} tokens...")
251
- # while windows_end <= input_ids.shape[1]:#璁板緱鏀规垚windstart
252
- while windows_start < input_ids.shape[1]:#璁板緱鏀规垚windstart
253
  windows_end = windows_start + MAX_TOKENS - 2
254
  ids = torch.cat((CLS, input_ids[:, windows_start:windows_end], SEP), 1)
255
  ids = ids.to(model.device)
 
248
  best_logits = torch.finfo(torch.float32).min
249
  STEP = round(((MAX_TOKENS - 2)//2)*1.75 )
250
  print(f"Processing {input_ids.shape[1]} tokens...")
251
+ # while windows_end <= input_ids.shape[1]:
252
+ while windows_start < input_ids.shape[1]:
253
  windows_end = windows_start + MAX_TOKENS - 2
254
  ids = torch.cat((CLS, input_ids[:, windows_start:windows_end], SEP), 1)
255
  ids = ids.to(model.device)