MaximEremeev commited on
Commit
df44061
·
verified ·
1 Parent(s): 7bb228e

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -0
app.py CHANGED
@@ -332,6 +332,9 @@ async def restore_text(req: RestoreRequest) -> Dict[str, Any]:
332
  is_char = req.mode == "char"
333
  mask = "[MASK]" if is_char else tokenizer_bpe.mask_token
334
  text = req.text.replace("#", "[GAP]")
 
 
 
335
 
336
  n_gaps = text.count("-") + text.count("[GAP]")
337
 
 
332
  is_char = req.mode == "char"
333
  mask = "[MASK]" if is_char else tokenizer_bpe.mask_token
334
  text = req.text.replace("#", "[GAP]")
335
+ # Lowering everything except special tokens
336
+ parts = SPECIAL_RE.split(text)
337
+ text = "".join(p if SPECIAL_RE.fullmatch(p) else p.lower() for p in parts if p)
338
 
339
  n_gaps = text.count("-") + text.count("[GAP]")
340