Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
|
@@ -332,6 +332,9 @@ async def restore_text(req: RestoreRequest) -> Dict[str, Any]:
|
|
| 332 |
is_char = req.mode == "char"
|
| 333 |
mask = "[MASK]" if is_char else tokenizer_bpe.mask_token
|
| 334 |
text = req.text.replace("#", "[GAP]")
|
|
|
|
|
|
|
|
|
|
| 335 |
|
| 336 |
n_gaps = text.count("-") + text.count("[GAP]")
|
| 337 |
|
|
|
|
| 332 |
is_char = req.mode == "char"
|
| 333 |
mask = "[MASK]" if is_char else tokenizer_bpe.mask_token
|
| 334 |
text = req.text.replace("#", "[GAP]")
|
| 335 |
+
# Lowering everything except special tokens
|
| 336 |
+
parts = SPECIAL_RE.split(text)
|
| 337 |
+
text = "".join(p if SPECIAL_RE.fullmatch(p) else p.lower() for p in parts if p)
|
| 338 |
|
| 339 |
n_gaps = text.count("-") + text.count("[GAP]")
|
| 340 |
|