Spaces:
Running
on
Zero
Running
on
Zero
Update inference.py
Browse files- inference.py +2 -0
inference.py
CHANGED
|
@@ -115,6 +115,8 @@ def tokenizer_encode(text: str, append_bos: bool = True, normalize: bool = True)
|
|
| 115 |
text = text.replace('”', '"')
|
| 116 |
text = text.replace('’', "'")
|
| 117 |
text = text.replace('\n', " ")
|
|
|
|
|
|
|
| 118 |
|
| 119 |
b = list(text.encode('utf-8'))
|
| 120 |
if append_bos:
|
|
|
|
| 115 |
text = text.replace('”', '"')
|
| 116 |
text = text.replace('’', "'")
|
| 117 |
text = text.replace('\n', " ")
|
| 118 |
+
text = text.replace(':', ',')
|
| 119 |
+
text = text.replace(';', ',')
|
| 120 |
|
| 121 |
b = list(text.encode('utf-8'))
|
| 122 |
if append_bos:
|