Spaces:
Build error
Build error
Update clip/simple_tokenizer.py
Browse files- clip/simple_tokenizer.py +1 -1
clip/simple_tokenizer.py
CHANGED
|
@@ -62,7 +62,7 @@ class SimpleTokenizer(object):
|
|
| 62 |
def __init__(self, bpe_path: str = default_bpe()):
|
| 63 |
self.byte_encoder = bytes_to_unicode()
|
| 64 |
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
| 65 |
-
if not os.
|
| 66 |
txt_path = bpe_path.replace(".gz", "")
|
| 67 |
os.system(f"gzip {txt_path}")
|
| 68 |
merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
|
|
|
|
| 62 |
def __init__(self, bpe_path: str = default_bpe()):
|
| 63 |
self.byte_encoder = bytes_to_unicode()
|
| 64 |
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
| 65 |
+
if not os.path.exists(bpe_path):
|
| 66 |
txt_path = bpe_path.replace(".gz", "")
|
| 67 |
os.system(f"gzip {txt_path}")
|
| 68 |
merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
|