Spaces:
Build error
Build error
Update clip/simple_tokenizer.py
Browse files- clip/simple_tokenizer.py +1 -1
clip/simple_tokenizer.py
CHANGED
|
@@ -63,7 +63,7 @@ class SimpleTokenizer(object):
|
|
| 63 |
self.byte_encoder = bytes_to_unicode()
|
| 64 |
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
| 65 |
if not os.system(bpe_path):
|
| 66 |
-
txt_path = bpe_path.replace(".gz"
|
| 67 |
os.system(f"gzip {txt_path}")
|
| 68 |
merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
|
| 69 |
merges = merges[1:49152-256-2+1]
|
|
|
|
| 63 |
self.byte_encoder = bytes_to_unicode()
|
| 64 |
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
| 65 |
if not os.system(bpe_path):
|
| 66 |
+
txt_path = bpe_path.replace(".gz", "")
|
| 67 |
os.system(f"gzip {txt_path}")
|
| 68 |
merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
|
| 69 |
merges = merges[1:49152-256-2+1]
|