Yuchan
commited on
Update AlphaS2S.py
Browse files- AlphaS2S.py +6 -0
AlphaS2S.py
CHANGED
|
@@ -48,6 +48,7 @@ def download_file(url, save_path):
|
|
| 48 |
|
| 49 |
DATA_PATH = "converted.jsonl"
|
| 50 |
TOKENIZER_PATH = "ko_unigram.model"
|
|
|
|
| 51 |
|
| 52 |
if not os.path.exists(DATA_PATH):
|
| 53 |
download_file(
|
|
@@ -60,6 +61,11 @@ if not os.path.exists(TOKENIZER_PATH):
|
|
| 60 |
"https://huggingface.co/datasets/Yuchan5386/Multiturn/resolve/main/unigram.model?download=true",
|
| 61 |
TOKENIZER_PATH
|
| 62 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
|
| 64 |
sp = spm.SentencePieceProcessor(TOKENIZER_PATH)
|
| 65 |
sp_en = spm.SentencePieceProcessor(TOKENIZER_PATH1)
|
|
|
|
| 48 |
|
| 49 |
DATA_PATH = "converted.jsonl"
|
| 50 |
TOKENIZER_PATH = "ko_unigram.model"
|
| 51 |
+
TOKENIZER_PATH1 = "en_bpe.model"
|
| 52 |
|
| 53 |
if not os.path.exists(DATA_PATH):
|
| 54 |
download_file(
|
|
|
|
| 61 |
"https://huggingface.co/datasets/Yuchan5386/Multiturn/resolve/main/unigram.model?download=true",
|
| 62 |
TOKENIZER_PATH
|
| 63 |
)
|
| 64 |
+
if not os.path.exists(TOKENIZER_PATH1):
|
| 65 |
+
download_file(
|
| 66 |
+
"https://huggingface.co/datasets/Yuchan5386/Multiturn/resolve/main/bpe.model?download=true",
|
| 67 |
+
TOKENIZER_PATH1
|
| 68 |
+
)
|
| 69 |
|
| 70 |
sp = spm.SentencePieceProcessor(TOKENIZER_PATH)
|
| 71 |
sp_en = spm.SentencePieceProcessor(TOKENIZER_PATH1)
|