Yuchan commited on
Commit
ab6754e
·
verified ·
1 Parent(s): 41c502c

Update AlphaS2S.py

Browse files
Files changed (1) hide show
  1. AlphaS2S.py +6 -0
AlphaS2S.py CHANGED
@@ -48,6 +48,7 @@ def download_file(url, save_path):
48
 
49
  DATA_PATH = "converted.jsonl"
50
  TOKENIZER_PATH = "ko_unigram.model"
 
51
 
52
  if not os.path.exists(DATA_PATH):
53
  download_file(
@@ -60,6 +61,11 @@ if not os.path.exists(TOKENIZER_PATH):
60
  "https://huggingface.co/datasets/Yuchan5386/Multiturn/resolve/main/unigram.model?download=true",
61
  TOKENIZER_PATH
62
  )
 
 
 
 
 
63
 
64
  sp = spm.SentencePieceProcessor(TOKENIZER_PATH)
65
  sp_en = spm.SentencePieceProcessor(TOKENIZER_PATH1)
 
48
 
49
  DATA_PATH = "converted.jsonl"
50
  TOKENIZER_PATH = "ko_unigram.model"
51
+ TOKENIZER_PATH1 = "en_bpe.model"
52
 
53
  if not os.path.exists(DATA_PATH):
54
  download_file(
 
61
  "https://huggingface.co/datasets/Yuchan5386/Multiturn/resolve/main/unigram.model?download=true",
62
  TOKENIZER_PATH
63
  )
64
+ if not os.path.exists(TOKENIZER_PATH1):
65
+ download_file(
66
+ "https://huggingface.co/datasets/Yuchan5386/Multiturn/resolve/main/bpe.model?download=true",
67
+ TOKENIZER_PATH1
68
+ )
69
 
70
  sp = spm.SentencePieceProcessor(TOKENIZER_PATH)
71
  sp_en = spm.SentencePieceProcessor(TOKENIZER_PATH1)