Spaces:

rkingery
/

dumb-language-model

Build error

Ryan Kingery commited on Aug 11, 2022

Commit

2db9276

1 Parent(s): 161b3ed

Update utils.py

Files changed (1) hide show

utils.py CHANGED Viewed

@@ -46,7 +46,10 @@ def clean_text(tokens):
             text.append(token)
         prev_token = token
     detokenizer = TreebankWordDetokenizer()
-    return detokenizer.detokenize(text).replace("' ", "'").replace(" . ", ". ").replace(" ? ", "? ").replace(" ! ", "! ")
 def generate_text(seed, model, vocab, max_len=20, temperature=1., device=device, skip_tokens=['<unk>'], top_k=50):
     stoi, itos = vocab.get_stoi(), vocab.get_itos()

             text.append(token)
         prev_token = token
     detokenizer = TreebankWordDetokenizer()
+    text = detokenizer.detokenize(text)
+    text = text.replace(" ' ", "' ").replace("' ", "'")
+    text = text.replace(" . . . ", "...").replace(" . ", ". ").replace(" ? ", "? ").replace(" ! ", "! ")
+    return text
 def generate_text(seed, model, vocab, max_len=20, temperature=1., device=device, skip_tokens=['<unk>'], top_k=50):
     stoi, itos = vocab.get_stoi(), vocab.get_itos()