fix parsing so spaces in front of numbers don't go missing

app.py

@@ -189,13 +189,11 @@ def token_generator(generator1, generator2, mapping_fn=None, fillvalue=None):
     if not mapping_fn:
         mapping_fn = lambda x: x
     for output1, output2 in itertools.zip_longest(generator1, generator2, fillvalue=fillvalue):
-        tokens1 = re.findall(r'\s*\S+\s*', mapping_fn(output1))
-        tokens2 = re.findall(r'\s*\S+\s*', mapping_fn(output2))
-        # tokens1 = re.findall(r'(.*?)(\s|$)', mapping_fn(output1))
-        # tokens2 = re.findall(r'(.*?)(\s|$)', mapping_fn(output2))
+        tokens1 = re.findall(r'(.*?)(\s|$)', mapping_fn(output1))
+        tokens2 = re.findall(r'(.*?)(\s|$)', mapping_fn(output2))
 
         for token1, token2 in itertools.zip_longest(tokens1, tokens2, fillvalue=''):
-            yield token1, token2
+            yield "".join(token1), "".join(token2)
 
 
 def chat(history1, history2, system_msg, state):