Include EOS token

app.py

@@ -86,18 +86,21 @@ dependencies.
 
 text = st.text_area("Input text", DEFAULT_TEXT)
 
+if tokenizer.eos_token:
+    text += tokenizer.eos_token
 inputs = tokenizer([text])
 [input_ids] = inputs["input_ids"]
+num_user_tokens = len(input_ids) - (1 if tokenizer.eos_token else 0)
 
-if len(input_ids) < 2 or len(input_ids) > max_tokens:
-    st.caption(f":red[{len(input_ids)}]/{max_tokens} tokens")
+if num_user_tokens < 1 or num_user_tokens > max_tokens:
+    st.caption(f":red[{num_user_tokens}]/{max_tokens} tokens")
 else:
-    st.caption(f"{len(input_ids)}/{max_tokens} tokens")
+    st.caption(f"{num_user_tokens}/{max_tokens} tokens")
 
-if len(input_ids) < 2:
-    st.error("Please enter at least 2 tokens.", icon="🚨")
+if num_user_tokens < 1:
+    st.error("Please enter at least one token.", icon="🚨")
     st.stop()
-if len(input_ids) > max_tokens:
+if num_user_tokens > max_tokens:
     st.error(
         f"Please enter at most {max_tokens} tokens or try reducing the window size.",
         icon="🚨"