Add app.py

app.py

@@ -0,0 +1,46 @@
+from gradio import Dataframe, Interface, Number, TextArea, Blocks, Row
+from pandas import DataFrame
+from transformers import CLIPTokenizer, PreTrainedTokenizer
+
+
+TOKENIZER: PreTrainedTokenizer = CLIPTokenizer.from_pretrained(
+    "openai/clip-vit-large-patch14"
+)
+
+
+def to_tokens(prompt: str) -> tuple[int, DataFrame]:
+    tokens = TOKENIZER.tokenize(prompt)
+    ids = TOKENIZER.convert_tokens_to_ids(tokens)
+
+    return (
+        len(tokens),
+        DataFrame({
+            "No": range(1, len(tokens)+1),
+            "Token": tokens,
+            "ID": ids
+        })
+    )
+
+
+def main():
+    app = Interface(
+        title="Token Counter",
+        fn=to_tokens,
+        inputs=TextArea(label="Prompt"),
+        outputs=[
+            Number(label="Token Count"),
+            Dataframe(
+                headers=[ "No", "Token", "ID" ],
+                datatype=[ "number", "str", "number" ],
+                label="Tokens"
+            )
+        ],
+        live=True,
+        allow_flagging="never"
+    )
+
+    app.launch()
+
+
+if __name__ == "__main__":
+    main()