initial commit

app.py

@@ -0,0 +1,38 @@
+import gradio as gr
+import pandas as pd
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+def tokenize_text(method, text, chunk_size, chunk_overlap, num_chunks):
+    num_chunks = int(num_chunks)
+    output = []
+    if method == "RecursiveCharacterTextSplitter":
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap, length_function=len, is_separator_regex=False)
+        tokenized_texts = text_splitter.split_text(text)[:num_chunks]
+        for i, chunk in enumerate(tokenized_texts):
+            output.append({
+                'chunk_num': i,
+                'text': chunk,
+                'tokens': len(chunk.split()),
+                'size': len(chunk)
+            })
+    df = pd.DataFrame(output)
+    return df
+
+iface = gr.Interface(
+    fn=tokenize_text,
+    inputs=[
+        gr.Dropdown(label="Select Tokenization Method", choices=["RecursiveCharacterTextSplitter"]),
+        gr.Textbox(label="Enter Text", lines=10, placeholder="Type or paste text here."),
+        gr.Number(label="Chunk Size", value=100),
+        gr.Number(label="Chunk Overlap", value=0),
+        gr.Number(label="Number of Chunks to Display", value=10)
+    ],
+    outputs=gr.Dataframe(headers=["Chunk #", "Text Chunk", "Character Count", "Token Count"]),
+    title="Text Tokenization Tool",
+    description="A tool for tokenizing text using different methods. Enter your text and choose your settings to see the results.",
+    theme="dark",
+    layout="vertical",
+    width=800  # Adjust this value as needed
+)
+
+iface.launch(share=True, inbrowser=True)

requirements.txt

@@ -0,0 +1,4 @@
+langchain
+gradio
+tiktoken
+sentence-transformers