clean

app.py

@@ -2,36 +2,51 @@ import gradio as gr
 import pandas as pd
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 
+# Constants for default values
+DEFAULT_CHUNK_SIZE = 100
+DEFAULT_CHUNK_OVERLAP = 0
+DEFAULT_NUM_CHUNKS = 10
+
 def tokenize_text(method, text, chunk_size, chunk_overlap, num_chunks):
+    """
+    Tokenizes the input text based on the selected method and provided parameters.
+    """
     num_chunks = int(num_chunks)
     output = []
+
+    # Ensure text is provided
+    if not text.strip():
+        return pd.DataFrame(columns=['Chunk #', 'Text Chunk', 'Character Count', 'Token Count'])
+
     if method == "RecursiveCharacterTextSplitter":
         text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap, length_function=len, is_separator_regex=False)
         tokenized_texts = text_splitter.split_text(text)[:num_chunks]
         for i, chunk in enumerate(tokenized_texts):
             output.append({
-                'chunk_num': i,
-                'text': chunk,
-                'tokens': len(chunk.split()),
-                'size': len(chunk)
+                'Chunk #': i,
+                'Text Chunk': chunk,
+                'Character Count': len(chunk),
+                'Token Count': len(chunk.split())
             })
+
     df = pd.DataFrame(output)
     return df
 
 iface = gr.Interface(
     fn=tokenize_text,
     inputs=[
-        gr.Dropdown(label="Select Tokenization Method", choices=["RecursiveCharacterTextSplitter"]),
+        gr.Dropdown(label="Select Tokenization Method", choices=["RecursiveCharacterTextSplitter"], default="RecursiveCharacterTextSplitter"),
         gr.Textbox(label="Enter Text", lines=10, placeholder="Type or paste text here."),
-        gr.Number(label="Chunk Size", value=100),
-        gr.Number(label="Chunk Overlap", value=0),
-        gr.Number(label="Number of Chunks to Display", value=10)
+        gr.Number(label="Chunk Size", value=DEFAULT_CHUNK_SIZE),
+        gr.Number(label="Chunk Overlap", value=DEFAULT_CHUNK_OVERLAP),
+        gr.Number(label="Number of Chunks to Display", value=DEFAULT_NUM_CHUNKS)
     ],
     outputs=gr.Dataframe(headers=["Chunk #", "Text Chunk", "Character Count", "Token Count"]),
     title="Text Tokenization Tool",
-    description="A tool for tokenizing text using different methods. Enter your text and choose your settings to see the results.",
+    description="A tool for tokenizing text using different methods. Enter your text and choose your settings to see the results. It splits the text into chunks based on the specified chunk size and overlap.",
     theme="dark",
+    layout="vertical",
     width=800  # Adjust this value as needed
 )
 
-iface.launch(share=True, inbrowser=True)
+iface.launch(share=True, inbrowser=True)