Update app.py

app.py

@@ -1,6 +1,7 @@
 import gradio as gr
 from langchain_community.document_loaders import YoutubeLoader
 from langchain.text_splitter import TokenTextSplitter
+from langchain.embeddings import OpenAIEmbeddings
 
 max_textboxes = 5
 
@@ -13,20 +14,24 @@ def process_youtube_url(url, language):
         )
         docs = loader.load()
         text = str(docs)
-        char_count = len(text)
-
-        text_splitter = TokenTextSplitter(chunk_size=10000, chunk_overlap=0)
+        
+        embeddings = OpenAIEmbeddings()
+        token_count = len(embeddings.encode(text))
+        
+        text_splitter = TokenTextSplitter(chunk_size=20_000, chunk_overlap=0)
         chunks = text_splitter.split_text(text)
-
+        
         output_textboxes = [chunk for i, chunk in enumerate(chunks)]
         output_textboxes += ["" for _ in range(max_textboxes - len(chunks))]
         
-        return *output_textboxes,[], text, char_count
+        return *output_textboxes, [], text, token_count
+    
     except Exception as e:
         error_msg = str(e)
         available_languages = extract_available_languages(error_msg)
         recommended_language = extract_recommended_language(error_msg)
-        return *[error_msg for _ in range(max_textboxes)],available_languages, recommended_language, 0
+        return *[error_msg for _ in range(max_textboxes)], available_languages, recommended_language, 0
+
 
 def extract_available_languages(error_msg):
     languages = []