Update app.py

app.py

@@ -1,6 +1,5 @@
 import gradio as gr
 from transformers import pipeline, AutoTokenizer
-from sentence_transformers import SentenceTransformer, util
 import nltk
 from nltk.tokenize import sent_tokenize
 
@@ -25,7 +24,7 @@ summarization_models = {
 tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn")
 
 # Initialize summarization pipelines
-summarizers = {model: pipeline("summarization", model=model) for model in summarization_models.values()}
+summarizers = {name: pipeline("summarization", model=model) for name, model in summarization_models.items()}
 
 # Initialize translation pipeline
 def get_translator(language):
@@ -36,23 +35,28 @@ def get_translator(language):
 
 # Helper function to split text into chunks
 def split_text(text, max_tokens=1024):
-    inputs = tokenizer(text, return_tensors='pt', truncation=False)
-    input_ids = inputs['input_ids'][0]
-    total_tokens = len(input_ids)
-    
+    sentences = sent_tokenize(text)
     chunks = []
-    start = 0
-    while start < total_tokens:
-        end = min(start + max_tokens, total_tokens)
-        chunk_ids = input_ids[start:end]
-        chunk_text = tokenizer.decode(chunk_ids, skip_special_tokens=True)
-        chunks.append(chunk_text)
-        start = end
-    
+    current_chunk = []
+    current_length = 0
+
+    for sentence in sentences:
+        sentence_length = len(tokenizer.tokenize(sentence))
+        if current_length + sentence_length <= max_tokens:
+            current_chunk.append(sentence)
+            current_length += sentence_length
+        else:
+            chunks.append(" ".join(current_chunk))
+            current_chunk = [sentence]
+            current_length = sentence_length
+
+    if current_chunk:
+        chunks.append(" ".join(current_chunk))
+
     return chunks
 
 # Helper function to summarize text
-def summarize_text(text, model):
+def summarize_text(text, model_name):
     if len(text) < 200:  # Adjust the threshold as needed
         print("Input text is too short for summarization. Please provide longer text.")
         return ""
@@ -60,7 +64,7 @@ def summarize_text(text, model):
     summaries = []
     for chunk in chunks:
         try:
-            summary = summarizers[model](chunk, max_length=150, min_length=20, do_sample=False)[0]['summary_text']
+            summary = summarizers[model_name](chunk, max_length=150, min_length=20, do_sample=False)[0]['summary_text']
             summaries.append(summary)
         except Exception as e:
             print(f"Error summarizing chunk: {chunk}\nError: {e}")
@@ -144,3 +148,4 @@ iface.launch()
 
 
 
+