Add sentence chunking

app.py

@@ -29,12 +29,47 @@ def transcribe_audio(audiofile):
     st.info('Done Transcription')
 
     return transcription
+def chunk_and_preprocess_text(text, model_name= 'philschmid/flan-t5-base-samsum'):
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    sentences = sent_tokenize(text)
+
+    length = 0
+    chunk = ""
+    chunks = []
+    count = -1
+    
+    for sentence in sentences:
+        count += 1
+        combined_length = len(tokenizer.tokenize(sentence)) + length # add the no. of sentence tokens to the length counter
+    
+        if combined_length  <= tokenizer.max_len_single_sentence: # if it doesn't exceed
+            chunk += sentence + " " # add the sentence to the chunk
+            length = combined_length # update the length counter
+    
+            # if it is the last sentence
+            if count == len(sentences) - 1:
+                chunks.append(chunk) # save the chunk
+      
+        else: 
+            chunks.append(chunk) # save the chunk
+            # reset 
+            length = 0 
+            chunk = ""
+        
+            # take care of the overflow sentence
+            chunk += sentence + " "
+            length = len(tokenizer.tokenize(sentence))
+
+    return chunks
 
 def summarize_podcast(audiotranscription):
     st.info("Summarizing...")
     summarizer = pipeline("summarization", model="philschmid/flan-t5-base-samsum", device=0)
 
-    summarized_text = summarizer(audiotranscription)
+    st.info("Chunking text")
+    text_chunks = chunk_and_preprocess_text(audiotranscription)
+
+    summarized_text = summarizer(text_chunks, max_len=200,min_len=50)
     st.session_state['summary'] = summarized_text
     return summarized_text