add requirements.txt

Summarizer.py

@@ -1,56 +0,0 @@
-import nltk
-
-from sumy.nlp.stemmers import Stemmer
-from sumy.summarizers.lsa import LsaSummarizer
-from sumy.utils import get_stop_words
-from transformers import Pipeline
-
-class Summarizer:
-    DEFAULT_LANGUAGE = "english"
-    DEFAULT_SENTENCE_LENGTH = 15
-
-    def __init__(self, pipeline: Pipeline):
-        self.pipeline = pipeline
-        stemmer = Stemmer(Summarizer.DEFAULT_LANGUAGE)
-        self.lsa_summarizer = LsaSummarizer(stemmer)
-        self.lsa_summarizer.stop_words = get_stop_words(language=Summarizer.DEFAULT_LANGUAGE)
-
-    @staticmethod
-    def sentence_list(summarized_sentences) -> list:
-        summarized_list = []
-        for sentence in summarized_sentences:
-            summarized_list.append(sentence._text)
-        return summarized_list
-
-    @staticmethod
-    def join_sentences(summarized_sentences: list) -> str:
-        return " ".join([sentence for sentence in summarized_sentences])
-
-    @staticmethod
-    def split_sentences_by_token_length(summary_sentences: list, split_token_length: int) -> list:
-        accumulated_list = []
-        result_list = []
-        cumulative_token_length = 0
-        for sentence in summary_sentences:
-            token_list = [token for token in nltk.word_tokenize(sentence) if token not in ["."]]
-            token_length = len(token_list)
-            if token_length + cumulative_token_length > split_token_length and result_list:
-                accumulated_list.append(Summarizer.join_sentences(result_list))
-                result_list = [sentence]
-                cumulative_token_length = token_length
-            else:
-                result_list.append(sentence)
-                cumulative_token_length += token_length
-
-        if result_list:
-            accumulated_list.append(Summarizer.join_sentences(result_list))
-
-        return accumulated_list
-
-    def abstractive_summary(self, summary_sentences: list) -> list:
-        wrapped_sentences = Summarizer.split_sentences_by_token_length(summary_sentences, split_token_length=600)
-        summary_list = []
-        for result in self.pipeline(wrapped_sentences, min_length=32, max_length=512):
-            summary_list.append(result['summary_text'])
-
-        return summary_list

app.py

@@ -8,8 +8,6 @@ import validators
 from transformers import pipeline
 from validators import ValidationFailure
 
-from Summarizer import Summarizer
-
 
 def main() -> None:
     nltk.download("punkt")
@@ -58,12 +56,10 @@ def main() -> None:
             text = file.read()
         return text
 
-    summarizer: Summarizer = Summarizer(create_pipeline())
-
     if "target_text" not in st.session_state:
         st.session_state.target_text = ""
     if "sentence_lenght" not in st.session_state:
-        st.session_state.sentence_length = Summarizer.DEFAULT_SENTENCE_LENGTH
+        st.session_state.sentence_length = 15
     if "sample_choice" not in st.session_state:
         st.session_state.sentence_length = ""
 
@@ -89,18 +85,19 @@ def main() -> None:
 
     summarize_button = st.button(label="Try it!")
 
-    @st.cache(suppress_st_warning=True,
-              show_spinner=False,
-              allow_output_mutation=True,
-              hash_funcs={"torch.nn.parameter.Parameter": lambda _: None,
-                          "tokenizers.Tokenizer": lambda _: None,
-                          "tokenizers.AddedToken": lambda _: None,
-                          })
+    # @st.cache(suppress_st_warning=True,
+    #           show_spinner=False,
+    #           allow_output_mutation=True,
+    #           hash_funcs={"torch.nn.parameter.Parameter": lambda _: None,
+    #                       "tokenizers.Tokenizer": lambda _: None,
+    #                       "tokenizers.AddedToken": lambda _: None,
+    #                       }
+    #           )
 
 
-    def summary_from_cache(summary_sentence: tuple) -> tuple:
-        with st.spinner("Summarizing in progress..."):
-            return tuple(summarizer.abstractive_summary(list(summary_sentence)))
+    # def summary_from_cache(summary_sentence: tuple) -> tuple:
+    #     with st.spinner("Summarizing in progress..."):
+    #         return tuple(summarizer.abstractive_summary(list(summary_sentence)))
 
     if summarize_button:
         output = pipeline(st.session_state.target_text)