Synced repo using 'sync_with_huggingface' Github Action

app.py

@@ -1,7 +1,8 @@
 import streamlit as st
+from summarize_doc.app.setupview import setup_view
 
 def main():
-    st.title('Summarize Document')
+    setup_view()
 
 
 if __name__ == '__main__':

pyproject.toml

@@ -8,6 +8,10 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = "^3.9.21"
 streamlit = "^1.41.1"
+pypdf2 = "^3.0.1"
+spacy = "3.7.5"
+langchain = "^0.3.14"
+nltk = "^3.9.1"
 
 
 [build-system]

summarize_doc/app/setupview.py

@@ -0,0 +1,13 @@
+import streamlit as st
+from .sidebar import sidebar_view
+def setup_view():
+    st.set_page_config(page_title='Summarize Document', page_icon='📑', layout='centered')
+    st.title('Summarize Document 📑')
+    st.write("This app summarizes text from a document. You can upload a text or PDF file and get a summary of the text.")
+    if 'extracted_text' not in st.session_state:
+        st.session_state.extracted_text = ""
+    sidebar_view()
+    st.write(st.session_state.extracted_text)
+
+
+

summarize_doc/app/sidebar.py

@@ -0,0 +1,29 @@
+import streamlit as st
+from summarize_doc.src.pdf_summarize.pdf_summarize import PDFSummarize
+from summarize_doc.src.txt_summarize.txt_summarize import TxtSummarize
+
+def sidebar_view():
+    with st.sidebar:
+        st.subheader("Your Document")
+        docs_file = st.file_uploader("Upload a document", type=['txt', 'pdf'])
+        summarization_type = st.radio("Set summarization type :", key="visiblity", options=["Sentences", "Bullet Points"])
+        summarization_length = st.number_input("Number of Sentences", key="num_sentences", min_value=1, max_value=10, value=5, step=1)
+
+        if st.button('Process'):
+            with st.spinner("Processing ... "):
+                if docs_file.type == 'application/pdf':
+                    summarized_text = PDFSummarize(docs_file, summarization_length=summarization_length).text
+                    if summarization_type == "Sentences":
+                        st.session_state.extracted_text = summarized_text
+                    else:
+                        summarized_text = summarized_text.replace(". ", ".\n\n")
+                        st.session_state.extracted_text = summarized_text
+                elif docs_file.type == 'text/plain':
+                    summarized_text = TxtSummarize(docs_file).text
+                    if summarization_type == "Sentences":
+                        st.session_state.extracted_text = summarized_text
+                    else:
+                        summarized_text = summarized_text.replace(". ", ".\n • \n")
+                        st.session_state.extracted_text = summarized_text
+                else:
+                    st.warning("Please upload a valid file type (txt or pdf)")

summarize_doc/src/pdf_summarize/pdf_summarize.py

@@ -0,0 +1,16 @@
+from ..summarize.helper_function import summarize_text
+from PyPDF2 import PdfReader
+
+class PDFSummarize:
+    def __init__(self, file_path, summarization_length=5):
+        self.file_path = file_path
+        raw_text = self.get_pdf_text()
+        self.text = summarize_text(raw_text, num_sentences=summarization_length)
+
+    def get_pdf_text(self):
+        text = ""
+        reader = PdfReader(self.file_path)
+        for page in reader.pages:
+            text += page.extract_text()
+        return text
+

summarize_doc/src/summarize/helper_function.py

@@ -0,0 +1,33 @@
+import re
+from collections import Counter
+import nltk
+from nltk.tokenize import sent_tokenize, word_tokenize
+from nltk.corpus import stopwords
+
+nltk.download('stopwords')
+nltk.download('punkt')
+nltk.download('punkt_tab')
+
+def preporcess_text(text):
+    stop_words = set(stopwords.words('english'))
+    words = word_tokenize(text)
+    words = [word.lower() for word in words if word.isalnum() and word.lower() not in stop_words]
+    return words
+
+def sentence_score(text):
+    sentences = sent_tokenize(text)
+    words = preporcess_text(text)
+    word_frequencies = Counter(words)
+    scores = {}
+
+    for sentence in sentences:
+        sentence_words = preporcess_text(sentence)
+        score = sum([word_frequencies[word] for word in sentence_words])
+        scores[sentence] = score
+    return scores
+
+def summarize_text(text, num_sentences=10):
+    scores = sentence_score(text)
+    ranked_sentences = sorted(scores, key=scores.get, reverse=True)
+    summary = " ".join(ranked_sentences[:num_sentences])
+    return summary

summarize_doc/src/txt_summarize/txt_summarize.py

@@ -0,0 +1,16 @@
+from ..summarize.helper_function import summarize_text
+
+class TxtSummarize:
+    def __init__(self, file_path):
+        self.file_path = file_path
+        raw_text = self.get_txt_text()
+        self.text = summarize_text(raw_text)
+
+    def get_txt_text(self):
+        try:
+            text = ""
+            text = self.file_path.read().decode("utf-8")
+            return text
+        except Exception as e:
+            raise RuntimeError(f"Error reading the file at {self.file_path}: {e}")
+