Update app.py

app.py

@@ -25,17 +25,23 @@ font_prop = FontProperties(fname=font_path)
 def jieba_tokenizer(text):
     return jieba.lcut(text)
 
-# Initialize KeyBERT model
+# Initialize KeyBERT models
 vectorizer = CountVectorizer(tokenizer=jieba_tokenizer)
 kw_model = KeyBERT()
+kw_model_multilingual = KeyBERT(model='distiluse-base-multilingual-cased-v1')
 
 # Extract keywords using MMR
 def extract_keywords(doc, diversity):
     keywords = kw_model.extract_keywords(doc, vectorizer=vectorizer, use_mmr=True, diversity=diversity)
     return keywords
 
+# Extract multilingual keywords
+def extract_multilingual_keywords(doc, diversity):
+    keywords = kw_model_multilingual.extract_keywords(doc, vectorizer=vectorizer, use_mmr=True, diversity=diversity)
+    return keywords
+
 # Plot keywords
-def plot_keywords(keywords, title):
+def plot_keywords(keywords, title, filename):
     words = [kw[0] for kw in keywords]
     scores = [kw[1] for kw in keywords]
     plt.figure(figsize=(10, 6))
@@ -45,8 +51,8 @@ def plot_keywords(keywords, title):
     plt.gca().invert_yaxis()
     plt.xticks(fontproperties=font_prop)
     plt.yticks(fontproperties=font_prop)
-    plt.savefig('/tmp/keywords_plot.png')
-    return '/tmp/keywords_plot.png'
+    plt.savefig(f'/tmp/{filename}.png')
+    return f'/tmp/{filename}.png'
 
 # Generate word cloud
 def generate_word_cloud(text):
@@ -67,23 +73,28 @@ def scrape_and_extract(url, diversity):
     content_div = soup.find('div', {'class': 'caas-body'})
     paragraphs = content_div.find_all('p')
     content = '\n'.join([p.text.strip() for p in paragraphs])
+    
     keywords = extract_keywords(content, diversity)
-    keyword_plot_path = plot_keywords(keywords, "Keyword Extraction Results")
+    keywords_multilingual = extract_multilingual_keywords(content, diversity)
+    
+    keyword_plot_path = plot_keywords(keywords, "Keyword Extraction Results", "keywords_plot")
+    keyword_plot_multilingual_path = plot_keywords(keywords_multilingual, "Multilingual Keyword Extraction Results", "keywords_multilingual_plot")
     wordcloud_path = generate_word_cloud(content)
-    return title, content, keywords, keyword_plot_path, wordcloud_path
+    
+    return title, content, keywords, keyword_plot_path, keywords_multilingual, keyword_plot_multilingual_path, wordcloud_path
 
 # Streamlit Interface
 st.set_page_config(page_title="Professional Keyword Extraction Tool", page_icon="🔍")
 
 st.title("🔍 Professional Keyword Extraction Tool")
-st.write("Extracts keywords from a given URL and displays a bar chart of the keywords with their respective scores. Additionally, a word cloud is generated based on TF-IDF scores.")
+st.write("Extracts keywords from a given URL and displays two bar charts of the keywords with their respective scores. Additionally, a word cloud is generated based on TF-IDF scores.")
 
 url = st.text_input("🌐 Enter the article URL here:")
 diversity = st.slider("Adjust Diversity (0.0: Most Relevant, 1.0: Most Diverse)", 0.0, 1.0, 0.5, step=0.01)
 
 if st.button("Extract Keywords"):
     if url:
-        title, content, keywords, keyword_plot_path, wordcloud_path = scrape_and_extract(url, diversity)
+        title, content, keywords, keyword_plot_path, keywords_multilingual, keyword_plot_multilingual_path, wordcloud_path = scrape_and_extract(url, diversity)
         
         st.subheader("📄 Article Title")
         st.write(title)
@@ -98,6 +109,13 @@ if st.button("Extract Keywords"):
         st.subheader("📊 Keywords Bar Chart")
         st.image(keyword_plot_path)
         
+        st.subheader("🔑 Multilingual Extracted Keywords")
+        keywords_multilingual_str = '\n'.join([f"{kw[0]}: {kw[1]:.4f}" for kw in keywords_multilingual])
+        st.text(keywords_multilingual_str)
+        
+        st.subheader("📊 Multilingual Keywords Bar Chart")
+        st.image(keyword_plot_multilingual_path)
+        
         st.subheader("☁️ Word Cloud")
         st.image(wordcloud_path)
     else: