Update keywords_processor.py

keywords_processor.py

@@ -1,18 +1,17 @@
 import gradio as gr
-from janome.tokenizer import Tokenizer
+import re
 from sklearn.feature_extraction.text import CountVectorizer
 import os
 
-def custom_tokenizer(text):
-    t = Tokenizer()
-    tokens = [token.surface for token in t.tokenize(text)]
-    return tokens
-
 def process_keywords(text):
     # カンマと改行を空白に変換
-    text = " ".join(text.split())
+    text = re.sub(r"[,\n]+", " ", text)
+    # 英数字と空白以外を削除
+    text = re.sub(r"[^\w\s]", "", text)
+    # 連続する空白を一つにする
+    text = re.sub(r"\s+", " ", text)
     try:
-        vectorizer = CountVectorizer(ngram_range=(1, 3), tokenizer=custom_tokenizer)
+        vectorizer = CountVectorizer(ngram_range=(1, 3), token_pattern=r"(?u)\b\w+\b")
         X = vectorizer.fit_transform([text])
         features = vectorizer.get_feature_names_out()
         if features.size > 0:
@@ -28,13 +27,23 @@ def save_keywords(keywords, filename="output1.txt"):
     with open(filename, 'w', encoding='utf-8') as file:
         if keywords:
             for keyword in keywords:
+                print(f"Saving keyword: {keyword}")  # 保存しようとしているキーワードをログに出力
                 file.write(keyword + "\n")
-    return "Keywords saved to {}".format(filename)
+        else:
+            print("No keywords to save.")  # 保存するキーワードがない場合のログ
+    return f"Keywords saved to {filename}"
 
 def process_and_save_keywords(text):
     keywords = process_keywords(text)
+    # キーワードが生成されたかどうかをチェック
+    if keywords:
+        print("Generated keywords:", ", ".join(keywords))
+    else:
+        print("No keywords generated from the input.")
+
     save_result = save_keywords(keywords)
     print(save_result)
+
     return ", ".join(keywords) if keywords else "No keywords", save_result
 
 with gr.Blocks() as demo: