add limits of length 10

lex_rank_util.py

@@ -8,6 +8,7 @@ from scipy.sparse.csgraph import connected_components
 from scipy.special import softmax
 import logging, math
 
+STOP_WORDS = ["作者", "版权所有", "版權所有", "投资有风险", "投資有風險", "http", "https", "来源"]
 logger = logging.getLogger(__name__)
 
 def degree_centrality_scores(
@@ -132,9 +133,18 @@ def find_siblings_by_index(sentences: [str], central_indices: [int], siblings: i
         head = max(idx - siblings, 0)
         tail = min(idx + siblings + 1, len(sentences))
         for i in range(head, tail):
-            if i not in ret:
+            if i not in ret and check_valid_sentence(sentences[i]):
                 ret.append(i)
                 num -= 1
 
     print(ret)
     return ret
+
+
+def check_valid_sentence(content: str):
+    if len(content) < 10:
+        return False
+    for stop_word in STOP_WORDS:
+        if stop_word in content:
+            return False
+    return True