updates

lrt/clustering/clusters.py

@@ -1,6 +1,32 @@
 from typing import List, Iterable, Union
 from pprint import pprint
 
+class KeyphraseCount:
+
+    def __init__(self, keyphrase: str, count: int) -> None:
+        super().__init__()
+        self.keyphrase = keyphrase
+        self.count = count
+
+    @classmethod
+    def reduce(cls, kcs: list) :
+        '''
+        kcs: List[KeyphraseCount]
+        '''
+        keys = ''
+        count = 0
+
+        for i in range(len(kcs)-1):
+            kc = kcs[i]
+            keys += kc.keyphrase + '/'
+            count += kc.count
+
+        keys += kcs[-1].keyphrase
+        count += kcs[-1].count
+        return KeyphraseCount(keys, count)
+
+
+
 class SingleCluster:
     def __init__(self):
         self.__container__ = []
@@ -12,7 +38,10 @@ class SingleCluster:
     def elements(self) -> List:
         return self.__container__
     def get_keyphrases(self):
-        return self.__keyphrases__
+        ret = []
+        for key, count in self.__keyphrases__.items():
+            ret.append(KeyphraseCount(key,count))
+        return ret
     def add_keyphrase(self, keyphrase:Union[str,Iterable]):
         if isinstance(keyphrase,str):
             if keyphrase not in self.__keyphrases__.keys():

lrt/lrt.py

@@ -5,6 +5,8 @@ from .utils import UnionFind, ArticleList
 from .academic_query import AcademicQuery
 import streamlit as st
 from tokenizers import Tokenizer
+from .clustering.clusters import KeyphraseCount
+
 
 
 class LiteratureResearchTool:
@@ -13,31 +15,40 @@ class LiteratureResearchTool:
         self.cluster_pipeline = ClusterPipeline(cluster_config)
 
 
-    def __postprocess_clusters__(self, clusters: ClusterList) ->ClusterList:
+    def __postprocess_clusters__(self, clusters: ClusterList,query: str) ->ClusterList:
         '''
         add top-5 keyphrases to each cluster
         :param clusters:
         :return: clusters
         '''
-        def condition(x, y):
-            return td.ratcliff_obershelp(x, y) > 0.8
+        def condition(x: KeyphraseCount, y: KeyphraseCount):
+            return td.ratcliff_obershelp(x.keyphrase, y.keyphrase) > 0.8
+
+        def valid_keyphrase(x:KeyphraseCount):
+            tmp = x.keyphrase
+            return tmp is not None and tmp != '' and not tmp.isspace() and  len(tmp)!=1\
+                 and  tmp != query
 
-        def valid_keyphrase(x:str):
-            return x is not None and x != '' and not x.isspace()
 
         for cluster in clusters:
-            cluster.top_5_keyphrases = []
-            keyphrases = cluster.get_keyphrases()
-            keyphrases = list(keyphrases.keys())
+
+            keyphrases = cluster.get_keyphrases() # [kc]
             keyphrases = list(filter(valid_keyphrase,keyphrases))
             unionfind = UnionFind(keyphrases, condition)
             unionfind.union_step()
 
-            keyphrases = sorted(list(unionfind.get_unions().values()), key=len, reverse=True)[:5]  # top-5 keyphrases: list
+            tmp = unionfind.get_unions() # dict(root_id = [kc])
+            tmp = tmp.values() # [[kc]]
+            # [[kc]] -> [ new kc] -> sorted
+            tmp = [KeyphraseCount.reduce(x) for x in tmp]
+            keyphrases = sorted(tmp,key= lambda x: x.count,reverse=True)[:5]
+            keyphrases = [x.keyphrase for x in keyphrases]
 
-            for i in keyphrases:
-                tmp = '/'.join(i)
-                cluster.top_5_keyphrases.append(tmp)
+            # keyphrases = sorted(list(unionfind.get_unions().values()), key=len, reverse=True)[:5]  # top-5 keyphrases: list
+            # for i in keyphrases:
+            #     tmp = '/'.join(i)
+            #     cluster.top_5_keyphrases.append(tmp)
+            cluster.top_5_keyphrases = keyphrases
 
         return clusters
 
@@ -85,7 +96,7 @@ class LiteratureResearchTool:
             self.literature_search.ieee(query, start_year, end_year, num_papers))  # ArticleList
             abstracts = articles.getAbstracts()  # List[str]
             clusters = self.cluster_pipeline(abstracts,max_k,standardization)
-            clusters = self.__postprocess_clusters__(clusters)
+            clusters = self.__postprocess_clusters__(clusters,query)
             return clusters, articles
 
         @st.cache(hash_funcs={Tokenizer: Tokenizer.__hash__},allow_output_mutation=True)
@@ -97,7 +108,7 @@ class LiteratureResearchTool:
             self.literature_search.arxiv(query, num_papers))  # ArticleList
             abstracts = articles.getAbstracts()  # List[str]
             clusters = self.cluster_pipeline(abstracts,max_k,standardization)
-            clusters = self.__postprocess_clusters__(clusters)
+            clusters = self.__postprocess_clusters__(clusters,query)
             return clusters, articles
 
         @st.cache(hash_funcs={Tokenizer: Tokenizer.__hash__},allow_output_mutation=True)
@@ -109,7 +120,7 @@ class LiteratureResearchTool:
             self.literature_search.paper_with_code(query, num_papers))  # ArticleList
             abstracts = articles.getAbstracts()  # List[str]
             clusters = self.cluster_pipeline(abstracts,max_k,standardization)
-            clusters = self.__postprocess_clusters__(clusters)
+            clusters = self.__postprocess_clusters__(clusters,query)
             return clusters, articles
 
         if platforn_name == 'IEEE':