print fix

app.py

@@ -28,7 +28,6 @@ def run_survey(surveyor, research_keywords, max_search, num_papers):
 
 
 def survey_space(surveyor):
-    st.container().title('Auto-Research V0.1 - Automated Survey generation from research keywords')
     form = st.sidebar.form(key='survey_form')
     research_keywords = form.text_input("What would you like to research in today?")
     max_search = form.number_input("num_papers_to_search", help="maximium number of papers to glance through - defaults to 20", 
@@ -42,6 +41,7 @@ def survey_space(surveyor):
 
 
 if __name__ == '__main__':
+    st.container().title('Auto-Research V0.1 - Automated Survey generation from research keywords')
     global surveyor
     surveyor_obj = Surveyor(print_fn=st.write)
     survey_space(surveyor_obj)

src/Surveyor.py

@@ -75,11 +75,10 @@ class Surveyor:
         self.print_fn = print
         if print_fn is not None:
             self.print_fn = print_fn
-            
+
         self.torch_device = 'cpu'
         self.print_fn("\nTorch_device: " + self.torch_device)
         if torch.cuda.is_available():
-            self.print_fn("\nloading defaults for gpu")
             self.torch_device = 'cuda'
             spacy.require_gpu()
 
@@ -153,7 +152,7 @@ class Surveyor:
             if not no_save_models:
                 self.embedder.save(models_dir + "/embedder")
         else:
-            self.print_fn("\nInitializing from previously saved models at" + models_dir)
+            self.print_fn("\n-Initializing from previously saved models at" + models_dir)
             self.title_tokenizer = AutoTokenizer.from_pretrained(title_model_name)
             self.title_model = AutoModelForSeq2SeqLM.from_pretrained(models_dir + "/title_model").to(self.torch_device)
             self.title_model.eval()
@@ -235,9 +234,9 @@ class Surveyor:
 
         papers = papers_meta[:self.num_papers]
         selected_papers = papers
-        self.print_fn("\nFirst stage paper collection...")
+        self.print_fn("\n-First stage paper collection...")
         ids_none, papers, cites = self.fetch_papers(dump_dir, img_dir, papers, pdf_dir, tab_dir, txt_dir)
-        self.print_fn("\nFirst stage paper collection complete, papers collected: \n" + ', '.join([p['id'] for p in papers]))
+        self.print_fn("\n-First stage paper collection complete, papers collected: \n" + ', '.join([p['id'] for p in papers]))
         new_papers = papers_meta[self.num_papers : self.num_papers + len(ids_none)]
         _ = self.get_freq_cited(cites)
         '''
@@ -248,16 +247,16 @@ class Surveyor:
         new_papers.extend(new_searched_papers)
         '''
         selected_papers.extend(new_papers)
-        self.print_fn("\nSecond stage paper collection...")
+        self.print_fn("\n-Second stage paper collection...")
         _, new_papers, _ = self.fetch_papers(dump_dir, img_dir, new_papers, pdf_dir, tab_dir, txt_dir, repeat=True)
-        self.print_fn("\nSecond stage paper collection complete, new papers collected: \n" + ', '.join([p['id'] for p in new_papers]))
+        self.print_fn("\n-Second stage paper collection complete, new papers collected: \n" + ', '.join([p['id'] for p in new_papers]))
         papers.extend(new_papers)
 
         joblib.dump(papers, dump_dir + 'papers_extracted_pdf_route.dmp')
         copy_tree(img_dir, dump_dir + os.path.basename(img_dir))
         copy_tree(tab_dir, dump_dir + os.path.basename(tab_dir))
 
-        self.print_fn("\nExtracting section-wise highlights.. ")
+        self.print_fn("\n-Extracting section-wise highlights.. ")
         papers = self.extract_highlights(papers)
 
         return papers, selected_papers
@@ -270,7 +269,7 @@ class Surveyor:
             [cites_list.append(val) for val in v]
         cite_freqs = {cite: cites_list.count(cite) for cite in set(cites_list)}
         sorted_cites = dict(sorted(cite_freqs.items(), key=lambda item: item[1], reverse=True)[:5])
-        self.print_fn("\nThe most cited paper ids are:\n" + str(sorted_cites))
+        self.print_fn("\n-The most cited paper ids are:\n" + str(sorted_cites))
 
         return sorted_cites.keys()
 
@@ -333,11 +332,11 @@ class Surveyor:
     def build_doc(self, research_sections, papers, query=None, filename='survey.txt'):
 
         import arxiv2bib
-        self.print_fn("\nbuilding bibliography entries.. ")
+        self.print_fn("\n-building bibliography entries.. ")
         bibentries = arxiv2bib.arxiv2bib([p['id'] for p in papers])
         bibentries = [r.bibtex() for r in bibentries]
 
-        self.print_fn("\nbuilding final survey file .. at "+ filename)
+        self.print_fn("\n-building final survey file .. at "+ filename)
         file = open(filename, 'w+')
         if query is None:
             query = 'Internal(existing) research'
@@ -768,7 +767,7 @@ class Surveyor:
             res = self.model(" ".join([l.lower() for l in lines]), ratio=0.5, )
         res_doc = self.nlp(res)
         res_lines = set([str(sent) for sent in list(res_doc.sents)])
-        # self.print_fn("\n".join(res_sents))
+        # self.print_fn("\n-".join(res_sents))
         with torch.no_grad():
             keywords = self.kw_model.extract_keywords(str(" ".join([l.lower() for l in lines])), stop_words='english')
             keyphrases = self.kw_model.extract_keywords(str(" ".join([l.lower() for l in lines])),
@@ -794,14 +793,14 @@ class Surveyor:
         return papers
 
     def extract_structure(self, papers, pdf_dir, txt_dir, img_dir, dump_dir, tab_dir, tables=False):
-        self.print_fn("\nextracting sections.. ")
+        self.print_fn("\n-extracting sections.. ")
         papers, ids_none = self.extract_parts(papers, txt_dir, dump_dir)
 
-        self.print_fn("\nextracting images.. for future correlation use-cases ")
+        self.print_fn("\n-extracting images.. for future correlation use-cases ")
         papers = self.extract_images(papers, pdf_dir, img_dir)
 
         if tables:
-            self.print_fn("\nextracting tables.. for future correlation use-cases ")
+            self.print_fn("\n-extracting tables.. for future correlation use-cases ")
             papers = self.extract_tables(papers, pdf_dir, tab_dir)
 
         return papers, ids_none
@@ -1057,7 +1056,7 @@ class Surveyor:
         for p in papers:
             if p['id'] == pid:
                 return p
-        self.print_fn("\npaper not found by file, \nfile: "+file+"\nall papers: "+', '.join([p['id'] for p in papers]))
+        self.print_fn("\n-paper not found by file, \nfile: "+file+"\nall papers: "+', '.join([p['id'] for p in papers]))
 
 
     def alpha_length(self, s):
@@ -1191,7 +1190,7 @@ class Surveyor:
             else:
                 discarded_ids.append(urlparse(result.entry_id).path.split('/')[-1].split('v')[0])
 
-        self.print_fn("\nPapers discarded due to id error [arxiv api bug: #74] :\n" + str(discarded_ids))
+        self.print_fn("\n-Papers discarded due to id error [arxiv api bug: #74] :\n" + str(discarded_ids))
 
         return results, searched_papers
 
@@ -1199,7 +1198,7 @@ class Surveyor:
         import arxiv
         from urllib.parse import urlparse
         ids = [p['id'] for p in papers]
-        self.print_fn("\ndownloading below selected papers: ")
+        self.print_fn("\n-downloading below selected papers: ")
         self.print_fn(ids)
         # asert(False)
         papers_filtered = arxiv.Search(id_list=ids).get()
@@ -1242,7 +1241,7 @@ class Surveyor:
 
 
         cites = internal_citations.citation_list_parallel(N=multiprocessing.cpu_count(), directory=txt_dir)
-        self.print_fn("\ncitation-network: ")
+        self.print_fn("\n-citation-network: ")
         self.print_fn(cites)
 
         for p in papers:
@@ -1354,10 +1353,10 @@ class Surveyor:
         if not num_papers:
             num_papers = self.DEFAULTS['num_papers']
         # arxiv api relevance search and data preparation
-        self.print_fn("\nsearching arXiv for top 100 papers.. ")
+        self.print_fn("\n-searching arXiv for top 100 papers.. ")
         results, searched_papers = self.search(query, max_search=max_search)
         joblib.dump(searched_papers, self.dump_dir + 'papers_metadata.dmp')
-        self.print_fn("\nfound " + str(len(searched_papers)) + " papers")
+        self.print_fn("\n-found " + str(len(searched_papers)) + " papers")
 
         # paper selection by scibert vector embedding relevance scores
         # papers_selected = select_papers(searched_papers, query, num_papers=num_papers)
@@ -1370,23 +1369,23 @@ class Surveyor:
 
         joblib.dump(papers_highlighted, self.dump_dir + 'papers_highlighted.dmp')
 
-        self.print_fn("\nStandardizing known section headings per paper.. ")
+        self.print_fn("\n-Standardizing known section headings per paper.. ")
         papers_standardized = self.standardize_headings(papers_highlighted)
         joblib.dump(papers_standardized, self.dump_dir + 'papers_standardized.dmp')
 
-        self.print_fn("\nBuilding paper-wise corpus.. ")
+        self.print_fn("\n-Building paper-wise corpus.. ")
         corpus = self.build_corpus(papers_highlighted, searched_papers)
         joblib.dump(corpus, self.dump_dir + 'corpus.dmp')
 
-        self.print_fn("\nBuilding section-wise corpus.. ")
+        self.print_fn("\n-Building section-wise corpus.. ")
         corpus_sectionwise = self.build_corpus_sectionwise(papers_standardized)
         joblib.dump(corpus_sectionwise, self.dump_dir + 'corpus_sectionwise.dmp')
 
-        self.print_fn("\nBuilding basic research highlights.. ")
+        self.print_fn("\n-Building basic research highlights.. ")
         research_blocks = self.build_basic_blocks(corpus_sectionwise, corpus)
         joblib.dump(research_blocks, self.dump_dir + 'research_blocks.dmp')
 
-        self.print_fn("\nReducing corpus to lines.. ")
+        self.print_fn("\n-Reducing corpus to lines.. ")
         corpus_lines = self.get_corpus_lines(corpus)
         joblib.dump(corpus_lines, self.dump_dir + 'corpus_lines.dmp')
 
@@ -1420,7 +1419,7 @@ class Surveyor:
         '''
         # self.print_fn("corpus types:"+ str(np.unique([type(txt) for k,txt in corpus.items()])))
 
-        self.print_fn("\nBuilding abstract.. ")
+        self.print_fn("\n-Building abstract.. ")
         abstract_block = self.get_abstract(corpus_lines, corpus_sectionwise, research_blocks)
         joblib.dump(abstract_block, self.dump_dir + 'abstract_block.dmp')
         '''
@@ -1429,7 +1428,7 @@ class Surveyor:
         self.print_fn(abstract_block)
         '''
 
-        self.print_fn("\nBuilding introduction.. ")
+        self.print_fn("\n-Building introduction.. ")
         intro_block = self.get_intro(corpus_sectionwise, research_blocks)
         joblib.dump(intro_block, self.dump_dir + 'intro_block.dmp')
         '''
@@ -1437,7 +1436,7 @@ class Surveyor:
         self.print_fn("intro_block:")
         self.print_fn(intro_block)
         '''
-        self.print_fn("\nBuilding custom sections.. ")
+        self.print_fn("\n-Building custom sections.. ")
         clustered_sections, clustered_sentences = self.get_clusters(papers_standardized, searched_papers)
         joblib.dump(clustered_sections, self.dump_dir + 'clustered_sections.dmp')
         joblib.dump(clustered_sentences, self.dump_dir + 'clustered_sentences.dmp')
@@ -1455,7 +1454,7 @@ class Surveyor:
         clustered_sections['introduction'] = intro_block
         joblib.dump(clustered_sections, self.dump_dir + 'research_sections.dmp')
 
-        self.print_fn("\nBuilding conclusion.. ")
+        self.print_fn("\n-Building conclusion.. ")
         conclusion_block = self.get_conclusion(clustered_sections)
         joblib.dump(conclusion_block, self.dump_dir + 'conclusion_block.dmp')
         clustered_sections['conclusion'] = conclusion_block
@@ -1472,7 +1471,7 @@ class Surveyor:
         shutil.copy(self.dump_dir + survey_file, survey_file)
         assert (os.path.exists(survey_file))
         output_zip = self.zip_outputs(self.dump_dir, query)
-        self.print_fn("\nSurvey complete.. \nSurvey file path :" + os.path.abspath(
+        self.print_fn("\n-Survey complete.. \nSurvey file path :" + os.path.abspath(
             survey_file) + "\nAll outputs zip path :" + os.path.abspath(self.dump_dir + output_zip))
 
         return os.path.abspath(self.dump_dir + output_zip), os.path.abspath(survey_file)