added beta deep research mode

app_gradio.py

@@ -480,6 +480,100 @@ def make_embedding_plot(papers_df, top_k, consensus_answer, arxiv_corpus=arxiv_c
     plt.axis('off')
     return fig
 
+
+def getsmallans(query, df):
+
+    allcontent = dr_smallans_prompt
+
+    smallauth = ''
+    linkstr = ''
+    for i, row in df.iterrows():
+        # content = f"Paper {i+1}: {row['title'].replace('\n',' ')}\n{row['abstract'].replace('\n',' ')}\n\n"
+        content = f"Paper ({row['authors'][0].split(',')[0]} et al. {row['date'].year}): {row['title']}\n{row['abstract']}\n\n"
+        smallauth = smallauth + f"({row['authors'][0].split(',')[0]} et al. {row['date'].year}) "
+        linkstr = linkstr + f"[{row['authors'][0].split(',')[0]} et al. {row['date'].year}](" + row['ADS Link'].split('](')[1] + ' \n\n'
+        allcontent = allcontent + content
+
+    # allcontent = allcontent + '\n Question: '+query
+
+    gen_client = openai_llm(temperature=0,model_name='gpt-4o-mini', openai_api_key = openai_key)
+
+    messages = [("system",allcontent,),("human", query),]
+    smallans = gen_client.invoke(messages).content
+
+    tmplnk = linkstr.split(' \n\n')
+    linkdict = {}
+    for i in range(len(tmplnk)-1):
+        linkdict[tmplnk[i].split('](')[0][1:]] = tmplnk[i]
+    
+    for key in linkdict.keys():
+        try:
+            smallans = smallans.replace(key, linkdict[key])
+            key2 = key[0:-4]+'('+key[-4:]+')'
+            smallans = smallans.replace(key2, linkdict[key])
+        except:
+            print('key not found', key)
+    
+    return smallans, smallauth, linkstr
+
+def compileinfo(query, atom_qns, atom_qn_ans, atom_qn_strs):
+
+    tmp = dr_compileinfo_prompt
+    links = ''
+    for i in range(len(atom_qn_ans)):
+        tmp = tmp + atom_qns[i] + '\n\n' + atom_qn_ans[i] + '\n\n'
+        links = links + atom_qn_strs[i] + '\n\n'
+
+    gen_client = openai_llm(temperature=0,model_name='gpt-4o-mini', openai_api_key = openai_key)
+
+    messages = [("system",tmp,),("human", query),]
+    smallans = gen_client.invoke(messages).content
+    return smallans, links
+
+def deep_research(question, top_k, ec):
+
+    full_answer = '## ' + question
+    
+    gen_client = openai_llm(temperature=0,model_name='gpt-4o-mini', openai_api_key = openai_key)
+    messages = [("system",prompt_qdec2,),("human", question),]
+    rscope_text = gen_client.invoke(messages).content
+
+    full_answer = full_answer +' \n'+ rscope_text   
+
+    rscope_messages = [("system","""In the given text, what are the main atomic questions being asked? Please answer as a concise list.""",),("human", rscope_text),]
+    rscope_qns = gen_client.invoke(rscope_messages).content
+
+    atom_qns = []
+    
+    temp = rscope_qns.split('\n')
+    for i in temp:
+        if i != '':
+            atom_qns.append(i)
+
+    atom_qn_dfs = []
+    atom_qn_ans = []
+    atom_qn_strs = []
+    for i in range(len(atom_qns)):
+        rs, small_df = ec.retrieve(atom_qns[i], top_k = top_k, return_scores=True)
+        formatted_df = ec.return_formatted_df(rs, small_df)
+        atom_qn_dfs.append(formatted_df)
+        smallans, smallauth, linkstr = getsmallans(atom_qns[i], atom_qn_dfs[i])
+
+        atom_qn_ans.append(smallans)
+        atom_qn_strs.append(linkstr)
+        full_answer = full_answer +' \n### '+atom_qns[i]
+        full_answer = full_answer +' \n'+smallans
+
+    finalans, finallinks = compileinfo(question, atom_qns, atom_qn_ans, atom_qn_strs)
+    full_answer = full_answer +' \n'+'### Summary:\n'+finalans
+    
+    full_df = pd.concat(atom_qn_dfs)
+    
+    rag_answer = {}
+    rag_answer['answer'] = full_answer
+    
+    return full_df, rag_answer
+
 def run_pathfinder(query, top_k, extra_keywords, toggles, prompt_type, rag_type, ec=ec, progress=gr.Progress()):
 
     yield None, None, None, None, None
@@ -507,21 +601,26 @@ def run_pathfinder(query, top_k, extra_keywords, toggles, prompt_type, rag_type,
             ec.hyde = True
             ec.rerank = True
     
-        progress(0.2, desc=search_text_list[np.random.choice(len(search_text_list))])
-        rs, small_df = ec.retrieve(query, top_k = top_k, return_scores=True)
-        formatted_df = ec.return_formatted_df(rs, small_df)
-        yield formatted_df, None, None, None, None
-    
-        progress(0.4, desc=gen_text_list[np.random.choice(len(gen_text_list))])
-        rag_answer = run_rag_qa(query, formatted_df, prompt_type)
-        yield formatted_df, rag_answer['answer'], None, None, None
+        if prompt_type == "Deep Research (BETA)":
+            formatted_df, rag_answer = deep_research(query, top_k = top_k, ec=ec)
+            yield formatted_df, rag_answer['answer'], None, None, None
+
+        else:
+            # progress(0.2, desc=search_text_list[np.random.choice(len(search_text_list))])
+            rs, small_df = ec.retrieve(query, top_k = top_k, return_scores=True)
+            formatted_df = ec.return_formatted_df(rs, small_df)
+            yield formatted_df, None, None, None, None
+        
+            # progress(0.4, desc=gen_text_list[np.random.choice(len(gen_text_list))])
+            rag_answer = run_rag_qa(query, formatted_df, prompt_type)
+            yield formatted_df, rag_answer['answer'], None, None, None
     
-        progress(0.6, desc="Generating consensus")
+        # progress(0.6, desc="Generating consensus")
         consensus_answer = evaluate_overall_consensus(query, [formatted_df['abstract'][i+1] for i in range(len(formatted_df))])
         consensus = '## Consensus \n'+consensus_answer.consensus + '\n\n'+consensus_answer.explanation + '\n\n > Relevance of retrieved papers to answer: %.1f' %consensus_answer.relevance_score
         yield formatted_df, rag_answer['answer'], consensus, None, None
     
-        progress(0.8, desc="Analyzing question type")
+        # progress(0.8, desc="Analyzing question type")
         question_type_gen = guess_question_type(query)
         if '<categorization>' in question_type_gen:
             question_type_gen = question_type_gen.split('<categorization>')[1]
@@ -531,7 +630,7 @@ def run_pathfinder(query, top_k, extra_keywords, toggles, prompt_type, rag_type,
         qn_type = question_type_gen
         yield formatted_df, rag_answer['answer'], consensus, qn_type, None
     
-        progress(1.0, desc="Visualizing embeddings")
+        # progress(1.0, desc="Visualizing embeddings")
         fig = make_embedding_plot(formatted_df, top_k, consensus_answer)
     
         yield formatted_df, rag_answer['answer'], consensus, qn_type, fig
@@ -551,6 +650,7 @@ def create_interface():
             with gr.Tab("pathfinder"):
                 with gr.Accordion("What is Pathfinder? / How do I use it?", open=False):
                     gr.Markdown(pathfinder_text)
+                    img2 = gr.Image("local_files/galaxy_worldmap_kiyer-min.png")
 
                 with gr.Row():
                     query = gr.Textbox(label="Ask me anything")
@@ -559,7 +659,7 @@ def create_interface():
                         top_k = gr.Slider(1, 30, step=1, value=10, label="top-k", info="Number of papers to retrieve")
                         keywords = gr.Textbox(label="Optional Keywords (comma-separated)",value="")
                         toggles = gr.CheckboxGroup(["Keywords", "Time", "Citations"], label="Weight by", info="weighting retrieved papers",value=['Keywords'])
-                        prompt_type = gr.Radio(choices=["Single-paper", "Multi-paper", "Bibliometric", "Broad but nuanced"], label="Prompt Specialization", value='Multi-paper')
+                        prompt_type = gr.Radio(choices=["Single-paper", "Multi-paper", "Bibliometric", "Broad but nuanced","Deep Research (BETA)"], label="Prompt Specialization", value='Multi-paper')
                         rag_type = gr.Radio(choices=["Semantic Search", "Semantic + HyDE", "Semantic + CoHERE", "Semantic + HyDE + CoHERE"], label="RAG Method",value='Semantic + HyDE + CoHERE')
                     with gr.Column(scale=2, min_width=300):
                         img1 = gr.Image("local_files/pathfinder_logo.png")