Update utils.py

utils.py

@@ -1,168 +1,168 @@
-import os
-import pdfplumber
-import pandas as pd
-import re
-from transformers import pipeline
-import pandas as pd
-
-
-# Load the Hugging Face token from the environment
-token = os.getenv("HUGGINGFACE_TOKEN")
-
-# Initialize the pipeline with the token
-pipe = pipeline("text-generation", model="meta-llama/Llama-2-7b-chat-hf", use_auth_token=token)
-
-
-# Function to extract text from PDF using pdfplumber
-def extract_text_from_pdf(pdf_file_path):
-    text = ""
-    with pdfplumber.open(pdf_file_path) as pdf:
-        for page in pdf.pages:
-            text += page.extract_text()  # Extract text from each page
-    return text
-
-# Define the function to structure the model output into required fields
-def structure_summary_output(text):
-    global pipe
-    prompt = (
-        f"Please summarize the following information from the academic paper:\n"
-        f"1. Context: Specify whether the study is focused on a specific industry, task or a broader, conceptual scope.\n"
-        f"2. Research Question and Findings: Identify the main research question and summarise the key findings.\n"
-        f"3. Theme of Research:\n"
-        f"   - Human vs. AI: Highlight any comparative advantages between humans and AI.\n"
-        f"   - Human + AI Collaboration: Indicate the type of collaboration discussed.\n"
-        f"   Note that the output for this field can only be either 'Human vs. AI' or 'Human + AI Collaboration'\n"
-        f"4. Method: Classify the study method as one of the following:\n"
-        f"   - Conceptual/Case Study\n"
-        f"   - Modeling: Either Stylized Modeling or Operations Research (OR) Model\n"
-        f"   - Empirical Study: Lab/Field Experiment or Secondary Data Analysis\n"
-        f"   Note that the output for this field can only be either 'Conceptual/Case Study' or 'Modeling' or 'Empirical Study'\n"
-        f"5. Contribution: Identify the primary contribution (theoretical, managerial, or methodological).\n"
-        f"6. Future Potential and Limitations: Summarize future research directions or limitations.\n\n"
-        f"Paper content:\n{text}\n\n"
-        f"Respond with the answers formatted in the following structure:\n"
-        f"- Context:\n- Research Question and Findings:\n- Theme of Research:\n- Method:\n"
-        f"- Contribution:\n- Future Potential and Limitations:\n"
-    )
-    output = pipe(prompt, max_new_tokens=512)
-    
-    # Extract structured text from model output
-    summary_text = output[0]["generated_text"]
-
-    # Split the text into structured sections
-    sections = {
-        "Context": "",
-        "Research Question and Findings": "",
-        "Theme of Research": "",
-        "Method": "",
-        "Contribution": "",
-        "Future Potential and Limitations": ""
-    }
-
-    # Regular expression to match each section header and text that follows
-    for section in sections.keys():
-        match = re.search(rf"- {section}:(.*?)(?=- [A-Z]|$)", summary_text, re.DOTALL)
-        if match:
-            sections[section] = match.group(1).strip()
-
-    # Return the extracted sections
-    return sections
-
-# Process each PDF and summarize
-def process_all_papers(pdf_directory, reference):
-    paper_summaries = []
-    
-    for paper_index in range(1, 33):
-        pdf_file_path = os.path.join(pdf_directory, f"{paper_index}.pdf")
-        
-        if os.path.exists(pdf_file_path):
-            # Extract text from the PDF using pdfplumber
-            text = extract_text_from_pdf(pdf_file_path)
-            
-            # Get structured summary of the paper
-            summary = structure_summary_output(text)
-            
-            # Find citation (assuming the file name contains the citation or we can get it from the reference dataframe)
-            citation = reference.loc[reference['Index'] == paper_index, 'Citation'].values[0]
-            
-            # Store the information in the dictionary
-            paper_summary = {
-                "ID": paper_index,
-                "Citation": citation,
-                "Context": summary["Context"],
-                "Research Question and Findings": summary["Research Question and Findings"],
-                "Theme of Research": summary["Theme of Research"],
-                "Method": summary["Method"],
-                "Contribution": summary["Contribution"],
-                "Future Potential and Limitations": summary["Future Potential and Limitations"]
-            }
-            
-            paper_summaries.append(paper_summary)
-    
-    return paper_summaries
-
-
-
-def interpret_search_criteria(user_input):
-    """
-    Determines search criteria based on user input text.
-    """
-    theme = ""
-    method = ""
-    
-    if "human vs ai" in user_input.lower():
-        theme = "Human vs. AI"
-    elif "human + ai" in user_input.lower() or "collaboration" in user_input.lower():
-        theme = "Human + AI Collaboration"
-    
-    if "empirical" in user_input.lower():
-        method = "Empirical Study"
-    elif "conceptual" in user_input.lower() or "case study" in user_input.lower():
-        method = "Conceptual/Case Study"
-    elif "modeling" in user_input.lower():
-        method = "Modeling"
-
-    return {"Theme": theme, "Method": method}
-
-
-def search_and_summarize_with_llm(paper_summaries, user_input):
-    """
-    Retrieves relevant articles based on interpreted search criteria and generates a cohesive summary.
-    """
-    global pipe
-    # Interpret the search criteria from the user input
-    search_criteria = interpret_search_criteria(user_input)
-    
-    # Collect all relevant summaries and citations
-    relevant_summaries = []
-    citation_list = []
-    
-    for summary in paper_summaries:
-        # paper_id, citation, context, rqf, theme, method, contribution, future = summary["ID"], 
-        
-        # Check if this summary matches all search criteria
-        if search_criteria['Theme'].lower() in summary["Theme of Research"].lower() and search_criteria['Method'].lower() in summary["Method"].lower():
-            # Append the full information of this paper summary
-            relevant_summaries.append(
-            f"Paper ID: {summary['ID']}\nCitation: {summary['Citation']}\nContext: {summary['Context']}\n"
-            f"Research Question and Findings: {summary['Research Question and Findings']}\nTheme of Research: {summary['Theme of Research']}\n"
-            f"Method: {summary['Method']}\nContribution: {summary['Contribution']}\n"
-            f"Future Potential and Limitations: {summary['Future Potential and Limitations']}\n\n"
-            )
-
-            citation_list.append(summary["Citation"])
-    
-    # Generate a cohesive summary using the LLM
-    combined_text = "\n".join(relevant_summaries)
-
-    prompt = (
-        f"Based on the following research summaries related to {search_criteria['Theme']} and {search_criteria['Method']},"
-        f"provide a cohesive summary discussing connections, common themes, trends, and future directions:\n\n{combined_text}"
-    )
-    summary_output = pipe(prompt, max_new_tokens=512)
-    cohesive_summary = summary_output[0]["generated_text"]
-    
-    # Format the citation list
-    formatted_citations = "Citations:\n" + "\n".join(f"- {citation}" for citation in citation_list)
-    
-    return cohesive_summary, formatted_citations
+import os
+import pdfplumber
+import pandas as pd
+import re
+from transformers import pipeline
+import pandas as pd
+
+
+# Load the Hugging Face token from the environment
+token = os.getenv("HUGGINGFACE_TOKEN")
+
+# Initialize the pipeline with the token
+pipe = pipeline("text-generation", model="meta-llama/Llama-2-7b-chat-hf", use_auth_token=token, device_map="auto")
+
+
+# Function to extract text from PDF using pdfplumber
+def extract_text_from_pdf(pdf_file_path):
+    text = ""
+    with pdfplumber.open(pdf_file_path) as pdf:
+        for page in pdf.pages:
+            text += page.extract_text()  # Extract text from each page
+    return text
+
+# Define the function to structure the model output into required fields
+def structure_summary_output(text):
+    global pipe
+    prompt = (
+        f"Please summarize the following information from the academic paper:\n"
+        f"1. Context: Specify whether the study is focused on a specific industry, task or a broader, conceptual scope.\n"
+        f"2. Research Question and Findings: Identify the main research question and summarise the key findings.\n"
+        f"3. Theme of Research:\n"
+        f"   - Human vs. AI: Highlight any comparative advantages between humans and AI.\n"
+        f"   - Human + AI Collaboration: Indicate the type of collaboration discussed.\n"
+        f"   Note that the output for this field can only be either 'Human vs. AI' or 'Human + AI Collaboration'\n"
+        f"4. Method: Classify the study method as one of the following:\n"
+        f"   - Conceptual/Case Study\n"
+        f"   - Modeling: Either Stylized Modeling or Operations Research (OR) Model\n"
+        f"   - Empirical Study: Lab/Field Experiment or Secondary Data Analysis\n"
+        f"   Note that the output for this field can only be either 'Conceptual/Case Study' or 'Modeling' or 'Empirical Study'\n"
+        f"5. Contribution: Identify the primary contribution (theoretical, managerial, or methodological).\n"
+        f"6. Future Potential and Limitations: Summarize future research directions or limitations.\n\n"
+        f"Paper content:\n{text}\n\n"
+        f"Respond with the answers formatted in the following structure:\n"
+        f"- Context:\n- Research Question and Findings:\n- Theme of Research:\n- Method:\n"
+        f"- Contribution:\n- Future Potential and Limitations:\n"
+    )
+    output = pipe(prompt, max_new_tokens=512)
+    
+    # Extract structured text from model output
+    summary_text = output[0]["generated_text"]
+
+    # Split the text into structured sections
+    sections = {
+        "Context": "",
+        "Research Question and Findings": "",
+        "Theme of Research": "",
+        "Method": "",
+        "Contribution": "",
+        "Future Potential and Limitations": ""
+    }
+
+    # Regular expression to match each section header and text that follows
+    for section in sections.keys():
+        match = re.search(rf"- {section}:(.*?)(?=- [A-Z]|$)", summary_text, re.DOTALL)
+        if match:
+            sections[section] = match.group(1).strip()
+
+    # Return the extracted sections
+    return sections
+
+# Process each PDF and summarize
+def process_all_papers(pdf_directory, reference):
+    paper_summaries = []
+    
+    for paper_index in range(1, 33):
+        pdf_file_path = os.path.join(pdf_directory, f"{paper_index}.pdf")
+        
+        if os.path.exists(pdf_file_path):
+            # Extract text from the PDF using pdfplumber
+            text = extract_text_from_pdf(pdf_file_path)
+            
+            # Get structured summary of the paper
+            summary = structure_summary_output(text)
+            
+            # Find citation (assuming the file name contains the citation or we can get it from the reference dataframe)
+            citation = reference.loc[reference['Index'] == paper_index, 'Citation'].values[0]
+            
+            # Store the information in the dictionary
+            paper_summary = {
+                "ID": paper_index,
+                "Citation": citation,
+                "Context": summary["Context"],
+                "Research Question and Findings": summary["Research Question and Findings"],
+                "Theme of Research": summary["Theme of Research"],
+                "Method": summary["Method"],
+                "Contribution": summary["Contribution"],
+                "Future Potential and Limitations": summary["Future Potential and Limitations"]
+            }
+            
+            paper_summaries.append(paper_summary)
+    
+    return paper_summaries
+
+
+
+def interpret_search_criteria(user_input):
+    """
+    Determines search criteria based on user input text.
+    """
+    theme = ""
+    method = ""
+    
+    if "human vs ai" in user_input.lower():
+        theme = "Human vs. AI"
+    elif "human + ai" in user_input.lower() or "collaboration" in user_input.lower():
+        theme = "Human + AI Collaboration"
+    
+    if "empirical" in user_input.lower():
+        method = "Empirical Study"
+    elif "conceptual" in user_input.lower() or "case study" in user_input.lower():
+        method = "Conceptual/Case Study"
+    elif "modeling" in user_input.lower():
+        method = "Modeling"
+
+    return {"Theme": theme, "Method": method}
+
+
+def search_and_summarize_with_llm(paper_summaries, user_input):
+    """
+    Retrieves relevant articles based on interpreted search criteria and generates a cohesive summary.
+    """
+    global pipe
+    # Interpret the search criteria from the user input
+    search_criteria = interpret_search_criteria(user_input)
+    
+    # Collect all relevant summaries and citations
+    relevant_summaries = []
+    citation_list = []
+    
+    for summary in paper_summaries:
+        # paper_id, citation, context, rqf, theme, method, contribution, future = summary["ID"], 
+        
+        # Check if this summary matches all search criteria
+        if search_criteria['Theme'].lower() in summary["Theme of Research"].lower() and search_criteria['Method'].lower() in summary["Method"].lower():
+            # Append the full information of this paper summary
+            relevant_summaries.append(
+            f"Paper ID: {summary['ID']}\nCitation: {summary['Citation']}\nContext: {summary['Context']}\n"
+            f"Research Question and Findings: {summary['Research Question and Findings']}\nTheme of Research: {summary['Theme of Research']}\n"
+            f"Method: {summary['Method']}\nContribution: {summary['Contribution']}\n"
+            f"Future Potential and Limitations: {summary['Future Potential and Limitations']}\n\n"
+            )
+
+            citation_list.append(summary["Citation"])
+    
+    # Generate a cohesive summary using the LLM
+    combined_text = "\n".join(relevant_summaries)
+
+    prompt = (
+        f"Based on the following research summaries related to {search_criteria['Theme']} and {search_criteria['Method']},"
+        f"provide a cohesive summary discussing connections, common themes, trends, and future directions:\n\n{combined_text}"
+    )
+    summary_output = pipe(prompt, max_new_tokens=512)
+    cohesive_summary = summary_output[0]["generated_text"]
+    
+    # Format the citation list
+    formatted_citations = "Citations:\n" + "\n".join(f"- {citation}" for citation in citation_list)
+    
+    return cohesive_summary, formatted_citations