Upload 20 files

utils/api_utils.py

@@ -0,0 +1,225 @@
+import asyncio
+
+from loguru import logger
+from openai import OpenAI
+from functools import partial
+from typing import Callable
+
+CLIENTS = {
+    "glm-4-plus": {
+        "api_key": "3a3c9f497e34a0514da974a4ccb886e.urkW20Nz3aklp3Mk",
+        "base_url": "https://open.bigmodel.cn/api/paas/v4",
+    },
+    "glm-4": {
+        "api_key": "3a3c9f497e34a0514da974a4ccb886e.urkW20Nz3aklp3Mk",
+        "base_url": "https://open.bigmodel.cn/api/paas/v4",
+    },
+    "glm-4-airx": {
+        "api_key": "3a3c9f497e34a0514da974a4ccb886e.urkW20Nz3aklp3Mk",
+        "base_url": "https://open.bigmodel.cn/api/paas/v4",
+    },
+    "glm-4-flash": {
+        "api_key": "4541d6f6421cb131eba8c8390d956237.V1W9TkfzupwCQmOU",
+        "base_url": "https://open.bigmodel.cn/api/paas/v4",
+    },
+    "gpt-4o-mini": {
+        "api_key": "sk-RqmH8qL4MUxDlvJtE6045a9931474629B11015Df08D3C915",
+        "base_url": "https://api.qqslyx.com/v1",
+    },
+    "deepseek-chat": {
+        "api_key": "sk-253d4686221e41618f239b064ada3d21",
+        "base_url": "https://api.deepseek.com/v1"
+    },
+    "deepseek/deepseek-chat-v3-0324:free": {
+        "api_key": "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",
+        "base_url": "https://openrouter.ai/api/v1"
+    }    
+}
+
+
+def get_chat_func(model_names: list[str]):
+    """
+    Get a list of chat functions for the specified model names.
+    
+    Args:
+        model_names (list[str]): A list of model names.
+
+    Returns:
+        list[Callable]: A list of chat functions.
+    """
+    chat_funcs = []
+    for model_name in model_names:
+        if model_name not in list(CLIENTS.keys()):
+            continue
+        chat_funcs.append(partial(chat_completion, model_name=model_name))
+    return chat_funcs
+
+
+async def chat_completion(prompt: str, model_name: str) -> str:
+    """
+    Perform a chat completion using the specified model.
+    
+    Args:
+        prompt (str): The prompt to send to the model.
+        model_name (str): The name of the model to use.
+        client (OpenAI, optional): The OpenAI client to use. Defaults to None.
+
+    Returns:
+        str: The response from the model.
+    
+    """
+    assert model_name in list(CLIENTS.keys()), f"Model {model_name} not found"
+    
+    API_KEY = CLIENTS[model_name]["api_key"]
+    BASE_URL = CLIENTS[model_name]["base_url"]
+    
+    client = OpenAI(api_key=API_KEY, base_url=BASE_URL)
+    completion = client.chat.completions.create(
+        model=model_name,
+        messages=[
+            {"role": "user", "content": prompt}
+        ]
+    )
+    return completion
+
+
+async def retry_operation(func, task, max_retries=5, delay=0.5, *args, **kwargs):
+    """
+    Retry an operation asynchronously with exponential backoff.
+    
+    Args:
+        func (Callable): The function to be retried.
+        task (Task): The task object to update the status.
+        max_retries (int, optional): The maximum number of retries. Defaults to 5.
+        delay (float, optional): The initial delay between retries. Defaults to 0.5.
+        *args: Additional positional arguments to pass to the function.
+        **kwargs: Additional keyword arguments to pass to the function.
+        
+    Returns:
+        Any: The result of the operation.
+    
+    """
+    retries = 0
+    exceptions = []
+    while retries < max_retries:
+        # return await func(*args, **kwargs)
+        try:
+            return await func(*args, **kwargs), None
+        except Exception as e:
+            exceptions.append(f"retry {retries}: {e}")
+            retries += 1
+            logger.error(e)
+            await asyncio.sleep(delay * retries)
+            continue
+    return None, "\n".join(exceptions)
+
+
+async def chat_completion_multiple_models(
+    prompt: str, 
+    model_names: list[str] = [],
+    chat_funcs: list[Callable] = []
+):
+    """
+    Perform a chat completion using multiple models asynchronously.
+    
+    Args:
+        prompt (str): The prompt to send to the models.
+        model_names (list[str], optional): A list of model names. Defaults to [].
+        chat_funcs (list[Callable], optional): A list of chat functions. Defaults to [].
+
+    Returns:
+        list[Any]: A list of results from the chat completions.
+    
+    """
+    if not chat_funcs or len(chat_funcs) == 0:
+        chat_funcs = get_chat_func(model_names)
+    return await asyncio.gather(
+        *(chat_func(prompt=prompt) 
+          for chat_func in chat_funcs)
+    )
+
+
+async def func_wrap_multiple_models(
+    wrap_func: Callable,
+    model_names: list[str] = [],
+    chat_funcs: list[Callable] = [],
+    model_weights: list[float] = [],
+    *args,
+):
+    """
+    Wrap a function to be executed asynchronously with multiple models.
+    
+    Args:
+        func (Callable): The function to be wrapped.
+        model_names (list[str], optional): A list of model names. Defaults to [].
+        chat_funcs (list[Callable], optional): A list of chat functions. Defaults to [].
+        model_weights (list[float], optional): A list of model weights. Defaults to [].
+        *args: Additional positional arguments to pass to the function.
+
+    Returns:
+        list[Any]: A list of results from the function.
+    
+    """
+    if not chat_funcs or len(chat_funcs) == 0:
+        chat_funcs = get_chat_func(model_names)
+    if not model_weights:
+        model_weights = [1.0 for _ in range(len(chat_funcs))]
+    assert len(chat_funcs) == len(model_weights), \
+        "model_weights must be same length as chat_funcs"
+    
+    return await asyncio.gather(
+        *(wrap_func(*args, chat_func=chat_func)
+          for chat_func in chat_funcs)
+    )
+
+
+async def compare_chat_chocies(
+    contents: list[str],
+    model_names: list[Callable] = [],
+    chat_funcs: list[Callable] = [],
+    model_weights: list[float] = []
+):
+    if not chat_funcs or len(chat_funcs) == 0:
+        chat_funcs = get_chat_func(model_names)
+    if not model_weights:
+        model_weights = [1.0 for _ in range(len(chat_funcs))]
+    assert len(chat_funcs) == len(model_weights), \
+        "model_weights must be same length as chat_funcs"
+    
+    prompts = []
+    eval_chat_funcs = []
+    for i in range(len(contents)):
+        prompt = f"""
+        You are provided with {len(contents)-1} choices, and you are asked to rank them based on the quality and relevance.
+        Rank 1 is the best.
+        Just Output Index and Corresponding Rank in format Index:Rank. 
+        Just Number, no text. For example: "0:1" is correct, "Index 0:1" and "0: 1" are wrong. 
+        One Line for Each Rank.
+        Just output like "Index:Rank\nIndex:Rank\nIndex:Rank\n"
+        No other output is allowed.
+        
+        """
+        for j, content in enumerate(contents):
+            if i == j: # skip self evaluation
+                continue
+            else:
+                prompt += f"""
+                Index {j}: 
+                {content}
+                ----------
+                
+                """
+        prompts.append(prompt)
+        eval_chat_funcs.append(chat_funcs[i])
+    compares = await asyncio.gather(
+        *(chat_func(prompt=prompt)
+          for prompt, chat_func in zip(prompts, eval_chat_funcs))
+    )
+    
+    rank_scores = {i: 0 for i in range(len(contents))}
+    for i, comp in enumerate(compares):
+        for rank in comp.choices[0].message.content.strip().split("\n"):
+            index, rank = rank.split(":")
+            rank_scores[int(index)] += int(rank) * model_weights[i]
+    return rank_scores
+    

utils/arxiv_utils.py

@@ -0,0 +1,738 @@
+import re
+import asyncio
+import aiohttp
+
+from minio import Minio
+from loguru import logger
+from lxml import etree
+
+from utils.api_utils import (
+    retry_operation,
+    get_chat_func,
+    compare_chat_chocies
+)
+from utils.r2_utils import (
+    get_client,
+    get_file_from_minio,
+    get_dataframe_from_minio,
+    upload_text_to_minio,
+    upload_task_json_to_minio,
+)
+from utils.common_utils import escape_csv_field
+from utils.paper_utils import (
+    process_papers,
+    generate_subheadings,
+    assign_subheadings_to_summaries,
+    create_paragraphs_by_subheading,
+    enhance_language_readability,
+    translate_to_chinese_before_references
+)
+from entities.task import ArxivTask
+
+
+BUCKET_NAME = "ai-scientist"
+
+
+# =================================
+# Function Groups: Pipeline for Arxiv
+#  
+# 1. pipeline
+# 2. single model chat
+# =================================
+
+async def arxiv_pipeline(
+    task: ArxivTask,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Arxiv pipeline
+    
+    Args:
+        task: ArxivTask, the task object
+        client: Minio client, the Minio client object
+        max_retries: int, the maximum number of retries
+        delay: float, the delay between retries
+        
+    Returns:
+        None
+    
+    """
+    if client is None:
+        client = get_client()
+        
+    customer_name = task.customer_name
+    uuid = task.uuid
+    model_names = task.model_names
+    
+    task.status_string["overall"] = "processing"
+    
+    await asyncio.gather(
+        *(process_arxiv_single_chat(
+            task, model_name, client, max_retries, delay
+        ) for model_name in model_names)
+    )
+    
+    # if compare between models
+    # at least 3 models should be selected
+    logger.info("Check Compare...")
+    if task.do_compare and len(task.model_names) >= 3:
+        if task.status.get("compare", 0) == 0:
+            contents = await asyncio.gather(
+                *(get_file_from_minio(
+                    bucket_name=BUCKET_NAME,
+                    object_name=f"{customer_name}/{uuid}/{model_name}/review_paper.txt",
+                ) for model_name in model_names)
+            )
+            contents = [c.data.decode("utf-8") for c in contents]
+            task.status_string["overall"] = "Start Compare"
+            
+            rank_scores = await compare_chat_chocies(
+                contents=contents,
+                model_names=model_names
+            )
+            best_content = contents[min(rank_scores, key=rank_scores.get)]
+            await upload_text_to_minio(
+                bucket_name=BUCKET_NAME,
+                object_name=f"{customer_name}/{uuid}/compared_review_paper.txt",
+                file_content=best_content
+            )
+            task.status_string["overall"] = "Finished"
+            await upload_task_json_to_minio(task, client)
+        else:
+            task.status_string["overall"] = "Finished"
+            await upload_task_json_to_minio(task, client)
+    else:
+        task.status_string["overall"] = "Finished"
+        await upload_task_json_to_minio(task, client)
+
+async def process_arxiv_single_chat(
+    task: ArxivTask,
+    model_name: str,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process Arxiv Task
+    
+    Args:
+        task: ArxivTask, the task object
+        model_name: str, the model name
+        client: Minio client, the Minio client object
+        max_retries: int, the maximum number of retries
+        delay: float, the delay between retries
+        
+    Returns:
+        None
+    
+    """
+    
+    # get minio client
+    if client is None:
+        client = get_client()
+        
+    # add status for <model_name>
+    if model_name not in task.status.keys():
+        task.status[model_name] = 0
+    
+    # set task status string
+    task.status_string["overall"] = "processing"
+    
+    process_steps = {
+        0: process_arxiv_fetch_arxiv_data,
+        1: process_arxiv_process_papers,
+        2: process_arxiv_generate_subheadings,
+        3: process_arxiv_assign_subheadings_to_summaries,
+        4: process_arxiv_create_paragraphs_by_subheading,
+        5: process_arxiv_enhance_language_readability,
+        6: process_arxiv_translate
+    }
+    
+    state_description = {
+        0: "Finished fetching data.",
+        1: "Finished paper processing.",
+        2: "Finished subheading generation.",
+        3: "Finished subheading assignment.",
+        4: "Finished paragraph generation.",
+        5: "Finished review language readability enhancement.",
+        6: "Finished review translation.",
+    }
+    
+    # Execute Phase
+    current_state = task.status[model_name]
+    for state in range(current_state, len(process_steps.keys())):
+        await process_steps[state](
+            task=task,
+            model_name=model_name,
+            save_name=model_name,
+            prev_name=model_name,
+            client=client,
+            max_retries=max_retries, delay=delay
+        )
+        task.status_string[model_name] = state_description[state]
+        task.status[model_name] = state + 1
+        await upload_task_json_to_minio(task, client)
+    
+    task.status_string[model_name] = "Finished."
+    await upload_task_json_to_minio(task, client)
+        
+
+# =================================
+# Function Groups: process_arxiv_* 
+# 1. _fetch_arxiv_data
+# 2. _process_papers
+# 3. _create_review_paper
+# =================================
+
+async def process_arxiv_fetch_arxiv_data(
+    task: ArxivTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Fetch Arxiv Data
+    
+    Args:
+        task: ArxivTask, the task object
+        model_name: str, the model name
+        save_name: str, the save name
+        prev_name: str, the previous name
+        client: Minio client, the Minio client object
+        max_retries: int, the maximum number of retries
+        delay: float, the delay between retries
+        
+    Returns:
+        None
+    
+    """
+
+    if client is None:
+        client = get_client()
+        
+    if prev_name is not None:
+        logger.warning("For first step, prev_model_name is not used.")
+        
+    query = task.query
+    customer_name = task.customer_name
+    uuid = task.uuid
+    start_date = task.start_date
+    end_date = task.end_date
+    total_page = task.size / 50
+    
+    results, exceptions = await retry_operation(
+        get_arxiv_df, task,
+        model_name=save_name,
+        start_date=start_date, end_date=end_date,
+        initial_query=query, total_page=total_page,
+        uuid=uuid, customer_name=customer_name, 
+        max_retries=max_retries, delay=delay
+    )
+    if results is None: # no valid result after max retries
+        task.status_string[model_name] = exceptions # store exception strings in status
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Arxiv Paper Crawl Failed.") # exit
+
+
+async def process_arxiv_process_papers(
+    task: ArxivTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process Arxiv Process Papers
+    
+    Args:
+        task: ArxivTask, the task object
+        model_name: str, the model name
+        save_name: str, the save name
+        prev_name: str, the previous name
+        client: Minio client, the Minio client object
+        max_retries: int, the maximum number of retries
+        delay: float, the delay between retries
+        
+    Returns:
+        None
+    
+    """
+    if client is None:
+        client = get_client()
+
+    query = task.query
+    direction = task.query
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func(model_names=[model_name])[0]
+
+    review_arxiv_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/arxiv_results.csv",
+        client=client
+    )
+    results, exceptions = await retry_operation(
+        process_papers, task,
+        dataframe=review_arxiv_df, 
+        topic=query, direction=direction, 
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        max_retries=max_retries, delay=delay,
+        chat_func=chat_func
+    )
+    if results is None: # no valid result after max retries
+        task.status_string[model_name] = exceptions # store exception strings in status
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Arxiv Paper Crawl Failed.") # exit
+
+
+async def process_arxiv_generate_subheadings(
+    task: ArxivTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Generate Subheadings
+
+    Args:
+        task: ArxivTask, the task object
+        model_name: str, the model name
+        save_name: str, the save name
+        prev_name: str, the previous name
+        client: Minio client, the Minio client object
+        max_retries: int, the maximum number of retries
+        delay: float, the delay between retries
+        
+    Returns:
+        None
+    """
+    if client is None:
+        client = get_client()
+        
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func(model_names=[model_name])[0]
+    
+    review_arxiv_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/arxiv_results.csv",
+        client=client
+    )
+    
+    results, exceptions = await retry_operation(
+        generate_subheadings, task,
+        dataframe=review_arxiv_df,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        max_retries=max_retries, delay=delay,
+        chat_func=chat_func
+    )
+    if results is None: # no valid result after max retries
+        task.status_string[model_name] = exceptions # store exception strings in status
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Arxiv Generate Subheadings Failed.") # exit
+    
+
+async def process_arxiv_assign_subheadings_to_summaries(
+    task: ArxivTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Assign Subheadings to Summaries
+    Args:
+        task: ArxivTask, the task object
+        model_name: str, the model name
+        save_name: str, the save name
+        prev_name: str, the previous name
+        client: Minio client, the Minio client object
+        max_retries: int, the maximum number of retries
+        delay: float, the delay between retries
+    
+    Returns:
+        None
+    """
+    if client is None:
+        client = get_client()
+        
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func(model_names=[model_name])[0]
+    
+    subheadings = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/generated_subheadings.txt",
+        client=client
+    )
+    subheadings = subheadings.data.decode("utf-8").split("\n")
+    
+    review_arxiv_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/arxiv_results.csv",
+        client=client
+    )
+    
+    results, exceptions = await retry_operation(
+        assign_subheadings_to_summaries, task,
+        subheadings=subheadings,
+        relevant_papers_df=review_arxiv_df,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        max_retries=max_retries, delay=delay,
+        chat_func=chat_func
+    )
+    if results is None: # no valid result after max retries
+        task.status_string[model_name] = exceptions # store exception strings in status
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Arxiv Assign Subheadings Failed.") # exit
+
+
+async def process_arxiv_create_paragraphs_by_subheading(
+    task: ArxivTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Create Paragraphs by Subheading
+
+    Args:
+        task: ArxivTask, the task object
+        model_name: str, the model name
+        save_name: str, the save name
+        prev_name: str, the previous name
+        client: Minio client, the Minio client object
+        max_retries: int, the maximum number of retries
+        delay: float, the delay between retries
+    
+    Returns:
+        None
+    """
+    if client is None:
+        client = get_client()
+        
+    query = task.query
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func(model_names=[model_name])[0]
+
+    subheadings = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/generated_subheadings.txt",
+        client=client
+    )
+    subheadings = subheadings.data.decode("utf-8").split("\n")
+    
+    review_arxiv_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/arxiv_results.csv",
+        client=client
+    )
+    
+    results, exceptions = await retry_operation(
+        create_paragraphs_by_subheading, task,
+        subheadings=subheadings, main_topic=query,
+        relevant_papers_df=review_arxiv_df,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        max_retries=max_retries, delay=delay,
+        chat_func=chat_func
+    )
+    if results is None: # no valid result after max retries
+        task.status_string[model_name] = exceptions # store exception strings in status
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Arxiv Create Paragraphs Failed.") # exit
+    
+
+async def process_arxiv_enhance_language_readability(
+    task: ArxivTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Enhance Language Readability
+    Args:
+        task: ArxivTask, the task object
+        prev_name: str, the previous name
+        model_name: str, the model name
+        save_name: str, the save name
+        client: Minio client, the Minio client object
+        max_retries: int, the maximum number of retries
+        delay: float, the delay between retries
+    
+    Returns:
+        None
+    """
+    if client is None:
+        client = get_client()
+        
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func(model_names=[model_name])[0]
+    
+    review_content = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/review_non_refined.txt",
+        client=client
+    )
+    review_content = review_content.data.decode("utf-8")
+    
+    results, exceptions = await retry_operation(
+        enhance_language_readability, task,
+        content=review_content,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        max_retries=max_retries, delay=delay,
+        chat_func=chat_func
+    )
+    if results is None: # no valid result after max retries
+        task.status_string[model_name] = exceptions # store exception strings in status
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Arxiv Enhance Language Failed.") # exit
+    
+    
+async def process_arxiv_translate(
+    task: ArxivTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Translate
+    Args:
+        task: ArxivTask, the task object
+        prev_name: str, the previous name
+        model_name: str, the model name
+        save_name: str, the save name
+        client: Minio client, the Minio client object
+        max_retries: int, the maximum number of retries
+        delay: float, the delay between retries
+    
+    Returns:
+        None
+    """
+    if client is None:
+        client = get_client()
+    
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func(model_names=[model_name])[0]
+    
+    review_content = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/review_paper.txt",
+        client=client
+    )
+    review_content = review_content.data.decode("utf-8")
+    
+    results, exceptions = await retry_operation(
+        translate_to_chinese_before_references, task,
+        text=review_content,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        max_retries=max_retries, delay=delay,
+        chat_func=chat_func
+    )
+    if results is None: # no valid result after max retries
+        task.status_string[model_name] = exceptions # store exception strings in status
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Arxiv Translate Failed.") # exit
+
+
+# =================================
+# Function Groups: Arxiv Task
+#
+# functions specific for arxiv task
+# =================================
+
+async def get_arxiv_df(
+    start_date, end_date,
+    initial_query, total_page,
+    uuid, customer_name, model_name
+):
+    cookie = \
+        'browser=117.174.233.206.1731117480203659; arxiv_labs={%22sameSite%22:%22strict%22%2C%22expires%22:365%2C%22last_tab%22:%22tabone%22}; arxiv-search-parameters="{\"order\": \"-announced_date_first\"\054 \"size\": \"50\"\054 \"abstracts\": \"show\"}"'
+    headers = {
+        "authority": "arxiv.org",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+        "accept-language": "zh-CN,zh;q=0.9",
+        "cache-control": "no-cache",
+        "pragma": "no-cache",
+        "referer": "https://arxiv.org/search/physics?query=^%^28^%^28^%^27deep+learning^%^27^%^29+OR+^%^28^%^27machine+learning^%^27^%^29^%^29+AND+^%^28^%^27antibody^%^27^%^29+earch+v0.5.6+released+2020+^%^28^%^28^%^27deep+learning^%^27^%^29+OR+^%^28^%^27machine+learning^%^27^%^29^%^29+AND^%^28^%^27antibody^%^27^%^29&searchtype=all&abstracts=show&order=-announced_date_first&size=50",
+        "sec-ch-ua": "^\\^Chromium^^;v=^\\^104^^, ^\\^",
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "^\\^Windows^^",
+        "sec-fetch-dest": "document",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-site": "same-origin",
+        "sec-fetch-user": "?1",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
+        'cookie': cookie
+    }
+
+    url = "https://arxiv.org/search/advanced"
+    csv_filename = f"{customer_name}/{uuid}/{model_name}/arxiv_results.csv"
+
+    texts = ""
+
+    fieldnames = ['JT', 'DCOM', 'PMID', 'TI',
+                  'FAU', 'FAU-frist', 'AB', 'Full_Text_Links']
+
+    texts += ",".join(fieldnames) + "\n"
+    res_count = 0
+    for page in range(0, int(total_page)+1):
+        offset = page * 50
+        params = {
+            "advanced": "",
+            "terms-0-operator": "AND",
+            "terms-0-term": initial_query,
+            "terms-0-field": "all",
+            "classification-physics_archives": "all",
+            "classification-include_cross_list": "include",
+            "date-year": "",
+            "date-filter_by": "date_range",
+            "date-from_date": start_date,
+            "date-to_date": end_date,
+            "date-date_type": "submitted_date",
+            "abstracts": "show",
+            "size": "50",
+            'start': offset,
+            "order": "-announced_date_first"
+        }
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, headers=headers, params=params) as resp:
+                if resp.status != 200:
+                    logger.error("Failed to retrieve data from arxiv")
+                    raise ConnectionError("Failed to retrieve data from arxiv")
+                res = await resp.text()
+
+        if "produced no results" in res:
+            logger.warning("No results found")
+            break
+        else:
+            tree = etree.HTML(res.encode('utf-8'))
+            li_list = tree.xpath('//*[@class="breathe-horizontal"]/li')
+            if len(li_list) > 0:
+                for aa in li_list:
+                    # 提取论文信息
+                    tid = ''.join(
+                        aa.xpath(
+                            './/*[@class="list-title is-inline-block"]/a/text()'
+                        )
+                    ).strip()
+                    authors = ''.join(
+                        aa.xpath('.//*[@class="authors"]/a/text()')
+                    ).strip()
+                    first_authors = aa.xpath(
+                        './/*[@class="authors"]/a/text()'
+                    )[0] if len(aa.xpath(
+                        './/*[@class="authors"]/a/text()')
+                    ) > 0 else ''
+                    title = ''.join(
+                        aa.xpath(
+                            './/*[@class="title is-5 mathjax"]//text()')
+                    ).strip()
+                    abstract = ','.join(aa.xpath(
+                        './/*[@class="abstract-full has-text-grey-dark mathjax"]//text()')
+                    ).strip()
+                    # pdate = aa.xpath(".//p[@class='is-size-7']/text()")[0] if len(aa.xpath(".//p[@class='is-size-7']/text()")) > 0 else ''
+                    pdate = aa.xpath(
+                        ".//p[@class='is-size-7']/text()"
+                    )[0].strip() if len(
+                        aa.xpath(".//p[@class='is-size-7']/text()")
+                    ) > 0 else ''
+                    pdate = re.sub(r'\s*;.*$', '', pdate)
+                    purl = ''.join(
+                        aa.xpath('.//*[@class="list-title is-inline-block"]/a/@href')).strip()
+                    subjects = await get_more_detail(purl)  # 获取更多细节
+                    texts += ",".join([
+                        escape_csv_field(x) for x in [
+                            subjects, pdate, tid, title, authors,
+                            first_authors, abstract, purl
+                        ]
+                    ]) + "\n"
+                res_count += len(li_list)
+            else:
+                break
+        await upload_text_to_minio(
+            bucket_name=BUCKET_NAME,
+            object_name=csv_filename,
+            file_content=texts,
+        )
+
+    logger.info(f'已成功保存至{csv_filename}, 共获取到结果：{res_count}个')
+    return csv_filename
+    
+
+async def get_more_detail(url):
+    """
+    获取论文的更多细节信息，如主题。
+
+    :param url: 论文的链接
+    :return: 主题字符串
+    """
+    headers = {
+        "authority": "arxiv.org",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+        "accept-language": "zh-CN,zh;q=0.9",
+        "cache-control": "no-cache",
+        "pragma": "no-cache",
+        "referer": "https://arxiv.org/search/advanced",
+        "sec-ch-ua": "^\\^Chromium^^;v=^\\^104^^, ^\\^",
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "^\\^Windows^^",
+        "sec-fetch-dest": "document",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-site": "same-origin",
+        "sec-fetch-user": "?1",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
+    }
+    cookies = {
+        "browser": "117.174.233.206.1731117480203659",
+        "arxiv-search-parameters": "^\\^^{^^^^order^\\^\\^\\^:"
+    }
+
+    # 发送请求获取论文详情
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, headers=headers, cookies=cookies) as resp:
+            if resp.status != 200:
+                logger.error("Failed to get detail from arxiv")
+                raise ConnectionError("Failed to get detail from arxiv")
+            res = await resp.text()
+
+    tree = etree.HTML(res.encode("utf-8"))
+
+    # 提取主题信息
+    subjects_list = tree.xpath('//*[@class="tablecell subjects"]//text()')
+    subjects = ''
+    if subjects_list:
+        subjects = ''.join([tags.strip() for tags in subjects_list if tags])
+    
+    return subjects

utils/common_utils.py

@@ -0,0 +1,41 @@
+import time
+import asyncio
+
+from functools import wraps
+
+
+def execution_time(func):
+    @wraps(func)
+    def sync_wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        end_time = time.time()
+        elapsed_time = end_time - start_time
+        print(f"Execution time of {func.__name__}: {elapsed_time:.4f} seconds")
+        return result
+
+    @wraps(func)
+    async def async_wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = await func(*args, **kwargs)
+        end_time = time.time()
+        elapsed_time = end_time - start_time
+        print(f"Execution time of {func.__name__}: {elapsed_time:.4f} seconds")
+        return result
+    
+    if asyncio.iscoroutinefunction(func):
+        return async_wrapper
+    else:
+        return sync_wrapper
+
+
+def escape_csv_field(field):
+    """
+    Escapes fields to ensure proper CSV formatting.
+    - Wraps the field in double quotes if it contains a comma, double quote, or newline.
+    - Escapes double quotes inside the field by doubling them.
+    """
+    field_str = str(field)  # Convert the field to a string
+    if ',' in field_str or '"' in field_str or '\n' in field_str:
+        field_str = '"' + field_str.replace('"', '""') + '"'
+    return field_str

utils/minio_utils.py

@@ -0,0 +1,256 @@
+import io
+import json
+import asyncio
+import pandas as pd
+
+from docx import Document
+from loguru import logger
+from minio import Minio
+from entities.task import Task, task_factory
+
+
+BUCKET_NAME = "ai-scientist"
+
+
+# def get_client():
+#     return Minio(
+#         endpoint="0.0.0.0:9000",
+#         access_key="9o5pg5tBkjZlsvp4tegE",
+#         secret_key="YQqCRAlBygHSy7Rh3qZ2kTLqo9WcTQiqttHLQaPE",
+#         secure=False
+#     )
+
+def get_client():
+    return Minio(
+        endpoint="0.0.0.0:9000",
+        access_key="minioadmin",
+        secret_key="minioadmin",
+        secure=False
+    )    
+    
+async def get_task_from_minio(
+    uuid: str, 
+    customer_name: str, 
+    client: Minio = None
+) -> Task:
+    """
+    Asynchronously retrieve a task from MinIO.
+
+    Args:
+        uuid (str): Task UUID.
+        customer_name (str): Customer name.
+        client (Minio, optional): MinIO client instance.
+
+    Returns:
+        Task: The task object.
+
+    Raises:
+        FileNotFoundError: If the task or customer data is not found.
+    """
+    if client is None:
+        client = get_client()
+
+    objects = await asyncio.to_thread(
+        lambda: list(client.list_objects(
+            bucket_name=BUCKET_NAME, 
+            prefix=f"{customer_name}/"
+        ))
+    )
+    
+    logger.info(objects)
+
+    # Check if customer exists
+    if len(objects) <= 0:
+        raise FileNotFoundError(f"No task found for customer {customer_name}")
+
+    # Check if task exists
+    object_names = [obj.object_name.split("/")[1] for obj in objects]
+    if uuid not in object_names:
+        raise FileNotFoundError(f"No task found for customer {customer_name} with uuid {uuid}")
+
+    # If task found
+    json_file = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/task.json",
+        client=client
+    )
+    
+    json_data = json_file.data.decode("utf-8")
+    json_data = json.loads(json_data)
+    print(json_data)
+    return task_factory[json_data["task_type"]].load_from_json(json_data)
+
+
+async def get_all_tasks_from_minio(
+    customer_name: str, 
+    client: Minio = None
+) -> list[Task]:
+    """
+
+    Asynchronously retrieve all tasks for a customer from MinIO.
+
+    Args:
+        customer_name (str): Customer name. 
+        client (Minio, optional): MinIO client instance.
+
+    Returns:
+        list[Task]: List of task objects.
+    """
+
+    if client is None:
+        client = get_client()
+        
+    objects = await asyncio.to_thread(
+        lambda: list(client.list_objects(
+            bucket_name=BUCKET_NAME, 
+            prefix=f"{customer_name}/"
+        ))
+    )
+    
+    # Check if customer exists
+    if len(objects) <= 0:
+        # raise FileNotFoundError(f"No task found for customer {customer_name}")
+        return []
+        
+    task_ids = [obj.object_name.split("/")[1] for obj in objects]
+    task_jsons = await asyncio.gather(
+        *(get_task_from_minio(
+            uuid=task_id, customer_name=customer_name
+        ) for task_id in task_ids)
+    )
+    return task_jsons
+    
+
+async def upload_task_json_to_minio(task: Task, client: Minio = None) -> Task:
+    """
+    Asynchronously upload a task's JSON representation to MinIO.
+
+    Args:
+        task (Task): The task object to upload.
+        client (Minio, optional): MinIO client instance.
+
+    Returns:
+        Task: The uploaded task object.
+    """
+    if client is None:
+        client = get_client()
+
+    json_data = task.save_to_json()
+    byte_data = io.BytesIO(json_data.encode("utf-8"))
+
+    await asyncio.to_thread(
+        lambda: client.put_object(
+            bucket_name=BUCKET_NAME,
+            object_name=f"{task.customer_name}/{task.uuid}/task.json",
+            data=byte_data,
+            length=len(byte_data.getvalue()),
+            content_type="application/json"
+        )
+    )
+    return task
+
+
+async def upload_text_to_minio(
+    bucket_name: str,
+    object_name: str,
+    file_content: str,
+    client: Minio = None,
+):
+    if client is None:
+        client = get_client()
+    
+    file_data = io.BytesIO(file_content.encode("utf-8"))
+    
+    try:
+        await asyncio.to_thread(
+            client.put_object,
+            bucket_name=bucket_name,
+            object_name=object_name,
+            data=file_data,
+            length=len(file_data.getvalue()),
+        )
+    except Exception as e:
+        raise Exception(f"Error uploading file to MinIO: {e}")
+    
+
+async def upload_dataframe_to_minio(
+    bucket_name: str,
+    object_name: str,
+    df: pd.DataFrame,
+    client: Minio = None,
+):
+    if client is None:
+        client = get_client()
+        
+    buffer = io.BytesIO()
+    df.to_csv(buffer, index=False)
+    
+    await upload_text_to_minio(
+        bucket_name=bucket_name,
+        object_name=object_name,
+        file_content=buffer.getvalue().decode("utf-8")
+    )
+
+
+async def upload_document_to_minio(
+    bucket_name: str,
+    object_name: str,
+    document: Document,
+    client: Minio = None,
+):
+    if client is None:
+        client = get_client()
+    
+    buffer = io.BytesIO()
+    document.save(buffer)
+    buffer.seek(0)
+    
+    await asyncio.to_thread(
+        lambda: client.put_object(
+            bucket_name=bucket_name,
+            object_name=object_name,
+            data=buffer,
+            length=buffer.getbuffer().nbytes,
+            content_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+        )
+    )
+        
+        
+async def get_file_from_minio(
+    bucket_name: str,
+    object_name: str,
+    client: Minio = None,
+):
+    if client is None:
+        client = get_client()
+        
+    try:
+        file_data = await asyncio.to_thread(
+            client.get_object,
+            bucket_name=bucket_name,
+            object_name=object_name
+        )
+        return file_data
+    except Exception as e:
+        raise Exception(f"Error getting file from MinIO: {e}")
+    
+
+async def get_dataframe_from_minio(
+    bucket_name: str,
+    object_name: str,
+    client: Minio = None,
+):
+    if client is None:
+        client = get_client()
+    
+    file_data = await get_file_from_minio(
+        bucket_name=bucket_name,
+        object_name=object_name,
+        client=client
+    )
+    
+    if object_name.endswith(".csv"):
+        df = pd.read_csv(io.BytesIO(file_data.data))
+    elif object_name.endswith(".xlsx") or object_name.endswith("xls"):
+        df = pd.read_excel(io.BytesIO(file_data.data))
+    return df

utils/paper_plus_utils.py

@@ -0,0 +1,1265 @@
+import io
+import os
+import re
+import math
+import random
+import asyncio
+import textwrap
+import pandas as pd
+
+from docx import Document
+from loguru import logger
+
+from .minio_utils import (
+    upload_text_to_minio,
+    upload_dataframe_to_minio,
+    upload_document_to_minio,
+    get_file_from_minio
+)
+from .common_utils import escape_csv_field
+
+
+BUCKET_NAME = "ai-scientist"
+
+
+# Function to check relevance and obtain keywords as reason
+async def is_relevant(title, abstract, topic, direction, chat_func):
+    """ 
+    Check if a paper is relevant to a topic and obtain keywords as reason.
+    
+    Args:
+        title (str): Title of the paper.
+        abstract (str): Abstract of the paper.
+        topic (str): Topic to check relevance against.
+        direction (str): Direction to check relevance against.
+        chat_func (function): Function to call the chat model.
+
+    Returns:
+        bool: True if the paper is relevant, False otherwise.
+        str: Keywords that indicate relevance.
+    
+    """
+    relevance_prompt = (
+        f"You are an academic expert specializing in the field of {topic}. Your task is to determine if the following paper is relevant to the research direction described as '{direction}'.\n\n"
+        "Please follow this reasoning process:\n"
+        "1. Carefully read the paper's title and abstract.\n"
+        "2. Identify the core research area, methodology, results, or focal points presented in the paper.\n"
+        "3. Compare these core elements to the given research direction. Consider whether the paper directly addresses, contributes to, or is closely aligned with the stated direction.\n"
+        "4. If the paper aligns conceptually, methodologically, or thematically with the direction, then it is relevant. If it is only tangential or unrelated, it is not relevant.\n"
+        "5. From the text, select the main keywords that strongly indicate relevance (if relevant). These keywords should be key concepts, terms, or phrases that link the paper’s content to the given research direction.\n"
+        "6. If not relevant, you can provide no keywords or give a brief note indicating no strong linkage.\n\n"
+        "You must provide the answer in the following exact format:\n"
+        "Relevance: True or False\n"
+        "Keywords: [Comma-separated keywords]\n\n"
+        f"Title: {title}\n"
+        f"Abstract: {abstract}\n"
+    )
+    response = await chat_func(relevance_prompt)
+    if response is None:
+        return False, "Relevance check unavailable due to server error."
+
+    try:
+        response_text = response.choices[0].message.content
+        relevance = "True" in response_text
+        keywords = response_text.split(
+            "Keywords:")[-1].strip() if "Keywords:" in response_text else ""
+        return relevance, keywords
+    except AttributeError:
+        logger.error("Error in chat_func response format:", response)
+        return False, "Relevance check failed"
+
+
+# Modified summarize_abstract function with error handling for failed completion requests
+async def summarize_abstract(title, abstract, first_author, chat_func):
+    """
+    Summarize the abstract of a research paper.
+    
+    Args:
+        title (str): Title of the paper.
+        abstract (str): Abstract of the paper.
+        first_author (str): Name of the first author.
+        chat_func (function): Function to call the chat model.
+
+    Returns:
+        str: Summary of the abstract.
+    
+    """
+    formatted_author = reformat_author_name(first_author)
+    
+    # decision_prompt仍然维持原有逻辑，用于判断摘要类型
+    decision_prompt = (
+        f"Your task is to decide the type of summary needed based on the abstract.\n\n"
+        f"Instructions:\n"
+        f"- If the study primarily introduces, describes, or refines a method, technique, model, or computational approach, "
+        f"with its main contribution being methodological rather than a discovery about a phenomenon, then output:\n"
+        f"Output: full\n\n"
+        f"- If the study primarily reports a new discovery, finding, result, or empirical outcome about a certain phenomenon, "
+        f"biological entity, material property, or theoretical insight, then output:\n"
+        f"Output: concise\n\n"
+        f"Make your decision strictly based on the abstract content. Do not provide explanations or reasoning, "
+        f"only the exact output word as instructed.\n\n"
+        f"Title: {title}\nAbstract: {abstract}\n"
+    )
+    
+    # full_summary_prompt不再要求使用第一作者信息，只需要两句话总结主要发现
+    full_summary_prompt = (
+        "In exactly two sentences, provide a high-level summary of the study’s key findings, "
+        "while maintaining concrete technical terms, methodologies, and specific entities. "
+        # "Do not use 'this study', 'the authors', or similar phrases as the subject; instead, use a proper noun or specific entity mentioned or implied in the abstract as the subject. "
+        "Use clear and advanced language without generalizing or replacing specific methods with vague terms.\n\n"
+        f"The summary should use clear, advanced language and mention the first author {formatted_author} followed by 'et al.':\n\n"
+        f"Title: {title}\nAbstract: {abstract}\n\n"
+        f"Summary by {formatted_author} et al.:"
+    )
+    
+    # concise_summary_prompt不再要求使用第一作者信息，只需要一句话总结主要发现
+    concise_summary_prompt = (
+        "In two sentence, provide a precise statement of the study’s main finding without generalizing and without making the study itself the subject. "
+        "Do not use 'this study', 'the authors', or similar phrases as the subject; instead, use a proper noun or specific entity mentioned or implied in the abstract as the subject of the sentence. "
+        "Directly present the finding as the sentence’s focus, using advanced and specific language.\n\n"
+        f"Title: {title}\nAbstract: {abstract}\n\n"
+    )
+
+    response_decision = await chat_func(decision_prompt)
+    response_decision = response_decision.choices[0].message.content.strip().lower()
+    
+    if response_decision and "full" in response_decision:
+        prompt_summary = full_summary_prompt
+    else:
+        prompt_summary = concise_summary_prompt
+        
+    response = await chat_func(prompt_summary)
+    
+    if response is None:
+        return "Summary unavailable due to server error."
+
+    try:
+        result = response.choices[0].message.content.strip()
+        result_words = result.split()
+        summary = " ".join(result_words)
+        return summary
+    except AttributeError:
+        logger.error("Error in chat_func response format:", response)
+        return "Summary unavailable"
+
+
+# Function to reformat first author name
+def reformat_author_name(author_name):
+    """
+    Reformat the first author name by removing commas.
+    
+    Args:
+        author_name (str): Name of the first author.
+        
+    Returns:
+        str: Reformatted name of the first author.
+        
+    """
+    try:
+        return author_name.replace(",", "")
+    except AttributeError:
+        return "Unknown Author"
+
+
+# Function to generate 3-5 hierarchical subheadings related to the main topic
+async def generate_subheadings(
+    relevant_papers_df, main_topic, 
+    uuid, customer_name, model_name,
+    chat_func
+):
+    """
+    Generate 3-5 hierarchical subheadings related to the main topic based on the summaries of relevant papers.
+    
+    Args:
+        relevant_papers_df: DataFrame containing relevant papers.
+        main_topic: Main topic of the research.
+        chat_func: Function to send chat messages to the chatbot.
+        
+    Returns:
+        List[str]: List of generated subheadings.
+    
+    """
+    # Determine the number of subheadings based on the number of rows
+    num_papers = len(relevant_papers_df)
+    if num_papers < 10:
+        num_subheadings = 1
+    elif num_papers <= 20:
+        num_subheadings = 2
+    elif num_papers <= 40:
+        num_subheadings = 3
+    elif num_papers <= 60:
+        num_subheadings = 4
+    elif num_papers <= 100:
+        num_subheadings = 5
+    else:
+        num_subheadings = 6
+
+    # Generate the summaries for the prompt
+    summaries = " ".join(relevant_papers_df['Summary'].tolist())
+    
+    # Create the improved prompt
+    prompt = (
+        f"Consider the following main topic: '{main_topic}'. You are given a set of summaries extracted from relevant research papers related to this topic. Your goal is to generate {num_subheadings} hierarchical subheadings that clearly reflect and logically organize the key concepts and themes found in these summaries.\n\n"
+        "Instructions:\n"
+        "1. Carefully read and analyze the provided summaries.\n"
+        "2. Identify broad thematic categories directly mentioned or strongly implied by the summaries. These should serve as the starting points for the subheadings.\n"
+        "3. Arrange the subheadings in a hierarchical manner: start with the most general or foundational aspects of the main topic, then move progressively towards more specific, nuanced, or advanced themes.\n"
+        "4. Ensure that each subheading is distinct and does not overlap in scope or content with the others. Every subheading should be directly supported by information present in the summaries.\n"
+        "5. Do not introduce concepts that are not reflected in the summaries. All subheadings must be grounded in the text provided.\n"
+        "6. The final output should be a simple list of subheadings, each preceded by a hyphen, without additional explanation or commentary.\n\n"
+        f"Summaries:\n{summaries}\n\n"
+        "Output format:\n- Subheading 1\n- Subheading 2\n- Subheading 3\n..."
+    )
+    
+    response = await chat_func(prompt)
+    subheadings = response.choices[0].message.content.strip().splitlines()
+    subheadings = [subheading.replace(r"[-*']", '').strip() for subheading in subheadings]
+    subheadings = [subheading.replace(r"- ", '').strip() for subheading in subheadings]
+    subheadings = [re.sub(r"^[^\w]+|[^\w]+$", '', subheading).strip() 
+                   for subheading in subheadings]
+    subheadings = subheadings[:num_subheadings]
+    logger.info("Generated Subheadings:\n" + "\n".join(subheadings))
+    
+    output_filename = f"{customer_name}/{uuid}/{model_name}/generated_subheadings.txt"
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_filename,
+        file_content="\n".join(subheadings)
+    )
+    logger.info(f"Subheadings saved to {output_filename}")
+    return subheadings
+
+
+# Function to assign summaries to subheadings with minimum allocation of references per subheading
+async def assign_subheadings_to_summaries(
+    relevant_papers_df, 
+    subheadings, 
+    uuid, customer_name, model_name,
+    chat_func
+):
+    """
+    Assign summaries to subheadings with minimum allocation of references per subheading.
+    
+    Args:
+        relevant_papers_df: DataFrame containing relevant papers.
+        subheadings: List of subheadings.
+        uuid: Unique identifier for the task.
+        customer_name: Name of the customer.
+        chat_func: Function to send chat messages to the chatbot.
+
+    Returns:
+        DataFrame with assigned subheadings.
+    
+    """
+    total_papers = len(relevant_papers_df)
+    min_papers_per_subheading = math.ceil(total_papers / (len(subheadings) + 1))
+
+    assigned_subheadings = []
+    prompts = []
+    for summary in relevant_papers_df['Summary']:
+        prompt = (
+            # 对模型的指令明确化
+            f"Given the following subheadings and a research paper summary, identify the single most appropriate subheading for the provided summary. "
+            f"You must carefully analyze the semantic content, thematic focus, and logical structure within the summary. "
+            f"Ensure that the chosen subheading closely matches the core topic, key findings, research objectives, or main arguments of the paper summary. "
+            f"Do not select a subheading that only partially fits; the chosen subheading should represent a strong and direct thematic alignment with the summary's central ideas. "
+            f"Each subheading covers a distinct aspect or theme. Avoid overlaps by choosing the one that best captures the essence of the summary. "
+            f"If a subheading does not logically or semantically align with the main theme or content described in the summary, it should not be chosen.\n\n"
+        
+            # 提供小标题列表
+            f"Subheadings:\n{subheadings}\n\n"
+        
+            # 提供文献摘要
+            f"Summary:\n{summary}\n\n"
+        
+            # 请求结果格式
+            "Output format:\nSubheading: [Chosen subheading]"
+        )
+        prompts.append(prompt)
+    responses = await asyncio.gather(
+        *(chat_func(prompt) for prompt in prompts)
+    )
+    for response in responses:
+        assigned_subheading = response.choices[0].message.content.split(": ", 1)[1]
+        assigned_subheadings.append(assigned_subheading)
+
+    relevant_papers_df['Assigned Subheading'] = assigned_subheadings
+
+    # Ensure minimum papers per subheading
+    counts = relevant_papers_df['Assigned Subheading'].value_counts().to_dict()
+    for subheading in subheadings:
+        if counts.get(subheading, 0) < min_papers_per_subheading:
+            extra_summaries = relevant_papers_df[relevant_papers_df['Assigned Subheading'] != subheading].sample(
+                min_papers_per_subheading - counts.get(subheading, 0)
+            )
+            relevant_papers_df.loc[extra_summaries.index,
+                                   'Assigned Subheading'] = subheading
+            
+    relevant_papers_df['Assigned Subheading'] = (
+        relevant_papers_df['Assigned Subheading']
+        .str.replace(r"^[^\w]+|[^\w]+$", '', regex=True)  # 去除开头和结尾的非字母数字字符
+        .str.strip()  # 去除字符串两端的空格
+    )
+
+    prefix = f"{customer_name}/{uuid}/{model_name}/"
+    output_dir = prefix
+
+    csv_filename = os.path.join(output_dir, f"assigned_subheadings.csv")
+
+    # relevant_papers_df.to_csv(csv_filename, index=False, encoding='utf-8')
+    await upload_dataframe_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=csv_filename,
+        df=relevant_papers_df,
+    )
+
+    logger.info(f"Assigned subheadings saved to {csv_filename}")
+    logger.info(f"Found {len(relevant_papers_df)} related papers")
+
+    return relevant_papers_df
+
+
+async def get_sorting_suggestions(subheading, sub_df, chat_func):
+    # Add original index column to sub_df to retain original paper number
+    sub_df = sub_df.copy()  # Avoid SettingWithCopyWarning
+    sub_df.reset_index(drop=True, inplace=True)
+    sub_df.index = sub_df.index + 1
+    sub_df['Original Index'] = sub_df.index
+
+    paper_num = sub_df.shape[0]
+    logger.info(paper_num)
+
+    if paper_num > 1:
+        # Combine summaries into one string, appending author information
+        summaries_text = '\n'.join(
+            [f"Paper {row['Original Index']} by {row['First Author']}:\nSummary: {row['Summary']}\nRelevance Keywords: {row['Relevance Keywords']}" 
+            for _, row in sub_df.iterrows()]
+        )
+        logger.info(summaries_text)
+        
+        prompt = (
+            f"You are an experienced scientist tasked with organizing a collection of {paper_num} papers under the subheading '{subheading}' for a scientific review article.\n\n"
+        
+            "You have the following input:\n"
+            "1. A set of papers, each with a summary and relevance keywords.\n"
+            "2. A need to arrange these papers in a coherent and logical order that supports a narrative flow in a review article.\n\n"
+        
+            "Please address the following tasks:\n\n"
+            "1. **Identify Key Themes and Group Papers:**\n"
+            "- First, thoroughly read the summaries and relevance keywords of all the provided papers.\n"
+            "- Determine distinct thematic groups or categories. A thematic group can be based on shared methodology, a common theoretical framework, a particular type of material, organism, phenomenon, or a progressive line of inquiry.\n"
+            "- The grouping should reflect logical subdivisions that a reader of a review article could follow. For instance:\n"
+            "  - Start with foundational or broadly relevant studies that introduce key concepts, contexts, or basic methods.\n"
+            "  - Follow with papers that build upon these foundations, introducing more advanced techniques, deeper investigations, specialized findings, or novel approaches.\n"
+            "  - Conclude with cutting-edge, most specialized, or recently introduced concepts that push the boundaries of the field.\n"
+            "- If certain papers align well as a stepping stone from one theme to another, position them accordingly to create a smooth thematic transition.\n\n"
+        
+            "2. **Determine the Logical Order Within Each Group:**\n"
+            "- Within each thematic group, arrange the papers in an order that naturally builds understanding. Consider:\n"
+            "  - Present foundational or earlier conceptual frameworks before more advanced or derivative studies.\n"
+            "  - Highlight any chronological clues (if provided) or logical sequences, such as a method introduced in one paper being applied or expanded in a later paper.\n"
+            "  - Move from general to specific, from simpler methodologies to more complex analyses, or from well-established concepts to more tentative or innovative ones.\n\n"
+        
+            "3. **Combine Groups into a Cohesive Narrative:**\n"
+            "- After organizing papers within their groups, merge the groups into a single final list.\n"
+            "- The final list should read like a storyline: start with a broad, conceptual or methodological foundation, then move through intermediate studies that expand and refine these ideas, and end with the most advanced, specialized, or novel findings.\n"
+            "- Ensure that transitions between groups make sense, helping a reader follow a narrative where each section logically paves the way for the next.\n\n"
+        
+            "4. **Provide the Final Ordered List:**\n"
+            "- Present the final ordered list as a numbered list from 1 to {paper_num}.\n"
+            "- Each entry should include the original paper number and the first author's name in the following format:\n"
+            "   <Final Position>. <Original Paper Number>. (<First Author's Last Name>)\n\n"
+            "For example:\n"
+            "1. 3. (Smith)\n"
+            "2. 1. (Johnson)\n"
+            "3. 5. (Williams)\n\n"
+            "All papers must appear once, and each final position should be unique. Do not omit any papers.\n\n"
+        
+            "Below are the papers:\n\n"
+            f"{summaries_text}\n\n"
+        
+            "Please reflect on the thematic connections and carefully arrange the papers according to the instructions above."
+        )
+
+        # Retry mechanism to handle mismatches
+        sorting_order = []
+        sorting_response = await chat_func(prompt)  # Replace with your chat model interface
+        sorting_suggestion = sorting_response.choices[0].message.content.strip()
+        logger.info(f'Sorting suggestion:{sorting_suggestion}')
+        matches = re.findall(r'(\d+)\.\s*(\d+)\.\s*\((.*?)\)', sorting_suggestion)
+
+        # Debugging: print out raw matches to verify correctness
+        logger.info(f"Matches found: {matches}")
+
+        for match in matches:
+            original_num = int(match[0])  # Original number
+            new_num = int(match[1])       # Recommended number
+            author = match[2].strip()     # Author name
+            sorting_order.append((original_num, new_num, author))
+    else:
+        author = sub_df["Fisrt Author"].values[0]
+        sorting_order.append((1, 1, author))
+    
+    # Ensure no duplicate new numbers and correct count
+    new_nums = [x[1] for x in sorting_order]
+    if len(sorting_order) == paper_num and len(set(new_nums)) == paper_num:
+        pass  # Sorting succeeded, break the loop
+    elif abs(len(sorting_order) - paper_num) <= 2:
+        logger.info(f"Warning: Sorting order mismatch, difference of {abs(len(sorting_order) - paper_num)}. Assigning missing positions.")
+        existing_sorted_numbers = [x[1] for x in sorting_order]
+        missing_numbers = set(range(1, paper_num + 1)) - set(existing_sorted_numbers)
+        
+        for idx, original_num in enumerate(range(1, paper_num + 1)):
+            if original_num not in existing_sorted_numbers:
+                random_new_num = random.choice(list(missing_numbers))
+                sorting_order.append((original_num, random_new_num, "Unknown Author"))  # Placeholder author
+                missing_numbers.remove(random_new_num)
+
+    # Sort by recommended number
+    sorting_order.sort(key=lambda x: x[1])  # Sort by new number
+
+    # Extract sorted original indices
+    final_sorted_order = [item[0] for item in sorting_order]
+
+    logger.info(f"Final sorted order: {final_sorted_order}")
+
+    # Reorder sub_df based on the sorted order
+    try:
+        sorted_indices = [sub_df[sub_df['Original Index'] == idx].index[0] for idx in final_sorted_order]
+        sorted_sub_df = sub_df.loc[sorted_indices].reset_index(drop=True)
+    except Exception as e:
+        logger.error(f"Error in sorting DataFrame: {e}")
+        raise ValueError("Reordering of DataFrame failed.")
+    
+    return sorted_sub_df
+
+
+# Function to create expanded paragraphs with required reference count and consistent reference indexing
+async def create_paragraphs_by_subheading(
+    relevant_papers_df, subheadings, main_topic, 
+    uuid, customer_name, model_name,
+    chat_func
+):
+    """
+    Create expanded paragraphs by subheading with required reference count and consistent reference indexing.
+    
+    Args:
+        relevant_papers_df (pd.DataFrame): DataFrame containing relevant papers and their summaries.
+        subheadings (list): List of subheadings for the review paper.
+        main_topic (str): Main topic of the review paper.
+        uuid (str): UUID of the task.
+        customer_name (str): Name of the customer.
+        chat_func (function): Function to send chat messages to the chatbot.
+        
+    Returns:
+        list: List of paragraphs with subheadings and consistent reference indexing.
+    
+    """
+    paragraphs = []
+
+    # Reorder relevant_papers_df based on the subheadings order
+    subheading_order = {subheading: idx for idx, subheading in enumerate(subheadings)}
+    relevant_papers_df['Subheading Order'] = \
+        relevant_papers_df['Assigned Subheading'].map(subheading_order)
+    
+    # Remove rows where 'Subheading Order' is NA
+    relevant_papers_df = relevant_papers_df.dropna(subset=['Subheading Order'])
+    
+    relevant_papers_df = relevant_papers_df.sort_values(by='Subheading Order')
+    
+    relevant_papers_df.reset_index(drop=True, inplace=True)
+    await upload_dataframe_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{model_name}/relevant_papers_sort.csv",
+        df=relevant_papers_df,
+    )
+    
+    # Split relevant_papers_df by 'Assigned Subheading' into separate sub-dataframes
+    subheading_groups = relevant_papers_df.groupby('Assigned Subheading')
+    
+    sub_dfs = []
+    sorted_sub_dataframes = []
+    for subheading in subheadings:
+        # Check if subheading exists in subheading_groups
+        if subheading in subheading_groups.groups:
+            sub_df = subheading_groups.get_group(subheading)
+            sub_dfs.append(sub_df)
+            
+    sorted_sub_dataframes = await asyncio.gather(
+        *(get_sorting_suggestions(subheading, sub_df, chat_func)
+          for sub_df in sub_dfs)
+    )
+    
+    sorted_sub_dataframes = [x for x in sorted_sub_dataframes if not x.empty]
+    
+    # Concatenate all sorted sub-dataframes and reset index
+    if sorted_sub_dataframes:
+        final_relevant_papers_df = pd.concat(sorted_sub_dataframes).reset_index(drop=True)
+        final_relevant_papers_df.index = final_relevant_papers_df.index + 1  # Start from index 1
+        final_relevant_papers_df['ref_index'] = final_relevant_papers_df.index  # Add ref_index column
+    else:
+        logger.error("Error: No valid sub-dataframes to concatenate.")
+        final_relevant_papers_df = pd.DataFrame()  # Create an empty DataFrame in case of error
+
+    final_relevant_papers_df = final_relevant_papers_df.drop_duplicates()
+    logger.info(final_relevant_papers_df.head())
+    
+    # Introduction
+    intro_prompt = (
+        f"Write a concise and advanced introductory paragraph for a scientific review paper on '{main_topic}'. "
+        "Introduce the topic, its importance, and the scope of the review. The introduction should provide a logical "
+        "setup for the following subheadings.\n\n"
+        "Output format:\n[Write introduction here]"
+    )
+    intro_response = await chat_func(intro_prompt)
+    intro_paragraph = intro_response.choices[0].message.content.strip()
+    paragraphs.append(f"**Introduction**\n{intro_paragraph}\n")
+
+    used_titles = set()
+    summaries_text_by_subheading = {subheading: [] for subheading in subheadings}
+    ref_index_map = {}
+    
+    for subheading in subheadings:
+        relevant_summaries = final_relevant_papers_df[
+            final_relevant_papers_df['Assigned Subheading'] == subheading
+        ]
+
+        for idx, (summary, title, author, pub_date, ref_index) in relevant_summaries[
+            ['Summary', 'Title', 'First Author', 'Publication Date', 'ref_index']
+        ].iterrows():
+            if title in used_titles:
+                continue
+            used_titles.add(title)
+            ref_index_map[title] = ref_index
+            summaries_text_by_subheading[subheading].append(
+                f"{summary} [Ref: {ref_index}]"
+            )
+    
+    logger.info(summaries_text_by_subheading)
+    paragraph_prompts = []
+    for subheading in subheadings:
+        summaries_text = summaries_text_by_subheading[subheading]
+
+        # Adjust word_size based on the number of summaries
+        num_summaries = len(summaries_text)
+        if num_summaries < 10:
+            word_size = num_summaries * 200 + 200  # If fewer than 10 summaries
+        elif num_summaries > 30:
+            word_size = num_summaries * 400 + 800  # If more than 20 summaries
+        elif num_summaries > 20:
+            word_size = num_summaries * 350 + 500  # If more than 20 summaries
+        else:
+            word_size = num_summaries * 250 + 300  # Otherwise, the default case
+        
+        # Generate the detailed paragraph for the subheading
+        paragraph_prompt = (
+            # f"Write a {word_size}-word thematically focused and critical paragraph under the subheading '{subheading}' for a scientific review on '{subheading}'. "
+            f"Write a {word_size}-word thematically focused and critical paragraph for a scientific review on '{subheading}'. "
+            "please do the following:\n"
+            "1.Begin the paragraph with 100-word sentences that summarize the main findings and objectives of the following studies, providing a clear context for the discussion.You may supplement this introduction with additional relevant knowledge to enhance understanding."
+            "2.Before introducing each piece of literature, you need to come up with a sentence or conjunction that connects the context"
+            "3.For each study, provide a overview, analyzing its objectives, methodologies, findings, and broader significance. "
+            "Ensure that the analysis of each study is presented in sequence, without skipping any, and maintain a logical flow."
+            "4.Relevant literature should be critically discussed, highlighting how it contributes to the field and emphasizing its strengths and limitations. "
+            "5.After discussing all studies, provide a concluding paragraph that offers a deep analysis of the collective progress represented by the studies, "
+            "identifying overarching trends, advancements, and gaps. Conclude with insightful suggestions for future directions and research areas that need further exploration. "
+            "please Meet the following requirements:\n"
+            "1.Maintain clear academic language in the style of *Nature*, with a focus on the relationships between studies and their contributions to the subheading's topic. "
+            "2.Ensure in-text citations are included in the format [Ref: number], avoid repetition, and provide a critical, objective comparison where relevant. "
+            "3.The entire paragraph should be coherent, without empty lines between studies, and flow logically from one point to the next. Each study must be fully represented,with no omission or skipping.\n "
+            "4.To prevent the simple stacking of literature, you need to think about how to make the article more readable, logical, and professional."
+            # f"Summaries:{' '.join(summaries_text)}"
+            f"Summaries:{' '.join(s.strip() for s in summaries_text)}"
+            "Output format:[Write paragraph here]"
+        )
+        paragraph_prompts.append(paragraph_prompt)
+
+    paragraph_responses = await asyncio.gather(
+        *(chat_func(para_prompt)
+          for para_prompt in paragraph_prompts)
+    )
+    for subheading, paragraph_response in \
+            zip(subheadings, paragraph_responses):
+        paragraph_text = paragraph_response.choices[0].message.content.strip()
+        paragraph_text = re.sub(r'\(Ref:\s*(\d+)\)', r'[Ref: \1]', paragraph_text)
+        paragraph_text = re.sub(r'\n\s*\n', '\n', paragraph_text)
+        paragraph_text = paragraph_text.replace('\n', ' ')
+        paragraph = f"**{subheading}**\n{paragraph_text}\n"
+        paragraphs.append(paragraph)
+
+    # Conclusion
+    conclusion_prompt = (
+        f"Write a concluding paragraph for a scientific review on '{main_topic}'. Summarize the main points discussed in the previous sections, "
+        "highlight the significance of the research, and suggest possible future directions or applications.\n\n"
+        "Output format:\n[Write conclusion here]"
+    )
+    conclusion_response = await chat_func(conclusion_prompt)
+    conclusion_paragraph = conclusion_response.choices[0].message.content.strip()
+    paragraphs.append(f"**Conclusion**\n{conclusion_paragraph}\n")
+
+    used_references = final_relevant_papers_df[
+        ['Title', 'First Author', 'Journal Title','Publication Date', 'ref_index']
+    ].sort_values(by='ref_index')
+
+    # References section (only used references)
+    references = "\n".join([
+        f"[Ref:{idx}]. {author} et al. {title}{Journal_Title}({pub_date})."
+        for idx, (author, title, Journal_Title, pub_date, ref_index) 
+            in enumerate(used_references[
+                ['First Author','Title', 'Journal Title', 'Publication Date', 'ref_index']
+            ].values, 1
+        )
+    ])
+    paragraphs.append(f"**References**\n{references}")
+
+    # Compile paragraphs into final content
+    final_content = "\n".join(paragraphs)
+
+    # Save grouped summaries to CSV with customer_name and current date
+    prefix = f"{customer_name}/{uuid}/{model_name}/"
+    output_dir = prefix
+
+    review_file = os.path.join(output_dir, f"review_non_refined.txt")
+
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=review_file,
+        file_content=final_content
+    )
+    
+    logger.info(f"Non-refined review saved to {review_file}")
+    return final_content
+
+
+# Function to enhance language and readability to meet Nature journal style
+async def enhance_language_readability(content, chat_func):
+    """
+    Enhance the language and readability of the given content to meet the style of the *Nature* journal.
+    
+    Args:
+        content (str): The content to enhance.
+        chat_func (function): The function to use for the chat completion.
+        
+    Returns:
+        str: The enhanced content.
+    
+    """
+    # Separate sections based on paragraph breaks
+    sections = content.split("\n\n")
+    enhanced_sections = []
+    prompts = []
+    for section in sections:
+        prompt = (
+            "Enhance the following text to align with the writing style of *Nature* journal. Refine language to be sophisticated and objective, "
+            "using advanced vocabulary and a factual tone. Ensure a high level of lexical diversity and rhythm, with alternating sentence lengths "
+            "and varied structures for readability. Avoid emotional, speculative, or conversational language, focusing on objective analysis.\n\n"
+            f"Text:\n{section}\n\n"
+            "Output format:\n[Enhanced text here]"
+        )
+        prompts.append(prompt)
+
+    responses = await asyncio.gather(
+        *(chat_func(prompt) for prompt in prompts)
+    )
+    for response in responses:
+        enhanced_section = response.choices[0].message.content.strip()
+        enhanced_sections.append(enhanced_section)
+
+    return "\n\n".join(enhanced_sections)
+
+
+async def split_by_section(content):
+    """
+    Split the given content into sections based on paragraph breaks.
+
+    Args:
+        content (str): The content to split.
+
+    Returns:
+        list: The list of sections.
+
+    """
+    # Split the content into sections based on paragraph breaks
+    subheading_pattern = r"(?m)^\*\*(.*?)\*\*$"
+    matches = list(re.finditer(subheading_pattern, content))
+    
+    sections = []
+    references_found = False
+    for i, match in enumerate(matches):
+        subheading = match.group(1).strip()  # Get the subheading text
+        if subheading.lower() == "references":
+            references_found = True
+        
+        start = match.end()  # End of the subheading line
+        end = matches[i + 1].start() if i + 1 < len(matches) else len(content)
+        paragraph_text = content[start:end].strip()
+        
+        if references_found:  # Add everything under "References" as is
+            sections.append((subheading, paragraph_text))
+            break  # Stop further processing
+        
+        sections.append((subheading, paragraph_text))
+    
+    return sections
+
+
+async def process_sections(sections, chat_func):
+    """
+    Processes each section (subheading and corresponding text) through the AI model.
+    Skips processing the "Introduction", "Conclusion", and "References" sections.
+    """
+    refined_sections = []
+    seen_subheadings = set()
+    skip_subheadings = {"introduction", "conclusion", "references"}  # Sections to skip
+
+    prompts = []
+    for idx, (subheading, text) in enumerate(sections):
+        subheading_clean = subheading.strip("*").strip()
+        logger.info(f"Processing section {idx + 1} of {len(sections)}: {subheading_clean}")
+        
+        if subheading_clean.lower() in skip_subheadings:
+            logger.info(f"Skipping '{subheading_clean}' section.")
+            # refined_sections.append((subheading, text))  # Keep these sections as is
+            continue
+        
+        if subheading_clean in seen_subheadings:
+            logger.info(f"Duplicate subheading detected: {subheading_clean}. Skipping.")
+            continue
+        
+        seen_subheadings.add(subheading_clean)
+        if text.strip():  # Skip empty sections
+            # Remove extra newlines and ensure no empty lines in the text
+            text = re.sub(r'\n\s*\n', ' ', text)  # Replace multiple newlines with a single space
+            text = text.replace('\n', ' ')        # Replace remaining newlines with spaces
+            text = re.sub(r'\s+', ' ', text).strip()  # Ensure no extra spaces
+            
+            # Updated prompt for higher review quality
+            prompt = textwrap.dedent(f"""
+            Your task is to refine the following academic section for clarity, depth, and suitability for publication in a high-impact journal.
+
+            Please adhere to these guidelines:
+
+            **1. Structure and Organization:**
+            - Identify and emphasize key themes or topics within the section.
+            - Group related studies together to enhance coherence and logical flow.
+            - Reorganize the content to ensure a clear progression of ideas.
+            - Use smooth transitions to connect paragraphs and concepts without relying on explicit subheadings.
+
+            **2. Integration and Analysis of Literature:**
+            - Synthesize findings from cited studies, highlighting connections, similarities, and differences.
+            - Avoid merely listing studies; focus on comparative analysis and critical evaluation.
+            - Highlight significant contributions, novel findings, or implications of each study.
+            - Discuss any controversies, differing perspectives, or gaps in the current research.
+
+            **3. Depth and Critical Insight:**
+            - Deepen analytical insights by going beyond surface-level summarization.
+            - Provide critical evaluations, discussing strengths, limitations, and areas needing further exploration.
+            - Highlight the significance of trends or shifts in the field.
+
+            **4. Language and Clarity:**
+            - Use precise and concise language appropriate for an academic audience.
+            - Vary sentence structures to enhance readability and engagement.
+            - Eliminate redundant or repetitive statements to streamline the content.
+            - Maintain a formal academic tone while ensuring the text is accessible.
+
+            **5. Consistency and Terminology:**
+            - Ensure consistency in terminology, style, and formatting throughout the section.
+            - Use technical terms accurately and define specialized terms if necessary.
+            - Avoid unnecessary acronyms unless commonly understood in the field.
+
+            **6. Accuracy and Detail:**
+            - Verify that descriptions of studies are accurate and that key findings are correctly represented.
+            - Emphasize the most relevant and impactful information from each study.
+            - Provide context where needed to aid understanding for a multidisciplinary audience.
+
+            **7. Conclusion and Future Directions:**
+            - Summarize main points and discuss how findings align or diverge from prior work.
+            - Suggest areas for future research based on identified gaps or limitations.
+            - Discuss practical implications or potential applications if relevant.
+
+            **8. Citation and Formatting:**
+            - Ensure citations are formatted accurately (e.g., [Ref: number]) and integrated smoothly into the text.
+            - Do not alter the "References" section or the citation order.
+            - Maintain the existing citation positions within the text.
+
+            **Section to refine:**
+            {text}
+            """)
+
+            prompts.append(prompt)
+    
+    # Call the AI model with the updated prompt
+    index = 0
+    refined_texts = await asyncio.gather(
+        *(chat_func(prompt) for prompt in prompts)
+    )
+    
+    logger.info(len(refined_texts))
+    logger.info(len(prompts))
+    
+    seen_subheadings = set()
+    for idx, (subheading, text) in enumerate(sections):
+        subheading_clean = subheading.strip("*").strip()
+        logger.info(f"Processing section {idx + 1} of {len(sections)}: {subheading_clean}")
+        
+        if subheading_clean.lower() in skip_subheadings:
+            refined_sections.append((subheading, text))
+            continue
+            
+        if subheading_clean in seen_subheadings:
+            logger.info(f"Duplicate subheading detected: {subheading_clean}. Skipping.")
+            continue
+        
+        seen_subheadings.add(subheading_clean)
+        if text.strip():
+            refined_text = refined_texts[index].choices[0].message.content.strip()
+            refined_text = re.sub(r'\n\s*\n', ' ', refined_text)  # Replace extra newlines with a single space
+            refined_text = refined_text.replace('\n', ' ')        # Replace remaining newlines with spaces
+            refined_text = re.sub(r'\s+', ' ', refined_text).strip()  # Ensure no extra spaces
+            refined_sections.append((subheading, refined_text))
+            index += 1
+    
+    return refined_sections
+
+
+async def process_papers(
+    dataframe, topic, direction, 
+    uuid, customer_name, model_name,
+    chat_func
+):
+    """
+    Process the given papers to extract relevant information and save it to a CSV file.
+    
+    Args:
+        dataframe (pandas.DataFrame): The DataFrame containing the papers.
+        topic (str): The topic to filter the papers by.
+        direction (str): The direction to filter the papers by.
+        uuid (str): The UUID of the task.
+        customer_name (str): The name of the customer.
+        chat_func (function): The function to use for the chat completion.
+        
+    Returns:
+        pandas.DataFrame: The DataFrame containing the relevant papers.
+    
+    """
+    # Duplicate, no need
+    # relevant_rows = []  # List to collect relevant rows for DataFrame creation
+
+    # Set up the output directory and CSV file
+    # output_dir = os.path.join(customer_name)
+    # os.makedirs(output_dir, exist_ok=True)
+    prefix = f"{customer_name}/{uuid}/{model_name}/"
+    output_dir = prefix
+
+    output_path = os.path.join(output_dir, "relevant_papers.csv")
+
+    # Create or clear the output file at the beginning
+    # with open(output_path, 'w', newline='', encoding='utf-8') as f:
+    #     writer = csv.writer(f, quoting=csv.QUOTE_ALL)
+    #     writer.writerow(["Journal Title", "Publication Date", "Title", "First Author", "Summary", "Is Relevant", "Relevance Keywords"])  # Writing header
+    texts = ""
+    fieldnames = ["Journal Title", "Publication Date", "Title",
+                  "First Author", "Summary", "Is Relevant", "Relevance Keywords"]
+    texts += ",".join([escape_csv_field(x) for x in fieldnames]) + "\n"
+
+    titles = []
+    abstracts = []
+    journal_titles = []
+    pubd_dates = []
+    first_authors = []
+    summaries = []
+    for idx, row in dataframe.iterrows():
+        title = row["TI"]
+        abstract = row["AB"]
+        journal_title = row["JT"]
+        pub_date = row["DCOM"]
+        first_author = row["FAU-frist"]
+
+        titles.append(title)
+        abstracts.append(abstract)
+        journal_titles.append(journal_title)
+        pubd_dates.append(pub_date)
+        first_authors.append(first_author)
+
+    relevants = await asyncio.gather(
+        *(is_relevant(
+            title, abstract, topic, direction, chat_func
+        ) for title, abstract in zip(titles, abstracts))
+    )
+
+    is_relevant_flags = [relevant[0] for relevant in relevants]
+    relevance_keywords = [relevant[1] for relevant in relevants]
+
+    rtitles = []
+    rabstracts = []
+    rjournal_titles = [] 
+    rpubd_dates = []
+    rfirst_authors = []
+    rflags = []
+    rkeywords = []
+
+    for (
+        rflag, rkeyword, title, abstarct, first_author, journal_title, pub_date
+    ) in zip(
+        is_relevant_flags, relevance_keywords,
+        titles, abstracts, first_authors, journal_titles, pubd_dates
+    ):
+        if rflag:
+            rtitles.append(title)
+            rabstracts.append(abstarct)
+            rfirst_authors.append(first_author)
+            rjournal_titles.append(journal_title)
+            rpubd_dates.append(pub_date)
+            rflags.append(rflag)
+            rkeywords.append(rkeyword)
+
+    summaries = await asyncio.gather(
+        *(summarize_abstract(
+            title, abstract, first_author, chat_func
+        ) for title, abstract, first_author in
+            zip(rtitles, rabstracts, rfirst_authors)
+        )
+    )
+
+    for (
+        summary,
+        journal_title, pub_date, title, first_author,
+        rflag, rkeyword
+    ) in zip(
+        summaries,
+        rjournal_titles, rpubd_dates, rtitles, rfirst_authors,
+        rflags, rkeywords
+    ):
+        journal_title = escape_csv_field(journal_title)
+        pub_date = escape_csv_field(pub_date)
+        title = escape_csv_field(title)
+        first_author = escape_csv_field(first_author)
+        summary = escape_csv_field(summary)
+        rkeyword = escape_csv_field(rkeyword)
+
+        texts += ",".join([
+            str(x) for x in [
+                journal_title, pub_date, title, first_author,
+                summary, rflag, rkeyword
+            ]
+        ]) + "\n"
+
+        # Print the added summary and keywords
+        logger.info(f"Added summary: {summary}")
+        logger.info(f"Relevance Keywords: {rkeyword}")
+
+    # Create the relevant DataFrame to return
+    # relevant_df = pd.DataFrame(relevant_rows)
+    # return relevant_df
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_path,
+        file_content=texts
+    )
+
+    return output_path
+
+
+async def translate_to_chinese_before_references(
+    text, 
+    uuid, customer_name, model_name, 
+    chat_func
+):
+    """
+    Translates the content of a text file to Chinese, keeping the '**References**' section in English.
+    
+    Args:
+        text (str): The content of the text file.
+        output_filename (str): The name of the output file.
+        chat_func (function): The function to use for translation.
+        
+    Returns:
+        str: The translated content.
+    
+    """
+    lines = text.split("\n")
+
+    # Step 3: 找到 '**References**' 行的索引
+    references_index = None
+    for i, line in enumerate(lines):
+        if line.strip() == "**References**":
+            references_index = i
+            break
+
+    # Step 4: 根据找到的索引分割内容
+    if references_index is not None:
+        main_content_lines = lines[:references_index]
+        references_content_lines = lines[references_index:]
+    else:
+        # 如果没有找到 '**References**'，则认为整个内容为正文
+        main_content_lines = lines
+        references_content_lines = []
+
+    # 将正文内容拼接为一个字符串
+    main_content = "\n".join(main_content_lines)
+
+    # Step 5: 分段处理正文内容进行翻译
+    sections = main_content.split("\n\n")
+    translated_sections = []
+
+    prompts = []
+    
+    for section in sections:
+        # 简化 prompt，只要求翻译正文内容
+        prompt = (
+            "Translate the following text to academic Chinese:\n\n"
+            f"Text:\n{section}\n\n"
+            "Output format:\n[Translated Chinese text here]"
+        )
+        prompts.append(prompt)
+    
+    responses = await asyncio.gather(
+        *(chat_func(prompt) for prompt in prompts)
+    )
+    for response in responses:
+        translated_section = response.choices[0].message.content.strip()
+        translated_sections.append(translated_section)
+
+    # Step 6: 将翻译后的正文拼接
+    translated_content = "\n\n".join(translated_sections)
+
+    # Step 7: 合并翻译后的正文和 References 部分
+    if references_content_lines:
+        references_content = "\n".join(references_content_lines)
+        final_content = translated_content + "\n\n" + references_content
+    else:
+        final_content = translated_content
+
+    # Step 8: 保存结果到新的文件
+    output_filename = f"{customer_name}/{uuid}/{model_name}/review_non_refined_translated.txt"
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_filename,
+        file_content=final_content
+    )
+
+    logger.info(f"\nTranslated content saved to {output_filename}")
+
+
+async def translate_refined_review_to_chinese(
+    refined_review_content,
+    uuid, customer_name, model_name,
+    chat_func
+):
+    
+    # Read the Word document
+    doc = Document(refined_review_content)
+    
+    # Prepare to create a new document for the translated content
+    translated_doc = Document()
+    
+    # Set of subheadings to skip translation
+    skip_subheadings = {"references"}
+    
+    # Keep track of the current section heading
+    current_heading = None
+    in_references_section = False
+    
+    prompts = []
+    for para in doc.paragraphs:
+        # Check if the paragraph is a heading
+        if para.style.name.startswith('Heading'):
+            # Get the heading text
+            current_heading = para.text.strip()
+            # Get the heading level
+            heading_level_match = re.findall(r'\d+', para.style.name)
+            heading_level = int(heading_level_match[0]) if heading_level_match else 1
+            
+            # Check if the heading text is in skip_subheadings
+            if current_heading.lower() in skip_subheadings:
+                in_references_section = True
+                # Add the heading as is
+                # translated_doc.add_heading(current_heading, level=heading_level)
+            else:
+                in_references_section = False
+                # Translate the heading
+                prompt = f"Translate the following heading to Chinese:\n\n{current_heading}"
+                prompts.append(prompt)
+                # translated_heading = chat_func(prompt)
+                # Add the translated heading
+                # translated_doc.add_heading(translated_heading, level=heading_level)
+        else:
+            if in_references_section:
+                # Add the paragraph as is
+                # translated_doc.add_paragraph(para.text)
+                pass
+            else:
+                # Translate the paragraph text to Chinese, preserving in-text citations
+                text_to_translate = para.text
+                if text_to_translate.strip() == '':
+                    # If the paragraph is empty, skip translation
+                    translated_doc.add_paragraph('')
+                else:
+                    # We need to preserve in-text citations, e.g., [Ref: 38]
+                    # Instruct the AI to keep the in-text citations in English
+                    prompt = f"""
+                    Translate the following text to academic Chinese. Keep any in-text citations (e.g., [Ref: number]) in English.
+
+                    Text:
+                    {text_to_translate}
+                    """
+                    prompts.append(prompt)
+    
+    translated_texts = await asyncio.gather(
+        *(chat_func(prompt) for prompt in prompts)
+    )
+    translated_texts = [
+        t.choices[0].message.content.strip() for t in translated_texts
+    ]
+    
+    index = 0
+    for para in doc.paragraphs:
+        # Check if the paragraph is a heading
+        if para.style.name.startswith('Heading'):
+            # Get the heading text
+            current_heading = para.text.strip()
+            # Get the heading level
+            heading_level_match = re.findall(r'\d+', para.style.name)
+            heading_level = int(heading_level_match[0]) if heading_level_match else 1
+            
+            # Check if the heading text is in skip_subheadings
+            if current_heading.lower() in skip_subheadings:
+                in_references_section = True
+                # Add the heading as is
+                translated_doc.add_heading(current_heading, level=heading_level)
+            else:
+                in_references_section = False
+                translated_doc.add_heading(translated_texts[index], level=heading_level)
+                index += 1
+        else:
+            if in_references_section:
+                # Add the paragraph as is
+                translated_doc.add_paragraph(para.text)
+            else:
+                # Translate the paragraph text to Chinese, preserving in-text citations
+                text_to_translate = para.text
+                if text_to_translate.strip() == '':
+                    # If the paragraph is empty, skip translation
+                    translated_doc.add_paragraph('')
+                else:
+                    translated_text = translated_texts[index]
+                    translated_doc.add_paragraph(translated_text)
+                    index += 1
+    
+    output_file_path = f"{customer_name}/{uuid}/{model_name}/review_paper_refined_translated.docx"
+    await upload_document_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_file_path,
+        document=translated_doc
+    )
+    return output_file_path
+
+
+async def refine_review_content(
+    non_refine_content,
+    uuid, customer_name, model_name,
+    chat_func
+):
+    sections = await split_by_section(non_refine_content)
+    refined_sections = await process_sections(sections, chat_func)
+    
+    prompt_title = f"""
+    Based on the following literature review, generate an appropriate and concise title:
+    {non_refine_content}
+    """
+    title = await chat_func(prompt_title)
+    title = title.choices[0].message.content.strip()
+    logger.info(f"Generated Title: {title}")
+    
+    doc = Document()
+    doc.add_heading(title, level=1)
+    
+    for subheading, content in refined_sections:
+        doc.add_heading(subheading, level=2)
+        doc.add_paragraph(content)
+    
+    output_file = f"{customer_name}/{uuid}/{model_name}/review_paper_refined.docx"
+    await upload_document_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_file,
+        document=doc
+    )
+    return output_file
+
+
+# Main function to automate the review paper creation process with language enhancement step
+async def create_review_paper(
+    relevant_papers_df,
+    main_topic,
+    uuid, customer_name, model_name,
+    chat_func,
+    translate_to_cn=False,
+    do_refine=False,
+):
+    """
+    Main function to automate the review paper creation process with language enhancement step.
+    
+    Args:
+        relevant_papers_df (pd.DataFrame): DataFrame containing relevant papers.
+        main_topic (str): Main topic of the review paper.
+        uuid (str): Unique identifier for the review paper.
+        customer_name (str): Name of the customer.
+        chat_func (function): Function to handle chat interactions.
+        translate_to_cn (bool): Flag to indicate if translation to Chinese is required.
+    
+    Returns:
+        None
+    
+    """
+    
+    # Step 1: Generate subheadings related to the main topic
+    subheadings = await generate_subheadings(
+        relevant_papers_df, main_topic,
+        chat_func
+    )
+
+    # Step 2: Assign each summary to a subheading
+    relevant_papers_df = await assign_subheadings_to_summaries(
+        relevant_papers_df, subheadings, 
+        uuid, customer_name, model_name,
+        chat_func
+    )
+
+    # Step 3: Create paragraphs by subheading, with introductory and concluding sections, and references
+    review_content = await create_paragraphs_by_subheading(
+        relevant_papers_df, subheadings, main_topic, 
+        uuid, customer_name, model_name,
+        chat_func
+    )
+
+    output_filename = f"{customer_name}/{uuid}/{model_name}/review_non_refined.txt"
+    
+    if do_refine:
+        # Step 4: Refine Review Content to a Word Document
+        await refine_review_content(
+            review_content,
+            uuid, customer_name, model_name,
+            chat_func
+        )
+        refined_review_content = await get_file_from_minio(
+            bucket_name=BUCKET_NAME,
+            object_name=f"{customer_name}/{uuid}/{model_name}/review_paper_refined.docx",
+        )
+        refined_review_content = io.BytesIO(refined_review_content.data)
+        
+    if translate_to_cn:
+        if do_refine:
+            await translate_refined_review_to_chinese(
+                refined_review_content,
+                uuid, customer_name, model_name,
+                chat_func
+            )
+            output_filename = f"{customer_name}/{uuid}/{model_name}/review_paper_refined_translated.txt"
+        else:
+            await translate_to_chinese_before_references(
+                review_content,
+                uuid, customer_name, model_name,
+                chat_func
+            )
+        output_filename = f"{customer_name}/{uuid}/{model_name}/review_non_refined_translated.txt"
+    return output_filename

utils/paper_utils.py

@@ -0,0 +1,694 @@
+import os
+import math
+import asyncio
+
+from loguru import logger
+
+from .minio_utils import (
+    upload_text_to_minio,
+    upload_dataframe_to_minio,
+)
+from .common_utils import escape_csv_field
+
+
+BUCKET_NAME = "ai-scientist"
+
+
+# Function to check relevance and obtain keywords as reason
+async def is_relevant(title, abstract, topic, direction, chat_func):
+    """ 
+    Check if a paper is relevant to a topic and obtain keywords as reason.
+    
+    Args:
+        title (str): Title of the paper.
+        abstract (str): Abstract of the paper.
+        topic (str): Topic to check relevance against.
+        direction (str): Direction to check relevance against.
+        chat_func (function): Function to call the chat model.
+
+    Returns:
+        bool: True if the paper is relevant, False otherwise.
+        str: Keywords that indicate relevance.
+    
+    """
+    relevance_prompt = (
+        f"You are an academic expert in {topic}. Identify if the following paper is "
+        f"related to '{direction}' and list only the main keywords that indicate relevance:\n\n"
+        f"Title: {title}\nAbstract: {abstract}\n\n"
+        "Answer format:\n"
+        "Relevance: True or False\n"
+        "Keywords: [Comma-separated keywords]"
+    )
+    response = await chat_func(relevance_prompt)
+    if response is None:
+        return False, "Relevance check unavailable due to server error."
+
+    try:
+        response_text = response.choices[0].message.content
+        relevance = "True" in response_text
+        keywords = response_text.split(
+            "Keywords:")[-1].strip() if "Keywords:" in response_text else ""
+        return relevance, keywords
+    except AttributeError:
+        logger.error("Error in chat_func response format:", response)
+        return False, "Relevance check failed"
+
+
+# Modified summarize_abstract function with error handling for failed completion requests
+async def summarize_abstract(title, abstract, first_author, chat_func):
+    """
+    Summarize the abstract of a research paper.
+    
+    Args:
+        title (str): Title of the paper.
+        abstract (str): Abstract of the paper.
+        first_author (str): Name of the first author.
+        chat_func (function): Function to call the chat model.
+
+    Returns:
+        str: Summary of the abstract.
+    
+    """
+    formatted_author = reformat_author_name(first_author)
+    summary_prompt = (
+        f"Write a concise, high-level summary in 2-3 sentences, highlighting the study's "
+        f"purpose, specific methodology, main findings, and significance. Avoid generalizing "
+        f"or replacing specific method names or entities with vague language. Retain concrete terms "
+        f"and clear descriptions of methodology and findings.\n\n"
+        f"Title: {title}\nAbstract: {abstract}\n\n"
+        f"Summary by {formatted_author} et al.:"
+    )
+
+    response = await chat_func(summary_prompt)
+    if response is None:
+        return "Summary unavailable due to server error."
+
+    try:
+        result = response.choices[0].message.content
+        result_words = result.split()
+        summary = " ".join(result_words)
+        return summary
+    except AttributeError:
+        logger.error("Error in chat_func response format:", response)
+        return "Summary unavailable"
+
+
+# Function to reformat first author name
+def reformat_author_name(author_name):
+    """
+    Reformat the first author name by removing commas.
+    
+    Args:
+        author_name (str): Name of the first author.
+        
+    Returns:
+        str: Reformatted name of the first author.
+        
+    """
+    try:
+        return author_name.replace(",", "")
+    except AttributeError:
+        return "Unknown Author"
+
+
+# Function to generate 3-5 hierarchical subheadings related to the main topic
+async def generate_subheadings(
+    relevant_papers_df, main_topic, 
+    uuid, customer_name, model_name,
+    chat_func
+):
+    """
+    Generate 3-5 hierarchical subheadings related to the main topic based on the summaries of relevant papers.
+    
+    Args:
+        relevant_papers_df: DataFrame containing relevant papers.
+        main_topic: Main topic of the research.
+        chat_func: Function to send chat messages to the chatbot.
+        
+    Returns:
+        List[str]: List of generated subheadings.
+    
+    """
+    summaries = " ".join(relevant_papers_df['Summary'].tolist())
+    prompt = (
+        f"The main topic is '{main_topic}'. Based on this topic and the following summaries from relevant research papers, "
+        "generate 3-5 hierarchical subheadings that progressively explore the topic. Begin with broader subheadings and "
+        "move towards more specific themes, avoiding overlap in scope or content. Subheadings should be distinct and arranged "
+        "in a logical order suitable for a structured review.\n\n"
+        f"Summaries:\n{summaries}\n\n"
+        "Output format:\n- Subheading 1\n- Subheading 2\n- Subheading 3\n..."
+    )
+    response = await chat_func(prompt)
+    subheadings = response.choices[0].message.content.strip().splitlines()
+    logger.info("Generated Subheadings:\n" + "\n".join(subheadings))
+    
+    output_filename = f"{customer_name}/{uuid}/{model_name}/generated_subheadings.txt"
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_filename,
+        file_content="\n".join(subheadings)
+    )
+    logger.info(f"Subheadings saved to {output_filename}")
+    return subheadings
+
+
+# Function to assign summaries to subheadings with minimum allocation of references per subheading
+async def assign_subheadings_to_summaries(
+    relevant_papers_df, 
+    subheadings, 
+    uuid, customer_name, model_name,
+    chat_func
+):
+    """
+    Assign summaries to subheadings with minimum allocation of references per subheading.
+    
+    Args:
+        relevant_papers_df: DataFrame containing relevant papers.
+        subheadings: List of subheadings.
+        uuid: Unique identifier for the task.
+        customer_name: Name of the customer.
+        chat_func: Function to send chat messages to the chatbot.
+
+    Returns:
+        DataFrame with assigned subheadings.
+    
+    """
+    total_papers = len(relevant_papers_df)
+    min_papers_per_subheading = math.ceil(
+        total_papers / (len(subheadings) + 1))
+
+    assigned_subheadings = []
+    prompts = []
+    for summary in relevant_papers_df['Summary']:
+        prompt = (
+            "Given the following subheadings and a research paper summary, determine the most appropriate subheading "
+            "for this summary. Each subheading should cover a unique aspect of the main topic without overlap. "
+            "Select the best-fitting subheading based on thematic relevance and coherence with similar studies.\n\n"
+            f"Subheadings:\n{subheadings}\n\n"
+            f"Summary:\n{summary}\n\n"
+            "Output format:\nSubheading: [Chosen subheading]"
+        )
+        prompts.append(prompt)
+    responses = await asyncio.gather(
+        *(chat_func(prompt) for prompt in prompts)
+    )
+    for response in responses:
+        assigned_subheading = response.choices[0].message.content.split(": ")[1]
+        assigned_subheadings.append(assigned_subheading)
+
+    relevant_papers_df['Assigned Subheading'] = assigned_subheadings
+
+    # Ensure minimum papers per subheading
+    counts = relevant_papers_df['Assigned Subheading'].value_counts().to_dict()
+    for subheading in subheadings:
+        if counts.get(subheading, 0) < min_papers_per_subheading:
+            extra_summaries = relevant_papers_df[relevant_papers_df['Assigned Subheading'] != subheading].sample(
+                min_papers_per_subheading - counts.get(subheading, 0)
+            )
+            relevant_papers_df.loc[extra_summaries.index,
+                                   'Assigned Subheading'] = subheading
+
+    prefix = f"{customer_name}/{uuid}/{model_name}/"
+    output_dir = prefix
+
+    csv_filename = os.path.join(output_dir, f"assigned_subheadings.csv")
+
+    # relevant_papers_df.to_csv(csv_filename, index=False, encoding='utf-8')
+    await upload_dataframe_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=csv_filename,
+        df=relevant_papers_df,
+    )
+
+    logger.info(f"Assigned subheadings saved to {csv_filename}")
+    logger.info(f"Found {len(relevant_papers_df)} related papers")
+
+    return relevant_papers_df
+
+
+# Function to create expanded paragraphs with required reference count and consistent reference indexing
+async def create_paragraphs_by_subheading(
+    relevant_papers_df, subheadings, main_topic, 
+    uuid, customer_name, model_name,
+    chat_func
+):
+    """
+    Create expanded paragraphs by subheading with required reference count and consistent reference indexing.
+    
+    Args:
+        relevant_papers_df (pd.DataFrame): DataFrame containing relevant papers and their summaries.
+        subheadings (list): List of subheadings for the review paper.
+        main_topic (str): Main topic of the review paper.
+        uuid (str): UUID of the task.
+        customer_name (str): Name of the customer.
+        chat_func (function): Function to send chat messages to the chatbot.
+        
+    Returns:
+        list: List of paragraphs with subheadings and consistent reference indexing.
+    
+    """
+    paragraphs = []
+
+    # Introduction
+    intro_prompt = (
+        f"Write a concise and advanced introductory paragraph for a scientific review paper on '{main_topic}'. "
+        "Introduce the topic, its importance, and the scope of the review. The introduction should provide a logical "
+        "setup for the following subheadings.\n\n"
+        "Output format:\n[Write introduction here]"
+    )
+    intro_response = await chat_func(intro_prompt)
+    intro_paragraph = intro_response.choices[0].message.content.strip()
+    paragraphs.append(f"**Introduction**\n{intro_paragraph}\n")
+
+    # Body paragraphs based on subheadings with consistent reference numbering
+    reference_map = {}
+    used_references = []
+    total_papers = len(relevant_papers_df)
+    min_papers_per_subheading = math.ceil(
+        total_papers / (len(subheadings) + 1))
+    ref_counter = 1
+
+    paragraph_prompts = []
+    for subheading in subheadings:
+        relevant_summaries = relevant_papers_df[relevant_papers_df['Assigned Subheading'] == subheading]
+
+        new_references = []
+        summaries_text = []
+        for idx, (summary, title, author, pub_date) in relevant_summaries[['Summary', 'Title', 'First Author', 'Publication Date']].iterrows():
+            if title not in reference_map:
+                reference_map[title] = ref_counter
+                ref_counter += 1
+            ref_index = reference_map[title]
+            summaries_text.append(f"{summary} [Ref: {ref_index}]")
+            new_references.append((title, author, pub_date))
+
+        # Compose prompt to generate an extended paragraph with at least 800 words
+        paragraph_prompt = (
+            f"Write an 800-word thematic and critical paragraph under the subheading '{subheading}' for a scientific review on '{main_topic}'. "
+            f"Combine the following summaries into a coherent, well-structured paragraph discussing the studies’ objectives, findings, "
+            "and methodologies. Use advanced academic language, include in-text citations in the format [Ref: number], and avoid repeating "
+            "content from previous sections. Provide critical insights and comparative analysis where relevant.\n\n"
+            f"Summaries:\n{' '.join(summaries_text)}\n\n"
+            "Output format:\n[Write paragraph here]"
+        )
+
+        paragraph_prompts.append(paragraph_prompt)
+        used_references.extend(new_references)
+
+    paragraph_responses = await asyncio.gather(
+        *(chat_func(para_prompt)
+          for para_prompt in paragraph_prompts)
+    )
+    for subheading, paragraph_response in \
+            zip(subheadings, paragraph_responses):
+        paragraph = f"**{subheading}**\n{paragraph_response.choices[0].message.content.strip()}\n"
+        paragraphs.append(paragraph)
+
+    # Conclusion
+    conclusion_prompt = (
+        f"Write a concluding paragraph for a scientific review on '{main_topic}'. Summarize the main points discussed in the previous sections, "
+        "highlight the significance of the research, and suggest possible future directions or applications.\n\n"
+        "Output format:\n[Write conclusion here]"
+    )
+    conclusion_response = await chat_func(conclusion_prompt)
+    conclusion_paragraph = conclusion_response.choices[0].message.content.strip()
+    paragraphs.append(f"**Conclusion**\n{conclusion_paragraph}\n")
+
+    # References section (only used references)
+    references = "\n".join(
+        [f"[Ref: {reference_map[title]}] {title}, {author}, {pub_date}" 
+         for title, author, pub_date in used_references]
+    )
+    paragraphs.append(f"**References**\n{references}")
+
+    # Compile paragraphs into final content
+    final_content = "\n\n".join(paragraphs)
+
+    # Save grouped summaries to CSV with customer_name and current date
+    prefix = f"{customer_name}/{uuid}/{model_name}/"
+    output_dir = prefix
+
+    csv_filename = os.path.join(output_dir, f"grouped_summaries.csv")
+    output_filename = os.path.join(output_dir, f"review_non_refined.txt")
+    # Prepare data for CSV
+    grouped_data = relevant_papers_df[['Assigned Subheading', 'Summary']]
+    # grouped_data.to_csv(csv_filename, index=False, encoding='utf-8')
+    await upload_dataframe_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=csv_filename,
+        df=grouped_data
+    )
+    
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_filename,
+        file_content=final_content
+    )
+
+    logger.info(f"\nGrouped summaries saved to {csv_filename}")
+    logger.info(f"Non-refined review saved to {output_filename}")
+    return final_content
+
+
+# Function to enhance language and readability to meet Nature journal style
+async def enhance_language_readability(
+    content, 
+    uuid, customer_name, model_name,
+    chat_func
+):
+    """
+    Enhance the language and readability of the given content to meet the style of the *Nature* journal.
+    
+    Args:
+        content (str): The content to enhance.
+        chat_func (function): The function to use for the chat completion.
+        
+    Returns:
+        str: The enhanced content.
+    
+    """
+    # Separate sections based on paragraph breaks
+    sections = content.split("\n\n")
+    enhanced_sections = []
+    prompts = []
+    for section in sections:
+        prompt = (
+            "Enhance the following text to align with the writing style of *Nature* journal. Refine language to be sophisticated and objective, "
+            "using advanced vocabulary and a factual tone. Ensure a high level of lexical diversity and rhythm, with alternating sentence lengths "
+            "and varied structures for readability. Avoid emotional, speculative, or conversational language, focusing on objective analysis.\n\n"
+            f"Text:\n{section}\n\n"
+            "Output format:\n[Enhanced text here]"
+        )
+        prompts.append(prompt)
+
+    responses = await asyncio.gather(
+        *(chat_func(prompt) for prompt in prompts)
+    )
+    for response in responses:
+        enhanced_section = response.choices[0].message.content.strip()
+        enhanced_sections.append(enhanced_section)
+
+    enhanced_content = "\n\n".join(enhanced_sections)
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{model_name}/review_paper.txt",
+        file_content=enhanced_content
+    )
+    
+    return enhanced_content
+
+
+async def process_papers(
+    dataframe, topic, direction, 
+    uuid, customer_name, model_name,
+    chat_func
+):
+    """
+    Process the given papers to extract relevant information and save it to a CSV file.
+    
+    Args:
+        dataframe (pandas.DataFrame): The DataFrame containing the papers.
+        topic (str): The topic to filter the papers by.
+        direction (str): The direction to filter the papers by.
+        uuid (str): The UUID of the task.
+        customer_name (str): The name of the customer.
+        chat_func (function): The function to use for the chat completion.
+        
+    Returns:
+        pandas.DataFrame: The DataFrame containing the relevant papers.
+    
+    """
+    # Duplicate, no need
+    # relevant_rows = []  # List to collect relevant rows for DataFrame creation
+
+    # Set up the output directory and CSV file
+    # output_dir = os.path.join(customer_name)
+    # os.makedirs(output_dir, exist_ok=True)
+    prefix = f"{customer_name}/{uuid}/{model_name}/"
+    output_dir = prefix
+
+    output_path = os.path.join(output_dir, "relevant_papers.csv")
+
+    # Create or clear the output file at the beginning
+    # with open(output_path, 'w', newline='', encoding='utf-8') as f:
+    #     writer = csv.writer(f, quoting=csv.QUOTE_ALL)
+    #     writer.writerow(["Journal Title", "Publication Date", "Title", "First Author", "Summary", "Is Relevant", "Relevance Keywords"])  # Writing header
+    texts = ""
+    fieldnames = ["Journal Title", "Publication Date", "Title",
+                  "First Author", "Summary", "Is Relevant", "Relevance Keywords"]
+    texts += ",".join([escape_csv_field(x) for x in fieldnames]) + "\n"
+
+    titles = []
+    abstracts = []
+    journal_titles = []
+    pubd_dates = []
+    first_authors = []
+    summaries = []
+    for idx, row in dataframe.iterrows():
+        title = row["TI"]
+        abstract = row["AB"]
+        journal_title = row["JT"]
+        pub_date = row["DCOM"]
+        first_author = row["FAU-frist"]
+
+        titles.append(title)
+        abstracts.append(abstract)
+        journal_titles.append(journal_title)
+        pubd_dates.append(pub_date)
+        first_authors.append(first_author)
+
+    relevants = await asyncio.gather(
+        *(is_relevant(
+            title, abstract, topic, direction, chat_func
+        ) for title, abstract in zip(titles, abstracts))
+    )
+
+    is_relevant_flags = [relevant[0] for relevant in relevants]
+    relevance_keywords = [relevant[1] for relevant in relevants]
+
+    rtitles = []
+    rabstracts = []
+    rjournal_titles = [] 
+    rpubd_dates = []
+    rfirst_authors = []
+    rflags = []
+    rkeywords = []
+
+    for (
+        rflag, rkeyword, title, abstarct, first_author, journal_title, pub_date
+    ) in zip(
+        is_relevant_flags, relevance_keywords,
+        titles, abstracts, first_authors, journal_titles, pubd_dates
+    ):
+        if rflag:
+            rtitles.append(title)
+            rabstracts.append(abstarct)
+            rfirst_authors.append(first_author)
+            rjournal_titles.append(journal_title)
+            rpubd_dates.append(pub_date)
+            rflags.append(rflag)
+            rkeywords.append(rkeyword)
+
+    summaries = await asyncio.gather(
+        *(summarize_abstract(
+            title, abstract, first_author, chat_func
+        ) for title, abstract, first_author in
+            zip(rtitles, rabstracts, rfirst_authors)
+        )
+    )
+
+    for (
+        summary,
+        journal_title, pub_date, title, first_author,
+        rflag, rkeyword
+    ) in zip(
+        summaries,
+        rjournal_titles, rpubd_dates, rtitles, rfirst_authors,
+        rflags, rkeywords
+    ):
+        journal_title = escape_csv_field(journal_title)
+        pub_date = escape_csv_field(pub_date)
+        title = escape_csv_field(title)
+        first_author = escape_csv_field(first_author)
+        summary = escape_csv_field(summary)
+        rkeyword = escape_csv_field(rkeyword)
+
+        texts += ",".join([
+            str(x) for x in [
+                journal_title, pub_date, title, first_author,
+                summary, rflag, rkeyword
+            ]
+        ]) + "\n"
+
+        # Print the added summary and keywords
+        logger.info(f"Added summary: {summary}")
+        logger.info(f"Relevance Keywords: {rkeyword}")
+
+    # Create the relevant DataFrame to return
+    # relevant_df = pd.DataFrame(relevant_rows)
+    # return relevant_df
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_path,
+        file_content=texts
+    )
+
+    return output_path
+
+
+async def translate_to_chinese_before_references(
+    text,
+    uuid, customer_name, model_name,
+    chat_func
+):
+    """
+    Translates the content of a text file to Chinese, keeping the '**References**' section in English.
+    
+    Args:
+        text (str): The content of the text file.
+        output_filename (str): The name of the output file.
+        chat_func (function): The function to use for translation.
+        
+    Returns:
+        str: The translated content.
+    
+    """
+    lines = text.split("\n")
+
+    # Step 3: 找到 '**References**' 行的索引
+    references_index = None
+    for i, line in enumerate(lines):
+        if line.strip() == "**References**":
+            references_index = i
+            break
+
+    # Step 4: 根据找到的索引分割内容
+    if references_index is not None:
+        main_content_lines = lines[:references_index]
+        references_content_lines = lines[references_index:]
+    else:
+        # 如果没有找到 '**References**'，则认为整个内容为正文
+        main_content_lines = lines
+        references_content_lines = []
+
+    # 将正文内容拼接为一个字符串
+    main_content = "\n".join(main_content_lines)
+
+    # Step 5: 分段处理正文内容进行翻译
+    sections = main_content.split("\n\n")
+    translated_sections = []
+
+    prompts = []
+    
+    for section in sections:
+        # 简化 prompt，只要求翻译正文内容
+        prompt = (
+            "Translate the following text to academic Chinese:\n\n"
+            f"Text:\n{section}\n\n"
+            "Output format:\n[Translated Chinese text here]"
+        )
+        prompts.append(prompt)
+    
+    responses = await asyncio.gather(
+        *(chat_func(prompt) for prompt in prompts)
+    )
+    for response in responses:
+        translated_section = response.choices[0].message.content.strip()
+        translated_sections.append(translated_section)
+
+    # Step 6: 将翻译后的正文拼接
+    translated_content = "\n\n".join(translated_sections)
+
+    # Step 7: 合并翻译后的正文和 References 部分
+    if references_content_lines:
+        references_content = "\n".join(references_content_lines)
+        final_content = translated_content + "\n\n" + references_content
+    else:
+        final_content = translated_content
+
+    # Step 8: 保存结果到新的文件
+    output_filename = f"{customer_name}/{uuid}/{model_name}/review_paper_translated.txt"
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_filename,
+        file_content=final_content
+    )
+
+    logger.info(f"\nTranslated content saved to {output_filename}")
+    return output_filename
+
+
+# Main function to automate the review paper creation process with language enhancement step
+async def create_review_paper(
+    relevant_papers_df,
+    main_topic,
+    uuid, customer_name, model_name,
+    chat_func,
+    translate_to_cn=False
+):
+    """
+    Main function to automate the review paper creation process with language enhancement step.
+    
+    Args:
+        relevant_papers_df (pd.DataFrame): DataFrame containing relevant papers.
+        main_topic (str): Main topic of the review paper.
+        uuid (str): Unique identifier for the review paper.
+        customer_name (str): Name of the customer.
+        chat_func (function): Function to handle chat interactions.
+        translate_to_cn (bool): Flag to indicate if translation to Chinese is required.
+    
+    Returns:
+        None
+    
+    """
+    
+    # Step 1: Generate subheadings related to the main topic
+    subheadings = await generate_subheadings(
+        relevant_papers_df, main_topic,
+        chat_func
+    )
+
+    # Step 2: Assign each summary to a subheading
+    relevant_papers_df = await assign_subheadings_to_summaries(
+        relevant_papers_df, subheadings, 
+        uuid, customer_name, model_name,
+        chat_func
+    )
+
+    # Step 3: Create paragraphs by subheading, with introductory and concluding sections, and references
+    review_content = await create_paragraphs_by_subheading(
+        relevant_papers_df, subheadings, main_topic, 
+        uuid, customer_name, model_name,
+        chat_func
+    )
+
+    # Step 4: Enhance language and readability
+    enhanced_content = await enhance_language_readability(
+        review_content,
+        chat_func
+    )
+
+    prefix = f"{customer_name}/{uuid}/{model_name}/"
+    output_dir = prefix
+
+    output_filename = os.path.join(output_dir, "review_paper.txt")
+
+    # Step: Translate to Chinese
+    if translate_to_cn:
+        await translate_to_chinese_before_references(
+            enhanced_content,
+            output_filename.replace(".txt", "_cn.txt"),
+            chat_func
+        )
+
+    # Step 6: Save the generated content to a text file
+    # with open(output_filename, "w", encoding="utf-8") as f:
+    #     f.write(enhanced_content)
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_filename,
+        file_content=enhanced_content
+    )
+
+    logger.info(f"\nReview paper saved to {output_filename}")
+    return output_filename

utils/pubmed_plus_utils.py

@@ -0,0 +1,665 @@
+import io
+import asyncio
+
+from minio import Minio
+from loguru import logger
+
+from entities.task import PubMedPlusTask
+from utils.api_utils import (
+    retry_operation,
+    get_chat_func,
+    compare_chat_chocies
+)
+from utils.r2_utils import (
+    get_client,
+    get_file_from_minio,
+    get_dataframe_from_minio,
+    upload_text_to_minio,
+    upload_task_json_to_minio,
+)
+from utils.paper_plus_utils import (
+    process_papers,
+    generate_subheadings,
+    assign_subheadings_to_summaries,
+    create_paragraphs_by_subheading,
+    refine_review_content,
+    translate_refined_review_to_chinese,
+    translate_to_chinese_before_references
+)
+from utils.pubmed_utils import (
+    generate_pubmed_search_string,
+    process_pubmed_data
+)
+
+
+BUCKET_NAME = "ai-scientist"
+
+
+# =================================
+# Function Groups: Pipeline for PubMed
+#
+# 1. pipeline
+# 2. single model chat
+# =================================
+
+async def pubmed_plus_pipeline(
+    task: PubMedPlusTask,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Pubmed pipeline
+
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        client: Minio, minio client
+        max_retries: int, max retries for each step
+        delay: float, delay between each retry
+
+    Returns:
+        None
+
+    """
+    if client is None:
+        client = get_client()
+
+    customer_name = task.customer_name
+    uuid = task.uuid
+    model_names = task.model_names
+
+    task.status_string["overall"] = "processing"
+
+    await asyncio.gather(
+        *(process_pubmed_single_chat(
+            task, model_name, client, max_retries, delay
+        ) for model_name in model_names)
+    )
+
+    # if compare between models
+    # at least 3 models should be selected
+    logger.info("Check Compare...")
+    if task.do_compare and len(task.model_names) >= 3:
+        if task.status.get("compare", 0) == 0:
+            contents = await asyncio.gather(
+                *(get_file_from_minio(
+                    bucket_name=BUCKET_NAME,
+                    object_name=f"{customer_name}/{uuid}/{model_name}/review_paper.txt",
+                    client=client
+                ) for model_name in model_names)
+            )
+            contents = [c.data.decode("utf-8") for c in contents]
+            task.status_string["overall"] = "Start Compare"
+
+            rank_scores = await compare_chat_chocies(
+                contents=contents,
+                model_names=model_names
+            )
+            best_content = contents[min(rank_scores, key=rank_scores.get)]
+            await upload_text_to_minio(
+                bucket_name=BUCKET_NAME,
+                object_name=f"{customer_name}/{uuid}/compared_reveiw_paper.txt",
+                file_content=best_content
+            )
+            task.status_string["overall"] = "Finished"
+            task.status["compare"] = 1
+            await upload_task_json_to_minio(task, client)
+        else:
+            task.status_string["overall"] = "Finished"
+            await upload_task_json_to_minio(task, client)
+    else:
+        logger.info("No Compare.")
+        task.status_string["overall"] = "Finished"
+        await upload_task_json_to_minio(task, client)
+
+
+async def process_pubmed_single_chat(
+    task: PubMedPlusTask,
+    model_name: str,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Task
+
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        model_name: str, model name, refer to the model used at this step
+        client: Minio, minio client
+        max_retries: int, max retries for each step
+        delay: float, delay between each retry
+
+    Returns:
+        None
+
+    """
+
+    # get minio client
+    if client is None:
+        client = get_client()
+
+    # add status for <model_name>
+    if model_name not in task.status.keys():
+        task.status[model_name] = 0
+
+    # set task status string
+    task.status_string["overall"] = "processing"
+    
+    process_steps = {
+        0: process_pubmed_generate_pubmed_string,
+        1: process_pubmed_fetch_data,
+        2: process_pubmed_process_papers,
+        3: process_pubmed_generate_subheadings,
+        4: process_pubmed_assign_subheadings_to_summaries,
+        5: process_pubmed_create_paragraphs_by_subheading,
+        6: process_pubmed_refine,
+        7: process_pubmed_translate,
+    }
+    
+    state_description = {
+        0: "Finished pubmed string generation.",
+        1: "Finished fetching data.",
+        2: "Finished paper processing.",
+        3: "Finished subheading generation.",
+        4: "Finished subheading assignment.",
+        5: "Finished paragraph generation.",
+        6: "Finished review refine.",
+        7: "Finished review translate.",
+    }
+
+    # Execute Phase
+    current_state = task.status[model_name]
+    for state in range(current_state, len(process_steps.keys())):
+        await process_steps[state](
+            task=task,
+            model_name=model_name,
+            save_name=model_name,
+            prev_name=model_name,
+            client=client,
+            max_retries=max_retries, delay=delay
+        )
+        task.status_string[model_name] = state_description[state]
+        task.status[model_name] = state + 1
+        await upload_task_json_to_minio(task, client)
+    
+    task.status_string[model_name] = "Finished."
+    await upload_task_json_to_minio(task, client)
+
+
+# =================================
+# Function Groups: process_pubmed_*
+# 1. _generate_pubmed_string
+# 2. _fetch_data
+# 3. _process_papers
+# 3. _generate_subheadings
+# 4. _assign_subheadings_to_summaries
+# 5. _create_paragraphs_by_subheading
+# 6. _refine
+# 7. _translate
+# =================================
+
+async def process_pubmed_generate_pubmed_string(
+    task: PubMedPlusTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Generate pubmed search string step
+
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+        max_retries: int, max retries for each step
+        delay: float, delay between each retry
+
+    Returns:
+        path to save results
+
+    """
+
+    if client is None:
+        client = get_client()
+
+    if prev_name is not None:
+        logger.warning("For first step, prev_model_name is not used.")
+
+    query = task.query
+    customer_name = task.customer_name
+    uuid = task.uuid
+
+    chat_func = get_chat_func(model_names=[model_name])[0]
+
+    pubmed_search_string, exceptions = await retry_operation(
+        generate_pubmed_search_string, task,
+        query=query,
+        max_retries=max_retries, delay=delay,
+        chat_func=chat_func
+    )
+    if pubmed_search_string is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Search String Generation Failed.")  # exit
+
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{save_name}/pubmed_search_string.txt",
+        file_content=pubmed_search_string
+    )
+
+
+async def process_pubmed_fetch_data(
+    task: PubMedPlusTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Fetch Data
+
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+
+    Returns:
+        path to save results
+
+    """
+
+    if client is None:
+        client = get_client()
+
+    customer_name = task.customer_name
+    uuid = task.uuid
+    start_year = task.start_year
+    end_year = task.end_year
+    size = task.size
+
+    pubmed_search_string = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/pubmed_search_string.txt",
+        client=client
+    )
+    pubmed_search_string = pubmed_search_string.data.decode("utf-8")
+    results, exceptions = await retry_operation(
+        process_pubmed_data, task,
+        query=pubmed_search_string,
+        model_name=save_name,
+        start_year=start_year, end_year=end_year,
+        size=size,
+        uuid=uuid, customer_name=customer_name,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise ConnectionError("Pubmed Data Fetch Failed.")  # exit
+
+
+async def process_pubmed_process_papers(
+    task: PubMedPlusTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Process Papers
+
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+
+    Returns:
+        path to save results
+
+    """
+    if client is None:
+        client = get_client()
+
+    query = task.query
+    direction = task.direction
+    customer_name = task.customer_name
+    uuid = task.uuid
+
+    chat_func = get_chat_func(model_names=[model_name])[0]
+
+    non_review_pubmed_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/pubmed_results_non_reviews.csv",
+        client=client
+    )
+    results, exceptions = await retry_operation(
+        process_papers, task,
+        dataframe=non_review_pubmed_df,
+        topic=query, direction=direction,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        max_retries=max_retries, delay=delay,
+        chat_func=chat_func
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Paper Processing Failed.")  # exit
+
+
+async def process_pubmed_generate_subheadings(
+    task: PubMedPlusTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Generate Subheadings
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        
+    Returns:
+        path to save results
+    """
+    if client is None:
+        client = get_client()
+    
+    query = task.query
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func([model_name])[0]
+    
+    relevant_papers_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/relevant_papers.csv",
+        client=client
+    )
+    
+    results, exceptions = await retry_operation(
+        generate_subheadings, task,
+        relevant_papers_df=relevant_papers_df,
+        main_topic=query,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        chat_func=chat_func,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Generate Subheadings Failed.")  # exit
+    
+
+async def process_pubmed_assign_subheadings_to_summaries(
+    task: PubMedPlusTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Assign Subheadings to Summaries
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+
+    Returns:
+        path to save results
+    """
+    
+    if client is None:
+        client = get_client()
+        
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func([model_name])[0]
+        
+    subheadings = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/generated_subheadings.txt",
+        client=client
+    )
+    subheadings = subheadings.data.decode("utf-8").split("\n")
+    
+    relevant_papers_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/relevant_papers.csv",
+        client=client
+    )
+    
+    results, exceptions = await retry_operation(
+        assign_subheadings_to_summaries, task,
+        subheadings=subheadings,
+        relevant_papers_df=relevant_papers_df,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        chat_func=chat_func,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Assign Subheadings Failed.")  # exit
+
+
+async def process_pubmed_create_paragraphs_by_subheading(
+    task: PubMedPlusTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Create Paragraphs by Subheading
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+        max_retries: int, max retries for the operation
+        delay: float, delay between retries
+
+    Returns:
+        path to save results
+    """
+    
+    if client is None:
+        client = get_client()
+    
+    query = task.query     
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func([model_name])[0]
+    
+    subheadings = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/generated_subheadings.txt",
+        client=client
+    )
+    subheadings = subheadings.data.decode("utf-8").split("\n")
+    
+    relevant_papers_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/assigned_subheadings.csv",
+        client=client
+    )
+    
+    results, exceptions = await retry_operation(
+        create_paragraphs_by_subheading, task,
+        subheadings=subheadings, main_topic=query,
+        relevant_papers_df=relevant_papers_df,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        chat_func=chat_func,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Create Paragraphs Failed.")  # exit
+
+
+async def process_pubmed_translate(
+    task: PubMedPlusTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Translate
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+        max_retries: int, max retries for the operation
+        delay: float, delay between retries
+        
+    Returns:
+        path to save results 
+    """
+    
+    if client is None:
+        client = get_client()
+        
+    customer_name = task.customer_name
+    uuid = task.uuid
+    do_refine = task.do_refine
+    
+    chat_func = get_chat_func([model_name])[0]
+    
+    if do_refine:
+        refined_review_content = await get_file_from_minio(
+            bucket_name=BUCKET_NAME,
+            object_name=f"{customer_name}/{uuid}/{prev_name}/review_paper_refined.docx",
+            client=client
+        )
+        refined_review_content = io.BytesIO(refined_review_content.data)
+        
+        results, exceptions = await retry_operation(
+            translate_refined_review_to_chinese, task,
+            refined_review_content=refined_review_content,
+            uuid=uuid, customer_name=customer_name, model_name=save_name,
+            chat_func=chat_func,
+            max_retries=max_retries, delay=delay
+        )
+        if results is None:  # no valid result after max retries
+            # store exception strings in status
+            task.status_string[model_name] = exceptions
+            await upload_task_json_to_minio(task, client)
+            raise RuntimeError("Pubmed Translate Refined Review Failed.")  # exit
+    else:
+        review_content = await get_file_from_minio(
+            bucket_name=BUCKET_NAME,
+            object_name=f"{customer_name}/{uuid}/{prev_name}/review_non_refined.txt",
+            client=client
+        )
+        results, exceptions = await retry_operation(
+            translate_to_chinese_before_references, task,
+            text=review_content,
+            uuid=uuid, customer_name=customer_name, model_name=save_name,
+            chat_func=chat_func,
+            max_retries=max_retries, delay=delay
+        )
+        if results is None:  # no valid result after max retries
+            # store exception strings in status
+            task.status_string[model_name] = exceptions
+            await upload_task_json_to_minio(task, client)
+            raise RuntimeError("Pubmed Translate Failed.")  # exit
+        
+
+async def process_pubmed_refine(
+    task: PubMedPlusTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Refine
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+        max_retries: int, max retries for the operation
+        delay: float, delay between retries
+        
+    Returns:
+        path to save results
+    """
+    
+    # additional check on if do_refine
+    # if not refine, exit here with 1
+    if not task.do_refine:
+        return 1
+    
+    if client is None:
+        client = get_client()
+        
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func([model_name])[0]
+    
+    review_content = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/review_non_refined.txt",
+        client=client
+    )
+    review_content = review_content.data.decode("utf-8")
+    
+    results, exceptions = await retry_operation(
+        refine_review_content, task,
+        non_refine_content=review_content,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        chat_func=chat_func,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Refine Failed.")  # exit
+    
+    

utils/pubmed_utils.py

@@ -0,0 +1,1078 @@
+import os
+import asyncio
+import aiohttp
+import requests
+import pandas as pd
+
+from minio import Minio
+from loguru import logger
+from bs4 import BeautifulSoup
+
+from entities.task import PubMedTask
+from utils.api_utils import (
+    retry_operation,
+    get_chat_func,
+    compare_chat_chocies
+)
+from utils.r2_utils import (
+    get_client,
+    get_file_from_minio,
+    get_dataframe_from_minio,
+    upload_text_to_minio,
+    upload_dataframe_to_minio,
+    upload_task_json_to_minio,
+)
+from utils.common_utils import escape_csv_field
+from utils.paper_utils import (
+    process_papers,
+    generate_subheadings,
+    assign_subheadings_to_summaries,
+    create_paragraphs_by_subheading,
+    enhance_language_readability,
+    translate_to_chinese_before_references
+)
+
+
+BUCKET_NAME = "ai-scientist"
+
+
+# =================================
+# Function Groups: Pipeline for PubMed
+#
+# 1. pipeline
+# 2. single model chat
+# =================================
+
+async def pubmed_pipeline(
+    task: PubMedTask,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Pubmed pipeline
+
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        client: Minio, minio client
+        max_retries: int, max retries for each step
+        delay: float, delay between each retry
+
+    Returns:
+        None
+
+    """
+    if client is None:
+        client = get_client()
+
+    customer_name = task.customer_name
+    uuid = task.uuid
+    model_names = task.model_names
+
+    task.status_string["overall"] = "processing"
+
+    await asyncio.gather(
+        *(process_pubmed_single_chat(
+            task, model_name, client, max_retries, delay
+        ) for model_name in model_names)
+    )
+
+    # if compare between models
+    # at least 3 models should be selected
+    logger.info("Check Compare...")
+    if task.do_compare and len(task.model_names) >= 3:
+        if task.status.get("compare", 0) == 0:
+            contents = await asyncio.gather(
+                *(get_file_from_minio(
+                    bucket_name=BUCKET_NAME,
+                    object_name=f"{customer_name}/{uuid}/{model_name}/review_paper.txt",
+                    client=client
+                ) for model_name in model_names)
+            )
+            contents = [c.data.decode("utf-8") for c in contents]
+            task.status_string["overall"] = "Start Compare"
+
+            rank_scores = await compare_chat_chocies(
+                contents=contents,
+                model_names=model_names
+            )
+            best_content = contents[min(rank_scores, key=rank_scores.get)]
+            await upload_text_to_minio(
+                bucket_name=BUCKET_NAME,
+                object_name=f"{customer_name}/{uuid}/compared_reveiw_paper.txt",
+                file_content=best_content
+            )
+            task.status_string["overall"] = "Finished"
+            task.status["compare"] = 1
+            await upload_task_json_to_minio(task, client)
+        else:
+            task.status_string["overall"] = "Finished"
+            await upload_task_json_to_minio(task, client)
+    else:
+        logger.info("No Compare.")
+        task.status_string["overall"] = "Finished"
+        await upload_task_json_to_minio(task, client)
+
+
+async def process_pubmed_single_chat(
+    task: PubMedTask,
+    model_name: str,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Task
+
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        model_name: str, model name, refer to the model used at this step
+        client: Minio, minio client
+        max_retries: int, max retries for each step
+        delay: float, delay between each retry
+
+    Returns:
+        None
+
+    """
+
+    # get minio client
+    if client is None:
+        client = get_client()
+
+    # add status for <model_name>
+    if model_name not in task.status.keys():
+        task.status[model_name] = 0
+
+    # set task status string
+    task.status_string["overall"] = "processing"
+
+    process_steps = {
+        0: process_pubmed_generate_pubmed_string,
+        1: process_pubmed_fetch_data,
+        2: process_pubmed_process_papers,
+        3: process_pubmed_generate_subheadings,
+        4: process_pubmed_assign_subheadings_to_summaries,
+        5: process_pubmed_create_paragraphs_by_subheading,
+        6: process_pubmed_enhance_language_readability,
+        7: process_pubmed_translate
+    }
+
+    state_description = {
+        0: "Finished pubmed string generation.",
+        1: "Finished fetching data.",
+        2: "Finished paper processing.",
+        3: "Finished subheading generation.",
+        4: "Finished subheading assignment.",
+        5: "Finished paragraph generation.",
+        6: "Finished review language readability enhancement.",
+        7: "Finished review translation."
+    }
+
+    # Execute Phase
+    current_state = task.status[model_name]
+    for state in range(current_state, len(process_steps.keys())):
+        await process_steps[state](
+            task=task,
+            model_name=model_name,
+            save_name=model_name,
+            prev_name=model_name,
+            client=client,
+            max_retries=max_retries, delay=delay
+        )
+        task.status_string[model_name] = state_description[state]
+        task.status[model_name] = state + 1
+        await upload_task_json_to_minio(task, client)
+    
+    task.status_string[model_name] = "Finished."
+    await upload_task_json_to_minio(task, client)
+    
+
+# =================================
+# Function Groups: process_pubmed_*
+# 1. _generate_pubmed_string
+# 2. _fetch_data
+# 3. _process_papers
+# 4. _generate_subheadings
+# 5. _assign_subheadings_to_summaries
+# 6. _create_paragraphs_by_subheading
+# 7. _enhance_language_readability
+# 8. _translate
+# =================================
+
+async def process_pubmed_generate_pubmed_string(
+    task: PubMedTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Generate pubmed search string step
+
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+        max_retries: int, max retries for each step
+        delay: float, delay between each retry
+
+    Returns:
+        path to save results
+
+    """
+
+    if client is None:
+        client = get_client()
+
+    if prev_name is not None:
+        logger.warning("For first step, prev_model_name is not used.")
+
+    query = task.query
+    customer_name = task.customer_name
+    uuid = task.uuid
+
+    chat_func = get_chat_func(model_names=[model_name])[0]
+
+    pubmed_search_string, exceptions = await retry_operation(
+        generate_pubmed_search_string, task,
+        query=query,
+        max_retries=max_retries, delay=delay,
+        chat_func=chat_func
+    )
+    if pubmed_search_string is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Search String Generation Failed.")  # exit
+
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{save_name}/pubmed_search_string.txt",
+        file_content=pubmed_search_string
+    )
+
+
+async def process_pubmed_fetch_data(
+    task: PubMedTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Fetch Data
+
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+
+    Returns:
+        path to save results
+
+    """
+
+    if client is None:
+        client = get_client()
+
+    customer_name = task.customer_name
+    uuid = task.uuid
+    start_year = task.start_year
+    end_year = task.end_year
+    size = task.size
+
+    pubmed_search_string = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/pubmed_search_string.txt",
+        client=client
+    )
+    pubmed_search_string = pubmed_search_string.data.decode("utf-8")
+    results, exceptions = await retry_operation(
+        process_pubmed_data, task,
+        query=pubmed_search_string,
+        model_name=save_name,
+        start_year=start_year, end_year=end_year,
+        size=size,
+        uuid=uuid, customer_name=customer_name,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise ConnectionError("Pubmed Data Fetch Failed.")  # exit
+
+
+async def process_pubmed_process_papers(
+    task: PubMedTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Process Papers
+
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+
+    Returns:
+        path to save results
+
+    """
+    if client is None:
+        client = get_client()
+
+    query = task.query
+    direction = task.direction
+    customer_name = task.customer_name
+    uuid = task.uuid
+
+    chat_func = get_chat_func(model_names=[model_name])[0]
+
+    non_review_pubmed_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/pubmed_results_non_reviews.csv",
+        client=client
+    )
+    results, exceptions = await retry_operation(
+        process_papers, task,
+        dataframe=non_review_pubmed_df,
+        topic=query, direction=direction,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        max_retries=max_retries, delay=delay,
+        chat_func=chat_func
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Paper Processing Failed.")  # exit
+
+
+async def process_pubmed_generate_subheadings(
+    task: PubMedTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Generate Subheadings
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        
+    Returns:
+        path to save results
+    """
+    if client is None:
+        client = get_client()
+    
+    query = task.query
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func([model_name])[0]
+    
+    relevant_papers_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/relevant_papers.csv",
+        client=client
+    )
+    
+    results, exceptions = await retry_operation(
+        generate_subheadings, task,
+        relevant_papers_df=relevant_papers_df,
+        main_topic=query,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        chat_func=chat_func,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Generate Subheadings Failed.")  # exit
+    
+
+async def process_pubmed_assign_subheadings_to_summaries(
+    task: PubMedTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Assign Subheadings to Summaries
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+
+    Returns:
+        path to save results
+    """
+    
+    if client is None:
+        client = get_client()
+        
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func([model_name])[0]
+        
+    subheadings = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/generated_subheadings.txt",
+        client=client
+    )
+    subheadings = subheadings.data.decode("utf-8").split("\n")
+    
+    relevant_papers_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/relevant_papers.csv",
+        client=client
+    )
+    
+    results, exceptions = await retry_operation(
+        assign_subheadings_to_summaries, task,
+        subheadings=subheadings,
+        relevant_papers_df=relevant_papers_df,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        chat_func=chat_func,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Assign Subheadings Failed.")  # exit
+
+
+async def process_pubmed_create_paragraphs_by_subheading(
+    task: PubMedTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Create Paragraphs by Subheading
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+        max_retries: int, max retries for the operation
+        delay: float, delay between retries
+
+    Returns:
+        path to save results
+    """
+    if client is None:
+        client = get_client()
+    
+    query = task.query     
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func([model_name])[0]
+    
+    subheadings = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/generated_subheadings.txt",
+        client=client
+    )
+    subheadings = subheadings.data.decode("utf-8").split("\n")
+    
+    relevant_papers_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/assigned_subheadings.csv",
+        client=client
+    )
+    
+    results, exceptions = await retry_operation(
+        create_paragraphs_by_subheading, task,
+        subheadings=subheadings, main_topic=query,
+        relevant_papers_df=relevant_papers_df,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        chat_func=chat_func,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Create Paragraphs Failed.")  # exit
+
+
+async def process_pubmed_enhance_language_readability(
+    task: PubMedTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Enhance Language Readability
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+        max_retries: int, max retries for the operation
+        delay: float, delay between retries
+        
+    Returns:
+        path to save results
+    """
+    if client is None:
+        client = get_client()
+        
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func([model_name])[0]
+    
+    review_content = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/review_non_refined.txt",
+        client=client
+    )
+    review_content = review_content.data.decode("utf-8")
+
+    results, exceptions = await retry_operation(
+        enhance_language_readability, task,
+        content=review_content,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        chat_func=chat_func,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Enhance Language Readability Failed.")  # exit
+    
+
+async def process_pubmed_translate(
+    task: PubMedTask,
+    model_name: str,
+    save_name: str,
+    prev_name: str = None,
+    client: Minio = None,
+    max_retries: int = 5,
+    delay: float = 0.5
+):
+    """
+    Process PubMed Translate
+    Args:
+        task: PubMedTask object, containig basic information for PubMedTask
+        prev_model_name: str, previous model name, refer to previous step result
+        model_name: str, next model name, refer to the model used at this step
+        save_name: str, save name for minio path
+        client: Minio, minio client
+        max_retries: int, max retries for the operation
+        delay: float, delay between retries
+        
+    Returns:
+        path to save results 
+    """
+    
+    if client is None:
+        client = get_client()
+        
+    customer_name = task.customer_name
+    uuid = task.uuid
+    
+    chat_func = get_chat_func([model_name])[0]
+    
+    review_content = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/{prev_name}/review_paper.txt",
+        client=client
+    )
+    review_content = review_content.data.decode("utf-8")
+    
+    results, exceptions = await retry_operation(
+        translate_to_chinese_before_references, task,
+        text=review_content,
+        uuid=uuid, customer_name=customer_name, model_name=save_name,
+        chat_func=chat_func,
+        max_retries=max_retries, delay=delay
+    )
+    if results is None:  # no valid result after max retries
+        # store exception strings in status
+        task.status_string[model_name] = exceptions
+        await upload_task_json_to_minio(task, client)
+        raise RuntimeError("Pubmed Translate Failed.")  # exit
+
+
+# =================================
+# Function Groups: PubMed Task
+#
+# functions specific for pubmed task
+# =================================
+
+async def generate_pubmed_search_string(query: str, chat_func) -> str:
+    # Construct the improved prompt using triple single quotes
+    prompt = f'''
+### Objective
+Your task is to generate a precise PubMed search string based on the input query: "{query}". You should:
+
+1. **Extract Critical Keywords**: Identify the main entities and concepts that have independent and specific meanings, avoiding overly general terms commonly found in many articles (e.g., "analysis", "study"). Focus on terms central to the topic.
+
+2. **Understand Keyword Relationships**: Analyze the logical relationship between keywords. If two or more keywords are conceptually similar or interchangeable, connect them using the OR operator. If they represent distinct concepts that must co-exist, connect them using the AND operator.
+
+3. **Expand Synonyms Thoughtfully**: For each critical keyword, generate at least 6 relevant English synonyms or related terms used in academic research. Ensure they align with the context of the query, including synonyms that may look different but are relevant based on the keyword's definition and hierarchy.
+
+4. **Include MeSH Terms**: Find the corresponding MeSH (Medical Subject Headings) terms for each critical keyword if available.
+
+5. **Construct the PubMed Search String**: Combine the critical keywords, their synonyms, and MeSH terms using Boolean operators. Ensure correct grouping using parentheses to reflect the logical relationships:
+   - If a group of terms is interchangeable (e.g., synonyms), use OR within parentheses.
+   - Use AND between distinct keyword groups.
+
+### Instructions
+- **Language**: All words must be in English.
+- **Avoid Stop Words**: Do not include stop words (e.g., 'a', 'an', 'the').
+- **Synonym Requirement**: For each critical keyword, generate **at least 6 synonyms** or related terms.
+- **Logical Operator Selection**: Adjust the Boolean logic based on the relationship between terms to accurately represent (A OR B) AND C patterns.
+- **Term Length**: Each term should be concise, with phrases containing at most two words.
+- **Formatting**:
+  - Use Boolean operators (AND, OR) to connect terms and use parentheses where necessary.
+  - Format MeSH terms as: "Term"[MeSH Terms]
+  - Format other terms as: "Term"[All Fields]
+
+### Example
+**Input**: Role of AI in antimicrobial resistance and drug discovery
+
+**Process**:
+1. **Extract Critical Keywords**:
+   - AI
+   - Antimicrobial resistance
+   - Drug discovery
+
+2. **Analyze Keyword Relationships**:
+   - AI OR machine learning (similar concepts)
+   - Antimicrobial resistance AND drug discovery (distinct concepts)
+
+3. **Expand Synonyms Thoughtfully**:
+   - **AI**: machine learning, artificial intelligence, deep learning, neural networks, computational intelligence, data-driven algorithms
+   - **Antimicrobial resistance**: antibiotic resistance, drug resistance, microbial resistance, bacterial resistance, pathogen resistance, multidrug resistance
+   - **Drug discovery**: drug design, pharmaceutical research, drug development, lead discovery, molecular screening, target identification
+
+4. **Include MeSH Terms**:
+   - **AI**: "Artificial Intelligence"[MeSH Terms]
+   - **Antimicrobial resistance**: "Drug Resistance, Microbial"[MeSH Terms]
+   - **Drug discovery**: "Drug Discovery"[MeSH Terms]
+
+5. **Construct the PubMed Search String**:
+
+'(("Artificial Intelligence"[MeSH Terms] OR "machine learning"[All Fields] OR "deep learning"[All Fields] OR "neural networks"[All Fields] OR "computational intelligence"[All Fields] OR "data-driven algorithms"[All Fields]) AND ("Drug Resistance, Microbial"[MeSH Terms] OR "antibiotic resistance"[All Fields] OR "microbial resistance"[All Fields] OR "bacterial resistance"[All Fields] OR "pathogen resistance"[All Fields] OR "multidrug resistance"[All Fields])) AND ("Drug Discovery"[MeSH Terms] OR "drug design"[All Fields] OR "pharmaceutical research"[All Fields] OR "drug development"[All Fields] OR "lead discovery"[All Fields] OR "molecular screening"[All Fields])'
+
+### Now, generate the PubMed search string for the following query:
+
+**Query**: {query}
+
+Please provide only the final PubMed search string in the specified format.
+    '''
+
+    # Call the language model to get the PubMed search string
+    result = await chat_func(prompt)
+
+    # Extract the PubMed search string from the model's response
+    pubmed_search_string = result.choices[0].message.content.strip()
+
+    return pubmed_search_string
+
+
+async def process_pubmed_data(
+    query,
+    model_name,
+    start_year, end_year, size,
+    uuid, customer_name
+):
+    """
+    Process PubMed Data
+
+    Args:
+        query: str, query for PubMed search
+        model_name: str, model name
+        start_year: int, start year for PubMed search
+        end_year: int, end year for PubMed search
+        size: int, number of results per page
+        uuid: str, uuid for the task
+        customer_name: str, customer name for the task
+        client: Minio, minio client
+
+    Returns:
+        path to save results
+
+    """
+
+    # get prefix
+    prefix = f"{customer_name}/{uuid}/{model_name}/"
+    output_folder = prefix
+
+    # set file paths
+    combined_txt_filename = os.path.join(
+        output_folder, f'pubmed_page_combined.txt')
+    results_csv_filename = os.path.join(output_folder, f'pubmed_results.csv')
+    results_with_links_csv_filename = os.path.join(
+        output_folder, f'pubmed_results_with_full_text_links.csv')
+    impact_factors_csv_filename = os.path.join(
+        output_folder, f'pubmed_results_with_impact_factors.csv')
+    non_review_csv_filename = os.path.join(
+        output_folder, f'pubmed_results_non_reviews.csv')
+
+    # step 1: save pubmed pages
+    await save_combined_pubmed_page(query, start_year, end_year, size, output_filename=combined_txt_filename)
+
+    # step 2: process pubmed files
+    await process_pubmed_file(combined_txt_filename, results_csv_filename)
+
+    # step 3：添加全文链接
+    # pubmed_df = pd.read_csv(results_csv_filename)
+    pubmed_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=results_csv_filename
+    )
+    pubmed_df["Full_Text_Links"] = pubmed_df["PMID"].apply(get_full_text_links)
+    await upload_dataframe_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=results_with_links_csv_filename,
+        df=pubmed_df
+    )
+
+    # step 4: merge impact factor
+    impact_factors_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name='2023-JCR.xlsx'
+    )
+
+    # Standardize the case of the JT column in both dataframes to lowercase
+    pubmed_df['JT'] = pubmed_df['JT'].str.lower()
+    impact_factors_df['JT'] = impact_factors_df['JT'].str.lower()
+
+    # Perform the merge based on the JT column
+    merged_df = pd.merge(pubmed_df, impact_factors_df, on='JT', how='left')
+
+    # Save the merged dataframe to a new CSV file
+    await upload_dataframe_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=impact_factors_csv_filename,
+        df=merged_df
+    )
+    logger.info(f"Merged data saved to {impact_factors_csv_filename}")
+
+    # step 5: filter non review papers
+    pubmed_df = await get_dataframe_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=impact_factors_csv_filename
+    )
+    non_review_pubmed_df = pubmed_df[pubmed_df["Review"] == "No"]
+    await upload_dataframe_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=non_review_csv_filename,
+        df=non_review_pubmed_df
+    )
+
+    logger.info(f"非评论类文章已保存到 {non_review_csv_filename}")
+
+    return pubmed_df, non_review_pubmed_df
+
+
+async def save_combined_pubmed_page(query, start_year, end_year, size=200, output_filename='pubmed_page_combined.txt'):
+    content1 = await save_pubmed_page(query, start_year, end_year, size)
+    content2 = await save_pubmed_page_date(query, start_year, end_year, size)
+
+    combined_content = content1 + "\n" + content2
+
+    # 保存合并的网页内容到指定的txt文件
+    # async with aiofiles.open(output_filename, 'w', encoding='utf-8') as file:
+    #     await file.write(combined_content)
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_filename,
+        file_content=combined_content
+    )
+
+    logger.info(f"Page content saved to {output_filename}")
+
+
+async def save_pubmed_page(query, start_year, end_year, size=200):
+    base_url = "https://pubmed.ncbi.nlm.nih.gov/"
+    params = {
+        'term': query,
+        'filter': f'years.{start_year}-{end_year}',
+        'format': 'pubmed',
+        'size': size
+    }
+
+    # 构建检索网址
+    search_url = f"{base_url}?term={params['term']}&filter={params['filter']}&format={params['format']}&size={params['size']}"
+    logger.info(f"检索网址: {search_url}")
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(base_url, params=params) as response:
+            if response.status != 200:
+                logger.error("Failed to retrieve data from save_pubmed_page")
+                raise ConnectionError(
+                    "Failed to retrieve data from save_pubmed_page")
+            return await response.text()
+
+
+async def save_pubmed_page_date(query, start_year, end_year, size=200):
+    base_url = "https://pubmed.ncbi.nlm.nih.gov/"
+    params = {
+        'term': query,
+        'filter': f'years.{start_year}-{end_year}',
+        'format': 'pubmed',
+        'size': size,
+        'sort': 'date'
+    }
+
+    # 构建检索网址
+    search_url = f"{base_url}?term={params['term']}&filter={params['filter']}&format={params['format']}&size={params['size']}&sort={params['sort']}"
+    logger.info(f"检索网址: {search_url}")
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(base_url, params=params) as response:
+            if response.status != 200:
+                logger.error(
+                    "Failed to retrieve data from save_pubmed_page_date")
+                raise ConnectionError(
+                    "Failed to retrieve data from save_pubmed_page_date")
+            return await response.text()
+
+
+async def process_pubmed_file(input_file, output_file):
+    # Read the file and replace specific text
+    # async with aiofiles.open(input_file, 'r', encoding='utf-8') as file:
+    #     content = await file.read()
+
+    content = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=input_file
+    )
+    content = content.data.decode("utf-8")
+    content = content.replace(
+        '<pre class="search-results-chunk">PMID-', '<pre class="search-results-chunk">\nPMID-')
+
+    # Split the content into lines
+    lines = content.split('\n')
+    records = []
+    current_record = {}
+    collecting_abstract = False
+    collecting_title = False
+    collecting_pt = False
+    abstract_lines = []
+    title_lines = []
+    pt_lines = []
+    first_author_recorded = False  # Flag to capture the first occurrence of FAU
+
+    for line in lines:
+        if line.startswith("PMID- "):
+            if current_record:
+                # Finalize the current record before starting a new one
+                current_record['AB'] = ' '.join(
+                    abstract_lines).replace('\n', ' ')
+                current_record['TI'] = ' '.join(title_lines).replace('\n', ' ')
+                current_record['PT'] = ' '.join(pt_lines).replace('\n', ' ')
+
+                # Default Review to 'No' if not set to 'Yes' during PT or AB processing
+                if 'Review' not in current_record:
+                    current_record['Review'] = 'No'
+
+                # Check for mismatches between FAU-frist and the first entry in FAU list
+                if 'FAU-frist' in current_record and 'FAU' in current_record and current_record['FAU']:
+                    if current_record['FAU-frist'] != current_record['FAU'][0]:
+                        current_record['FAU'].insert(
+                            0, current_record['FAU-frist'])
+
+                if 'JT' not in current_record:
+                    # Ensure JT is present even if not found
+                    current_record['JT'] = ''
+                if 'DCOM' not in current_record:
+                    # Ensure DCOM is present even if not found
+                    current_record['DCOM'] = ''
+
+                # Add current record to list of records
+                records.append(current_record)
+
+            # Start a new record
+            current_record = {'PMID': line.split("PMID- ")[1].strip()}
+            collecting_abstract = False
+            collecting_title = False
+            collecting_pt = False
+            abstract_lines = []
+            title_lines = []
+            pt_lines = []
+            first_author_recorded = False  # Reset the flag for a new record
+
+        elif line.startswith("FAU - "):
+            # Append each FAU to a list and capture the first occurrence
+            author_name = line.split("FAU - ")[1].strip()
+            if 'FAU' not in current_record:
+                current_record['FAU'] = []
+            current_record['FAU'].append(author_name)
+
+            # Record the first occurrence in FAU-frist
+            if not first_author_recorded:
+                current_record['FAU-frist'] = author_name
+                first_author_recorded = True
+
+        elif line.startswith("JT  - "):
+            current_record['JT'] = line.split("JT  - ")[1].strip()
+
+        elif line.startswith("DCOM- "):
+            current_record['DCOM'] = line.split("DCOM- ")[1].strip()
+
+        elif line.startswith("TI  - "):
+            collecting_title = True
+            title_lines.append(line.split("TI  - ")[1].strip())
+
+        elif collecting_title:
+            if any(line.startswith(prefix) for prefix in ["LID - ", "AB  - ", "FAU - ", "PG  - "]):
+                collecting_title = False
+            else:
+                title_lines.append(line.strip())
+
+        elif line.startswith("LID - "):
+            lid = line.split("LID - ")[1].strip()
+            if '[doi]' in lid:
+                lid = lid.split(' [doi]')[0]
+            # 保留较长的LID
+            if 'LID' in current_record:
+                current_record['LID'] = lid if len(lid) > len(
+                    current_record['LID']) else current_record['LID']
+            else:
+                current_record['LID'] = lid
+
+        elif line.startswith("AB  - "):
+            collecting_abstract = True
+            abstract_text = line.split("AB  - ")[1].strip()
+            abstract_lines.append(abstract_text)
+            # Check if 'review' is in AB line (case insensitive)
+            if 'review' in abstract_text.lower():
+                current_record['Review'] = 'Yes'
+
+        elif collecting_abstract:
+            if any(line.startswith(prefix) for prefix in ["LID - ", "FAU - ", "PG  - "]):
+                collecting_abstract = False
+            else:
+                abstract_text = line.strip()
+                abstract_lines.append(abstract_text)
+                # Check if 'review' is in AB line (case insensitive)
+                if 'review' in abstract_text.lower():
+                    current_record['Review'] = 'Yes'
+
+        elif line.startswith("PT  - "):
+            pt_line = line.split("PT  - ")[1].strip()
+            pt_lines.append(pt_line)
+            # Check if 'review' is in PT line (case insensitive)
+            if 'review' in pt_line.lower():
+                current_record['Review'] = 'Yes'
+
+        elif collecting_pt:
+            if any(line.startswith(prefix) for prefix in ["LID - ", "AB  - ", "FAU - ", "PG  - "]):
+                collecting_pt = False
+            else:
+                pt_text = line.strip()
+                pt_lines.append(pt_text)
+                # Check if 'review' is in PT line (case insensitive)
+                if 'review' in pt_text.lower():
+                    current_record['Review'] = 'Yes'
+
+    # Final record handling after loop ends
+    if current_record:
+        current_record['AB'] = ' '.join(abstract_lines).replace('\n', ' ')
+        current_record['TI'] = ' '.join(title_lines).replace('\n', ' ')
+        current_record['PT'] = ' '.join(pt_lines).replace('\n', ' ')
+        if 'Review' not in current_record:
+            current_record['Review'] = 'No'
+
+        # Check for mismatches between FAU-frist and the first entry in FAU list
+        if 'FAU-frist' in current_record and 'FAU' in current_record and current_record['FAU']:
+            if current_record['FAU-frist'] != current_record['FAU'][0]:
+                current_record['FAU'].insert(0, current_record['FAU-frist'])
+
+        if 'JT' not in current_record:
+            current_record['JT'] = ''
+        if 'DCOM' not in current_record:
+            current_record['DCOM'] = ''
+
+        records.append(current_record)
+
+    # Remove duplicate records by PMID
+    unique_records = []
+    seen_pmids = set()
+    for record in records:
+        if record['PMID'] not in seen_pmids:
+            seen_pmids.add(record['PMID'])
+            unique_records.append(record)
+
+    # Write unique records to output CSV
+    # async with aiofiles.open(output_file, 'w', encoding='utf-8', newline='') as csvfile:
+    text = ""
+    fieldnames = ['JT', 'DCOM', 'PMID', 'TI', 'LID',
+                  'AB', 'FAU', 'FAU-frist', 'PT', 'Review']
+    header = ','.join(fieldnames) + '\n'
+    text += header
+
+    # Write each record
+    for record in unique_records:
+        # Join the FAU list as a single string
+        record['FAU'] = '; '.join(record.get('FAU', []))  # Safely get 'FAU'
+
+        # Prepare the row as a CSV string
+        row = ','.join([escape_csv_field(str(record.get(field, '')))
+                       for field in fieldnames]) + '\n'
+        text += row
+
+    await upload_text_to_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=output_file,
+        file_content=text
+    )
+
+
+def get_full_text_links(pmid):
+    url = f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/"
+    response = requests.get(url)
+    soup = BeautifulSoup(response.text, 'html.parser')
+
+    # 从页面中提取所有链接
+    links = [link['href'] for link in soup.find_all('a', href=True)]
+
+    # 如果存在第27个链接，则返回它，否则返回None
+    return links[26] if len(links) >= 27 else None

utils/r2_utils.py

@@ -0,0 +1,197 @@
+import boto3
+import io
+import json
+import asyncio
+import pandas as pd
+
+from docx import Document
+from loguru import logger
+from entities.task import Task, task_factory
+
+
+BUCKET_NAME = "ai-scientist"
+
+r2_endpoint = "https://468d92a3c903c841bc2de3b413e45072.r2.cloudflarestorage.com/ai-scientist"
+
+TOKEN = "KhGGD1ZJI_YTlLaZ0nSMfBJSLnOhgYN6cwq1De7G"
+R2_ACCESS_KEY_ID = "b9bc4becece838742ae1dc161be92de3"
+R2_SECRET_ACCESS_KEY = "f68eb82bd1c00528f26c6ac9b57d737fe0e4729ac7c429030fbc22a17dc8f105"
+
+def get_client():
+    return boto3.client(
+        "s3",
+        endpoint_url=r2_endpoint,
+        aws_access_key_id=R2_ACCESS_KEY_ID,
+        aws_secret_access_key=R2_SECRET_ACCESS_KEY,
+        region_name="auto"  # R2 需要设置为 auto
+    )
+
+
+async def get_task_from_minio(
+    uuid: str,
+    customer_name: str,
+    client=None
+) -> Task:
+    if client is None:
+        client = get_client()
+
+    response = await asyncio.to_thread(
+        lambda: client.list_objects_v2(
+            Bucket=BUCKET_NAME,
+            Prefix=f"{customer_name}/"
+        )
+    )
+
+    objects = response.get("Contents", [])
+    if not objects:
+        raise FileNotFoundError(f"No task found for customer {customer_name}")
+
+    object_names = [obj["Key"].split("/")[1] for obj in objects]
+    if uuid not in object_names:
+        raise FileNotFoundError(f"No task found for customer {customer_name} with uuid {uuid}")
+
+    json_file = await get_file_from_minio(
+        bucket_name=BUCKET_NAME,
+        object_name=f"{customer_name}/{uuid}/task.json",
+        client=client
+    )
+
+    json_data = json_file.decode("utf-8")
+    json_data = json.loads(json_data)
+    return task_factory[json_data["task_type"]].load_from_json(json_data)
+
+
+async def get_all_tasks_from_minio(
+    customer_name: str,
+    client=None
+) -> list[Task]:
+    if client is None:
+        client = get_client()
+
+    response = await asyncio.to_thread(
+        lambda: client.list_objects_v2(
+            Bucket=BUCKET_NAME,
+            Prefix=f"{customer_name}/"
+        )
+    )
+    objects = response.get("Contents", [])
+    if not objects:
+        return []
+
+    task_ids = list(set([obj["Key"].split("/")[1] for obj in objects]))
+    task_jsons = await asyncio.gather(
+        *(get_task_from_minio(uuid=task_id, customer_name=customer_name, client=client) for task_id in task_ids)
+    )
+    return task_jsons
+
+
+async def upload_task_json_to_minio(task: Task, client=None) -> Task:
+    if client is None:
+        client = get_client()
+
+    json_data = task.save_to_json()
+    byte_data = io.BytesIO(json_data.encode("utf-8"))
+
+    await asyncio.to_thread(
+        lambda: client.put_object(
+            Bucket=BUCKET_NAME,
+            Key=f"{task.customer_name}/{task.uuid}/task.json",
+            Body=byte_data,
+            ContentType="application/json"
+        )
+    )
+    return task
+
+
+async def upload_text_to_minio(
+    bucket_name: str,
+    object_name: str,
+    file_content: str,
+    client=None,
+):
+    if client is None:
+        client = get_client()
+
+    file_data = io.BytesIO(file_content.encode("utf-8"))
+
+    await asyncio.to_thread(
+        lambda: client.put_object(
+            Bucket=bucket_name,
+            Key=object_name,
+            Body=file_data
+        )
+    )
+
+
+async def upload_dataframe_to_minio(
+    bucket_name: str,
+    object_name: str,
+    df: pd.DataFrame,
+    client=None,
+):
+    buffer = io.BytesIO()
+    df.to_csv(buffer, index=False)
+    await upload_text_to_minio(
+        bucket_name=bucket_name,
+        object_name=object_name,
+        file_content=buffer.getvalue().decode("utf-8"),
+        client=client
+    )
+
+
+async def upload_document_to_minio(
+    bucket_name: str,
+    object_name: str,
+    document: Document,
+    client=None,
+):
+    if client is None:
+        client = get_client()
+
+    buffer = io.BytesIO()
+    document.save(buffer)
+    buffer.seek(0)
+
+    await asyncio.to_thread(
+        lambda: client.put_object(
+            Bucket=bucket_name,
+            Key=object_name,
+            Body=buffer,
+            ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+        )
+    )
+
+
+async def get_file_from_minio(
+    bucket_name: str,
+    object_name: str,
+    client=None,
+):
+    if client is None:
+        client = get_client()
+
+    try:
+        response = await asyncio.to_thread(
+            lambda: client.get_object(Bucket=bucket_name, Key=object_name)
+        )
+        return response["Body"].read()
+    except Exception as e:
+        raise Exception(f"Error getting file from minio: {e}")
+
+
+async def get_dataframe_from_minio(
+    bucket_name: str,
+    object_name: str,
+    client=None,
+):
+    file_data = await get_file_from_minio(
+        bucket_name=bucket_name,
+        object_name=object_name,
+        client=client
+    )
+
+    if object_name.endswith(".csv"):
+        df = pd.read_csv(io.BytesIO(file_data))
+    elif object_name.endswith(".xlsx") or object_name.endswith(".xls"):
+        df = pd.read_excel(io.BytesIO(file_data))
+    return df