src/data/chunk.py

import pandas as pd
from autorag.data.chunk import langchain_chunk


def chunk(raw_df: pd.DataFrame, method: str, lang: str = "en", **kwargs) -> pd.DataFrame:
	corpus_df = langchain_chunk(raw_df, chunk_method=method, add_file_name=lang, **kwargs)
	return corpus_df