Upload 11 files

scripts/crag.py

@@ -0,0 +1,153 @@
+import argparse
+
+from rag_pipelines.embeddings.dense import DenseEmbeddings
+from rag_pipelines.embeddings.sparse import SparseEmbeddings
+from rag_pipelines.llms.groq import ChatGroqGenerator
+from rag_pipelines.pipelines.crag import CorrectiveRAGPipeline
+from rag_pipelines.retrieval_evaluator.document_grader import DocumentGrader
+from rag_pipelines.retrieval_evaluator.retrieval_evaluator import RetrievalEvaluator
+from rag_pipelines.vectordb.pinecone_hybrid_index import PineconeHybridVectorDB
+from rag_pipelines.vectordb.pinecone_hybrid_retriever import PineconeHybridRetriever
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run the Corrective RAG pipeline.")
+
+    # Dense embeddings arguments
+    parser.add_argument(
+        "--dense_model_name",
+        type=str,
+        default="sentence-transformers/all-MiniLM-L6-v2",
+        help="Dense embedding model name.",
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="cpu",
+        help="Device to run the dense embedding model.",
+    )
+
+    # Sparse embeddings arguments
+    parser.add_argument(
+        "--sparse_max_seq_length",
+        type=int,
+        default=512,
+        help="Maximum sequence length for sparse embeddings.",
+    )
+
+    # Pinecone arguments
+    parser.add_argument("--pinecone_api_key", type=str, required=True, help="Pinecone API key.")
+    parser.add_argument("--index_name", type=str, default="edgar", help="Pinecone index name.")
+    parser.add_argument("--dimension", type=int, default=384, help="Dimension of embeddings.")
+    parser.add_argument("--metric", type=str, default="dotproduct", help="Metric for similarity search.")
+    parser.add_argument("--region", type=str, default="us-east-1", help="Pinecone region.")
+    parser.add_argument(
+        "--namespace",
+        type=str,
+        default="edgar-all",
+        help="Namespace for Pinecone retriever.",
+    )
+
+    # Retriever arguments
+    parser.add_argument("--alpha", type=float, default=0.5, help="Alpha parameter for hybrid retriever.")
+    parser.add_argument("--top_k", type=int, default=5, help="Number of top documents to retrieve.")
+
+    # LLM arguments
+    parser.add_argument(
+        "--llm_model",
+        type=str,
+        default="llama-3.2-90b-vision-preview",
+        help="Language model name.",
+    )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=0,
+        help="Temperature for the language model.",
+    )
+    parser.add_argument("--llm_api_key", type=str, required=True, help="API key for the language model.")
+
+    # Retrieval Evaluator and Document Grader arguments
+    parser.add_argument(
+        "--relevance_threshold",
+        type=float,
+        default=0.7,
+        help="Relevance threshold for document grading.",
+    )
+
+    # Query
+    parser.add_argument(
+        "--query",
+        type=str,
+        required=True,
+        help="Query to run through the Corrective RAG pipeline.",
+    )
+
+    args = parser.parse_args()
+
+    # Initialize embeddings
+    dense_embeddings = DenseEmbeddings(
+        model_name=args.dense_model_name,
+        model_kwargs={"device": args.device},
+        encode_kwargs={"normalize_embeddings": True},
+        show_progress=True,
+    )
+    sparse_embeddings = SparseEmbeddings(model_kwargs={"max_seq_length": args.sparse_max_seq_length})
+
+    dense_embedding_model = dense_embeddings.embedding_model
+    sparse_embedding_model = sparse_embeddings.sparse_embedding_model
+
+    # Initialize Pinecone vector DB
+    pinecone_vector_db = PineconeHybridVectorDB(
+        api_key=args.pinecone_api_key,
+        index_name=args.index_name,
+        dimension=args.dimension,
+        metric=args.metric,
+        region=args.region,
+    )
+
+    # Initialize Pinecone retriever
+    pinecone_retriever = PineconeHybridRetriever(
+        index=pinecone_vector_db.index,
+        dense_embedding_model=dense_embedding_model,
+        sparse_embedding_model=sparse_embedding_model,
+        alpha=args.alpha,
+        top_k=args.top_k,
+        namespace=args.namespace,
+    )
+
+    # Initialize RetrievalEvaluator and DocumentGrader
+    retrieval_evaluator = RetrievalEvaluator(
+        llm_model=args.llm_model,
+        llm_api_key=args.llm_api_key,
+        temperature=args.temperature,
+    )
+    document_grader = DocumentGrader(
+        evaluator=retrieval_evaluator,
+        threshold=args.relevance_threshold,
+    )
+
+    # Load the prompt and initialize the generator
+    generator = ChatGroqGenerator(
+        model=args.llm_model,
+        api_key=args.llm_api_key,
+        llm_params={"temperature": args.temperature},
+    )
+    llm = generator.llm
+
+    # Initialize the Corrective RAG pipeline
+    corrective_rag = CorrectiveRAGPipeline(
+        retriever=pinecone_retriever.hybrid_retriever,
+        prompt=retrieval_evaluator.prompt_template,
+        llm=llm,
+        document_grader=document_grader,
+        tracing_project_name="sec_corrective_rag",
+    )
+
+    # Run the pipeline
+    output = corrective_rag.run(args.query)
+    print(output)
+
+
+if __name__ == "__main__":
+    main()

scripts/crag_evaluation.py

@@ -0,0 +1,300 @@
+import argparse
+
+from dataloaders import (
+    ARCDataloader,
+    EdgarDataLoader,
+    FactScoreDataloader,
+    PopQADataloader,
+    TriviaQADataloader,
+)
+
+from rag_pipelines.embeddings.dense import DenseEmbeddings
+from rag_pipelines.embeddings.sparse import SparseEmbeddings
+from rag_pipelines.evaluation import (
+    AnswerRelevancyScorer,
+    ContextualPrecisionScorer,
+    ContextualRecallScorer,
+    ContextualRelevancyScorer,
+    Evaluator,
+    FaithfulnessScorer,
+    HallucinationScorer,
+    SummarizationScorer,
+)
+from rag_pipelines.evaluation.evaluator import Evaluator
+from rag_pipelines.llms.groq import ChatGroqGenerator
+from rag_pipelines.pipelines.crag import CorrectiveRAGPipeline
+from rag_pipelines.retrieval_evaluator.document_grader import DocumentGrader
+from rag_pipelines.retrieval_evaluator.retrieval_evaluator import RetrievalEvaluator
+from rag_pipelines.vectordb.pinecone_hybrid_index import PineconeHybridVectorDB
+from rag_pipelines.vectordb.pinecone_hybrid_retriever import PineconeHybridRetriever
+
+SUPPORTED_DATASETS = {
+    "arc": ARCDataloader,
+    "edgar": EdgarDataLoader,
+    "popqa": PopQADataloader,
+    "factscore": FactScoreDataloader,
+    "triviaqa": TriviaQADataloader,
+}
+
+SCORER_CLASSES = {
+    "contextual_precision": ContextualPrecisionScorer,
+    "contextual_recall": ContextualRecallScorer,
+    "contextual_relevancy": ContextualRelevancyScorer,
+    "answer_relevancy": AnswerRelevancyScorer,
+    "faithfulness": FaithfulnessScorer,
+    "summarization": SummarizationScorer,
+    "hallucination": HallucinationScorer,
+}
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run the Corrective RAG pipeline.")
+
+    # Dense embeddings arguments
+    parser.add_argument(
+        "--dense_model_name",
+        type=str,
+        default="sentence-transformers/all-MiniLM-L6-v2",
+        help="Dense embedding model name.",
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="cpu",
+        help="Device to run the dense embedding model.",
+    )
+
+    # Sparse embeddings arguments
+    parser.add_argument(
+        "--sparse_max_seq_length",
+        type=int,
+        default=512,
+        help="Maximum sequence length for sparse embeddings.",
+    )
+
+    # Pinecone arguments
+    parser.add_argument("--pinecone_api_key", type=str, required=True, help="Pinecone API key.")
+    parser.add_argument("--index_name", type=str, default="edgar", help="Pinecone index name.")
+    parser.add_argument("--dimension", type=int, default=384, help="Dimension of embeddings.")
+    parser.add_argument("--metric", type=str, default="dotproduct", help="Metric for similarity search.")
+    parser.add_argument("--region", type=str, default="us-east-1", help="Pinecone region.")
+    parser.add_argument(
+        "--namespace",
+        type=str,
+        default="edgar-all",
+        help="Namespace for Pinecone retriever.",
+    )
+
+    # Retriever arguments
+    parser.add_argument("--alpha", type=float, default=0.5, help="Alpha parameter for hybrid retriever.")
+    parser.add_argument("--top_k", type=int, default=5, help="Number of top documents to retrieve.")
+
+    # LLM arguments
+    parser.add_argument(
+        "--llm_model",
+        type=str,
+        default="llama-3.2-90b-vision-preview",
+        help="Language model name.",
+    )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=0,
+        help="Temperature for the language model.",
+    )
+    parser.add_argument("--llm_api_key", type=str, required=True, help="API key for the language model.")
+
+    # Retrieval Evaluator and Document Grader arguments
+    parser.add_argument(
+        "--relevance_threshold",
+        type=float,
+        default=0.7,
+        help="Relevance threshold for document grading.",
+    )
+
+    # Load evaluation data
+    parser = argparse.ArgumentParser(description="Load evaluation dataset and initialize the dataloader.")
+    parser.add_argument(
+        "--dataset_type",
+        type=str,
+        default="edgar",
+        choices=SUPPORTED_DATASETS.keys(),
+        help="Dataset to load from. Options: arc, edgar, popqa, factscore, triviaqa.",
+    )
+    parser.add_argument(
+        "--hf_dataset_path",
+        type=str,
+        default="lamini/earnings-calls-qa",
+        help="Path to the HuggingFace dataset.",
+    )
+    parser.add_argument(
+        "--dataset_split",
+        type=str,
+        default="test",
+        help="Split of the dataset to use (e.g., train, validation, test).",
+    )
+
+    # Scorer arguments
+    parser.add_argument(
+        "--scorer",
+        type=str,
+        default="contextual_precision",
+        choices=[
+            "contextual_precision",
+            "contextual_recall",
+            "contextual_relevancy",
+            "answer_relevancy",
+            "faithfulness",
+            "summarization",
+            "hallucination",
+        ],
+        help="Scorer to use.",
+    )
+
+    # Evaluation arguments
+    parser.add_argument(
+        "--evaluation_name",
+        type=str,
+        default="hybrid_rag",
+        help="Name of the evaluation.",
+    )
+
+    # Add argument for selecting scorers
+    parser.add_argument(
+        "--scorers",
+        type=str,
+        nargs="+",
+        choices=SCORER_CLASSES.keys(),
+        required=True,
+        help="List of scorers to use. Options: contextual_precision, contextual_recall, contextual_relevancy, "
+        "answer_relevancy, faithfulness, summarization, hallucination.",
+    )
+
+    # Add shared arguments for scorer parameters
+    parser.add_argument("--threshold", type=float, default=0.5, help="Threshold for evaluation.")
+    parser.add_argument("--model", type=str, default="gpt-4", help="Model to use for scoring.")
+    parser.add_argument("--include_reason", action="store_true", help="Include reasons in scoring.")
+    parser.add_argument(
+        "--assessment_questions",
+        type=str,
+        nargs="*",
+        help="List of assessment questions for scoring.",
+    )
+    parser.add_argument("--strict_mode", action="store_true", help="Enable strict mode for scoring.")
+    parser.add_argument("--async_mode", action="store_true", help="Enable asynchronous processing.")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose output.")
+    parser.add_argument(
+        "--truths_extraction_limit",
+        type=int,
+        default=None,
+        help="Limit for truth extraction in scoring.",
+    )
+
+    args = parser.parse_args()
+
+    # Initialize dataloader based on the dataset type
+    try:
+        DataLoaderClass = SUPPORTED_DATASETS[args.dataset_type]
+        dataloader = DataLoaderClass(dataset_name=args.hf_dataset_path, split=args.dataset_split)
+    except KeyError:
+        msg = (
+            f"Dataset '{args.dataset_type}' is not supported. "
+            f"Supported options are: {', '.join(SUPPORTED_DATASETS.keys())}."
+        )
+        raise ValueError(msg)
+
+    eval_dataset = dataloader.get_eval_data()
+
+    # Initialize embeddings
+    dense_embeddings = DenseEmbeddings(
+        model_name=args.dense_model_name,
+        model_kwargs={"device": args.device},
+        encode_kwargs={"normalize_embeddings": True},
+        show_progress=True,
+    )
+    sparse_embeddings = SparseEmbeddings(model_kwargs={"max_seq_length": args.sparse_max_seq_length})
+
+    dense_embedding_model = dense_embeddings.embedding_model
+    sparse_embedding_model = sparse_embeddings.sparse_embedding_model
+
+    # Initialize Pinecone vector DB
+    pinecone_vector_db = PineconeHybridVectorDB(
+        api_key=args.pinecone_api_key,
+        index_name=args.index_name,
+        dimension=args.dimension,
+        metric=args.metric,
+        region=args.region,
+    )
+
+    # Initialize Pinecone retriever
+    pinecone_retriever = PineconeHybridRetriever(
+        index=pinecone_vector_db.index,
+        dense_embedding_model=dense_embedding_model,
+        sparse_embedding_model=sparse_embedding_model,
+        alpha=args.alpha,
+        top_k=args.top_k,
+        namespace=args.namespace,
+    )
+
+    # Initialize RetrievalEvaluator and DocumentGrader
+    retrieval_evaluator = RetrievalEvaluator(
+        llm_model=args.llm_model,
+        llm_api_key=args.llm_api_key,
+        temperature=args.temperature,
+    )
+    document_grader = DocumentGrader(
+        evaluator=retrieval_evaluator,
+        threshold=args.relevance_threshold,
+    )
+
+    # Load the prompt and initialize the generator
+    generator = ChatGroqGenerator(
+        model=args.llm_model,
+        api_key=args.llm_api_key,
+        llm_params={"temperature": args.temperature},
+    )
+    llm = generator.llm
+
+    # Initialize the Corrective RAG pipeline
+    corrective_rag = CorrectiveRAGPipeline(
+        retriever=pinecone_retriever.hybrid_retriever,
+        prompt=retrieval_evaluator.prompt_template,
+        llm=llm,
+        document_grader=document_grader,
+        tracing_project_name="sec_corrective_rag",
+    )
+
+    # Initialize the scorers with the provided arguments
+    scorers = []
+    for scorer_name in args.scorers:
+        if scorer_name in SCORER_CLASSES:
+            ScorerClass = SCORER_CLASSES[scorer_name]
+            scorer = ScorerClass(
+                threshold=args.threshold,
+                model=args.model,
+                include_reason=args.include_reason,
+                assessment_questions=args.assessment_questions,
+                strict_mode=args.strict_mode,
+                async_mode=args.async_mode,
+                verbose=args.verbose,
+                truths_extraction_limit=args.truths_extraction_limit,
+            )
+            scorers.append(scorer)
+        else:
+            msg = f"Scorer '{scorer_name}' is not supported."
+            raise ValueError(msg)
+
+    # Run the pipeline
+    evaluator = Evaluator(
+        evaluation_name=args.evaluation_name,
+        pipeline=corrective_rag,
+        dataset=eval_dataset,
+        scorers=[scorers],
+    )
+
+    evaluation_results = evaluator.evaluate()
+    print(evaluation_results)
+
+
+if __name__ == "__main__":
+    main()

scripts/dspy_rag.py

@@ -0,0 +1,121 @@
+import os
+
+import weave
+from dataloaders.langchain import FinanceBenchDataloader
+from dspy import LM
+from langchain_huggingface import HuggingFaceEmbeddings
+
+from rag_pipelines.embeddings import SparseEmbeddingsMilvus as SparseEmbeddings
+from rag_pipelines.evaluation import AnswerRelevancyScorer, ContextualPrecisionScorer, Evaluator
+from rag_pipelines.pipelines import RAG, DSPyRAGPipeline
+from rag_pipelines.vectordb import DSPyMilvusRetriever as MilvusRetriever
+from rag_pipelines.vectordb import MilvusVectorDB, milvus_retriever
+
+os.environ["WEAVE_PARALLELISM"] = "1"
+os.environ["WEAVE_TRACE_LANGCHAIN"] = "false"
+
+dense_model = "intfloat/multilingual-e5-large"
+encode_kwargs = {"prompt": "query: "}
+model_kwargs = {
+    "device": "cpu",
+    "trust_remote_code": True,
+    "backend": "onnx",
+    "model_kwargs": {"file_name": "onnx/model.onnx"},
+}
+
+##### Use the e5-large-instruct model for everything now
+
+dense_field = "dense_vector"
+sparse_field = "sparse_vector"
+text_field = "text"
+metadata_field = "metadata"
+
+dense_search_params = {
+    "metric_type": "COSINE",
+}
+
+sparse_search_params = {
+    "metric_type": "IP",
+}
+
+milvus_uri = "https://in03-8aaa331b36bf39c.serverless.gcp-us-west1.cloud.zilliz.com"
+milvus_token = (
+    "cd567c8418a6b8fe4b438300cfc56212f22ef1347bc12031b0114bd72ba0aec3978ce8c107c11a4ae01239b010c15765358cdf37"
+)
+milvus_collection_name = "financebenchsub"
+
+
+tracing_project_name = "dspy_rag"
+weave_params = {}
+
+client = weave.init(tracing_project_name, **weave_params)
+
+dense_embeddings = HuggingFaceEmbeddings(model_name=dense_model, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs)
+sparse_embeddings = SparseEmbeddings(model_name="Splade_PP_en_v1")
+
+milvus_vector_db = MilvusVectorDB(
+    uri=milvus_uri,
+    token=milvus_token,
+    collection_name=milvus_collection_name,
+)
+
+milvus_retriever = MilvusRetriever(
+    collection=milvus_vector_db.collection,
+    dense_embedding_model=dense_embeddings,
+    sparse_embedding_model=sparse_embeddings,
+    anns_fields=[dense_field, sparse_field],
+    field_search_params=[dense_search_params, sparse_search_params],
+    text_field=text_field,
+    top_k=3,
+)
+
+llm = LM(
+    "groq/llama-3.3-70b-versatile",
+    api_key="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",
+    num_retries=120,
+)
+
+dspy_rag_module = RAG(milvus_retriever)
+
+pipeline = DSPyRAGPipeline(rag_module=dspy_rag_module, llm=llm)
+
+dataloader = FinanceBenchDataloader(
+    dataset_name="PatronusAI/financebench",
+    split="train[:2]",
+)
+
+data = dataloader.load_data()
+eval_data = dataloader.get_evaluation_data()
+
+evaluation_dataset = weave.Dataset(name="financebench_test_evaluation_dataset", rows=eval_data)
+
+questions = dataloader.get_questions()
+
+# dataloader.publish_to_weave(
+#     weave_project_name="financebench_test",
+#     dataset_name="financebench_test_dataset",
+#     evaluation_dataset_name="financebench_test_evaluation_dataset",
+# )
+
+answer_relevancy_scorer = AnswerRelevancyScorer(
+    threshold=0.5,
+    model="gpt-4",
+    include_reason=True,
+    verbose=True,
+)
+contextual_precision_scorer = ContextualPrecisionScorer(
+    threshold=0.5,
+    model="gpt-4",
+    include_reason=True,
+    verbose=True,
+)
+
+evaluator = Evaluator(
+    evaluation_name="financebench_test_subset_2",
+    evaluation_dataset=evaluation_dataset,
+    evaluation_scorers=[answer_relevancy_scorer, contextual_precision_scorer],
+    pipeline=pipeline,
+)
+
+evaluation_results = evaluator.evaluate()
+print(evaluation_results)

scripts/hybrid_rag.py

@@ -0,0 +1,144 @@
+import argparse
+
+from langchain_core.prompts import ChatPromptTemplate
+
+from rag_pipelines.embeddings.dense import DenseEmbeddings
+from rag_pipelines.embeddings.sparse import SparseEmbeddings
+from rag_pipelines.llms.groq import ChatGroqGenerator
+from rag_pipelines.pipelines.rag import RAGPipeline
+from rag_pipelines.prompts.rag_prompt import RAG_PROMPT
+from rag_pipelines.vectordb.pinecone_hybrid_index import PineconeHybridVectorDB
+from rag_pipelines.vectordb.pinecone_hybrid_retriever import PineconeHybridRetriever
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run the Hybrid RAG pipeline.")
+
+    # Dense embeddings arguments
+    parser.add_argument(
+        "--dense_model_name",
+        type=str,
+        default="sentence-transformers/all-MiniLM-L6-v2",
+        help="Dense embedding model name.",
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="cpu",
+        help="Device to run the dense embedding model.",
+    )
+
+    # Sparse embeddings arguments
+    parser.add_argument(
+        "--sparse_max_seq_length",
+        type=int,
+        default=512,
+        help="Maximum sequence length for sparse embeddings.",
+    )
+
+    # Pinecone arguments
+    parser.add_argument("--pinecone_api_key", type=str, required=True, help="Pinecone API key.")
+    parser.add_argument("--index_name", type=str, default="edgar", help="Pinecone index name.")
+    parser.add_argument("--dimension", type=int, default=384, help="Dimension of embeddings.")
+    parser.add_argument("--metric", type=str, default="dotproduct", help="Metric for similarity search.")
+    parser.add_argument("--region", type=str, default="us-east-1", help="Pinecone region.")
+    parser.add_argument("--cloud", type=str, default="aws", help="Pinecone cloud provider.")
+    parser.add_argument(
+        "--namespace",
+        type=str,
+        default="edgar-all",
+        help="Namespace for Pinecone retriever.",
+    )
+
+    # Retriever arguments
+    parser.add_argument("--alpha", type=float, default=0.5, help="Alpha parameter for hybrid retriever.")
+    parser.add_argument("--top_k", type=int, default=5, help="Number of top documents to retrieve.")
+
+    # LLM arguments
+    parser.add_argument(
+        "--llm_model",
+        type=str,
+        default="llama-3.2-90b-vision-preview",
+        help="Language model name.",
+    )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=0,
+        help="Temperature for the language model.",
+    )
+    parser.add_argument("--llm_api_key", type=str, required=True, help="API key for the language model.")
+
+    # Query
+    parser.add_argument(
+        "--query",
+        type=str,
+        required=True,
+        help="Query to run through the Hybrid RAG pipeline.",
+    )
+
+    args = parser.parse_args()
+
+    # Initialize embeddings
+    dense_embeddings = DenseEmbeddings(
+        model_name=args.dense_model_name,
+        model_kwargs={"device": args.device},
+        encode_kwargs={"normalize_embeddings": True},
+        show_progress=True,
+    )
+    sparse_embeddings = SparseEmbeddings(model_kwargs={"max_seq_length": args.sparse_max_seq_length})
+
+    dense_embedding_model = dense_embeddings.embedding_model
+    sparse_embedding_model = sparse_embeddings.sparse_embedding_model
+
+    # Initialize Pinecone vector DB
+    pinecone_vector_db = PineconeHybridVectorDB(
+        api_key=args.pinecone_api_key,
+        index_name=args.index_name,
+        dimension=args.dimension,
+        metric=args.metric,
+        region=args.region,
+        cloud=args.cloud,
+    )
+
+    # Initialize Pinecone retriever
+    pinecone_retriever = PineconeHybridRetriever(
+        index=pinecone_vector_db.index,
+        dense_embedding_model=dense_embedding_model,
+        sparse_embedding_model=sparse_embedding_model,
+        alpha=args.alpha,
+        top_k=args.top_k,
+        namespace=args.namespace,
+    )
+
+    # Load the prompt
+
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("human", RAG_PROMPT),
+        ]
+    )
+
+    # Initialize the LLM
+    generator = ChatGroqGenerator(
+        model=args.llm_model,
+        api_key=args.llm_api_key,
+        llm_params={"temperature": args.temperature},
+    )
+    llm = generator.llm
+
+    # Initialize the Hybrid RAG pipeline
+    hybrid_rag = RAGPipeline(
+        retriever=pinecone_retriever.hybrid_retriever,
+        prompt=prompt,
+        llm=llm,
+        tracing_project_name="sec_hybrid_rag",
+    )
+
+    # Run the pipeline
+    output = hybrid_rag.predict(args.query)
+    print(output)
+
+
+if __name__ == "__main__":
+    main()

scripts/indexing_financebench_milvus.py

@@ -0,0 +1,269 @@
+import argparse
+
+from dataloaders.langchain import FinanceBenchDataloader
+from langchain_huggingface import HuggingFaceEmbeddings
+from pymilvus import CollectionSchema, DataType, FieldSchema
+
+from rag_pipelines.embeddings import SparseEmbeddingsMilvus as SparseEmbeddings
+from rag_pipelines.unstructured import UnstructuredChunker, UnstructuredDocumentLoader
+from rag_pipelines.utils import dict_type
+from rag_pipelines.vectordb import MilvusVectorDB
+
+
+def parse_arguments() -> argparse.Namespace:
+    """Parse command-line arguments.
+
+    Returns:
+        argparse.Namespace: Parsed command-line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="Run the FinanceBench pipeline to load, process, chunk, embed, and index documents."
+    )
+
+    # FinanceBench dataset parameters
+    parser.add_argument(
+        "--dataset_name",
+        type=str,
+        default="PatronusAI/financebench",
+        help="HuggingFace dataset name.",
+    )
+    parser.add_argument(
+        "--split",
+        type=str,
+        default="train",
+        help="Dataset split to use (e.g., 'train').",
+    )
+
+    # PDF directory for unstructured document loader
+    parser.add_argument(
+        "--pdf_dir",
+        type=str,
+        default="pdfs/",
+        help="Directory path containing PDF files.",
+    )
+
+    # UnstructuredDocumentLoader parameters
+    parser.add_argument(
+        "--strategy",
+        type=str,
+        default="fast",
+        help="Processing strategy for the unstructured document loader.",
+    )
+    parser.add_argument(
+        "--mode",
+        type=str,
+        default="elements",
+        help="Extraction mode for the unstructured document loader.",
+    )
+
+    # Milvus connection parameters
+    parser.add_argument(
+        "--milvus_uri",
+        type=str,
+        help="URI for the Milvus server.",
+    )
+    parser.add_argument(
+        "--milvus_token",
+        type=str,
+        help="Authentication token for Milvus.",
+    )
+    parser.add_argument(
+        "--collection_name",
+        type=str,
+        default="financebench",
+        help="Name of the Milvus collection to create/use.",
+    )
+
+    # Dense embedding model parameters
+    parser.add_argument(
+        "--dense_embedding_model",
+        type=str,
+        default="sentence-transformers/all-mpnet-base-v2",
+        help="Model name for dense embeddings.",
+    )
+    parser.add_argument(
+        "--dense_model_kwargs",
+        type=dict_type,
+        default='{"device": "cpu", "trust_remote_code": true}',
+        help="Keyword arguments for dense embeddings model initialization.",
+    )
+    parser.add_argument(
+        "--dense_encode_kwargs",
+        type=dict_type,
+        default='{"normalize_embeddings": true}',
+        help="Keyword arguments for dense embeddings encoding.",
+    )
+
+    # Sparse embedding model parameters
+    parser.add_argument(
+        "--sparse_embedding_model",
+        type=str,
+        default="Splade_PP_en_v1",
+        help="Model name for sparse embeddings.",
+    )
+
+    # Schema configuration parameters
+
+    # Field names
+    parser.add_argument(
+        "--pk_field",
+        type=str,
+        default="doc_id",
+        help="Name of the primary key field.",
+    )
+    parser.add_argument(
+        "--dense_field",
+        type=str,
+        default="dense_vector",
+        help="Name of the dense vector field.",
+    )
+    parser.add_argument(
+        "--sparse_field",
+        type=str,
+        default="sparse_vector",
+        help="Name of the sparse vector field.",
+    )
+    parser.add_argument(
+        "--text_field",
+        type=str,
+        default="text",
+        help="Name of the text field.",
+    )
+    parser.add_argument(
+        "--metadata_field",
+        type=str,
+        default="metadata",
+        help="Name of the metadata field.",
+    )
+
+    parser.add_argument(
+        "--dense_dim",
+        type=int,
+        default=768,
+        help="Dimension of dense embeddings.",
+    )
+    parser.add_argument(
+        "--pk_max_length",
+        type=int,
+        default=100,
+        help="Max length for the primary key field.",
+    )
+    parser.add_argument(
+        "--text_max_length",
+        type=int,
+        default=65535,
+        help="Max length for the text field.",
+    )
+
+    # Index parameters
+    parser.add_argument(
+        "--dense_index_params",
+        type=dict_type,
+        default='{"index_type": "FLAT", "metric_type": "IP"}',
+        help="JSON string specifying dense index parameters.",
+    )
+    parser.add_argument(
+        "--sparse_index_params",
+        type=dict_type,
+        default='{"index_type": "SPARSE_INVERTED_INDEX", "metric_type": "IP"}',
+        help="JSON string specifying sparse index parameters.",
+    )
+
+    # Collection creation flag
+    parser.add_argument(
+        "--create_new_collection",
+        action="store_true",
+        help="Create a new collection or use existing. Defaults to False.",
+    )
+
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Run the FinanceBench document processing pipeline.
+
+    This function performs the following steps:
+    1. Loads the FinanceBench dataset.
+    2. Retrieves PDF documents from the specified directory.
+    3. Processes PDFs using the UnstructuredDocumentLoader.
+    4. Chunks documents using the UnstructuredChunker.
+    5. Generates dense and sparse embeddings with specified parameters.
+    6. Sets up a Milvus vector database and indexes the documents.
+    """
+    args = parse_arguments()
+
+    # Initialize FinanceBench dataloader and load the corpus PDFs
+    dataloader = FinanceBenchDataloader(
+        dataset_name=args.dataset_name,
+        split=args.split,
+    )
+
+    # Load and transform PDF documents from the provided directory
+    unstructured_document_loader = UnstructuredDocumentLoader(
+        strategy=args.strategy,
+        mode=args.mode,
+    )
+
+    # Chunk the documents using the UnstructuredChunker
+    chunker = UnstructuredChunker()
+
+    # Initialize dense and sparse embedding models with additional parameters
+    dense_embeddings = HuggingFaceEmbeddings(
+        model_name=args.dense_embedding_model,
+        model_kwargs=args.dense_model_kwargs,
+        encode_kwargs=args.dense_encode_kwargs,
+    )
+    sparse_embeddings = SparseEmbeddings(
+        model_name=args.sparse_embedding_model,
+    )
+
+    # Define Milvus collection fields and schema
+    pk_field = args.pk_field
+    dense_field = args.dense_field
+    sparse_field = args.sparse_field
+    text_field = args.text_field
+    metadata_field = args.metadata_field
+
+    fields = [
+        FieldSchema(
+            name=pk_field,
+            dtype=DataType.VARCHAR,
+            is_primary=True,
+            auto_id=True,
+            max_length=args.pk_max_length,
+        ),
+        FieldSchema(name=dense_field, dtype=DataType.FLOAT_VECTOR, dim=args.dense_dim),
+        FieldSchema(name=sparse_field, dtype=DataType.SPARSE_FLOAT_VECTOR),
+        FieldSchema(name=text_field, dtype=DataType.VARCHAR, max_length=args.text_max_length),
+        FieldSchema(name=metadata_field, dtype=DataType.JSON),
+    ]
+    schema = CollectionSchema(fields=fields, enable_dynamic_field=False)
+
+    # Initialize the Milvus vector database client
+    milvus_vector_db = MilvusVectorDB(
+        uri=args.milvus_uri,
+        token=args.milvus_token,
+        collection_name=args.collection_name,
+        collection_schema=schema,
+        dense_field=dense_field,
+        sparse_field=sparse_field,
+        text_field=text_field,
+        metadata_field=metadata_field,
+        dense_index_params=args.dense_index_params,
+        sparse_index_params=args.sparse_index_params,
+        create_new_collection=args.create_new_collection,
+    )
+
+    # Add documents to the Milvus vector database
+    dataloader.get_corpus_pdfs()
+    documents = unstructured_document_loader.transform_documents(args.pdf_dir)
+    chunked_documents = chunker.transform_documents(documents)
+    milvus_vector_db.add_documents(
+        documents=chunked_documents,
+        dense_embedding_model=dense_embeddings,
+        sparse_embedding_model=sparse_embeddings,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/indexing_financebench_milvus_voyage.py

@@ -0,0 +1,269 @@
+import argparse
+
+from dataloaders.langchain import FinanceBenchDataloader
+from langchain_huggingface import HuggingFaceEmbeddings
+from pymilvus import CollectionSchema, DataType, FieldSchema
+
+from rag_pipelines.embeddings import SparseEmbeddingsMilvus as SparseEmbeddings
+from rag_pipelines.unstructured import UnstructuredChunker, UnstructuredDocumentLoader
+from rag_pipelines.utils import dict_type
+from rag_pipelines.vectordb import MilvusVectorDB
+
+
+def parse_arguments() -> argparse.Namespace:
+    """Parse command-line arguments.
+
+    Returns:
+        argparse.Namespace: Parsed command-line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="Run the FinanceBench pipeline to load, process, chunk, embed, and index documents."
+    )
+
+    # FinanceBench dataset parameters
+    parser.add_argument(
+        "--dataset_name",
+        type=str,
+        default="PatronusAI/financebench",
+        help="HuggingFace dataset name.",
+    )
+    parser.add_argument(
+        "--split",
+        type=str,
+        default="train",
+        help="Dataset split to use (e.g., 'train').",
+    )
+
+    # PDF directory for unstructured document loader
+    parser.add_argument(
+        "--pdf_dir",
+        type=str,
+        default="pdfs/",
+        help="Directory path containing PDF files.",
+    )
+
+    # UnstructuredDocumentLoader parameters
+    parser.add_argument(
+        "--strategy",
+        type=str,
+        default="fast",
+        help="Processing strategy for the unstructured document loader.",
+    )
+    parser.add_argument(
+        "--mode",
+        type=str,
+        default="elements",
+        help="Extraction mode for the unstructured document loader.",
+    )
+
+    # Milvus connection parameters
+    parser.add_argument(
+        "--milvus_uri",
+        type=str,
+        help="URI for the Milvus server.",
+    )
+    parser.add_argument(
+        "--milvus_token",
+        type=str,
+        help="Authentication token for Milvus.",
+    )
+    parser.add_argument(
+        "--collection_name",
+        type=str,
+        default="financebench",
+        help="Name of the Milvus collection to create/use.",
+    )
+
+    # Dense embedding model parameters
+    parser.add_argument(
+        "--dense_embedding_model",
+        type=str,
+        default="sentence-transformers/all-mpnet-base-v2",
+        help="Model name for dense embeddings.",
+    )
+    parser.add_argument(
+        "--dense_model_kwargs",
+        type=dict_type,
+        default='{"device": "cpu", "trust_remote_code": true}',
+        help="Keyword arguments for dense embeddings model initialization.",
+    )
+    parser.add_argument(
+        "--dense_encode_kwargs",
+        type=dict_type,
+        default='{"normalize_embeddings": true}',
+        help="Keyword arguments for dense embeddings encoding.",
+    )
+
+    # Sparse embedding model parameters
+    parser.add_argument(
+        "--sparse_embedding_model",
+        type=str,
+        default="Splade_PP_en_v1",
+        help="Model name for sparse embeddings.",
+    )
+
+    # Schema configuration parameters
+
+    # Field names
+    parser.add_argument(
+        "--pk_field",
+        type=str,
+        default="doc_id",
+        help="Name of the primary key field.",
+    )
+    parser.add_argument(
+        "--dense_field",
+        type=str,
+        default="dense_vector",
+        help="Name of the dense vector field.",
+    )
+    parser.add_argument(
+        "--sparse_field",
+        type=str,
+        default="sparse_vector",
+        help="Name of the sparse vector field.",
+    )
+    parser.add_argument(
+        "--text_field",
+        type=str,
+        default="text",
+        help="Name of the text field.",
+    )
+    parser.add_argument(
+        "--metadata_field",
+        type=str,
+        default="metadata",
+        help="Name of the metadata field.",
+    )
+
+    parser.add_argument(
+        "--dense_dim",
+        type=int,
+        default=768,
+        help="Dimension of dense embeddings.",
+    )
+    parser.add_argument(
+        "--pk_max_length",
+        type=int,
+        default=100,
+        help="Max length for the primary key field.",
+    )
+    parser.add_argument(
+        "--text_max_length",
+        type=int,
+        default=65535,
+        help="Max length for the text field.",
+    )
+
+    # Index parameters
+    parser.add_argument(
+        "--dense_index_params",
+        type=dict_type,
+        default='{"index_type": "FLAT", "metric_type": "IP"}',
+        help="JSON string specifying dense index parameters.",
+    )
+    parser.add_argument(
+        "--sparse_index_params",
+        type=dict_type,
+        default='{"index_type": "SPARSE_INVERTED_INDEX", "metric_type": "IP"}',
+        help="JSON string specifying sparse index parameters.",
+    )
+
+    # Collection creation flag
+    parser.add_argument(
+        "--create_new_collection",
+        action="store_true",
+        help="Create a new collection or use existing. Defaults to False.",
+    )
+
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Run the FinanceBench document processing pipeline.
+
+    This function performs the following steps:
+    1. Loads the FinanceBench dataset.
+    2. Retrieves PDF documents from the specified directory.
+    3. Processes PDFs using the UnstructuredDocumentLoader.
+    4. Chunks documents using the UnstructuredChunker.
+    5. Generates dense and sparse embeddings with specified parameters.
+    6. Sets up a Milvus vector database and indexes the documents.
+    """
+    args = parse_arguments()
+
+    # Initialize FinanceBench dataloader and load the corpus PDFs
+    dataloader = FinanceBenchDataloader(
+        dataset_name=args.dataset_name,
+        split=args.split,
+    )
+
+    # Load and transform PDF documents from the provided directory
+    unstructured_document_loader = UnstructuredDocumentLoader(
+        strategy=args.strategy,
+        mode=args.mode,
+    )
+
+    # Chunk the documents using the UnstructuredChunker
+    chunker = UnstructuredChunker()
+
+    # Initialize dense and sparse embedding models with additional parameters
+    dense_embeddings = HuggingFaceEmbeddings(
+        model_name=args.dense_embedding_model,
+        model_kwargs=args.dense_model_kwargs,
+        encode_kwargs=args.dense_encode_kwargs,
+    )
+    sparse_embeddings = SparseEmbeddings(
+        model_name=args.sparse_embedding_model,
+    )
+
+    # Define Milvus collection fields and schema
+    pk_field = args.pk_field
+    dense_field = args.dense_field
+    sparse_field = args.sparse_field
+    text_field = args.text_field
+    metadata_field = args.metadata_field
+
+    fields = [
+        FieldSchema(
+            name=pk_field,
+            dtype=DataType.VARCHAR,
+            is_primary=True,
+            auto_id=True,
+            max_length=args.pk_max_length,
+        ),
+        FieldSchema(name=dense_field, dtype=DataType.FLOAT_VECTOR, dim=args.dense_dim),
+        FieldSchema(name=sparse_field, dtype=DataType.SPARSE_FLOAT_VECTOR),
+        FieldSchema(name=text_field, dtype=DataType.VARCHAR, max_length=args.text_max_length),
+        FieldSchema(name=metadata_field, dtype=DataType.JSON),
+    ]
+    schema = CollectionSchema(fields=fields, enable_dynamic_field=False)
+
+    # Initialize the Milvus vector database client
+    milvus_vector_db = MilvusVectorDB(
+        uri=args.milvus_uri,
+        token=args.milvus_token,
+        collection_name=args.collection_name,
+        collection_schema=schema,
+        dense_field=dense_field,
+        sparse_field=sparse_field,
+        text_field=text_field,
+        metadata_field=metadata_field,
+        dense_index_params=args.dense_index_params,
+        sparse_index_params=args.sparse_index_params,
+        create_new_collection=args.create_new_collection,
+    )
+
+    # Add documents to the Milvus vector database
+    dataloader.get_corpus_pdfs()
+    documents = unstructured_document_loader.transform_documents(args.pdf_dir)
+    chunked_documents = chunker.transform_documents(documents)
+    milvus_vector_db.add_documents(
+        documents=chunked_documents,
+        dense_embedding_model=dense_embeddings,
+        sparse_embedding_model=sparse_embeddings,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/indexing_pinecone.py

@@ -0,0 +1,193 @@
+import argparse
+
+import weave
+from dataloaders.langchain import FinanceBenchDataloader
+
+from rag_pipelines.embeddings.dense import DenseEmbeddings
+from rag_pipelines.embeddings.sparse_pinecone_text import SparseEmbeddings
+from rag_pipelines.vectordb.pinecone_hybrid_index import PineconeHybridVectorDB
+
+
+def parse_arguments() -> argparse.Namespace:
+    """Parse command-line arguments for the FinanceBench pipeline.
+
+    Returns:
+        argparse.Namespace: Parsed command-line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="Process FinanceBench data, generate embeddings, and add processed documents to a Pinecone hybrid index."
+    )
+
+    # Weave tracing project name
+    parser.add_argument(
+        "--project_name",
+        required=True,
+        help="Weave project name to initialize tracing.",
+    )
+
+    # FinanceBench dataloader arguments
+    parser.add_argument(
+        "--dataset_name",
+        type=str,
+        required=True,
+        help="Name of the FinanceBench dataset (e.g., 'PatronusAI/financebench').",
+    )
+    parser.add_argument(
+        "--split",
+        type=str,
+        default="train[:1]",
+        help="Dataset split to use (e.g., 'train[:1]').",
+    )
+
+    # Dense Embeddings arguments
+    parser.add_argument(
+        "--dense_model_name",
+        type=str,
+        required=True,
+        help="Dense embedding model name (e.g., 'sentence-transformers/all-MiniLM-L6-v2').",
+    )
+    parser.add_argument(
+        "--dense_device",
+        type=str,
+        default="cpu",
+        help="Device to run the dense embedding model (e.g., 'cpu' or 'cuda').",
+    )
+    parser.add_argument(
+        "--normalize_embeddings",
+        action="store_true",
+        help="Flag to normalize embeddings during encoding.",
+    )
+    parser.add_argument(
+        "--show_progress",
+        action="store_true",
+        help="Flag to show progress during embedding generation.",
+    )
+
+    # Sparse Embeddings arguments
+    parser.add_argument(
+        "--sparse_max_seq_length",
+        type=int,
+        required=True,
+        help="Maximum sequence length for sparse embeddings.",
+    )
+
+    # Semantic Chunking arguments (if applicable in your downstream pipeline)
+    parser.add_argument(
+        "--chunking_threshold_type",
+        type=str,
+        default="percentile",
+        help="Threshold type for semantic chunking (e.g., 'percentile' or 'absolute').",
+    )
+
+    # Pinecone configuration arguments
+    parser.add_argument(
+        "--pinecone_api_key",
+        type=str,
+        required=True,
+        help="API key for the Pinecone vector database.",
+    )
+    parser.add_argument(
+        "--pinecone_index_name",
+        type=str,
+        required=True,
+        help="Name of the Pinecone index.",
+    )
+    parser.add_argument(
+        "--pinecone_dimension",
+        type=int,
+        required=True,
+        help="Vector dimension in the Pinecone index.",
+    )
+    parser.add_argument(
+        "--pinecone_metric",
+        type=str,
+        required=True,
+        help="Similarity metric for the Pinecone index (e.g., 'dotproduct' or 'cosine').",
+    )
+    parser.add_argument(
+        "--pinecone_region",
+        type=str,
+        required=True,
+        help="Pinecone region (e.g., 'us-east-1').",
+    )
+    parser.add_argument(
+        "--pinecone_cloud",
+        type=str,
+        required=True,
+        help="Pinecone cloud provider (e.g., 'aws').",
+    )
+    parser.add_argument(
+        "--namespace",
+        type=str,
+        required=True,
+        help="Namespace for document storage in Pinecone.",
+    )
+
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Load FinanceBench data, generate dense and sparse embeddings, add processed documents to a Pinecone index.
+
+    The pipeline performs the following steps:
+    1. Initialize Weave tracing.
+    2. Load FinanceBench documents.
+    3. Generate dense and sparse embeddings for the documents.
+    4. Initialize and configure the Pinecone hybrid vector database.
+    5. Index the processed documents in Pinecone.
+    """
+    args = parse_arguments()
+
+    # Initialize Weave tracing
+    weave.init(args.project_name)
+
+    # Load FinanceBench dataset using FinanceBenchDataloader
+    data_loader = FinanceBenchDataloader(
+        dataset_name=args.dataset_name,
+        split=args.split,
+    )
+    # Download and prepare PDF documents from the dataset (if not already cached)
+    data_loader.get_corpus_pdfs()
+    # Create structured documents from the downloaded PDFs
+    documents = data_loader.create_documents()
+    print("Loaded Documents:")
+    print(documents)
+
+    # Initialize dense embedding model
+    dense_embeddings = DenseEmbeddings(
+        model_name=args.dense_model_name,
+        model_kwargs={"device": args.dense_device},
+        encode_kwargs={"normalize_embeddings": args.normalize_embeddings},
+        show_progress=args.show_progress,
+    )
+
+    # Initialize sparse embedding model
+    sparse_embeddings = SparseEmbeddings(model_kwargs={"max_seq_length": args.sparse_max_seq_length})
+
+    # Extract embedding models for use in the Pinecone vector database
+    dense_embedding_model = dense_embeddings.embedding_model
+    sparse_embedding_model = sparse_embeddings.sparse_embedding_model
+
+    # Initialize PineconeHybridVectorDB with specified configuration
+    pinecone_vector_db = PineconeHybridVectorDB(
+        api_key=args.pinecone_api_key,
+        index_name=args.pinecone_index_name,
+        dimension=args.pinecone_dimension,
+        metric=args.pinecone_metric,
+        region=args.pinecone_region,
+        cloud=args.pinecone_cloud,
+    )
+
+    # Add the processed documents to the Pinecone hybrid index using both dense and sparse embeddings
+    pinecone_vector_db.add_documents(
+        documents=documents,
+        dense_embedding_model=dense_embedding_model,
+        sparse_embedding_model=sparse_embedding_model,
+        namespace=args.namespace,
+    )
+
+    print("Documents have been indexed successfully in Pinecone.")
+
+
+if __name__ == "__main__":
+    main()

scripts/indexing_weaviate.py

@@ -0,0 +1,159 @@
+import argparse
+import logging
+
+import weave
+from dataloaders.langchain import FinanceBenchDataloader
+from langchain_huggingface import HuggingFaceEmbeddings
+
+from rag_pipelines.unstructured.unstructured_chunker import UnstructuredChunker
+from rag_pipelines.unstructured.unstructured_pdf_loader import UnstructuredDocumentLoader
+from rag_pipelines.utils.logging import LoggerFactory
+from rag_pipelines.vectordb.weaviate import (
+    WeaviateVectorDB,
+)  # Assumes the WeaviateVectorDB class is defined as shown above
+
+logger_factory = LoggerFactory(logger_name=__name__, log_level=logging.INFO)
+logger = logger_factory.get_logger()
+
+
+def parse_arguments() -> argparse.Namespace:
+    """Parse command-line arguments.
+
+    Returns:
+        argparse.Namespace: Parsed command-line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="Run the FinanceBench pipeline to load, process, chunk, embed, and index documents in Weaviate."
+    )
+
+    # FinanceBench dataset parameters
+    parser.add_argument(
+        "--dataset_name",
+        type=str,
+        default="PatronusAI/financebench",
+        help="Name of the FinanceBench dataset to use.",
+    )
+    parser.add_argument(
+        "--split",
+        type=str,
+        default="train[:1]",
+        help="Dataset split to use (e.g., 'train[:1]').",
+    )
+
+    # PDF directory for unstructured document loader
+    parser.add_argument(
+        "--pdf_dir",
+        type=str,
+        default="pdfs/",
+        help="Directory path containing PDF files.",
+    )
+
+    # UnstructuredDocumentLoader parameters
+    parser.add_argument(
+        "--strategy",
+        type=str,
+        default="fast",
+        help="Processing strategy for the unstructured document loader.",
+    )
+    parser.add_argument(
+        "--mode",
+        type=str,
+        default="elements",
+        help="Extraction mode for the unstructured document loader.",
+    )
+
+    # Weaviate connection parameters
+    parser.add_argument(
+        "--cluster_url",
+        type=str,
+        required=True,
+        help="URL of the Weaviate cluster.",
+    )
+    parser.add_argument(
+        "--api_key",
+        type=str,
+        required=True,
+        help="API key for Weaviate authentication.",
+    )
+    parser.add_argument(
+        "--collection_name",
+        type=str,
+        default="financebench",
+        help="Name of the Weaviate collection to create/use.",
+    )
+    parser.add_argument(
+        "--text_field",
+        type=str,
+        default="text",
+        help="Field name that contains document text in Weaviate.",
+    )
+
+    # Dense embedding model parameters
+    parser.add_argument(
+        "--dense_model_name",
+        type=str,
+        default="sentence-transformers/all-mpnet-base-v2",
+        help="Dense embedding model name.",
+    )
+
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Run the FinanceBench document processing pipeline using Weaviate.
+
+    The pipeline performs the following steps:
+      1. Initializes Weave tracing.
+      2. Loads a subset of the FinanceBench dataset.
+      3. Retrieves PDF documents from the specified directory.
+      4. Processes PDFs using the UnstructuredDocumentLoader.
+      5. Chunks documents using the UnstructuredChunker.
+      6. Generates dense embeddings.
+      7. Sets up a Weaviate vector database and indexes the documents.
+    """
+    args = parse_arguments()
+
+    # Initialize Weave tracing
+    weave.init("financebench_test")
+
+    # Load FinanceBench dataset and retrieve corpus PDFs
+    dataloader = FinanceBenchDataloader(
+        dataset_name=args.dataset_name,
+        split=args.split,
+    )
+    dataloader.get_corpus_pdfs()
+
+    # Load and transform PDF documents from the specified directory
+    unstructured_document_loader = UnstructuredDocumentLoader(
+        strategy=args.strategy,
+        mode=args.mode,
+    )
+    documents = unstructured_document_loader.transform_documents(args.pdf_dir)
+    logger.info("Loaded Documents:")
+    logger.info(documents)
+
+    # Chunk the documents using the UnstructuredChunker
+    chunker = UnstructuredChunker()
+    chunked_documents = chunker.transform_documents(documents)
+    logger.info("Chunked Documents:")
+    logger.info(chunked_documents)
+
+    # Initialize the dense embedding model
+    embeddings = HuggingFaceEmbeddings(model_name=args.dense_model_name)
+
+    # Initialize the Weaviate vector database client
+    weaviate_vector_db = WeaviateVectorDB(
+        cluster_url=args.cluster_url,
+        api_key=args.api_key,
+        collection_name=args.collection_name,
+        text_field=args.text_field,
+        dense_embedding_model=embeddings,
+    )
+
+    # Index the chunked documents in Weaviate using the dense embeddings
+    weaviate_vector_db.add_documents(documents=chunked_documents)
+    logger.info("Documents have been indexed successfully in Weaviate.")
+
+
+if __name__ == "__main__":
+    main()

scripts/rag_evaluation.py

@@ -0,0 +1,289 @@
+import argparse
+
+from dataloaders import (
+    ARCDataloader,
+    EdgarDataLoader,
+    FactScoreDataloader,
+    PopQADataloader,
+    TriviaQADataloader,
+)
+from langchain_core.prompts import ChatPromptTemplate
+
+from rag_pipelines.embeddings.dense import DenseEmbeddings
+from rag_pipelines.embeddings.sparse import SparseEmbeddings
+from rag_pipelines.evaluation import (
+    AnswerRelevancyScorer,
+    ContextualPrecisionScorer,
+    ContextualRecallScorer,
+    ContextualRelevancyScorer,
+    Evaluator,
+    FaithfulnessScorer,
+    HallucinationScorer,
+    SummarizationScorer,
+)
+from rag_pipelines.llms.groq import ChatGroqGenerator
+from rag_pipelines.pipelines.rag import RAGPipeline
+from rag_pipelines.prompts.rag_prompt import RAG_PROMPT
+from rag_pipelines.vectordb.pinecone_hybrid_index import PineconeHybridVectorDB
+from rag_pipelines.vectordb.pinecone_hybrid_retriever import PineconeHybridRetriever
+
+SUPPORTED_DATASETS = {
+    "arc": ARCDataloader,
+    "edgar": EdgarDataLoader,
+    "popqa": PopQADataloader,
+    "factscore": FactScoreDataloader,
+    "triviaqa": TriviaQADataloader,
+}
+
+SCORER_CLASSES = {
+    "contextual_precision": ContextualPrecisionScorer,
+    "contextual_recall": ContextualRecallScorer,
+    "contextual_relevancy": ContextualRelevancyScorer,
+    "answer_relevancy": AnswerRelevancyScorer,
+    "faithfulness": FaithfulnessScorer,
+    "summarization": SummarizationScorer,
+    "hallucination": HallucinationScorer,
+}
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run the Hybrid RAG pipeline.")
+
+    # Dense embeddings arguments
+    parser.add_argument(
+        "--dense_model_name",
+        type=str,
+        default="sentence-transformers/all-MiniLM-L6-v2",
+        help="Dense embedding model name.",
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="cpu",
+        help="Device to run the dense embedding model.",
+    )
+
+    # Sparse embeddings arguments
+    parser.add_argument(
+        "--sparse_max_seq_length",
+        type=int,
+        default=512,
+        help="Maximum sequence length for sparse embeddings.",
+    )
+
+    # Pinecone arguments
+    parser.add_argument("--pinecone_api_key", type=str, required=True, help="Pinecone API key.")
+    parser.add_argument("--index_name", type=str, default="edgar", help="Pinecone index name.")
+    parser.add_argument("--dimension", type=int, default=384, help="Dimension of embeddings.")
+    parser.add_argument("--metric", type=str, default="dotproduct", help="Metric for similarity search.")
+    parser.add_argument("--region", type=str, default="us-east-1", help="Pinecone region.")
+    parser.add_argument("--cloud", type=str, default="aws", help="Pinecone cloud provider.")
+    parser.add_argument(
+        "--namespace",
+        type=str,
+        default="edgar-all",
+        help="Namespace for Pinecone retriever.",
+    )
+
+    # Retriever arguments
+    parser.add_argument("--alpha", type=float, default=0.5, help="Alpha parameter for hybrid retriever.")
+    parser.add_argument("--top_k", type=int, default=5, help="Number of top documents to retrieve.")
+
+    # LLM arguments
+    parser.add_argument(
+        "--llm_model",
+        type=str,
+        default="llama-3.2-90b-vision-preview",
+        help="Language model name.",
+    )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=0,
+        help="Temperature for the language model.",
+    )
+    parser.add_argument("--llm_api_key", type=str, required=True, help="API key for the language model.")
+
+    # Load evaluation data
+    parser = argparse.ArgumentParser(description="Load evaluation dataset and initialize the dataloader.")
+    parser.add_argument(
+        "--dataset_type",
+        type=str,
+        default="edgar",
+        choices=SUPPORTED_DATASETS.keys(),
+        help="Dataset to load from. Options: arc, edgar, popqa, factscore, triviaqa.",
+    )
+    parser.add_argument(
+        "--hf_dataset_path",
+        type=str,
+        default="lamini/earnings-calls-qa",
+        help="Path to the HuggingFace dataset.",
+    )
+    parser.add_argument(
+        "--dataset_split",
+        type=str,
+        default="test",
+        help="Split of the dataset to use (e.g., train, validation, test).",
+    )
+
+    # Scorer arguments
+    parser.add_argument(
+        "--scorer",
+        type=str,
+        default="contextual_precision",
+        choices=[
+            "contextual_precision",
+            "contextual_recall",
+            "contextual_relevancy",
+            "answer_relevancy",
+            "faithfulness",
+            "summarization",
+            "hallucination",
+        ],
+        help="Scorer to use.",
+    )
+
+    # Evaluation arguments
+    parser.add_argument(
+        "--evaluation_name",
+        type=str,
+        default="hybrid_rag",
+        help="Name of the evaluation.",
+    )
+
+    # Add argument for selecting scorers
+    parser.add_argument(
+        "--scorers",
+        type=str,
+        nargs="+",
+        choices=SCORER_CLASSES.keys(),
+        required=True,
+        help="List of scorers to use. Options: contextual_precision, contextual_recall, contextual_relevancy, "
+        "answer_relevancy, faithfulness, summarization, hallucination.",
+    )
+
+    # Add shared arguments for scorer parameters
+    parser.add_argument("--threshold", type=float, default=0.5, help="Threshold for evaluation.")
+    parser.add_argument("--model", type=str, default="gpt-4", help="Model to use for scoring.")
+    parser.add_argument("--include_reason", action="store_true", help="Include reasons in scoring.")
+    parser.add_argument(
+        "--assessment_questions",
+        type=str,
+        nargs="*",
+        help="List of assessment questions for scoring.",
+    )
+    parser.add_argument("--strict_mode", action="store_true", help="Enable strict mode for scoring.")
+    parser.add_argument("--async_mode", action="store_true", help="Enable asynchronous processing.")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose output.")
+    parser.add_argument(
+        "--truths_extraction_limit",
+        type=int,
+        default=None,
+        help="Limit for truth extraction in scoring.",
+    )
+
+    args = parser.parse_args()
+
+    # Initialize dataloader based on the dataset type
+    try:
+        DataLoaderClass = SUPPORTED_DATASETS[args.dataset_type]
+        dataloader = DataLoaderClass(dataset_name=args.hf_dataset_path, split=args.dataset_split)
+    except KeyError:
+        msg = (
+            f"Dataset '{args.dataset_type}' is not supported. "
+            f"Supported options are: {', '.join(SUPPORTED_DATASETS.keys())}."
+        )
+        raise ValueError(msg)
+
+    eval_dataset = dataloader.get_eval_data()
+
+    # Initialize embeddings
+    dense_embeddings = DenseEmbeddings(
+        model_name=args.dense_model_name,
+        model_kwargs={"device": args.device},
+        encode_kwargs={"normalize_embeddings": True},
+        show_progress=True,
+    )
+    sparse_embeddings = SparseEmbeddings(model_kwargs={"max_seq_length": args.sparse_max_seq_length})
+
+    dense_embedding_model = dense_embeddings.embedding_model
+    sparse_embedding_model = sparse_embeddings.sparse_embedding_model
+
+    # Initialize Pinecone vector DB
+    pinecone_vector_db = PineconeHybridVectorDB(
+        api_key=args.pinecone_api_key,
+        index_name=args.index_name,
+        dimension=args.dimension,
+        metric=args.metric,
+        region=args.region,
+        cloud=args.cloud,
+    )
+
+    # Initialize Pinecone retriever
+    pinecone_retriever = PineconeHybridRetriever(
+        index=pinecone_vector_db.index,
+        dense_embedding_model=dense_embedding_model,
+        sparse_embedding_model=sparse_embedding_model,
+        alpha=args.alpha,
+        top_k=args.top_k,
+        namespace=args.namespace,
+    )
+
+    # Load the prompt
+
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("human", RAG_PROMPT),
+        ]
+    )
+
+    # Initialize the LLM
+    generator = ChatGroqGenerator(
+        model=args.llm_model,
+        api_key=args.llm_api_key,
+        llm_params={"temperature": args.temperature},
+    )
+    llm = generator.llm
+
+    # Initialize the Hybrid RAG pipeline
+    hybrid_rag = RAGPipeline(
+        retriever=pinecone_retriever.hybrid_retriever,
+        prompt=prompt,
+        llm=llm,
+        tracing_project_name="sec_hybrid_rag",
+    )
+
+    # Initialize the scorers with the provided arguments
+    scorers = []
+    for scorer_name in args.scorers:
+        if scorer_name in SCORER_CLASSES:
+            ScorerClass = SCORER_CLASSES[scorer_name]
+            scorer = ScorerClass(
+                threshold=args.threshold,
+                model=args.model,
+                include_reason=args.include_reason,
+                assessment_questions=args.assessment_questions,
+                strict_mode=args.strict_mode,
+                async_mode=args.async_mode,
+                verbose=args.verbose,
+                truths_extraction_limit=args.truths_extraction_limit,
+            )
+            scorers.append(scorer)
+        else:
+            msg = f"Scorer '{scorer_name}' is not supported."
+            raise ValueError(msg)
+
+    # Run the pipeline
+    evaluator = Evaluator(
+        evaluation_name=args.evaluation_name,
+        pipeline=hybrid_rag,
+        dataset=eval_dataset,
+        scorers=[scorers],
+    )
+
+    evaluation_results = evaluator.evaluate()
+    print(evaluation_results)
+
+
+if __name__ == "__main__":
+    main()

scripts/self_rag.py

@@ -0,0 +1,136 @@
+import argparse
+
+from langchain_community.retrievers import PineconeHybridSearchRetriever
+from langchain_core.prompts.chat import ChatPromptTemplate
+from langchain_groq import ChatGroq
+
+from rag_pipelines.pipelines.self_rag import SelfRAGPipeline
+from rag_pipelines.query_transformer.query_transformer import QueryTransformer
+from rag_pipelines.retrieval_evaluator.document_grader import DocumentGrader
+from rag_pipelines.retrieval_evaluator.retrieval_evaluator import RetrievalEvaluator
+from rag_pipelines.websearch.web_search import WebSearch
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run the Self-RAG pipeline.")
+
+    # Pinecone retriever arguments
+    parser.add_argument("--pinecone_api_key", type=str, required=True, help="Pinecone API key.")
+    parser.add_argument("--index_name", type=str, default="edgar", help="Pinecone index name.")
+    parser.add_argument("--dimension", type=int, default=384, help="Dimension of embeddings.")
+    parser.add_argument("--metric", type=str, default="dotproduct", help="Metric for similarity search.")
+    parser.add_argument("--region", type=str, default="us-east-1", help="Pinecone region.")
+    parser.add_argument(
+        "--namespace",
+        type=str,
+        default="edgar-all",
+        help="Namespace for Pinecone retriever.",
+    )
+
+    # Query Transformer arguments
+    parser.add_argument(
+        "--query_transformer_model",
+        type=str,
+        default="t5-small",
+        help="Model used for query transformation.",
+    )
+
+    # Retrieval Evaluator arguments
+    parser.add_argument(
+        "--llm_model",
+        type=str,
+        default="llama-3.2-90b-vision-preview",
+        help="Language model name for retrieval evaluator.",
+    )
+    parser.add_argument("--llm_api_key", type=str, required=True, help="API key for the language model.")
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=0.7,
+        help="Temperature for the language model.",
+    )
+    parser.add_argument(
+        "--relevance_threshold",
+        type=float,
+        default=0.7,
+        help="Relevance threshold for document grading.",
+    )
+
+    # Web Search arguments
+    parser.add_argument("--web_search_api_key", type=str, required=True, help="API key for web search.")
+
+    # Prompt arguments
+    parser.add_argument(
+        "--prompt_template_path",
+        type=str,
+        required=True,
+        help="Path to the prompt template for LLM.",
+    )
+
+    # Query
+    parser.add_argument(
+        "--query",
+        type=str,
+        required=True,
+        help="Query to run through the Self-RAG pipeline.",
+    )
+
+    args = parser.parse_args()
+
+    # Initialize Pinecone retriever
+    retriever = PineconeHybridSearchRetriever(
+        api_key=args.pinecone_api_key,
+        index_name=args.index_name,
+        dimension=args.dimension,
+        metric=args.metric,
+        region=args.region,
+        namespace=args.namespace,
+    )
+
+    # Initialize Query Transformer
+    query_transformer = QueryTransformer(model_name=args.query_transformer_model)
+
+    # Initialize Retrieval Evaluator and Document Grader
+    retrieval_evaluator = RetrievalEvaluator(
+        llm_model=args.llm_model,
+        llm_api_key=args.llm_api_key,
+        temperature=args.temperature,
+    )
+    document_grader = DocumentGrader(
+        evaluator=retrieval_evaluator,
+        threshold=args.relevance_threshold,
+    )
+
+    # Initialize Web Search
+    web_search = WebSearch(api_key=args.web_search_api_key)
+
+    # Load the prompt template
+    with open(args.prompt_template_path) as file:
+        prompt_template_str = file.read()
+    prompt = ChatPromptTemplate.from_template(prompt_template_str)
+
+    # Initialize the LLM
+    llm = ChatGroq(
+        model=args.llm_model,
+        api_key=args.llm_api_key,
+        llm_params={"temperature": args.temperature},
+    )
+
+    # Initialize Self-RAG Pipeline
+    self_rag_pipeline = SelfRAGPipeline(
+        retriever=retriever,
+        query_transformer=query_transformer,
+        retrieval_evaluator=retrieval_evaluator,
+        document_grader=document_grader,
+        web_search=web_search,
+        prompt=prompt,
+        llm=llm,
+    )
+
+    # Run the pipeline
+    output = self_rag_pipeline.run(args.query)
+    print(output)
+
+
+if __name__ == "__main__":
+    main()

scripts/self_rag_evaluation.py

@@ -0,0 +1,281 @@
+import argparse
+
+from dataloaders.langchain import (
+    ARCDataloader,
+    EdgarDataLoader,
+    FactScoreDataloader,
+    PopQADataloader,
+    TriviaQADataloader,
+)
+from langchain_community.retrievers import PineconeHybridSearchRetriever
+from langchain_core.prompts.chat import ChatPromptTemplate
+from langchain_groq import ChatGroq
+
+from rag_pipelines.evaluation import (
+    AnswerRelevancyScorer,
+    ContextualPrecisionScorer,
+    ContextualRecallScorer,
+    ContextualRelevancyScorer,
+    Evaluator,
+    FaithfulnessScorer,
+    HallucinationScorer,
+    SummarizationScorer,
+)
+from rag_pipelines.pipelines.self_rag import SelfRAGPipeline
+from rag_pipelines.query_transformer.query_transformer import QueryTransformer
+from rag_pipelines.retrieval_evaluator.document_grader import DocumentGrader
+from rag_pipelines.retrieval_evaluator.retrieval_evaluator import RetrievalEvaluator
+from rag_pipelines.websearch.web_search import WebSearch
+
+SUPPORTED_DATASETS = {
+    "arc": ARCDataloader,
+    "edgar": EdgarDataLoader,
+    "popqa": PopQADataloader,
+    "factscore": FactScoreDataloader,
+    "triviaqa": TriviaQADataloader,
+}
+
+SCORER_CLASSES = {
+    "contextual_precision": ContextualPrecisionScorer,
+    "contextual_recall": ContextualRecallScorer,
+    "contextual_relevancy": ContextualRelevancyScorer,
+    "answer_relevancy": AnswerRelevancyScorer,
+    "faithfulness": FaithfulnessScorer,
+    "summarization": SummarizationScorer,
+    "hallucination": HallucinationScorer,
+}
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run the Self-RAG pipeline.")
+
+    # Pinecone retriever arguments
+    parser.add_argument("--pinecone_api_key", type=str, required=True, help="Pinecone API key.")
+    parser.add_argument("--index_name", type=str, default="edgar", help="Pinecone index name.")
+    parser.add_argument("--dimension", type=int, default=384, help="Dimension of embeddings.")
+    parser.add_argument("--metric", type=str, default="dotproduct", help="Metric for similarity search.")
+    parser.add_argument("--region", type=str, default="us-east-1", help="Pinecone region.")
+    parser.add_argument(
+        "--namespace",
+        type=str,
+        default="edgar-all",
+        help="Namespace for Pinecone retriever.",
+    )
+
+    # Query Transformer arguments
+    parser.add_argument(
+        "--query_transformer_model",
+        type=str,
+        default="t5-small",
+        help="Model used for query transformation.",
+    )
+
+    # Retrieval Evaluator arguments
+    parser.add_argument(
+        "--llm_model",
+        type=str,
+        default="llama-3.2-90b-vision-preview",
+        help="Language model name for retrieval evaluator.",
+    )
+    parser.add_argument("--llm_api_key", type=str, required=True, help="API key for the language model.")
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=0.7,
+        help="Temperature for the language model.",
+    )
+    parser.add_argument(
+        "--relevance_threshold",
+        type=float,
+        default=0.7,
+        help="Relevance threshold for document grading.",
+    )
+
+    # Web Search arguments
+    parser.add_argument("--web_search_api_key", type=str, required=True, help="API key for web search.")
+
+    # Prompt arguments
+    parser.add_argument(
+        "--prompt_template_path",
+        type=str,
+        required=True,
+        help="Path to the prompt template for LLM.",
+    )
+
+    # Load evaluation data
+    parser = argparse.ArgumentParser(description="Load evaluation dataset and initialize the dataloader.")
+    parser.add_argument(
+        "--dataset_type",
+        type=str,
+        default="edgar",
+        choices=SUPPORTED_DATASETS.keys(),
+        help="Dataset to load from. Options: arc, edgar, popqa, factscore, triviaqa.",
+    )
+    parser.add_argument(
+        "--hf_dataset_path",
+        type=str,
+        default="lamini/earnings-calls-qa",
+        help="Path to the HuggingFace dataset.",
+    )
+    parser.add_argument(
+        "--dataset_split",
+        type=str,
+        default="test",
+        help="Split of the dataset to use (e.g., train, validation, test).",
+    )
+
+    # Scorer arguments
+    parser.add_argument(
+        "--scorer",
+        type=str,
+        default="contextual_precision",
+        choices=[
+            "contextual_precision",
+            "contextual_recall",
+            "contextual_relevancy",
+            "answer_relevancy",
+            "faithfullness",
+            "summarization",
+            "hallucination",
+        ],
+        help="Scorer to use.",
+    )
+
+    # Evaluation arguments
+    parser.add_argument(
+        "--evaluation_name",
+        type=str,
+        default="hybrid_rag",
+        help="Name of the evaluation.",
+    )
+
+    # Add argument for selecting scorers
+    parser.add_argument(
+        "--scorers",
+        type=str,
+        nargs="+",
+        choices=SCORER_CLASSES.keys(),
+        required=True,
+        help="List of scorers to use. Options: contextual_precision, contextual_recall, contextual_relevancy, "
+        "answer_relevancy, faithfulness, summarization, hallucination.",
+    )
+
+    # Add shared arguments for scorer parameters
+    parser.add_argument("--threshold", type=float, default=0.5, help="Threshold for evaluation.")
+    parser.add_argument("--model", type=str, default="gpt-4", help="Model to use for scoring.")
+    parser.add_argument("--include_reason", action="store_true", help="Include reasons in scoring.")
+    parser.add_argument(
+        "--assessment_questions",
+        type=str,
+        nargs="*",
+        help="List of assessment questions for scoring.",
+    )
+    parser.add_argument("--strict_mode", action="store_true", help="Enable strict mode for scoring.")
+    parser.add_argument("--async_mode", action="store_true", help="Enable asynchronous processing.")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose output.")
+    parser.add_argument(
+        "--truths_extraction_limit",
+        type=int,
+        default=None,
+        help="Limit for truth extraction in scoring.",
+    )
+
+    args = parser.parse_args()
+
+    # Initialize dataloader based on the dataset type
+    try:
+        DataLoaderClass = SUPPORTED_DATASETS[args.dataset_type]
+        dataloader = DataLoaderClass(dataset_name=args.hf_dataset_path, split=args.dataset_split)
+    except KeyError:
+        msg = (
+            f"Dataset '{args.dataset_type}' is not supported. "
+            f"Supported options are: {', '.join(SUPPORTED_DATASETS.keys())}."
+        )
+        raise ValueError(msg)
+
+    eval_dataset = dataloader.get_eval_data()
+
+    # Initialize Pinecone retriever
+    retriever = PineconeHybridSearchRetriever(
+        api_key=args.pinecone_api_key,
+        index_name=args.index_name,
+        dimension=args.dimension,
+        metric=args.metric,
+        region=args.region,
+        namespace=args.namespace,
+    )
+
+    # Initialize Query Transformer
+    query_transformer = QueryTransformer(model_name=args.query_transformer_model)
+
+    # Initialize Retrieval Evaluator and Document Grader
+    retrieval_evaluator = RetrievalEvaluator(
+        llm_model=args.llm_model,
+        llm_api_key=args.llm_api_key,
+        temperature=args.temperature,
+    )
+    document_grader = DocumentGrader(
+        evaluator=retrieval_evaluator,
+        threshold=args.relevance_threshold,
+    )
+
+    # Initialize Web Search
+    web_search = WebSearch(api_key=args.web_search_api_key)
+
+    # Load the prompt template
+    with open(args.prompt_template_path) as file:
+        prompt_template_str = file.read()
+    prompt = ChatPromptTemplate.from_template(prompt_template_str)
+
+    # Initialize the LLM
+    llm = ChatGroq(
+        model=args.llm_model,
+        api_key=args.llm_api_key,
+        llm_params={"temperature": args.temperature},
+    )
+
+    # Initialize Self-RAG Pipeline
+    self_rag_pipeline = SelfRAGPipeline(
+        retriever=retriever,
+        query_transformer=query_transformer,
+        retrieval_evaluator=retrieval_evaluator,
+        document_grader=document_grader,
+        web_search=web_search,
+        prompt=prompt,
+        llm=llm,
+    )
+
+    # Initialize the scorers with the provided arguments
+    scorers = []
+    for scorer_name in args.scorers:
+        if scorer_name in SCORER_CLASSES:
+            scorer_class = SCORER_CLASSES[scorer_name]
+            scorer = scorer_class(
+                threshold=args.threshold,
+                model=args.model,
+                include_reason=args.include_reason,
+                assessment_questions=args.assessment_questions,
+                strict_mode=args.strict_mode,
+                async_mode=args.async_mode,
+                verbose=args.verbose,
+                truths_extraction_limit=args.truths_extraction_limit,
+            )
+            scorers.append(scorer)
+        else:
+            msg = f"Scorer '{scorer_name}' is not supported."
+            raise ValueError(msg)
+
+    # Run the pipeline
+    evaluator = Evaluator(
+        evaluation_name=args.evaluation_name,
+        pipeline=self_rag_pipeline,
+        dataset=eval_dataset,
+        scorers=[scorers],
+    )
+
+    evaluation_results = evaluator.evaluate()
+    print(evaluation_results)
+
+
+if __name__ == "__main__":
+    main()