batch_run.py

import json
import subprocess
from multiprocessing import Pool
from tqdm import tqdm

def run_command(item):
    data_source = item["data_source"]
    chart_variation = item["chart_variation"]
    
    cmd = [
        "python", "pipeline.py",
        "--modules", "infographics_generator",
        "--input", data_source,
        "--output", "./test_output_new_new",
        "--chart-name", chart_variation
    ]
    
    subprocess.run(cmd)
    return item

start_id = 0

with open("extracted_info.json", "r") as f:
    data = json.load(f)[start_id:start_id+1]

with Pool(processes=8) as pool:
    list(tqdm(pool.imap(run_command, data), total=len(data)))