readFlow

import random as rnd
import tarfile
import zipfile

import decord
import webdataset as wds
import numpy as np
import torch
from torch.utils.data.dataset import IterableDataset, ChainDataset
from decord import VideoReader
from lavis.common.registry import registry
from lavis.datasets.datasets.base_dataset import ConcatDataset
from tqdm import tqdm

from PIL import Image

decord.bridge.set_bridge("torch")
MAX_INT = registry.get("MAX_INT")

def readFlow(filename, new_size=(256, 256)):
    """ Read optical flow from file.

    The function reads optical flow from a .flo file and returns it as a numpy array.

    Args:
    filename (str): The path to the .flo file.

    Returns:
    numpy.ndarray: A numpy array containing the optical flow with u and v components stacked in depth.
    """
    # Open the file in binary mode
    with open(filename, 'rb') as f:
        # Check the magic number in the header matches the expected .flo format
        magic = np.fromfile(f, np.float32, count=1)[0]
        assert magic == 202021.25, 'Invalid .flo file'

        # Read the width and height of the flow field
        width = np.fromfile(f, np.int32, count=1)[0]
        height = np.fromfile(f, np.int32, count=1)[0]

        # Read the flow field data
        flow_data = np.fromfile(f, np.float32, count=2 * width * height)

        # Reshape the flow field data into a 3D numpy array with shape (height, width, 2)
        flow = np.reshape(flow_data, (height, width, 2))
        flow = np.transpose(flow, (1, 0, 2))

    flow_resized = cv2.resize(flow, new_size[::-1], interpolation=cv2.INTER_LINEAR)
    # min_val = flow_resized.min()
    # max_val = flow_resized.max()
    # flow_resized = 2 * (flow_resized - min_val) / (max_val - min_val) - 1

    return flow_resized

def load_flow(frames_dir, indices,
               height=-1, width=-1):

    #print(frames_dir)
    frame_files = sorted([os.path.join(frames_dir, f) for f in os.listdir(frames_dir) if f.split('.')[-1] in ['flo']])
    frms = []
    for idx in indices:
        # print(idx, frame_files[idx])
        flow = readFlow(frame_files[idx], (height, width))
        min_val = flow.min()
        max_val = flow.max()
        flow = 2 * (flow - min_val) / (max_val - min_val) - 1
        frms.append(np.asarray(flow))

    frms = np.stack(frms).astype(np.float32)  # (C, T, H, W)
    # print('frms',frms.shape)
    # print(frms.shape)
    frms = torch.from_numpy(frms) #.unsqueeze(1)  # (T, 1, H, W)
    frms = frms.permute(0,3,1,2)


        assert magic == 202021.25, 'Invalid .flo file'

        # Read the width and height of the flow field
        width = np.fromfile(f, np.int32, count=1)[0]
        height = np.fromfile(f, np.int32, count=1)[0]

        # Read the flow field data
        flow_data = np.fromfile(f, np.float32, count=2 * width * height)

        # Reshape the flow field data into a 3D numpy array with shape (height, width, 2)
        flow = np.reshape(flow_data, (height, width, 2))
        flow = np.transpose(flow, (1, 0, 2))

    flow_resized = cv2.resize(flow, new_size[::-1], interpolation=cv2.INTER_LINEAR)
    # min_val = flow_resized.min()
    # max_val = flow_resized.max()
    # flow_resized = 2 * (flow_resized - min_val) / (max_val - min_val) - 1

    return flow_resized

def load_flow(frames_dir, indices,
               height=-1, width=-1):

    #print(frames_dir)
    frame_files = sorted([os.path.join(frames_dir, f) for f in os.listdir(frames_dir) if f.split('.')[-1] in ['flo']])
    frms = []
    for idx in indices:
        # print(idx, frame_files[idx])
        flow = readFlow(frame_files[idx], (height, width))
        min_val = flow.min()
        max_val = flow.max()
        flow = 2 * (flow - min_val) / (max_val - min_val) - 1
        frms.append(np.asarray(flow))

    frms = np.stack(frms).astype(np.float32)  # (C, T, H, W)
    # print('frms',frms.shape)
    # print(frms.shape)
    frms = torch.from_numpy(frms) #.unsqueeze(1)  # (T, 1, H, W)
    frms = frms.permute(0,3,1,2)
    return frms

def load_depth(frames_dir, indices,
               height=-1, width=-1, invalid_val=-99):

    frame_files = sorted([os.path.join(frames_dir, f) for f in os.listdir(frames_dir) if f.split('.')[-1] in ['jpg', 'png']])
    # print(frame_files)
    # print(len(indices), indices)
    frms = []
    for idx in indices:
        depth = Image.open(frame_files[idx])
        depth = depth.resize((width, height))
        depth = np.asarray(depth)

        invalid_mask = depth == invalid_val
        mask = np.logical_not(invalid_mask)
        vmin = np.percentile(depth[mask],2)
        vmax = np.percentile(depth[mask],85)
        if vmin != vmax:
            depth = (depth - vmin) / (vmax - vmin)
        else:


               height=-1, width=-1):

    #print(frames_dir)
    frame_files = sorted([os.path.join(frames_dir, f) for f in os.listdir(frames_dir) if f.split('.')[-1] in ['flo']])
    frms = []
    for idx in indices:
        # print(idx, frame_files[idx])
        flow = readFlow(frame_files[idx], (height, width))
        min_val = flow.min()
        max_val = flow.max()
        flow = 2 * (flow - min_val) / (max_val - min_val) - 1
        frms.append(np.asarray(flow))

    frms = np.stack(frms).astype(np.float32)  # (C, T, H, W)
    # print('frms',frms.shape)
    # print(frms.shape)
    frms = torch.from_numpy(frms) #.unsqueeze(1)  # (T, 1, H, W)
    frms = frms.permute(0,3,1,2)
    return frms

def load_depth(frames_dir, indices,
               height=-1, width=-1, invalid_val=-99):

    frame_files = sorted([os.path.join(frames_dir, f) for f in os.listdir(frames_dir) if f.split('.')[-1] in ['jpg', 'png']])
    # print(frame_files)
    # print(len(indices), indices)
    frms = []
    for idx in indices:
        depth = Image.open(frame_files[idx])
        depth = depth.resize((width, height))
        depth = np.asarray(depth)

        invalid_mask = depth == invalid_val
        mask = np.logical_not(invalid_mask)
        vmin = np.percentile(depth[mask],2)
        vmax = np.percentile(depth[mask],85)
        if vmin != vmax:
            depth = (depth - vmin) / (vmax - vmin)
        else:
            depth = depth * 0.

        frms.append(np.asarray(depth))

    frms = np.stack(frms).astype(np.float32)  # (C, T, H, W)
    frms = torch.from_numpy(frms).unsqueeze(1)  # (1, C, T, H, W)

    return frms

def load_frames(frames_dir, n_frms=MAX_INT, height=-1, width=-1,
                sampling="uniform", clip_proposal=None, type='rgb', indices=None):

    frame_files = sorted([os.path.join(frames_dir, f) for f in os.listdir(frames_dir) if f.split('.')[-1] in ['jpg', 'png']])
    # print('frame_files', len(frame_files))
    vlen = len(frame_files) - 1 # for flow
    n_frms = n_frms #min(n_frms, vlen)

    if clip_proposal is None:
        start, end = 0, vlen
    else:
        start, end = int(clip_proposal[0]*vlen), int(clip_proposal[1]*vlen)
        if start < 0:
            start = 0
        if end > vlen:
            end = vlen

    intervals = np.linspace(start=start, stop=end, num=n_frms + 1).astype(int)
    ranges = [(intervals[i], intervals[i + 1]) for i in range(len(intervals) - 1)]


        depth = Image.open(frame_files[idx])
        depth = depth.resize((width, height))
        depth = np.asarray(depth)

        invalid_mask = depth == invalid_val
        mask = np.logical_not(invalid_mask)
        vmin = np.percentile(depth[mask],2)
        vmax = np.percentile(depth[mask],85)
        if vmin != vmax:
            depth = (depth - vmin) / (vmax - vmin)
        else:
            depth = depth * 0.

        frms.append(np.asarray(depth))

    frms = np.stack(frms).astype(np.float32)  # (C, T, H, W)
    frms = torch.from_numpy(frms).unsqueeze(1)  # (1, C, T, H, W)

    return frms

def load_frames(frames_dir, n_frms=MAX_INT, height=-1, width=-1,
                sampling="uniform", clip_proposal=None, type='rgb', indices=None):

    frame_files = sorted([os.path.join(frames_dir, f) for f in os.listdir(frames_dir) if f.split('.')[-1] in ['jpg', 'png']])
    # print('frame_files', len(frame_files))
    vlen = len(frame_files) - 1 # for flow
    n_frms = n_frms #min(n_frms, vlen)

    if clip_proposal is None:
        start, end = 0, vlen
    else:
        start, end = int(clip_proposal[0]*vlen), int(clip_proposal[1]*vlen)
        if start < 0:
            start = 0
        if end > vlen:
            end = vlen

    intervals = np.linspace(start=start, stop=end, num=n_frms + 1).astype(int)
    ranges = [(intervals[i], intervals[i + 1]) for i in range(len(intervals) - 1)]

    if indices is None:
        if sampling == 'random':
            indices_ = [rnd.choice(range(x[0], x[1])) if x[0] != x[1] else x[0] for x in ranges]
        elif sampling == 'uniform':
            indices_ = [(x[0] + x[1]) // 2 for x in ranges]
        # elif sampling == "headtail":
        #     indices_h = sorted(rnd.sample(range(vlen // 2), n_frms // 2))
        #     indices_t = sorted(rnd.sample(range(vlen // 2, vlen), n_frms // 2))
        #     indices = indices_h + indices_t
        else:
            raise NotImplementedError
    else:
        indices_ = indices

    # print(len(indices_) , n_frms, indices_)

    if len(indices_) < n_frms:
        extra = [indices_[-1] for i in range(n_frms - len(indices_))]
        indices_.extend(extra)
    frms = []
    # debug = [frame_files[idx] for idx in indices]
    # print(debug)
    # print('frame_files',len(frame_files))
    # print('indices_', indices_)
    for idx in indices_:
        # print(frame_files[idx])
        with Image.open(frame_files[idx]) as img:
            if type == 'norm':
                img = img.convert('RGB')
            if type == 'flow':
                img = img.transpose(Image.FLIP_LEFT_RIGHT)
                img = img.transpose(2) # ROTATE_90
                # print('here')
            if type == 'depth':
                 img = img.convert('RGB')

            if height > 0 and width > 0:
                img = img.resize((width, height))
            frms.append(np.asarray(img))

    frms = np.stack(frms).transpose(3, 0, 1, 2).astype(np.float32)  # (C, T, H, W)
    frms = torch.from_numpy(frms)
    # print('frms', frms.shape) # 3, 4, 224, 224 for rgb,  4, 4, 224, 224 for norm, 3, 4, 224, 224 for norm.convert('RGB')

    return frms, indices_

# add for loading video
def load_video(video_path, n_frms=MAX_INT, height=-1, width=-1, sampling="uniform", clip_proposal=None):
    vr = VideoReader(uri=video_path, height=height, width=width)
    vlen = len(vr)
    n_frms = min(n_frms, vlen)
    fps = vr.get_avg_fps()
    if clip_proposal is None:
        start, end = 0, vlen
    else:
        start, end = int(clip_proposal[0]*fps), int(clip_proposal[1]*fps)
        if start < 0:
            start = 0
        if end > vlen:
            end = vlen

    intervals = np.linspace(start=start, stop=end, num=n_frms + 1).astype(int)
    ranges = []
    for idx, interv in enumerate(intervals[:-1]):
        ranges.append((interv, intervals[idx + 1]))


            if type == 'norm':
                img = img.convert('RGB')
            if type == 'flow':
                img = img.transpose(Image.FLIP_LEFT_RIGHT)
                img = img.transpose(2) # ROTATE_90
                # print('here')
            if type == 'depth':
                 img = img.convert('RGB')

            if height > 0 and width > 0:
                img = img.resize((width, height))
            frms.append(np.asarray(img))

    frms = np.stack(frms).transpose(3, 0, 1, 2).astype(np.float32)  # (C, T, H, W)
    frms = torch.from_numpy(frms)
    # print('frms', frms.shape) # 3, 4, 224, 224 for rgb,  4, 4, 224, 224 for norm, 3, 4, 224, 224 for norm.convert('RGB')

    return frms, indices_

# add for loading video
def load_video(video_path, n_frms=MAX_INT, height=-1, width=-1, sampling="uniform", clip_proposal=None):
    vr = VideoReader(uri=video_path, height=height, width=width)
    vlen = len(vr)
    n_frms = min(n_frms, vlen)
    fps = vr.get_avg_fps()
    if clip_proposal is None:
        start, end = 0, vlen
    else:
        start, end = int(clip_proposal[0]*fps), int(clip_proposal[1]*fps)
        if start < 0:
            start = 0
        if end > vlen:
            end = vlen

    intervals = np.linspace(start=start, stop=end, num=n_frms + 1).astype(int)
    ranges = []
    for idx, interv in enumerate(intervals[:-1]):
        ranges.append((interv, intervals[idx + 1]))

    if sampling == 'random':
        indices = []
        for x in ranges:
            if x[0] == x[1]:
                indices.append(x[0])
            else:
                indices.append(rnd.choice(range(x[0], x[1])))
    elif sampling == 'uniform':

        indices = [(x[0] + x[1]) // 2 for x in ranges]

    elif sampling == "headtail":
        indices_h = sorted(rnd.sample(range(vlen // 2), n_frms // 2))
        indices_t = sorted(rnd.sample(range(vlen // 2, vlen), n_frms // 2))
        indices = indices_h + indices_t
    else:
        raise NotImplementedError

    if len(indices) < n_frms:
        rest = [indices[-1] for i in range(n_frms - len(indices))]
        indices = indices + rest
    # get_batch -> T, H, W, C
    frms = vr.get_batch(indices).permute(3, 0, 1, 2).float()  # (C, T, H, W)

    return frms, indices, fps

def load_video_demo(video_path, n_frms=MAX_INT, height=-1, width=-1, sampling="uniform", clip_proposal=None):
    vr = VideoReader(uri=video_path, height=height, width=width)
    vlen = len(vr)
    n_frms = min(n_frms, vlen)
    fps = vr.get_avg_fps()
    if clip_proposal is None:
        start, end = 0, vlen
    else:
        start, end = int(clip_proposal[0]*fps), int(clip_proposal[1]*fps)
        if start < 0:
            start = 0
        if end > vlen:
            end = vlen

    intervals = np.linspace(start=start, stop=end, num=n_frms + 1).astype(int)
    ranges = []
    for idx, interv in enumerate(intervals[:-1]):
        ranges.append((interv, intervals[idx + 1]))


                indices.append(rnd.choice(range(x[0], x[1])))
    elif sampling == 'uniform':

        indices = [(x[0] + x[1]) // 2 for x in ranges]

    elif sampling == "headtail":
        indices_h = sorted(rnd.sample(range(vlen // 2), n_frms // 2))
        indices_t = sorted(rnd.sample(range(vlen // 2, vlen), n_frms // 2))
        indices = indices_h + indices_t
    else:
        raise NotImplementedError

    if len(indices) < n_frms:
        rest = [indices[-1] for i in range(n_frms - len(indices))]
        indices = indices + rest
    # get_batch -> T, H, W, C
    frms = vr.get_batch(indices).permute(3, 0, 1, 2).float()  # (C, T, H, W)

    return frms, indices, fps

def load_video_demo(video_path, n_frms=MAX_INT, height=-1, width=-1, sampling="uniform", clip_proposal=None):
    vr = VideoReader(uri=video_path, height=height, width=width)
    vlen = len(vr)
    n_frms = min(n_frms, vlen)
    fps = vr.get_avg_fps()
    if clip_proposal is None:
        start, end = 0, vlen
    else:
        start, end = int(clip_proposal[0]*fps), int(clip_proposal[1]*fps)
        if start < 0:
            start = 0
        if end > vlen:
            end = vlen

    intervals = np.linspace(start=start, stop=end, num=n_frms + 1).astype(int)
    ranges = []
    for idx, interv in enumerate(intervals[:-1]):
        ranges.append((interv, intervals[idx + 1]))

    if sampling == 'random':
        indices = []
        for x in ranges:
            if x[0] == x[1]:
                indices.append(x[0])
            else:
                indices.append(rnd.choice(range(x[0], x[1])))
    elif sampling == 'uniform':

        indices = [(x[0] + x[1]) // 2 for x in ranges]

    elif sampling == "headtail":
        indices_h = sorted(rnd.sample(range(vlen // 2), n_frms // 2))
        indices_t = sorted(rnd.sample(range(vlen // 2, vlen), n_frms // 2))
        indices = indices_h + indices_t
    else:
        raise NotImplementedError

    if len(indices) < n_frms:
        rest = [indices[-1] for i in range(n_frms - len(indices))]
        indices = indices + rest
    # get_batch -> T, H, W, C

    frms = vr.get_batch(indices)
    frms = frms.asnumpy()
    frms = torch.from_numpy(frms)
    frms = frms.permute(3, 0, 1, 2).float()  # (C, T, H, W)

    return frms, indices, fps, vlen

def apply_to_sample(f, sample):
    if len(sample) == 0:
        return {}

    def _apply(x):
        if torch.is_tensor(x):
            return f(x)
        elif isinstance(x, dict):
            return {key: _apply(value) for key, value in x.items()}
        elif isinstance(x, list):
            return [_apply(x) for x in x]
        else:
            return x

    return _apply(sample)

def move_to_cuda(sample):
    def _move_to_cuda(tensor):


    elif sampling == "headtail":
        indices_h = sorted(rnd.sample(range(vlen // 2), n_frms // 2))
        indices_t = sorted(rnd.sample(range(vlen // 2, vlen), n_frms // 2))
        indices = indices_h + indices_t
    else:
        raise NotImplementedError

    if len(indices) < n_frms:
        rest = [indices[-1] for i in range(n_frms - len(indices))]
        indices = indices + rest
    # get_batch -> T, H, W, C

    frms = vr.get_batch(indices)
    frms = frms.asnumpy()
    frms = torch.from_numpy(frms)
    frms = frms.permute(3, 0, 1, 2).float()  # (C, T, H, W)

    return frms, indices, fps, vlen

def apply_to_sample(f, sample):
    if len(sample) == 0:
        return {}

    def _apply(x):
        if torch.is_tensor(x):
            return f(x)
        elif isinstance(x, dict):
            return {key: _apply(value) for key, value in x.items()}
        elif isinstance(x, list):
            return [_apply(x) for x in x]
        else:
            return x

    return _apply(sample)

def move_to_cuda(sample):
    def _move_to_cuda(tensor):
        return tensor.cuda()

    return apply_to_sample(_move_to_cuda, sample)

def prepare_sample(samples, cuda_enabled=True):
    if cuda_enabled:
        samples = move_to_cuda(samples)

    # TODO fp16 support

    return samples

def reorg_datasets_by_split(datasets):
    """


    return frms, indices, fps, vlen

def apply_to_sample(f, sample):
    if len(sample) == 0:
        return {}

    def _apply(x):
        if torch.is_tensor(x):
            return f(x)
        elif isinstance(x, dict):
            return {key: _apply(value) for key, value in x.items()}
        elif isinstance(x, list):
            return [_apply(x) for x in x]
        else:
            return x

    return _apply(sample)

def move_to_cuda(sample):
    def _move_to_cuda(tensor):
        return tensor.cuda()

    return apply_to_sample(_move_to_cuda, sample)

def prepare_sample(samples, cuda_enabled=True):
    if cuda_enabled:
        samples = move_to_cuda(samples)

    # TODO fp16 support

    return samples

def reorg_datasets_by_split(datasets):
    """
    Organizes datasets by split.

    Args:
        datasets: dict of torch.utils.data.Dataset objects by name.

    Returns:
        Dict of datasets by split {split_name: List[Datasets]}.


    def _apply(x):
        if torch.is_tensor(x):
            return f(x)
        elif isinstance(x, dict):
            return {key: _apply(value) for key, value in x.items()}
        elif isinstance(x, list):
            return [_apply(x) for x in x]
        else:
            return x

    return _apply(sample)

def move_to_cuda(sample):
    def _move_to_cuda(tensor):
        return tensor.cuda()

    return apply_to_sample(_move_to_cuda, sample)

def prepare_sample(samples, cuda_enabled=True):
    if cuda_enabled:
        samples = move_to_cuda(samples)

    # TODO fp16 support

    return samples

def reorg_datasets_by_split(datasets):
    """
    Organizes datasets by split.

    Args:
        datasets: dict of torch.utils.data.Dataset objects by name.

    Returns:
        Dict of datasets by split {split_name: List[Datasets]}.
    """
    # if len(datasets) == 1:
    #     return datasets[list(datasets.keys())[0]]
    # else:
    reorg_datasets = dict()

    # reorganize by split
    for _, dataset in datasets.items():
        for split_name, dataset_split in dataset.items():


    return _apply(sample)

def move_to_cuda(sample):
    def _move_to_cuda(tensor):
        return tensor.cuda()

    return apply_to_sample(_move_to_cuda, sample)

def prepare_sample(samples, cuda_enabled=True):
    if cuda_enabled:
        samples = move_to_cuda(samples)

    # TODO fp16 support

    return samples

def reorg_datasets_by_split(datasets):
    """
    Organizes datasets by split.

    Args:
        datasets: dict of torch.utils.data.Dataset objects by name.

    Returns:
        Dict of datasets by split {split_name: List[Datasets]}.
    """
    # if len(datasets) == 1:
    #     return datasets[list(datasets.keys())[0]]
    # else:
    reorg_datasets = dict()

    # reorganize by split
    for _, dataset in datasets.items():
        for split_name, dataset_split in dataset.items():
            if split_name not in reorg_datasets:
                reorg_datasets[split_name] = [dataset_split]
            else:
                reorg_datasets[split_name].append(dataset_split)

    return reorg_datasets

def concat_datasets(datasets):
    """
    Concatenates multiple datasets into a single dataset.

    It supports may-style datasets and DataPipeline from WebDataset. Currently, does not support
    generic IterableDataset because it requires creating separate samplers.

    Now only supports conctenating training datasets and assuming validation and testing
    have only a single dataset. This is because metrics should not be computed on the concatenated
    datasets.

    Args:
        datasets: dict of torch.utils.data.Dataset objects by split.

    Returns:
        Dict of concatenated datasets by split, "train" is the concatenation of multiple datasets,
        "val" and "test" remain the same.


    Returns:
        Dict of datasets by split {split_name: List[Datasets]}.
    """
    # if len(datasets) == 1:
    #     return datasets[list(datasets.keys())[0]]
    # else:
    reorg_datasets = dict()

    # reorganize by split
    for _, dataset in datasets.items():
        for split_name, dataset_split in dataset.items():
            if split_name not in reorg_datasets:
                reorg_datasets[split_name] = [dataset_split]
            else:
                reorg_datasets[split_name].append(dataset_split)

    return reorg_datasets

def concat_datasets(datasets):
    """
    Concatenates multiple datasets into a single dataset.

    It supports may-style datasets and DataPipeline from WebDataset. Currently, does not support
    generic IterableDataset because it requires creating separate samplers.

    Now only supports conctenating training datasets and assuming validation and testing
    have only a single dataset. This is because metrics should not be computed on the concatenated
    datasets.

    Args:
        datasets: dict of torch.utils.data.Dataset objects by split.

    Returns:
        Dict of concatenated datasets by split, "train" is the concatenation of multiple datasets,
        "val" and "test" remain the same.

        If the input training datasets contain both map-style and DataPipeline datasets, returns
        a tuple, where the first element is a concatenated map-style dataset and the second
        element is a chained DataPipeline dataset.

    """
    # concatenate datasets in the same split
    for split_name in datasets:
        if split_name != "train":
            assert (
                len(datasets[split_name]) == 1
            ), "Do not support multiple {} datasets.".format(split_name)
            datasets[split_name] = datasets[split_name][0]
        else:
            iterable_datasets, map_datasets = [], []
            for dataset in datasets[split_name]:
                if isinstance(dataset, wds.DataPipeline):
                    logging.info(
                        "Dataset {} is IterableDataset, can't be concatenated.".format(
                            dataset
                        )
                    )
                    iterable_datasets.append(dataset)
                elif isinstance(dataset, IterableDataset):
                    raise NotImplementedError(
                        "Do not support concatenation of generic IterableDataset."
                    )
                else:
                    map_datasets.append(dataset)

            # if len(iterable_datasets) > 0:
            # concatenate map-style datasets and iterable-style datasets separately
            chained_datasets = (
                ChainDataset(iterable_datasets) if len(iterable_datasets) > 0 else None
            )
            concat_datasets = (
                ConcatDataset(map_datasets) if len(map_datasets) > 0 else None
            )

            train_datasets = concat_datasets, chained_datasets
            train_datasets = tuple([x for x in train_datasets if x is not None])
            train_datasets = (
                train_datasets[0] if len(train_datasets) == 1 else train_datasets
            )

            datasets[split_name] = train_datasets

    return datasets

def extract_archive(from_path, to_path=None, overwrite=False):
    """Extract archive.

    Args:
        from_path: the path of the archive.
        to_path: the root path of the extracted files (directory of from_path)
        overwrite: overwrite existing files (False)

    Returns:
        List of paths to extracted files even if not overwritten.

    Examples:
        >>> url = 'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz'
        >>> from_path = './validation.tar.gz'
        >>> to_path = './'
        >>> torchtext.utils.download_from_url(url, from_path)
        >>> torchtext.utils.extract_archive(from_path, to_path)
        >>> ['.data/val.de', '.data/val.en']


            # if len(iterable_datasets) > 0:
            # concatenate map-style datasets and iterable-style datasets separately
            chained_datasets = (
                ChainDataset(iterable_datasets) if len(iterable_datasets) > 0 else None
            )
            concat_datasets = (
                ConcatDataset(map_datasets) if len(map_datasets) > 0 else None
            )

            train_datasets = concat_datasets, chained_datasets
            train_datasets = tuple([x for x in train_datasets if x is not None])
            train_datasets = (
                train_datasets[0] if len(train_datasets) == 1 else train_datasets
            )

            datasets[split_name] = train_datasets

    return datasets

def extract_archive(from_path, to_path=None, overwrite=False):
    """Extract archive.

    Args:
        from_path: the path of the archive.
        to_path: the root path of the extracted files (directory of from_path)
        overwrite: overwrite existing files (False)

    Returns:
        List of paths to extracted files even if not overwritten.

    Examples:
        >>> url = 'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz'
        >>> from_path = './validation.tar.gz'
        >>> to_path = './'
        >>> torchtext.utils.download_from_url(url, from_path)
        >>> torchtext.utils.extract_archive(from_path, to_path)
        >>> ['.data/val.de', '.data/val.en']
        >>> torchtext.utils.download_from_url(url, from_path)
        >>> torchtext.utils.extract_archive(from_path, to_path)
        >>> ['.data/val.de', '.data/val.en']

    """

    if to_path is None:
        to_path = os.path.dirname(from_path)

    if from_path.endswith((".tar.gz", ".tgz")):
        logging.info("Opening tar file {} to {}.".format(from_path, to_path))
        with tarfile.open(from_path, "r") as tar:
            files = []
            for file_ in tqdm(tar):
                file_path = os.path.join(to_path, file_.name)
                if file_.isfile():
                    files.append(file_path)
                    if os.path.exists(file_path):
                        logging.info("{} already extracted.".format(file_path))
                        if not overwrite:
                            continue
                tar.extract(file_, to_path)
            logging.info("Finished extracting tar file {}.".format(from_path))
            return files

    elif from_path.endswith(".zip"):
        assert zipfile.is_zipfile(from_path), from_path
        logging.info("Opening zip file {} to {}.".format(from_path, to_path))
        with zipfile.ZipFile(from_path, "r") as zfile:
            files = []
            for file_ in tqdm(zfile.namelist()):
                file_path = os.path.join(to_path, file_)
                files.append(file_path)
                if os.path.exists(file_path):
                    logging.info("{} already extracted.".format(file_path))
                    if not overwrite:
                        continue
                zfile.extract(file_, to_path)
        files = [f for f in files if os.path.isfile(f)]
        logging.info("Finished extracting zip file {}.".format(from_path))
        return files

    elif from_path.endswith(".gz"):
        logging.info("Opening gz file {} to {}.".format(from_path, to_path))
        default_block_size = 65536
        filename = from_path[:-3]
        files = [filename]
        with gzip.open(from_path, "rb") as gzfile, open(filename, "wb") as d_file:
            while True:
                block = gzfile.read(default_block_size)
                if not block:
                    break
                else:
                    d_file.write(block)
            d_file.write(block)
        logging.info("Finished extracting gz file {}.".format(from_path))
        return files

    else:
        raise NotImplementedError(
            "We currently only support tar.gz, .tgz, .gz and zip achives."
        )

def save_frames_grid(img_array, out_path):
    import torch
    from PIL import Image
    from torchvision.utils import make_grid

    if len(img_array.shape) == 3:
        img_array = img_array.unsqueeze(0)
    elif len(img_array.shape) == 5:
        b, t, c, h, w = img_array.shape
        img_array = img_array.view(-1, c, h, w)
    elif len(img_array.shape) == 4:
        pass
    else:
        raise NotImplementedError(
            "Supports only (b,t,c,h,w)-shaped inputs. First two dimensions can be ignored."
        )

    assert img_array.shape[1] == 3, "Exepcting input shape of (H, W, 3), i.e. RGB-only."


        default_block_size = 65536
        filename = from_path[:-3]
        files = [filename]
        with gzip.open(from_path, "rb") as gzfile, open(filename, "wb") as d_file:
            while True:
                block = gzfile.read(default_block_size)
                if not block:
                    break
                else:
                    d_file.write(block)
            d_file.write(block)
        logging.info("Finished extracting gz file {}.".format(from_path))
        return files

    else:
        raise NotImplementedError(
            "We currently only support tar.gz, .tgz, .gz and zip achives."
        )

def save_frames_grid(img_array, out_path):
    import torch
    from PIL import Image
    from torchvision.utils import make_grid

    if len(img_array.shape) == 3:
        img_array = img_array.unsqueeze(0)
    elif len(img_array.shape) == 5:
        b, t, c, h, w = img_array.shape
        img_array = img_array.view(-1, c, h, w)
    elif len(img_array.shape) == 4:
        pass
    else:
        raise NotImplementedError(
            "Supports only (b,t,c,h,w)-shaped inputs. First two dimensions can be ignored."
        )

    assert img_array.shape[1] == 3, "Exepcting input shape of (H, W, 3), i.e. RGB-only."

    grid = make_grid(img_array)
    ndarr = grid.permute(1, 2, 0).to("cpu", torch.uint8).numpy()

    img = Image.fromarray(ndarr)

    img.save(out_path)


        indices = indices_h + indices_t
    else:
        raise NotImplementedError

    if len(indices) < n_frms:
        rest = [indices[-1] for i in range(n_frms - len(indices))]
        indices = indices + rest
    # get_batch -> T, H, W, C

    frms = vr.get_batch(indices)
    frms = frms.asnumpy()
    frms = torch.from_numpy(frms)
    frms = frms.permute(3, 0, 1, 2).float()  # (C, T, H, W)

    return frms, indices, fps, vlen

def apply_to_sample(f, sample):
    if len(sample) == 0:
        return {}

    def _apply(x):
        if torch.is_tensor(x):
            return f(x)
        elif isinstance(x, dict):
            return {key: _apply(value) for key, value in x.items()}
        elif isinstance(x, list):
            return [_apply(x) for x in x]
        else:
            return x

    return _apply(sample)

def move_to_cuda(sample):
    def _move_to_cuda(tensor):
        return tensor.cuda()

    return apply_to_sample(_move_to_cuda, sample)

def prepare_sample(samples, cuda_enabled=True):
    if cuda_enabled:
        samples = move_to_cuda(samples)

    # TODO fp16 support

    return samples


    return frms, indices, fps, vlen

def apply_to_sample(f, sample):
    if len(sample) == 0:
        return {}

    def _apply(x):
        if torch.is_tensor(x):
            return f(x)
        elif isinstance(x, dict):
            return {key: _apply(value) for key, value in x.items()}
        elif isinstance(x, list):
            return [_apply(x) for x in x]
        else:
            return x

    return _apply(sample)

def move_to_cuda(sample):
    def _move_to_cuda(tensor):
        return tensor.cuda()

    return apply_to_sample(_move_to_cuda, sample)

def prepare_sample(samples, cuda_enabled=True):
    if cuda_enabled:
        samples = move_to_cuda(samples)

    # TODO fp16 support

    return samples

def reorg_datasets_by_split(datasets):
    """
    Organizes datasets by split.

    Args:
        datasets: dict of torch.utils.data.Dataset objects by name.


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

# TODO
# 1. Go to https://www.mediafire.com/file/czh8sezbo9s4692/test_videos.zip/file
#      and https://www.mediafire.com/file/x3rrbe4hwp04e6w/train_val_videos.zip/file
# 2. Right-click the Download button and copy the link address
#      e.g.
#    DATA_URL = {
#        "train": "https://download1602.mediafire.com/xxxxxxxxxxxx/x3rrbe4hwp04e6w/train_val_videos.zip",
#        "test": "https://download2390.mediafire.com/xxxxxxxxxxxx/czh8sezbo9s4692/test_videos.zip",
#    }
# 3. Paste the link address to DATA_URL

DATA_URL = {
    "train": "https://download2295.mediafire.com/4bb7p74xrbgg/x3rrbe4hwp04e6w/train_val_videos.zip",
    "test": "https://download2390.mediafire.com/79hfq3592lqg/czh8sezbo9s4692/test_videos.zip",
}

def download_datasets(root, url):
    """
    Download the Imagenet-R dataset archives and expand them
    in the folder provided as parameter
    """
    download_and_extract_archive(url=url, download_root=root)

def merge_datasets(download_path, storage_path):
    """
    Merge datasets in download_path to storage_path
    """

    # Merge train and test datasets
    train_path = os.path.join(download_path, "TrainValVideo")
    test_path = os.path.join(download_path, "TestVideo")
    train_test_path = storage_path

    print("Merging to {}".format(train_test_path))

    os.makedirs(train_test_path, exist_ok=True)

    for file_name in os.listdir(train_path):
        os.rename(
            os.path.join(train_path, file_name),
            os.path.join(train_test_path, file_name),
        )

    for file_name in os.listdir(test_path):
        os.rename(
            os.path.join(test_path, file_name),
            os.path.join(train_test_path, file_name),
        )

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/msrvtt/defaults_cap.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.msrvtt_cap.build_info.videos.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        for k, v in DATA_URL.items():
            print("Downloading {} to {}".format(v, k))
            download_datasets(download_dir, v)
    except Exception as e:
        # remove download dir if failed
        cleanup_dir(download_dir)
        print("Failed to download or extracting datasets. Aborting.")

    try:
        merge_datasets(download_dir, storage_dir)
    except Exception as e:
        # remove storage dir if failed
        cleanup_dir(download_dir)
        cleanup_dir(storage_dir)
        print("Failed to merging datasets. Aborting.")

    cleanup_dir(download_dir)


)

# TODO
# 1. Go to https://www.mediafire.com/file/czh8sezbo9s4692/test_videos.zip/file
#      and https://www.mediafire.com/file/x3rrbe4hwp04e6w/train_val_videos.zip/file
# 2. Right-click the Download button and copy the link address
#      e.g.
#    DATA_URL = {
#        "train": "https://download1602.mediafire.com/xxxxxxxxxxxx/x3rrbe4hwp04e6w/train_val_videos.zip",
#        "test": "https://download2390.mediafire.com/xxxxxxxxxxxx/czh8sezbo9s4692/test_videos.zip",
#    }
# 3. Paste the link address to DATA_URL

DATA_URL = {
    "train": "https://download2295.mediafire.com/4bb7p74xrbgg/x3rrbe4hwp04e6w/train_val_videos.zip",
    "test": "https://download2390.mediafire.com/79hfq3592lqg/czh8sezbo9s4692/test_videos.zip",
}

def download_datasets(root, url):
    """
    Download the Imagenet-R dataset archives and expand them
    in the folder provided as parameter
    """
    download_and_extract_archive(url=url, download_root=root)

def merge_datasets(download_path, storage_path):
    """
    Merge datasets in download_path to storage_path
    """

    # Merge train and test datasets
    train_path = os.path.join(download_path, "TrainValVideo")
    test_path = os.path.join(download_path, "TestVideo")
    train_test_path = storage_path

    print("Merging to {}".format(train_test_path))

    os.makedirs(train_test_path, exist_ok=True)

    for file_name in os.listdir(train_path):
        os.rename(
            os.path.join(train_path, file_name),
            os.path.join(train_test_path, file_name),


#    DATA_URL = {
#        "train": "https://download1602.mediafire.com/xxxxxxxxxxxx/x3rrbe4hwp04e6w/train_val_videos.zip",
#        "test": "https://download2390.mediafire.com/xxxxxxxxxxxx/czh8sezbo9s4692/test_videos.zip",
#    }
# 3. Paste the link address to DATA_URL

DATA_URL = {
    "train": "https://download2295.mediafire.com/4bb7p74xrbgg/x3rrbe4hwp04e6w/train_val_videos.zip",
    "test": "https://download2390.mediafire.com/79hfq3592lqg/czh8sezbo9s4692/test_videos.zip",
}

def download_datasets(root, url):
    """
    Download the Imagenet-R dataset archives and expand them
    in the folder provided as parameter
    """
    download_and_extract_archive(url=url, download_root=root)

def merge_datasets(download_path, storage_path):
    """
    Merge datasets in download_path to storage_path
    """

    # Merge train and test datasets
    train_path = os.path.join(download_path, "TrainValVideo")
    test_path = os.path.join(download_path, "TestVideo")
    train_test_path = storage_path

    print("Merging to {}".format(train_test_path))

    os.makedirs(train_test_path, exist_ok=True)

    for file_name in os.listdir(train_path):
        os.rename(
            os.path.join(train_path, file_name),
            os.path.join(train_test_path, file_name),
        )

    for file_name in os.listdir(test_path):
        os.rename(
            os.path.join(test_path, file_name),
            os.path.join(train_test_path, file_name),
        )

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/msrvtt/defaults_cap.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.msrvtt_cap.build_info.videos.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        for k, v in DATA_URL.items():
            print("Downloading {} to {}".format(v, k))


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    get_abs_path,
    get_cache_path,
)

import opendatasets as od

DATA_URL = "https://www.kaggle.com/datasets/hsankesara/flickr-image-dataset"

print(
    """
    To download the dataset, you need to have a Kaggle account and the associated key.
    See https://www.kaggle.com/docs/api to create account and a new API token.
    """
)

def move_directory(src_dir, dst_dir):
    """
    Move files from download_path to storage_path
    """
    print("Moving to {}".format(dst_dir))

    os.makedirs(dst_dir, exist_ok=True)

    for file_name in os.listdir(src_dir):
        os.rename(
            os.path.join(src_dir, file_name),
            os.path.join(dst_dir, file_name),
        )

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/flickr30k/defaults.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.flickr30k.build_info.images.storage

    storage_dir = Path(get_cache_path(storage_dir))
    download_dir = storage_dir.parent / "download"

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    os.makedirs(download_dir)

    try:
        print("Downloading {} to {}".format(DATA_URL, download_dir))
        od.download(DATA_URL, download_dir)
    except Exception as e:
        print(e)
        # remove download dir if failed
        cleanup_dir(download_dir)
        exit(1)

    move_directory(
        download_dir / "flickr-image-dataset" / "flickr30k_images" / "flickr30k_images",
        storage_dir / "flickr30k-images",
    )

    cleanup_dir(download_dir)


from lavis.common.utils import (
    cleanup_dir,
    get_abs_path,
    get_cache_path,
)

import opendatasets as od

DATA_URL = "https://www.kaggle.com/datasets/hsankesara/flickr-image-dataset"

print(
    """
    To download the dataset, you need to have a Kaggle account and the associated key.
    See https://www.kaggle.com/docs/api to create account and a new API token.
    """
)

def move_directory(src_dir, dst_dir):
    """
    Move files from download_path to storage_path
    """
    print("Moving to {}".format(dst_dir))

    os.makedirs(dst_dir, exist_ok=True)

    for file_name in os.listdir(src_dir):
        os.rename(
            os.path.join(src_dir, file_name),
            os.path.join(dst_dir, file_name),
        )

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/flickr30k/defaults.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.flickr30k.build_info.images.storage

    storage_dir = Path(get_cache_path(storage_dir))
    download_dir = storage_dir.parent / "download"

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    os.makedirs(download_dir)

    try:


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import io
import os
import pathlib
import urllib
import tqdm

from concurrent.futures import ThreadPoolExecutor

from lavis.common.utils import get_abs_path, get_cache_path
from lavis.datasets.builders import load_dataset
from omegaconf import OmegaConf
from PIL import Image

# DATA_URL = {"train": "http://www.cs.rice.edu/~vo9/sbucaptions/sbu_images.tar"}

USER_AGENT = (
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1"
)

def fetch_single_image(image_url, timeout=None, retries=0):
    for _ in range(retries + 1):
        try:
            request = urllib.request.Request(
                image_url,
                data=None,
                headers={"user-agent": USER_AGENT},
            )
            with urllib.request.urlopen(request, timeout=timeout) as req:
                image = Image.open(io.BytesIO(req.read()))
            break
        except Exception:
            image = None
    return image

def download_and_save_image(ann, save_dir, timeout=None, retries=0):
    image = fetch_single_image(ann["url"], timeout=timeout, retries=retries)

    if image is not None:
        image_path = os.path.join(save_dir, ann["image"])
        print(image_path)
        image.save(image_path)

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/sbu_caption/defaults.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.sbu_caption.build_info.images.storage

    storage_dir = pathlib.Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    storage_dir.mkdir(parents=True, exist_ok=True)

    num_threads = 20
    dset = load_dataset("sbu_caption")["train"].annotation

    print("Downloading dataset...")
    # multiprocessing
    with ThreadPoolExecutor(max_workers=num_threads) as executor:
        for ann in tqdm.tqdm(dset):
            executor.submit(
                download_and_save_image,
                ann,
                storage_dir,
                timeout=30,
                retries=10,
            )


import io
import os
import pathlib
import urllib
import tqdm

from concurrent.futures import ThreadPoolExecutor

from lavis.common.utils import get_abs_path, get_cache_path
from lavis.datasets.builders import load_dataset
from omegaconf import OmegaConf
from PIL import Image

# DATA_URL = {"train": "http://www.cs.rice.edu/~vo9/sbucaptions/sbu_images.tar"}

USER_AGENT = (
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1"
)

def fetch_single_image(image_url, timeout=None, retries=0):
    for _ in range(retries + 1):
        try:
            request = urllib.request.Request(
                image_url,
                data=None,
                headers={"user-agent": USER_AGENT},
            )
            with urllib.request.urlopen(request, timeout=timeout) as req:
                image = Image.open(io.BytesIO(req.read()))
            break
        except Exception:
            image = None
    return image

def download_and_save_image(ann, save_dir, timeout=None, retries=0):
    image = fetch_single_image(ann["url"], timeout=timeout, retries=retries)

    if image is not None:
        image_path = os.path.join(save_dir, ann["image"])
        print(image_path)
        image.save(image_path)

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/sbu_caption/defaults.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.sbu_caption.build_info.images.storage

    storage_dir = pathlib.Path(get_cache_path(storage_dir))


    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1"
)

def fetch_single_image(image_url, timeout=None, retries=0):
    for _ in range(retries + 1):
        try:
            request = urllib.request.Request(
                image_url,
                data=None,
                headers={"user-agent": USER_AGENT},
            )
            with urllib.request.urlopen(request, timeout=timeout) as req:
                image = Image.open(io.BytesIO(req.read()))
            break
        except Exception:
            image = None
    return image

def download_and_save_image(ann, save_dir, timeout=None, retries=0):
    image = fetch_single_image(ann["url"], timeout=timeout, retries=retries)

    if image is not None:
        image_path = os.path.join(save_dir, ann["image"])
        print(image_path)
        image.save(image_path)

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/sbu_caption/defaults.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.sbu_caption.build_info.images.storage

    storage_dir = pathlib.Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    storage_dir.mkdir(parents=True, exist_ok=True)

    num_threads = 20
    dset = load_dataset("sbu_caption")["train"].annotation


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = "https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/didemo/didemo_videos.tar.gz"

def download_datasets(root, url):
    """
    Download the Imagenet-R dataset archives and expand them
    in the folder provided as parameter
    """
    download_and_extract_archive(url=url, download_root=root)

def move_files(download_path, storage_path):
    """
    Move files from download_path to storage_path
    """
    print("Moving to {}".format(storage_path))

    os.makedirs(storage_path, exist_ok=True)

    for file_name in os.listdir(download_path):
        os.rename(
            os.path.join(download_path, file_name),
            os.path.join(storage_path, file_name),
        )

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/didemo/defaults_ret.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.didemo_retrieval.build_info.videos.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        print("Downloading {} to {}".format(DATA_URL, download_dir))
        download_datasets(download_dir, DATA_URL)
    except Exception as e:
        # remove download dir if failed
        cleanup_dir(download_dir)
        print("Failed to download or extracting datasets. Aborting.")

    move_files(download_dir / "videos", storage_dir)
    cleanup_dir(download_dir)


 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = "https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/didemo/didemo_videos.tar.gz"

def download_datasets(root, url):
    """
    Download the Imagenet-R dataset archives and expand them
    in the folder provided as parameter
    """
    download_and_extract_archive(url=url, download_root=root)

def move_files(download_path, storage_path):
    """
    Move files from download_path to storage_path
    """
    print("Moving to {}".format(storage_path))

    os.makedirs(storage_path, exist_ok=True)

    for file_name in os.listdir(download_path):
        os.rename(
            os.path.join(download_path, file_name),
            os.path.join(storage_path, file_name),
        )

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/didemo/defaults_ret.yaml")


from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = "https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/didemo/didemo_videos.tar.gz"

def download_datasets(root, url):
    """
    Download the Imagenet-R dataset archives and expand them
    in the folder provided as parameter
    """
    download_and_extract_archive(url=url, download_root=root)

def move_files(download_path, storage_path):
    """
    Move files from download_path to storage_path
    """
    print("Moving to {}".format(storage_path))

    os.makedirs(storage_path, exist_ok=True)

    for file_name in os.listdir(download_path):
        os.rename(
            os.path.join(download_path, file_name),
            os.path.join(storage_path, file_name),
        )

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/didemo/defaults_ret.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.didemo_retrieval.build_info.videos.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        print("Downloading {} to {}".format(DATA_URL, download_dir))
        download_datasets(download_dir, DATA_URL)


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = "https://www.cs.utexas.edu/users/ml/clamp/videoDescription/YouTubeClips.tar"

def download_datasets(root, url):
    download_and_extract_archive(url=url, download_root=root)

def move_files(download_path, storage_path):
    """
    Move files from download_path to storage_path
    """
    print("Moving to {}".format(storage_path))

    os.makedirs(storage_path, exist_ok=True)

    for file_name in os.listdir(download_path):
        os.rename(
            os.path.join(download_path, file_name),
            os.path.join(storage_path, file_name),
        )

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/msvd/defaults_cap.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.msvd_cap.build_info.videos.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        print("Downloading {}".format(DATA_URL))
        download_datasets(download_dir, DATA_URL)
    except Exception as e:
        # remove download dir if failed
        cleanup_dir(download_dir)
        print("Failed to download or extracting datasets. Aborting.")

    move_files(download_dir / "YouTubeClips", storage_dir)
    cleanup_dir(download_dir)


 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = "https://www.cs.utexas.edu/users/ml/clamp/videoDescription/YouTubeClips.tar"

def download_datasets(root, url):
    download_and_extract_archive(url=url, download_root=root)

def move_files(download_path, storage_path):
    """
    Move files from download_path to storage_path
    """
    print("Moving to {}".format(storage_path))

    os.makedirs(storage_path, exist_ok=True)

    for file_name in os.listdir(download_path):
        os.rename(
            os.path.join(download_path, file_name),
            os.path.join(storage_path, file_name),
        )

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/msvd/defaults_cap.yaml")


import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = "https://www.cs.utexas.edu/users/ml/clamp/videoDescription/YouTubeClips.tar"

def download_datasets(root, url):
    download_and_extract_archive(url=url, download_root=root)

def move_files(download_path, storage_path):
    """
    Move files from download_path to storage_path
    """
    print("Moving to {}".format(storage_path))

    os.makedirs(storage_path, exist_ok=True)

    for file_name in os.listdir(download_path):
        os.rename(
            os.path.join(download_path, file_name),
            os.path.join(storage_path, file_name),
        )

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/msvd/defaults_cap.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.msvd_cap.build_info.videos.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        print("Downloading {}".format(DATA_URL))
        download_datasets(download_dir, DATA_URL)


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = {
    "train": "https://cs.stanford.edu/people/rak248/VG_100K_2/images.zip",
    "train2": "https://cs.stanford.edu/people/rak248/VG_100K_2/images2.zip",
}

def download_datasets(root, url):
    download_and_extract_archive(url=url, download_root=root, extract_root=storage_dir)

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/vg/defaults_caption.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.vg_caption.build_info.images.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        for k, v in DATA_URL.items():
            print("Downloading {} to {}".format(v, k))
            download_datasets(download_dir, v)
    except Exception as e:
        # remove download dir if failed
        cleanup_dir(download_dir)
        print("Failed to download or extracting datasets. Aborting.")

    cleanup_dir(download_dir)


import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = {
    "train": "https://cs.stanford.edu/people/rak248/VG_100K_2/images.zip",
    "train2": "https://cs.stanford.edu/people/rak248/VG_100K_2/images2.zip",
}

def download_datasets(root, url):
    download_and_extract_archive(url=url, download_root=root, extract_root=storage_dir)

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/vg/defaults_caption.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.vg_caption.build_info.images.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        for k, v in DATA_URL.items():
            print("Downloading {} to {}".format(v, k))


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = "https://downloads.cs.stanford.edu/nlp/data/gqa/images.zip"

def download_datasets(root, url):
    download_and_extract_archive(url=url, download_root=root, extract_root=storage_dir.parent)

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/gqa/defaults.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.gqa.build_info.images.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        print("Downloading {}".format(DATA_URL))
        download_datasets(download_dir, DATA_URL)
    except Exception as e:
        # remove download dir if failed
        cleanup_dir(download_dir)
        print("Failed to download or extracting datasets. Aborting.")

    cleanup_dir(download_dir)


 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = "https://downloads.cs.stanford.edu/nlp/data/gqa/images.zip"

def download_datasets(root, url):
    download_and_extract_archive(url=url, download_root=root, extract_root=storage_dir.parent)

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/gqa/defaults.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.gqa.build_info.images.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        print("Downloading {}".format(DATA_URL))
        download_datasets(download_dir, DATA_URL)


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = {
    "train": "http://images.cocodataset.org/zips/train2014.zip",  # md5: 0da8c0bd3d6becc4dcb32757491aca88
    "val": "http://images.cocodataset.org/zips/val2014.zip",  # md5: a3d79f5ed8d289b7a7554ce06a5782b3
    "test": "http://images.cocodataset.org/zips/test2014.zip",  # md5: 04127eef689ceac55e3a572c2c92f264
    "test2015": "http://images.cocodataset.org/zips/test2015.zip",  # md5: 04127eef689ceac55e3a572c2c92f264
}

def download_datasets(root, url):
    download_and_extract_archive(url=url, download_root=root, extract_root=storage_dir)

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/coco/defaults_cap.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.coco_caption.build_info.images.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        for k, v in DATA_URL.items():
            print("Downloading {} to {}".format(v, k))
            download_datasets(download_dir, v)
    except Exception as e:
        # remove download dir if failed
        cleanup_dir(download_dir)
        print("Failed to download or extracting datasets. Aborting.")

    cleanup_dir(download_dir)


from pathlib import Path

from omegaconf import OmegaConf

from lavis.common.utils import (
    cleanup_dir,
    download_and_extract_archive,
    get_abs_path,
    get_cache_path,
)

DATA_URL = {
    "train": "http://images.cocodataset.org/zips/train2014.zip",  # md5: 0da8c0bd3d6becc4dcb32757491aca88
    "val": "http://images.cocodataset.org/zips/val2014.zip",  # md5: a3d79f5ed8d289b7a7554ce06a5782b3
    "test": "http://images.cocodataset.org/zips/test2014.zip",  # md5: 04127eef689ceac55e3a572c2c92f264
    "test2015": "http://images.cocodataset.org/zips/test2015.zip",  # md5: 04127eef689ceac55e3a572c2c92f264
}

def download_datasets(root, url):
    download_and_extract_archive(url=url, download_root=root, extract_root=storage_dir)

if __name__ == "__main__":

    config_path = get_abs_path("configs/datasets/coco/defaults_cap.yaml")

    storage_dir = OmegaConf.load(
        config_path
    ).datasets.coco_caption.build_info.images.storage

    download_dir = Path(get_cache_path(storage_dir)).parent / "download"
    storage_dir = Path(get_cache_path(storage_dir))

    if storage_dir.exists():
        print(f"Dataset already exists at {storage_dir}. Aborting.")
        exit(0)

    try:
        for k, v in DATA_URL.items():
            print("Downloading {} to {}".format(v, k))


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import json
import logging
import os
import time
from multiprocessing import Pool

import numpy as np
import requests
import tqdm
from lavis.common.utils import cleanup_dir, get_abs_path, get_cache_path
from omegaconf import OmegaConf

header_mzl = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
    # "User-Agent": "Googlebot-Image/1.0",  # Pretend to be googlebot
    # "X-Forwarded-For": "64.18.15.200",
}

header_gbot = {
    "User-Agent": "Googlebot-Image/1.0",  # Pretend to be googlebot
}

headers = [header_mzl, header_gbot]

# Setup
logging.basicConfig(filename="download_nocaps.log", filemode="w", level=logging.INFO)
requests.packages.urllib3.disable_warnings(
    requests.packages.urllib3.exceptions.InsecureRequestWarning
)

def download_file(url, filename):
    max_retries = 20
    cur_retries = 0

    header = headers[0]

    while cur_retries < max_retries:
        try:
            r = requests.get(url, headers=header, timeout=10)
            with open(filename, "wb") as f:
                f.write(r.content)

            break
        except Exception as e:
            logging.info(" ".join(repr(e).splitlines()))
            logging.error(url)
            cur_retries += 1

            # random sample a header from headers
            header = headers[np.random.randint(0, len(headers))]

    time.sleep(3 + cur_retries * 2)

def download_image_from_url_val(url):
    basename = os.path.basename(url)
    filename = os.path.join(storage_dir, "val", basename)

    download_file(url, filename)

def download_image_from_url_test(url):
    basename = os.path.basename(url)
    filename = os.path.join(storage_dir, "test", basename)

    download_file(url, filename)

if __name__ == "__main__":
    os.makedirs("tmp", exist_ok=True)

    # storage dir
    config_path = get_abs_path("configs/datasets/nocaps/defaults.yaml")

    storage_dir = OmegaConf.load(config_path).datasets.nocaps.build_info.images.storage
    storage_dir = get_cache_path(storage_dir)
    # make sure the storage dir exists
    os.makedirs(storage_dir, exist_ok=True)
    print("Storage dir:", storage_dir)

    # make sure the storage dir for val and test exists
    os.makedirs(os.path.join(storage_dir, "val"), exist_ok=True)
    os.makedirs(os.path.join(storage_dir, "test"), exist_ok=True)

    # download annotations
    val_url = "https://nocaps.s3.amazonaws.com/nocaps_val_4500_captions.json"
    tst_url = "https://s3.amazonaws.com/nocaps/nocaps_test_image_info.json"

    print("Downloading validation annotations from %s" % val_url)
    download_file(val_url, "tmp/nocaps_val_ann.json")
    print("Downloading testing annotations from %s" % tst_url)
    download_file(tst_url, "tmp/nocaps_tst_ann.json")

    # open annotations
    val_ann = json.load(open("tmp/nocaps_val_ann.json"))
    tst_ann = json.load(open("tmp/nocaps_tst_ann.json"))

    # collect image urls
    val_info = val_ann["images"]
    tst_info = tst_ann["images"]

    val_urls = [info["coco_url"] for info in val_info]
    tst_urls = [info["coco_url"] for info in tst_info]

    # setup multiprocessing
    # large n_procs possibly causes server to reject requests
    n_procs = 16

    with Pool(n_procs) as pool:
        print("Downloading validation images...")
        list(
            tqdm.tqdm(
                pool.imap(download_image_from_url_val, val_urls), total=len(val_urls)
            )
        )

    with Pool(n_procs) as pool:
        print("Downloading test images...")
        list(
            tqdm.tqdm(
                pool.imap(download_image_from_url_test, tst_urls), total=len(tst_urls)
            )
        )

    # clean tmp
    cleanup_dir("tmp")


header_mzl = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
    # "User-Agent": "Googlebot-Image/1.0",  # Pretend to be googlebot
    # "X-Forwarded-For": "64.18.15.200",
}

header_gbot = {
    "User-Agent": "Googlebot-Image/1.0",  # Pretend to be googlebot
}

headers = [header_mzl, header_gbot]

# Setup
logging.basicConfig(filename="download_nocaps.log", filemode="w", level=logging.INFO)
requests.packages.urllib3.disable_warnings(
    requests.packages.urllib3.exceptions.InsecureRequestWarning
)

def download_file(url, filename):
    max_retries = 20
    cur_retries = 0

    header = headers[0]

    while cur_retries < max_retries:
        try:
            r = requests.get(url, headers=header, timeout=10)
            with open(filename, "wb") as f:
                f.write(r.content)

            break
        except Exception as e:
            logging.info(" ".join(repr(e).splitlines()))
            logging.error(url)
            cur_retries += 1

            # random sample a header from headers
            header = headers[np.random.randint(0, len(headers))]

    time.sleep(3 + cur_retries * 2)

def download_image_from_url_val(url):
    basename = os.path.basename(url)
    filename = os.path.join(storage_dir, "val", basename)

    download_file(url, filename)

def download_image_from_url_test(url):
    basename = os.path.basename(url)
    filename = os.path.join(storage_dir, "test", basename)

    download_file(url, filename)

if __name__ == "__main__":
    os.makedirs("tmp", exist_ok=True)

    # storage dir


    header = headers[0]

    while cur_retries < max_retries:
        try:
            r = requests.get(url, headers=header, timeout=10)
            with open(filename, "wb") as f:
                f.write(r.content)

            break
        except Exception as e:
            logging.info(" ".join(repr(e).splitlines()))
            logging.error(url)
            cur_retries += 1

            # random sample a header from headers
            header = headers[np.random.randint(0, len(headers))]

    time.sleep(3 + cur_retries * 2)

def download_image_from_url_val(url):
    basename = os.path.basename(url)
    filename = os.path.join(storage_dir, "val", basename)

    download_file(url, filename)

def download_image_from_url_test(url):
    basename = os.path.basename(url)
    filename = os.path.join(storage_dir, "test", basename)

    download_file(url, filename)

if __name__ == "__main__":
    os.makedirs("tmp", exist_ok=True)

    # storage dir
    config_path = get_abs_path("configs/datasets/nocaps/defaults.yaml")

    storage_dir = OmegaConf.load(config_path).datasets.nocaps.build_info.images.storage
    storage_dir = get_cache_path(storage_dir)
    # make sure the storage dir exists
    os.makedirs(storage_dir, exist_ok=True)
    print("Storage dir:", storage_dir)


            break
        except Exception as e:
            logging.info(" ".join(repr(e).splitlines()))
            logging.error(url)
            cur_retries += 1

            # random sample a header from headers
            header = headers[np.random.randint(0, len(headers))]

    time.sleep(3 + cur_retries * 2)

def download_image_from_url_val(url):
    basename = os.path.basename(url)
    filename = os.path.join(storage_dir, "val", basename)

    download_file(url, filename)

def download_image_from_url_test(url):
    basename = os.path.basename(url)
    filename = os.path.join(storage_dir, "test", basename)

    download_file(url, filename)

if __name__ == "__main__":
    os.makedirs("tmp", exist_ok=True)

    # storage dir
    config_path = get_abs_path("configs/datasets/nocaps/defaults.yaml")

    storage_dir = OmegaConf.load(config_path).datasets.nocaps.build_info.images.storage
    storage_dir = get_cache_path(storage_dir)
    # make sure the storage dir exists
    os.makedirs(storage_dir, exist_ok=True)
    print("Storage dir:", storage_dir)

    # make sure the storage dir for val and test exists
    os.makedirs(os.path.join(storage_dir, "val"), exist_ok=True)
    os.makedirs(os.path.join(storage_dir, "test"), exist_ok=True)

    # download annotations
    val_url = "https://nocaps.s3.amazonaws.com/nocaps_val_4500_captions.json"


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import glob
from pathlib import Path
import time
from omegaconf import OmegaConf
import pandas as pd
import numpy as np
import requests
import zlib
import os
import io
import shelve
from lavis.common.utils import get_abs_path, get_cache_path
import magic  # pip install python-magic
import json
from multiprocessing import Pool
from tqdm import tqdm
from PIL import Image
from torchvision.transforms import functional as TF

headers = {
    #'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
    "User-Agent": "Googlebot-Image/1.0",  # Pretend to be googlebot
    "X-Forwarded-For": "64.18.15.200",
}

def _df_split_apply(tup_arg):
    split_ind, subset, func = tup_arg
    r = subset.apply(func, axis=1)
    return (split_ind, r)

def df_multiprocess(df, processes, chunk_size, func, dataset_name):
    print("Generating parts...")
    with shelve.open(
        "%s_%s_%s_results.tmp" % (dataset_name, func.__name__, chunk_size)
    ) as results:

        pbar = tqdm(total=len(df), position=0)
        # Resume:
        finished_chunks = set([int(k) for k in results.keys()])
        pbar.desc = "Resuming"
        for k in results.keys():
            pbar.update(len(results[str(k)][1]))

        pool_data = (
            (index, df[i : i + chunk_size], func)
            for index, i in enumerate(range(0, len(df), chunk_size))
            if index not in finished_chunks
        )
        print(
            int(len(df) / chunk_size),
            "parts.",
            chunk_size,
            "per part.",
            "Using",
            processes,
            "processes",
        )

        pbar.desc = "Downloading"
        with Pool(processes) as pool:
            for i, result in enumerate(
                pool.imap_unordered(_df_split_apply, pool_data, 2)
            ):
                results[str(result[0])] = result
                pbar.update(len(result[1]))
        pbar.close()

    print("Finished Downloading.")
    return

# Unique name based on url
def _file_name(row):
    name = (
        "%s/%s_%s"
        % (
            # row["folder"],
            storage_dir,
            row.name,
            (zlib.crc32(row["url"].encode("utf-8")) & 0xFFFFFFFF),
        )
        + ".jpg"
    )
    return name

# For checking mimetypes separately without download
def check_mimetype(row):
    if os.path.isfile(str(row["file"])):
        row["mimetype"] = magic.from_file(row["file"], mime=True)
        row["size"] = os.stat(row["file"]).st_size
    return row

# Don't download image, just check with a HEAD request, can't resume.
# Can use this instead of download_image to get HTTP status codes.
def check_download(row):
    fname = _file_name(row)
    try:
        # not all sites will support HEAD
        response = requests.head(
            row["url"], stream=False, timeout=5, allow_redirects=True, headers=headers
        )
        row["status"] = response.status_code
        row["headers"] = dict(response.headers)
    except:
        # log errors later, set error as 408 timeout
        row["status"] = 408
        return row
    if response.ok:
        row["file"] = fname
    return row

def resize_img(req):
    image = Image.open(req).convert("RGB")
    image = TF.resize(
        # image, size=(resize_size, resize_size)
        image,
        size=resize_size,
    )  # , interpolation=Image.LANCZOS)
    return image

def download_image(row):
    fname = _file_name(row)
    # Skip Already downloaded, retry others later
    if os.path.isfile(fname):
        row["status"] = 200
        row["file"] = fname
        row["mimetype"] = magic.from_file(row["file"], mime=True)
        row["size"] = os.stat(row["file"]).st_size
        return row

    try:
        # use smaller timeout to skip errors, but can result in failed downloads
        response = requests.get(
            row["url"], stream=False, timeout=5, allow_redirects=True, headers=headers
        )
        row["status"] = response.status_code
        # row['headers'] = dict(response.headers)
    except Exception as e:
        # log errors later, set error as 408 timeout
        row["status"] = 408
        return row

    if response.ok:
        try:
            # some sites respond with gzip transport encoding
            response.raw.decode_content = True
            img = resize_img(io.BytesIO(response.content))
            img.save(fname)

            row["mimetype"] = magic.from_file(fname, mime=True)
            row["size"] = os.stat(fname).st_size

        except Exception as e:
            #     # This is if it times out during a download or decode
            row["status"] = 408

    row["file"] = fname
    return row

def open_tsv(fname, folder):
    print("Opening %s Data File..." % fname)
    df = pd.read_csv(
        fname, sep="\t", names=["caption", "url"]
    )  # , usecols=range(1, 2))
    df["folder"] = folder
    print("Processing", len(df), " Images:")
    return df

def df_from_shelve(chunk_size, func, dataset_name):
    print("Generating Dataframe from results...")
    with shelve.open(
        "%s_%s_%s_results.tmp" % (dataset_name, func.__name__, chunk_size)
    ) as results:
        keylist = sorted([int(k) for k in results.keys()])
        df = pd.concat([results[str(k)][1] for k in keylist], sort=True)
    return df

resize_size = 384

config_path = get_abs_path("configs/datasets/conceptual_caption/defaults_3m.yaml")

storage_dir = OmegaConf.load(
    config_path
).datasets.conceptual_caption_3m.build_info.images.storage
storage_dir = Path(get_cache_path(storage_dir))

os.makedirs(storage_dir, exist_ok=True)

# number of processes in the pool can be larger than cores
num_processes = 32
# chunk_size is how many images per chunk per process - changing this resets progress when restarting.
images_per_part = 100

data_name = "cc3m"
df = open_tsv("/nas-ssd2/ziyang/data/dl-cc3m/Validation_GCC-1.1.0-Validation.tsv", data_name)
df_multiprocess(
    df=df,
    processes=num_processes,
    chunk_size=images_per_part,
    func=download_image,
    dataset_name=data_name,
)
df = df_from_shelve(
    chunk_size=images_per_part, func=download_image, dataset_name=data_name
)
df.to_csv(
    "downloaded_%s_report.tsv.gz" % data_name,
    compression="gzip",
    sep="\t",
    header=False,
    index=False,
)
print("Saved.")


import requests
import zlib
import os
import io
import shelve
from lavis.common.utils import get_abs_path, get_cache_path
import magic  # pip install python-magic
import json
from multiprocessing import Pool
from tqdm import tqdm
from PIL import Image
from torchvision.transforms import functional as TF

headers = {
    #'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
    "User-Agent": "Googlebot-Image/1.0",  # Pretend to be googlebot
    "X-Forwarded-For": "64.18.15.200",
}

def _df_split_apply(tup_arg):
    split_ind, subset, func = tup_arg
    r = subset.apply(func, axis=1)
    return (split_ind, r)

def df_multiprocess(df, processes, chunk_size, func, dataset_name):
    print("Generating parts...")
    with shelve.open(
        "%s_%s_%s_results.tmp" % (dataset_name, func.__name__, chunk_size)
    ) as results:

        pbar = tqdm(total=len(df), position=0)
        # Resume:
        finished_chunks = set([int(k) for k in results.keys()])
        pbar.desc = "Resuming"
        for k in results.keys():
            pbar.update(len(results[str(k)][1]))

        pool_data = (
            (index, df[i : i + chunk_size], func)
            for index, i in enumerate(range(0, len(df), chunk_size))
            if index not in finished_chunks
        )


import magic  # pip install python-magic
import json
from multiprocessing import Pool
from tqdm import tqdm
from PIL import Image
from torchvision.transforms import functional as TF

headers = {
    #'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
    "User-Agent": "Googlebot-Image/1.0",  # Pretend to be googlebot
    "X-Forwarded-For": "64.18.15.200",
}

def _df_split_apply(tup_arg):
    split_ind, subset, func = tup_arg
    r = subset.apply(func, axis=1)
    return (split_ind, r)

def df_multiprocess(df, processes, chunk_size, func, dataset_name):
    print("Generating parts...")
    with shelve.open(
        "%s_%s_%s_results.tmp" % (dataset_name, func.__name__, chunk_size)
    ) as results:

        pbar = tqdm(total=len(df), position=0)
        # Resume:
        finished_chunks = set([int(k) for k in results.keys()])
        pbar.desc = "Resuming"
        for k in results.keys():
            pbar.update(len(results[str(k)][1]))

        pool_data = (
            (index, df[i : i + chunk_size], func)
            for index, i in enumerate(range(0, len(df), chunk_size))
            if index not in finished_chunks
        )
        print(
            int(len(df) / chunk_size),
            "parts.",
            chunk_size,
            "per part.",
            "Using",
            processes,
            "processes",
        )

        pbar.desc = "Downloading"
        with Pool(processes) as pool:
            for i, result in enumerate(
                pool.imap_unordered(_df_split_apply, pool_data, 2)
            ):
                results[str(result[0])] = result
                pbar.update(len(result[1]))
        pbar.close()

    print("Finished Downloading.")
    return

# Unique name based on url
def _file_name(row):
    name = (
        "%s/%s_%s"
        % (
            # row["folder"],
            storage_dir,
            row.name,
            (zlib.crc32(row["url"].encode("utf-8")) & 0xFFFFFFFF),
        )
        + ".jpg"
    )
    return name

# For checking mimetypes separately without download
def check_mimetype(row):
    if os.path.isfile(str(row["file"])):


            "per part.",
            "Using",
            processes,
            "processes",
        )

        pbar.desc = "Downloading"
        with Pool(processes) as pool:
            for i, result in enumerate(
                pool.imap_unordered(_df_split_apply, pool_data, 2)
            ):
                results[str(result[0])] = result
                pbar.update(len(result[1]))
        pbar.close()

    print("Finished Downloading.")
    return

# Unique name based on url
def _file_name(row):
    name = (
        "%s/%s_%s"
        % (
            # row["folder"],
            storage_dir,
            row.name,
            (zlib.crc32(row["url"].encode("utf-8")) & 0xFFFFFFFF),
        )
        + ".jpg"
    )
    return name

# For checking mimetypes separately without download
def check_mimetype(row):
    if os.path.isfile(str(row["file"])):
        row["mimetype"] = magic.from_file(row["file"], mime=True)
        row["size"] = os.stat(row["file"]).st_size
    return row

# Don't download image, just check with a HEAD request, can't resume.
# Can use this instead of download_image to get HTTP status codes.
def check_download(row):
    fname = _file_name(row)
    try:
        # not all sites will support HEAD
        response = requests.head(
            row["url"], stream=False, timeout=5, allow_redirects=True, headers=headers
        )
        row["status"] = response.status_code


    print("Finished Downloading.")
    return

# Unique name based on url
def _file_name(row):
    name = (
        "%s/%s_%s"
        % (
            # row["folder"],
            storage_dir,
            row.name,
            (zlib.crc32(row["url"].encode("utf-8")) & 0xFFFFFFFF),
        )
        + ".jpg"
    )
    return name

# For checking mimetypes separately without download
def check_mimetype(row):
    if os.path.isfile(str(row["file"])):
        row["mimetype"] = magic.from_file(row["file"], mime=True)
        row["size"] = os.stat(row["file"]).st_size
    return row

# Don't download image, just check with a HEAD request, can't resume.
# Can use this instead of download_image to get HTTP status codes.
def check_download(row):
    fname = _file_name(row)
    try:
        # not all sites will support HEAD
        response = requests.head(
            row["url"], stream=False, timeout=5, allow_redirects=True, headers=headers
        )
        row["status"] = response.status_code
        row["headers"] = dict(response.headers)
    except:
        # log errors later, set error as 408 timeout
        row["status"] = 408
        return row
    if response.ok:
        row["file"] = fname
    return row


            # row["folder"],
            storage_dir,
            row.name,
            (zlib.crc32(row["url"].encode("utf-8")) & 0xFFFFFFFF),
        )
        + ".jpg"
    )
    return name

# For checking mimetypes separately without download
def check_mimetype(row):
    if os.path.isfile(str(row["file"])):
        row["mimetype"] = magic.from_file(row["file"], mime=True)
        row["size"] = os.stat(row["file"]).st_size
    return row

# Don't download image, just check with a HEAD request, can't resume.
# Can use this instead of download_image to get HTTP status codes.
def check_download(row):
    fname = _file_name(row)
    try:
        # not all sites will support HEAD
        response = requests.head(
            row["url"], stream=False, timeout=5, allow_redirects=True, headers=headers
        )
        row["status"] = response.status_code
        row["headers"] = dict(response.headers)
    except:
        # log errors later, set error as 408 timeout
        row["status"] = 408
        return row
    if response.ok:
        row["file"] = fname
    return row

def resize_img(req):
    image = Image.open(req).convert("RGB")
    image = TF.resize(
        # image, size=(resize_size, resize_size)
        image,
        size=resize_size,
    )  # , interpolation=Image.LANCZOS)
    return image

def download_image(row):
    fname = _file_name(row)
    # Skip Already downloaded, retry others later
    if os.path.isfile(fname):
        row["status"] = 200
        row["file"] = fname
        row["mimetype"] = magic.from_file(row["file"], mime=True)
        row["size"] = os.stat(row["file"]).st_size


# Don't download image, just check with a HEAD request, can't resume.
# Can use this instead of download_image to get HTTP status codes.
def check_download(row):
    fname = _file_name(row)
    try:
        # not all sites will support HEAD
        response = requests.head(
            row["url"], stream=False, timeout=5, allow_redirects=True, headers=headers
        )
        row["status"] = response.status_code
        row["headers"] = dict(response.headers)
    except:
        # log errors later, set error as 408 timeout
        row["status"] = 408
        return row
    if response.ok:
        row["file"] = fname
    return row

def resize_img(req):
    image = Image.open(req).convert("RGB")
    image = TF.resize(
        # image, size=(resize_size, resize_size)
        image,
        size=resize_size,
    )  # , interpolation=Image.LANCZOS)
    return image

def download_image(row):
    fname = _file_name(row)
    # Skip Already downloaded, retry others later
    if os.path.isfile(fname):
        row["status"] = 200
        row["file"] = fname
        row["mimetype"] = magic.from_file(row["file"], mime=True)
        row["size"] = os.stat(row["file"]).st_size
        return row

    try:
        # use smaller timeout to skip errors, but can result in failed downloads
        response = requests.get(
            row["url"], stream=False, timeout=5, allow_redirects=True, headers=headers
        )
        row["status"] = response.status_code
        # row['headers'] = dict(response.headers)
    except Exception as e:


        row["headers"] = dict(response.headers)
    except:
        # log errors later, set error as 408 timeout
        row["status"] = 408
        return row
    if response.ok:
        row["file"] = fname
    return row

def resize_img(req):
    image = Image.open(req).convert("RGB")
    image = TF.resize(
        # image, size=(resize_size, resize_size)
        image,
        size=resize_size,
    )  # , interpolation=Image.LANCZOS)
    return image

def download_image(row):
    fname = _file_name(row)
    # Skip Already downloaded, retry others later
    if os.path.isfile(fname):
        row["status"] = 200
        row["file"] = fname
        row["mimetype"] = magic.from_file(row["file"], mime=True)
        row["size"] = os.stat(row["file"]).st_size
        return row

    try:
        # use smaller timeout to skip errors, but can result in failed downloads
        response = requests.get(
            row["url"], stream=False, timeout=5, allow_redirects=True, headers=headers
        )
        row["status"] = response.status_code
        # row['headers'] = dict(response.headers)
    except Exception as e:
        # log errors later, set error as 408 timeout
        row["status"] = 408
        return row

    if response.ok:
        try:
            # some sites respond with gzip transport encoding
            response.raw.decode_content = True
            img = resize_img(io.BytesIO(response.content))
            img.save(fname)

            row["mimetype"] = magic.from_file(fname, mime=True)
            row["size"] = os.stat(fname).st_size

        except Exception as e:
            #     # This is if it times out during a download or decode
            row["status"] = 408

    row["file"] = fname
    return row

def open_tsv(fname, folder):
    print("Opening %s Data File..." % fname)
    df = pd.read_csv(
        fname, sep="\t", names=["caption", "url"]
    )  # , usecols=range(1, 2))
    df["folder"] = folder
    print("Processing", len(df), " Images:")
    return df

def df_from_shelve(chunk_size, func, dataset_name):
    print("Generating Dataframe from results...")
    with shelve.open(
        "%s_%s_%s_results.tmp" % (dataset_name, func.__name__, chunk_size)
    ) as results:
        keylist = sorted([int(k) for k in results.keys()])
        df = pd.concat([results[str(k)][1] for k in keylist], sort=True)
    return df


        return row

    if response.ok:
        try:
            # some sites respond with gzip transport encoding
            response.raw.decode_content = True
            img = resize_img(io.BytesIO(response.content))
            img.save(fname)

            row["mimetype"] = magic.from_file(fname, mime=True)
            row["size"] = os.stat(fname).st_size

        except Exception as e:
            #     # This is if it times out during a download or decode
            row["status"] = 408

    row["file"] = fname
    return row

def open_tsv(fname, folder):
    print("Opening %s Data File..." % fname)
    df = pd.read_csv(
        fname, sep="\t", names=["caption", "url"]
    )  # , usecols=range(1, 2))
    df["folder"] = folder
    print("Processing", len(df), " Images:")
    return df

def df_from_shelve(chunk_size, func, dataset_name):
    print("Generating Dataframe from results...")
    with shelve.open(
        "%s_%s_%s_results.tmp" % (dataset_name, func.__name__, chunk_size)
    ) as results:
        keylist = sorted([int(k) for k in results.keys()])
        df = pd.concat([results[str(k)][1] for k in keylist], sort=True)
    return df

resize_size = 384

config_path = get_abs_path("configs/datasets/conceptual_caption/defaults_3m.yaml")

storage_dir = OmegaConf.load(
    config_path
).datasets.conceptual_caption_3m.build_info.images.storage
storage_dir = Path(get_cache_path(storage_dir))


            row["size"] = os.stat(fname).st_size

        except Exception as e:
            #     # This is if it times out during a download or decode
            row["status"] = 408

    row["file"] = fname
    return row

def open_tsv(fname, folder):
    print("Opening %s Data File..." % fname)
    df = pd.read_csv(
        fname, sep="\t", names=["caption", "url"]
    )  # , usecols=range(1, 2))
    df["folder"] = folder
    print("Processing", len(df), " Images:")
    return df

def df_from_shelve(chunk_size, func, dataset_name):
    print("Generating Dataframe from results...")
    with shelve.open(
        "%s_%s_%s_results.tmp" % (dataset_name, func.__name__, chunk_size)
    ) as results:
        keylist = sorted([int(k) for k in results.keys()])
        df = pd.concat([results[str(k)][1] for k in keylist], sort=True)
    return df

resize_size = 384

config_path = get_abs_path("configs/datasets/conceptual_caption/defaults_3m.yaml")

storage_dir = OmegaConf.load(
    config_path
).datasets.conceptual_caption_3m.build_info.images.storage
storage_dir = Path(get_cache_path(storage_dir))

os.makedirs(storage_dir, exist_ok=True)

# number of processes in the pool can be larger than cores
num_processes = 32
# chunk_size is how many images per chunk per process - changing this resets progress when restarting.
images_per_part = 100

data_name = "cc3m"
df = open_tsv("/nas-ssd2/ziyang/data/dl-cc3m/Validation_GCC-1.1.0-Validation.tsv", data_name)