ConcatDataset

        self._add_instance_ids()

    def __len__(self):
        return len(self.annotation)

    def collater(self, samples):
        return default_collate(samples)

    def set_processors(self, vis_processor, text_processor):
        self.vis_processor = vis_processor
        self.text_processor = text_processor

    def _add_instance_ids(self, key="instance_id"):
        for idx, ann in enumerate(self.annotation):
            if isinstance(ann, str):
                pass
            else:
                ann[key] = str(idx)

class ConcatDataset(ConcatDataset):
    def __init__(self, datasets: Iterable[Dataset]) -> None:
        super().__init__(datasets)

    def collater(self, samples):
        # TODO For now only supports datasets with same underlying collater implementations

        all_keys = set()
        for s in samples:
            all_keys.update(s)

        shared_keys = all_keys
        for s in samples:
            shared_keys = shared_keys & set(s.keys())

        samples_shared_keys = []
        for s in samples:
            samples_shared_keys.append({k: s[k] for k in s.keys() if k in shared_keys})

        return self.datasets[0].collater(samples_shared_keys)


    def __len__(self):
        return len(self.annotation)

    def collater(self, samples):
        return default_collate(samples)

    def set_processors(self, vis_processor, text_processor):
        self.vis_processor = vis_processor
        self.text_processor = text_processor

    def _add_instance_ids(self, key="instance_id"):
        for idx, ann in enumerate(self.annotation):
            if isinstance(ann, str):
                pass
            else:
                ann[key] = str(idx)

class ConcatDataset(ConcatDataset):
    def __init__(self, datasets: Iterable[Dataset]) -> None:
        super().__init__(datasets)

    def collater(self, samples):
        # TODO For now only supports datasets with same underlying collater implementations

        all_keys = set()
        for s in samples:
            all_keys.update(s)

        shared_keys = all_keys
        for s in samples:
            shared_keys = shared_keys & set(s.keys())

        samples_shared_keys = []
        for s in samples:
            samples_shared_keys.append({k: s[k] for k in s.keys() if k in shared_keys})

        return self.datasets[0].collater(samples_shared_keys)


        # print(ann_paths)

        for ann_path in ann_paths:
            #print('herrrr', ann_path)
            if '.json' in ann_path:
                self.annotation.extend(json.load(open(ann_path, "r")))
                if 'train' in ann_path:
                    self.data_type = 'train'
                else:
                    self.data_type = 'val'
            else:
                raise AttributeError('Undefined data type')

        #self.annotation = self.annotation[:100]
        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __len__(self):
        return len(self.annotation)

    def collater(self, samples):
        return default_collate(samples)

    def set_processors(self, vis_processor, text_processor):
        self.vis_processor = vis_processor
        self.text_processor = text_processor

    def _add_instance_ids(self, key="instance_id"):
        for idx, ann in enumerate(self.annotation):
            if isinstance(ann, str):
                pass
            else:
                ann[key] = str(idx)

class ConcatDataset(ConcatDataset):
    def __init__(self, datasets: Iterable[Dataset]) -> None:
        super().__init__(datasets)


collater

    def collater(self, samples):
        return default_collate(samples)

    def set_processors(self, vis_processor, text_processor):
        self.vis_processor = vis_processor
        self.text_processor = text_processor

    def _add_instance_ids(self, key="instance_id"):
        for idx, ann in enumerate(self.annotation):
            if isinstance(ann, str):
                pass
            else:
                ann[key] = str(idx)

class ConcatDataset(ConcatDataset):
    def __init__(self, datasets: Iterable[Dataset]) -> None:
        super().__init__(datasets)

    def collater(self, samples):
        # TODO For now only supports datasets with same underlying collater implementations

        all_keys = set()
        for s in samples:
            all_keys.update(s)

        shared_keys = all_keys
        for s in samples:
            shared_keys = shared_keys & set(s.keys())

        samples_shared_keys = []
        for s in samples:
            samples_shared_keys.append({k: s[k] for k in s.keys() if k in shared_keys})

        return self.datasets[0].collater(samples_shared_keys)


                self.annotation.extend(json.load(open(ann_path, "r")))
                if 'train' in ann_path:
                    self.data_type = 'train'
                else:
                    self.data_type = 'val'
            else:
                raise AttributeError('Undefined data type')

        #self.annotation = self.annotation[:100]
        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __len__(self):
        return len(self.annotation)

    def collater(self, samples):
        return default_collate(samples)

    def set_processors(self, vis_processor, text_processor):
        self.vis_processor = vis_processor
        self.text_processor = text_processor

    def _add_instance_ids(self, key="instance_id"):
        for idx, ann in enumerate(self.annotation):
            if isinstance(ann, str):
                pass
            else:
                ann[key] = str(idx)

class ConcatDataset(ConcatDataset):
    def __init__(self, datasets: Iterable[Dataset]) -> None:
        super().__init__(datasets)

    def collater(self, samples):
        # TODO For now only supports datasets with same underlying collater implementations

        all_keys = set()
        for s in samples:
            all_keys.update(s)


                    self.data_type = 'val'
            else:
                raise AttributeError('Undefined data type')

        #self.annotation = self.annotation[:100]
        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __len__(self):
        return len(self.annotation)

    def collater(self, samples):
        return default_collate(samples)

    def set_processors(self, vis_processor, text_processor):
        self.vis_processor = vis_processor
        self.text_processor = text_processor

    def _add_instance_ids(self, key="instance_id"):
        for idx, ann in enumerate(self.annotation):
            if isinstance(ann, str):
                pass
            else:
                ann[key] = str(idx)

class ConcatDataset(ConcatDataset):
    def __init__(self, datasets: Iterable[Dataset]) -> None:
        super().__init__(datasets)

    def collater(self, samples):
        # TODO For now only supports datasets with same underlying collater implementations

        all_keys = set()
        for s in samples:
            all_keys.update(s)

        shared_keys = all_keys
        for s in samples:
            shared_keys = shared_keys & set(s.keys())

        samples_shared_keys = []
        for s in samples:
            samples_shared_keys.append({k: s[k] for k in s.keys() if k in shared_keys})


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
import json
import torch
import numpy as np
import copy

from PIL import Image
from PIL import ImageFile

ImageFile.LOAD_TRUNCATED_IMAGES = True
from lavis.datasets.datasets.base_dataset import BaseDataset

def get_qclass(question):
    lques = question
    if 'What' in lques:
        return 'What'
    if 'How' in lques:
        return 'How'
    if 'Can' in lques:
        return 'Can'
    if 'Is' in lques:
        return 'Is'
    if 'Which' in lques:
        return 'Which'
    return 'Other'

class ThreeDVQADataset(BaseDataset):
    def __init__(self, **kwargs):
        super().__init__(kwargs['vis_processor'], kwargs['text_processor'], kwargs['vis_root'], kwargs['ann_paths'], kwargs['modalities'])

        self.modalities = kwargs['modalities']

        self.scene_ids = {}
        n = 0
        new_annotation = []
        for ann in self.annotation:
            try:
                img_id = ann["scene_id"]
                if img_id not in self.scene_ids.keys():
                    self.scene_ids[img_id] = n
                    n += 1
                new_annotation.append(ann)
            except:
                pass
        self.annotation = new_annotation

        for modality in self.modalities:
            if 'pc' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                self.pc_feat_root = self.pc_root + '/voxelized_features_sam_nonzero_preprocess/'
                self.voxel_root = self.pc_root + '/voxelized_voxels_sam_nonzero_preprocess/'
                self.annotation = [
                ann for ann in self.annotation if os.path.exists(os.path.join(self.pc_feat_root, str(ann["scene_id"]) + ".pt"))
            ]
            if 'video' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])
                setattr(self, f"existing_{modality}_annotation",getattr(self, f'get_existing_{modality}_annotations')())

            if 'frame' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

            if 'depth' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

    def get_existing_video_annotations(self):
        return [f.split('.')[0] for f in os.listdir(self.video_root)]

    def get_video_path(self, ann):
        return os.path.join(self.video_root, f'{ann["scene_id"]}.mp4')

    def get_frame_path(self, ann):
        return os.path.join(self.frame_root, f'{ann["scene_id"]}/')

    def get_depth_path(self, ann):
        return os.path.join(self.depth_root, f'{ann["scene_id"]}/')

    def __getitem__(self, index):

        ann = copy.deepcopy(self.annotation[index])
        if 'question_id' in ann.keys(): # 3dqa data
            qa_input = self.text_processor(ann['situation']) + '. Question: ' + self.text_processor(ann["question"]) + ' Based on the frames and 3D Model information, answer the question using a single word or phase.'
            qtype = get_qclass(ann['question'])
            question_id = ann['question_id']
            answer = ann["answers"][0]
            question_id = qtype + '_' + str(question_id)
        else: # pre-training data
            qa_input = 'Question: ' + self.text_processor(ann["question"]) + ' Based on 3D Model information, answer the question.'
            answer = ann["answers"][0]
            answer = self.text_processor(answer)
            question_id = str(ann["scene_id"])

        scene_id = str(ann["scene_id"])

        out = {
            "qa_input": qa_input,
            "qa_output": answer,
            "scene_id": self.scene_ids[ann["scene_id"]],
            "question_id": question_id,
        }

        for modality in self.modalities:

            if modality == 'pc':
                pc_feat = torch.load(os.path.join(self.pc_feat_root, f"{scene_id}.pt"), map_location="cpu")
                if isinstance(pc_feat, np.ndarray):
                    pc_feat = torch.tensor(pc_feat).float()
                pc = np.load(os.path.join(self.voxel_root, f"{scene_id}.npy"))
                pc = torch.tensor(pc).float().cpu()
                if pc_feat.shape[0] > 5000:
                    idxes = torch.sort(torch.randperm(pc_feat.shape[0])[:5000])[1]
                    pc_feat = pc_feat[idxes]
                    pc = pc[idxes]
                else:
                    pc_feat = torch.cat([pc_feat, torch.zeros(5000 - pc_feat.shape[0], 1408)], dim=0)
                    pc = torch.cat([pc, torch.zeros(5000 - pc.shape[0], 3)], dim=0)
                out["pc_feat"] = pc_feat
                out['pc'] = pc

            if modality == 'video':
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                rgb, indices, fps = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"])
                out['rgb'] = rgb.to(torch.float32)

            if modality == 'frame':
                indices = None
                clip = None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                frms, indices = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                assert len(rgb) == getattr(self, f"{modality}_processor").n_frms
                out['rgb'] = rgb

            if modality == 'depth':
                assert indices is not None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                depth, _ = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices, type='depth')
                out['depth'] = depth

        return out

    def __len__(self):
        return len(self.annotation)

class ThreeDVQAEvalDataset(BaseDataset):
    def __init__(self, **kwargs):
        super().__init__(kwargs['vis_processor'], kwargs['text_processor'], kwargs['vis_root'], kwargs['ann_paths'], kwargs['modalities'])

        self.modalities = kwargs['modalities']

        self.scene_ids = {}
        n = 0
        new_annotation = []
        for ann in self.annotation:
            try:
                img_id = ann["scene_id"]
                if img_id not in self.scene_ids.keys():
                    self.scene_ids[img_id] = n
                    n += 1
                new_annotation.append(ann)
            except:
                pass
        self.annotation = new_annotation

        for modality in self.modalities:
            if 'pc' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                self.pc_feat_root = self.pc_root + '/voxelized_features_sam_nonzero_preprocess/'
                self.voxel_root = self.pc_root + '/voxelized_voxels_sam_nonzero_preprocess/'
                # pc_root = '/nas-ssd2/shoubin/datasets/scannet_feat/'
                # self.pc_feat_root = pc_root + '/voxelized_features_sam_nonzero_preprocess/'
                # self.voxel_root = pc_root + '/voxelized_voxels_sam_nonzero_preprocess/'
                self.annotation = [
                    ann for ann in self.annotation if os.path.exists(os.path.join(self.pc_feat_root, str(ann["scene_id"]) + ".pt"))
                ]
            if 'video' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])
                setattr(self, f"existing_{modality}_annotation",getattr(self, f'get_existing_{modality}_annotations')())

            if 'frame' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

            if 'depth' in modality:
                # todo
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

    def get_existing_video_annotations(self):
        return [f.split('.')[0] for f in os.listdir(self.video_root)]

    def get_video_path(self, ann):
        return os.path.join(self.video_root, f'{ann["scene_id"]}.mp4')

    def get_frame_path(self, ann):
        return os.path.join(self.frame_root, f'{ann["scene_id"]}/')

    def get_depth_path(self, ann):
        return os.path.join(self.depth_root, f'{ann["scene_id"]}/')

    def __getitem__(self, index):

        ann = copy.deepcopy(self.annotation[index])
        if 'question_id' in ann.keys(): # 3dqa data
            qa_input = self.text_processor(ann['situation']) + '. Question: ' + self.text_processor(ann["question"]) + ' Based on the frames and 3D Model information, answer the question using a single word or phase.'
            qtype = get_qclass(ann['question'])
            question_id = ann['question_id']
            answer = ann["answers"][0]
            question_id = qtype + '_' + str(question_id)
        else: # pre-training data
            # ann['qa_input'] =  'Question: ' + self.text_processor(question) + ' Based on the frames information, answer the question using a single word or phase.'
            qa_input = 'Question: ' + self.text_processor(ann["question"]) + ' Based on 3D Model information, answer the question using a single word or phase.'
            answer = ann["answers"][0]
            answer = self.text_processor(answer)
            question_id = str(ann["scene_id"])

        scene_id = str(ann["scene_id"])

        out = {
            "qa_input": qa_input,
            "qa_output": answer,
            "scene_id": self.scene_ids[ann["scene_id"]],
            "question_id": question_id,
        }

        for modality in self.modalities:

            if modality == 'pc':
                pc_feat = torch.load(os.path.join(self.pc_feat_root, f"{scene_id}.pt"), map_location="cpu")  # [N, 1408]
                if isinstance(pc_feat, np.ndarray):
                    pc_feat = torch.tensor(pc_feat).float()
                pc = np.load(os.path.join(self.voxel_root, f"{scene_id}.npy"))
                pc = torch.tensor(pc).float().cpu()
                # sample 10000 points: [N, 1408] -> [10000, 1408]
                if pc_feat.shape[0] > 5000:
                    idxes = torch.sort(torch.randperm(pc_feat.shape[0])[:5000])[1]
                    pc_feat = pc_feat[idxes]
                    pc = pc[idxes]
                else:
                    pc_feat = torch.cat([pc_feat, torch.zeros(5000 - pc_feat.shape[0], 1408)], dim=0)
                    pc = torch.cat([pc, torch.zeros(5000 - pc.shape[0], 3)], dim=0)

                out["pc_feat"] = pc_feat
                out['pc'] = pc
            if modality == 'video':
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                rgb, indices, fps = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"])
                out['rgb'] = rgb.to(torch.float32)

            if modality == 'frame':
                indices = None
                clip = None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                frms, indices = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                assert len(rgb) == getattr(self, f"{modality}_processor").n_frms
                out['rgb'] = rgb

            if modality == 'depth':
                assert indices is not None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                depth, _ = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices, type='depth')
                out['depth'] = depth

        return out

    def __len__(self):
        return len(self.annotation)


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
import json
import torch
import numpy as np
import copy

from PIL import Image
from PIL import ImageFile

ImageFile.LOAD_TRUNCATED_IMAGES = True
from lavis.datasets.datasets.base_dataset import BaseDataset

def get_qclass(question):
    lques = question
    if 'What' in lques:
        return 'What'
    if 'How' in lques:
        return 'How'
    if 'Can' in lques:
        return 'Can'
    if 'Is' in lques:
        return 'Is'
    if 'Which' in lques:
        return 'Which'
    return 'Other'

class ThreeDVQADataset(BaseDataset):
    def __init__(self, **kwargs):
        super().__init__(kwargs['vis_processor'], kwargs['text_processor'], kwargs['vis_root'], kwargs['ann_paths'], kwargs['modalities'])

        self.modalities = kwargs['modalities']

        self.scene_ids = {}
        n = 0
        new_annotation = []
        for ann in self.annotation:
            try:
                img_id = ann["scene_id"]
                if img_id not in self.scene_ids.keys():
                    self.scene_ids[img_id] = n
                    n += 1
                new_annotation.append(ann)
            except:
                pass
        self.annotation = new_annotation


ThreeDVQADataset

from PIL import ImageFile

ImageFile.LOAD_TRUNCATED_IMAGES = True
from lavis.datasets.datasets.base_dataset import BaseDataset

def get_qclass(question):
    lques = question
    if 'What' in lques:
        return 'What'
    if 'How' in lques:
        return 'How'
    if 'Can' in lques:
        return 'Can'
    if 'Is' in lques:
        return 'Is'
    if 'Which' in lques:
        return 'Which'
    return 'Other'

class ThreeDVQADataset(BaseDataset):
    def __init__(self, **kwargs):
        super().__init__(kwargs['vis_processor'], kwargs['text_processor'], kwargs['vis_root'], kwargs['ann_paths'], kwargs['modalities'])

        self.modalities = kwargs['modalities']

        self.scene_ids = {}
        n = 0
        new_annotation = []
        for ann in self.annotation:
            try:
                img_id = ann["scene_id"]
                if img_id not in self.scene_ids.keys():
                    self.scene_ids[img_id] = n
                    n += 1
                new_annotation.append(ann)
            except:
                pass
        self.annotation = new_annotation

        for modality in self.modalities:
            if 'pc' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                self.pc_feat_root = self.pc_root + '/voxelized_features_sam_nonzero_preprocess/'
                self.voxel_root = self.pc_root + '/voxelized_voxels_sam_nonzero_preprocess/'
                self.annotation = [
                ann for ann in self.annotation if os.path.exists(os.path.join(self.pc_feat_root, str(ann["scene_id"]) + ".pt"))
            ]
            if 'video' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])
                setattr(self, f"existing_{modality}_annotation",getattr(self, f'get_existing_{modality}_annotations')())

            if 'frame' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

            if 'depth' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

    def get_existing_video_annotations(self):
        return [f.split('.')[0] for f in os.listdir(self.video_root)]

    def get_video_path(self, ann):
        return os.path.join(self.video_root, f'{ann["scene_id"]}.mp4')

    def get_frame_path(self, ann):
        return os.path.join(self.frame_root, f'{ann["scene_id"]}/')

    def get_depth_path(self, ann):
        return os.path.join(self.depth_root, f'{ann["scene_id"]}/')

    def __getitem__(self, index):

        ann = copy.deepcopy(self.annotation[index])
        if 'question_id' in ann.keys(): # 3dqa data
            qa_input = self.text_processor(ann['situation']) + '. Question: ' + self.text_processor(ann["question"]) + ' Based on the frames and 3D Model information, answer the question using a single word or phase.'
            qtype = get_qclass(ann['question'])
            question_id = ann['question_id']
            answer = ann["answers"][0]
            question_id = qtype + '_' + str(question_id)
        else: # pre-training data
            qa_input = 'Question: ' + self.text_processor(ann["question"]) + ' Based on 3D Model information, answer the question.'
            answer = ann["answers"][0]
            answer = self.text_processor(answer)
            question_id = str(ann["scene_id"])

        scene_id = str(ann["scene_id"])

        out = {
            "qa_input": qa_input,
            "qa_output": answer,
            "scene_id": self.scene_ids[ann["scene_id"]],
            "question_id": question_id,
        }

        for modality in self.modalities:

            if modality == 'pc':
                pc_feat = torch.load(os.path.join(self.pc_feat_root, f"{scene_id}.pt"), map_location="cpu")
                if isinstance(pc_feat, np.ndarray):
                    pc_feat = torch.tensor(pc_feat).float()
                pc = np.load(os.path.join(self.voxel_root, f"{scene_id}.npy"))
                pc = torch.tensor(pc).float().cpu()
                if pc_feat.shape[0] > 5000:
                    idxes = torch.sort(torch.randperm(pc_feat.shape[0])[:5000])[1]
                    pc_feat = pc_feat[idxes]
                    pc = pc[idxes]
                else:
                    pc_feat = torch.cat([pc_feat, torch.zeros(5000 - pc_feat.shape[0], 1408)], dim=0)
                    pc = torch.cat([pc, torch.zeros(5000 - pc.shape[0], 3)], dim=0)
                out["pc_feat"] = pc_feat
                out['pc'] = pc

            if modality == 'video':
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                rgb, indices, fps = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"])
                out['rgb'] = rgb.to(torch.float32)

            if modality == 'frame':
                indices = None
                clip = None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                frms, indices = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                assert len(rgb) == getattr(self, f"{modality}_processor").n_frms
                out['rgb'] = rgb

            if modality == 'depth':
                assert indices is not None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                depth, _ = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices, type='depth')
                out['depth'] = depth

        return out

    def __len__(self):
        return len(self.annotation)

class ThreeDVQAEvalDataset(BaseDataset):
    def __init__(self, **kwargs):
        super().__init__(kwargs['vis_processor'], kwargs['text_processor'], kwargs['vis_root'], kwargs['ann_paths'], kwargs['modalities'])

        self.modalities = kwargs['modalities']

        self.scene_ids = {}
        n = 0
        new_annotation = []
        for ann in self.annotation:
            try:
                img_id = ann["scene_id"]
                if img_id not in self.scene_ids.keys():
                    self.scene_ids[img_id] = n
                    n += 1
                new_annotation.append(ann)
            except:
                pass


ThreeDVQAEvalDataset

                indices = None
                clip = None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                frms, indices = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                assert len(rgb) == getattr(self, f"{modality}_processor").n_frms
                out['rgb'] = rgb

            if modality == 'depth':
                assert indices is not None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                depth, _ = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices, type='depth')
                out['depth'] = depth

        return out

    def __len__(self):
        return len(self.annotation)

class ThreeDVQAEvalDataset(BaseDataset):
    def __init__(self, **kwargs):
        super().__init__(kwargs['vis_processor'], kwargs['text_processor'], kwargs['vis_root'], kwargs['ann_paths'], kwargs['modalities'])

        self.modalities = kwargs['modalities']

        self.scene_ids = {}
        n = 0
        new_annotation = []
        for ann in self.annotation:
            try:
                img_id = ann["scene_id"]
                if img_id not in self.scene_ids.keys():
                    self.scene_ids[img_id] = n
                    n += 1
                new_annotation.append(ann)
            except:
                pass
        self.annotation = new_annotation

        for modality in self.modalities:
            if 'pc' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                self.pc_feat_root = self.pc_root + '/voxelized_features_sam_nonzero_preprocess/'
                self.voxel_root = self.pc_root + '/voxelized_voxels_sam_nonzero_preprocess/'
                # pc_root = '/nas-ssd2/shoubin/datasets/scannet_feat/'
                # self.pc_feat_root = pc_root + '/voxelized_features_sam_nonzero_preprocess/'
                # self.voxel_root = pc_root + '/voxelized_voxels_sam_nonzero_preprocess/'
                self.annotation = [
                    ann for ann in self.annotation if os.path.exists(os.path.join(self.pc_feat_root, str(ann["scene_id"]) + ".pt"))
                ]
            if 'video' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])
                setattr(self, f"existing_{modality}_annotation",getattr(self, f'get_existing_{modality}_annotations')())

            if 'frame' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

            if 'depth' in modality:
                # todo
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

    def get_existing_video_annotations(self):
        return [f.split('.')[0] for f in os.listdir(self.video_root)]

    def get_video_path(self, ann):
        return os.path.join(self.video_root, f'{ann["scene_id"]}.mp4')

    def get_frame_path(self, ann):
        return os.path.join(self.frame_root, f'{ann["scene_id"]}/')

    def get_depth_path(self, ann):
        return os.path.join(self.depth_root, f'{ann["scene_id"]}/')

    def __getitem__(self, index):

        ann = copy.deepcopy(self.annotation[index])
        if 'question_id' in ann.keys(): # 3dqa data
            qa_input = self.text_processor(ann['situation']) + '. Question: ' + self.text_processor(ann["question"]) + ' Based on the frames and 3D Model information, answer the question using a single word or phase.'
            qtype = get_qclass(ann['question'])
            question_id = ann['question_id']
            answer = ann["answers"][0]
            question_id = qtype + '_' + str(question_id)
        else: # pre-training data
            # ann['qa_input'] =  'Question: ' + self.text_processor(question) + ' Based on the frames information, answer the question using a single word or phase.'
            qa_input = 'Question: ' + self.text_processor(ann["question"]) + ' Based on 3D Model information, answer the question using a single word or phase.'
            answer = ann["answers"][0]
            answer = self.text_processor(answer)
            question_id = str(ann["scene_id"])

        scene_id = str(ann["scene_id"])

        out = {
            "qa_input": qa_input,
            "qa_output": answer,
            "scene_id": self.scene_ids[ann["scene_id"]],
            "question_id": question_id,
        }

        for modality in self.modalities:

            if modality == 'pc':
                pc_feat = torch.load(os.path.join(self.pc_feat_root, f"{scene_id}.pt"), map_location="cpu")  # [N, 1408]
                if isinstance(pc_feat, np.ndarray):
                    pc_feat = torch.tensor(pc_feat).float()
                pc = np.load(os.path.join(self.voxel_root, f"{scene_id}.npy"))
                pc = torch.tensor(pc).float().cpu()
                # sample 10000 points: [N, 1408] -> [10000, 1408]
                if pc_feat.shape[0] > 5000:
                    idxes = torch.sort(torch.randperm(pc_feat.shape[0])[:5000])[1]
                    pc_feat = pc_feat[idxes]
                    pc = pc[idxes]
                else:
                    pc_feat = torch.cat([pc_feat, torch.zeros(5000 - pc_feat.shape[0], 1408)], dim=0)
                    pc = torch.cat([pc, torch.zeros(5000 - pc.shape[0], 3)], dim=0)

                out["pc_feat"] = pc_feat
                out['pc'] = pc
            if modality == 'video':
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                rgb, indices, fps = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"])
                out['rgb'] = rgb.to(torch.float32)

            if modality == 'frame':
                indices = None
                clip = None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                frms, indices = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                assert len(rgb) == getattr(self, f"{modality}_processor").n_frms
                out['rgb'] = rgb

            if modality == 'depth':
                assert indices is not None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                depth, _ = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices, type='depth')
                out['depth'] = depth

        return out

    def __len__(self):
        return len(self.annotation)


                clip = None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                frms, indices = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                assert len(rgb) == getattr(self, f"{modality}_processor").n_frms
                out['rgb'] = rgb

            if modality == 'depth':
                assert indices is not None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                depth, _ = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices, type='depth')
                out['depth'] = depth

        return out

    def __len__(self):
        return len(self.annotation)

class ThreeDVQAEvalDataset(BaseDataset):
    def __init__(self, **kwargs):
        super().__init__(kwargs['vis_processor'], kwargs['text_processor'], kwargs['vis_root'], kwargs['ann_paths'], kwargs['modalities'])

        self.modalities = kwargs['modalities']

        self.scene_ids = {}
        n = 0
        new_annotation = []
        for ann in self.annotation:
            try:
                img_id = ann["scene_id"]
                if img_id not in self.scene_ids.keys():
                    self.scene_ids[img_id] = n
                    n += 1
                new_annotation.append(ann)
            except:
                pass
        self.annotation = new_annotation

        for modality in self.modalities:
            if 'pc' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                self.pc_feat_root = self.pc_root + '/voxelized_features_sam_nonzero_preprocess/'
                self.voxel_root = self.pc_root + '/voxelized_voxels_sam_nonzero_preprocess/'
                # pc_root = '/nas-ssd2/shoubin/datasets/scannet_feat/'
                # self.pc_feat_root = pc_root + '/voxelized_features_sam_nonzero_preprocess/'
                # self.voxel_root = pc_root + '/voxelized_voxels_sam_nonzero_preprocess/'
                self.annotation = [
                    ann for ann in self.annotation if os.path.exists(os.path.join(self.pc_feat_root, str(ann["scene_id"]) + ".pt"))
                ]
            if 'video' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])
                setattr(self, f"existing_{modality}_annotation",getattr(self, f'get_existing_{modality}_annotations')())

            if 'frame' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

            if 'depth' in modality:
                # todo
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

    def get_existing_video_annotations(self):
        return [f.split('.')[0] for f in os.listdir(self.video_root)]

    def get_video_path(self, ann):
        return os.path.join(self.video_root, f'{ann["scene_id"]}.mp4')

    def get_frame_path(self, ann):
        return os.path.join(self.frame_root, f'{ann["scene_id"]}/')

    def get_depth_path(self, ann):
        return os.path.join(self.depth_root, f'{ann["scene_id"]}/')

    def __getitem__(self, index):

        ann = copy.deepcopy(self.annotation[index])
        if 'question_id' in ann.keys(): # 3dqa data
            qa_input = self.text_processor(ann['situation']) + '. Question: ' + self.text_processor(ann["question"]) + ' Based on the frames and 3D Model information, answer the question using a single word or phase.'
            qtype = get_qclass(ann['question'])
            question_id = ann['question_id']


                # pc_root = '/nas-ssd2/shoubin/datasets/scannet_feat/'
                # self.pc_feat_root = pc_root + '/voxelized_features_sam_nonzero_preprocess/'
                # self.voxel_root = pc_root + '/voxelized_voxels_sam_nonzero_preprocess/'
                self.annotation = [
                    ann for ann in self.annotation if os.path.exists(os.path.join(self.pc_feat_root, str(ann["scene_id"]) + ".pt"))
                ]
            if 'video' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])
                setattr(self, f"existing_{modality}_annotation",getattr(self, f'get_existing_{modality}_annotations')())

            if 'frame' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

            if 'depth' in modality:
                # todo
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

    def get_existing_video_annotations(self):
        return [f.split('.')[0] for f in os.listdir(self.video_root)]

    def get_video_path(self, ann):
        return os.path.join(self.video_root, f'{ann["scene_id"]}.mp4')

    def get_frame_path(self, ann):
        return os.path.join(self.frame_root, f'{ann["scene_id"]}/')

    def get_depth_path(self, ann):
        return os.path.join(self.depth_root, f'{ann["scene_id"]}/')

    def __getitem__(self, index):

        ann = copy.deepcopy(self.annotation[index])
        if 'question_id' in ann.keys(): # 3dqa data
            qa_input = self.text_processor(ann['situation']) + '. Question: ' + self.text_processor(ann["question"]) + ' Based on the frames and 3D Model information, answer the question using a single word or phase.'
            qtype = get_qclass(ann['question'])
            question_id = ann['question_id']
            answer = ann["answers"][0]
            question_id = qtype + '_' + str(question_id)
        else: # pre-training data


                self.annotation = [
                    ann for ann in self.annotation if os.path.exists(os.path.join(self.pc_feat_root, str(ann["scene_id"]) + ".pt"))
                ]
            if 'video' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])
                setattr(self, f"existing_{modality}_annotation",getattr(self, f'get_existing_{modality}_annotations')())

            if 'frame' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

            if 'depth' in modality:
                # todo
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

    def get_existing_video_annotations(self):
        return [f.split('.')[0] for f in os.listdir(self.video_root)]

    def get_video_path(self, ann):
        return os.path.join(self.video_root, f'{ann["scene_id"]}.mp4')

    def get_frame_path(self, ann):
        return os.path.join(self.frame_root, f'{ann["scene_id"]}/')

    def get_depth_path(self, ann):
        return os.path.join(self.depth_root, f'{ann["scene_id"]}/')

    def __getitem__(self, index):

        ann = copy.deepcopy(self.annotation[index])
        if 'question_id' in ann.keys(): # 3dqa data
            qa_input = self.text_processor(ann['situation']) + '. Question: ' + self.text_processor(ann["question"]) + ' Based on the frames and 3D Model information, answer the question using a single word or phase.'
            qtype = get_qclass(ann['question'])
            question_id = ann['question_id']
            answer = ann["answers"][0]
            question_id = qtype + '_' + str(question_id)
        else: # pre-training data
            # ann['qa_input'] =  'Question: ' + self.text_processor(question) + ' Based on the frames information, answer the question using a single word or phase.'
            qa_input = 'Question: ' + self.text_processor(ann["question"]) + ' Based on 3D Model information, answer the question using a single word or phase.'
            answer = ann["answers"][0]


            if 'video' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])
                setattr(self, f"existing_{modality}_annotation",getattr(self, f'get_existing_{modality}_annotations')())

            if 'frame' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

            if 'depth' in modality:
                # todo
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

    def get_existing_video_annotations(self):
        return [f.split('.')[0] for f in os.listdir(self.video_root)]

    def get_video_path(self, ann):
        return os.path.join(self.video_root, f'{ann["scene_id"]}.mp4')

    def get_frame_path(self, ann):
        return os.path.join(self.frame_root, f'{ann["scene_id"]}/')

    def get_depth_path(self, ann):
        return os.path.join(self.depth_root, f'{ann["scene_id"]}/')

    def __getitem__(self, index):

        ann = copy.deepcopy(self.annotation[index])
        if 'question_id' in ann.keys(): # 3dqa data
            qa_input = self.text_processor(ann['situation']) + '. Question: ' + self.text_processor(ann["question"]) + ' Based on the frames and 3D Model information, answer the question using a single word or phase.'
            qtype = get_qclass(ann['question'])
            question_id = ann['question_id']
            answer = ann["answers"][0]
            question_id = qtype + '_' + str(question_id)
        else: # pre-training data
            # ann['qa_input'] =  'Question: ' + self.text_processor(question) + ' Based on the frames information, answer the question using a single word or phase.'
            qa_input = 'Question: ' + self.text_processor(ann["question"]) + ' Based on 3D Model information, answer the question using a single word or phase.'
            answer = ann["answers"][0]
            answer = self.text_processor(answer)
            question_id = str(ann["scene_id"])


                setattr(self, f"existing_{modality}_annotation",getattr(self, f'get_existing_{modality}_annotations')())

            if 'frame' in modality:
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

            if 'depth' in modality:
                # todo
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

    def get_existing_video_annotations(self):
        return [f.split('.')[0] for f in os.listdir(self.video_root)]

    def get_video_path(self, ann):
        return os.path.join(self.video_root, f'{ann["scene_id"]}.mp4')

    def get_frame_path(self, ann):
        return os.path.join(self.frame_root, f'{ann["scene_id"]}/')

    def get_depth_path(self, ann):
        return os.path.join(self.depth_root, f'{ann["scene_id"]}/')

    def __getitem__(self, index):

        ann = copy.deepcopy(self.annotation[index])
        if 'question_id' in ann.keys(): # 3dqa data
            qa_input = self.text_processor(ann['situation']) + '. Question: ' + self.text_processor(ann["question"]) + ' Based on the frames and 3D Model information, answer the question using a single word or phase.'
            qtype = get_qclass(ann['question'])
            question_id = ann['question_id']
            answer = ann["answers"][0]
            question_id = qtype + '_' + str(question_id)
        else: # pre-training data
            # ann['qa_input'] =  'Question: ' + self.text_processor(question) + ' Based on the frames information, answer the question using a single word or phase.'
            qa_input = 'Question: ' + self.text_processor(ann["question"]) + ' Based on 3D Model information, answer the question using a single word or phase.'
            answer = ann["answers"][0]
            answer = self.text_processor(answer)
            question_id = str(ann["scene_id"])

        scene_id = str(ann["scene_id"])


                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

            if 'depth' in modality:
                # todo
                setattr(self, f"{modality}_root", kwargs[f"{modality}_root"])
                setattr(self, f"{modality}_processor", kwargs[f"{modality}_processor"])

    def get_existing_video_annotations(self):
        return [f.split('.')[0] for f in os.listdir(self.video_root)]

    def get_video_path(self, ann):
        return os.path.join(self.video_root, f'{ann["scene_id"]}.mp4')

    def get_frame_path(self, ann):
        return os.path.join(self.frame_root, f'{ann["scene_id"]}/')

    def get_depth_path(self, ann):
        return os.path.join(self.depth_root, f'{ann["scene_id"]}/')

    def __getitem__(self, index):

        ann = copy.deepcopy(self.annotation[index])
        if 'question_id' in ann.keys(): # 3dqa data
            qa_input = self.text_processor(ann['situation']) + '. Question: ' + self.text_processor(ann["question"]) + ' Based on the frames and 3D Model information, answer the question using a single word or phase.'
            qtype = get_qclass(ann['question'])
            question_id = ann['question_id']
            answer = ann["answers"][0]
            question_id = qtype + '_' + str(question_id)
        else: # pre-training data
            # ann['qa_input'] =  'Question: ' + self.text_processor(question) + ' Based on the frames information, answer the question using a single word or phase.'
            qa_input = 'Question: ' + self.text_processor(ann["question"]) + ' Based on 3D Model information, answer the question using a single word or phase.'
            answer = ann["answers"][0]
            answer = self.text_processor(answer)
            question_id = str(ann["scene_id"])

        scene_id = str(ann["scene_id"])

        out = {
            "qa_input": qa_input,
            "qa_output": answer,
            "scene_id": self.scene_ids[ann["scene_id"]],
            "question_id": question_id,
        }

        for modality in self.modalities:

            if modality == 'pc':
                pc_feat = torch.load(os.path.join(self.pc_feat_root, f"{scene_id}.pt"), map_location="cpu")  # [N, 1408]
                if isinstance(pc_feat, np.ndarray):
                    pc_feat = torch.tensor(pc_feat).float()
                pc = np.load(os.path.join(self.voxel_root, f"{scene_id}.npy"))
                pc = torch.tensor(pc).float().cpu()
                # sample 10000 points: [N, 1408] -> [10000, 1408]
                if pc_feat.shape[0] > 5000:
                    idxes = torch.sort(torch.randperm(pc_feat.shape[0])[:5000])[1]
                    pc_feat = pc_feat[idxes]
                    pc = pc[idxes]
                else:
                    pc_feat = torch.cat([pc_feat, torch.zeros(5000 - pc_feat.shape[0], 1408)], dim=0)
                    pc = torch.cat([pc, torch.zeros(5000 - pc.shape[0], 3)], dim=0)

                out["pc_feat"] = pc_feat
                out['pc'] = pc
            if modality == 'video':
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                rgb, indices, fps = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"])
                out['rgb'] = rgb.to(torch.float32)

            if modality == 'frame':
                indices = None
                clip = None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                frms, indices = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                assert len(rgb) == getattr(self, f"{modality}_processor").n_frms
                out['rgb'] = rgb

            if modality == 'depth':
                assert indices is not None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                depth, _ = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices, type='depth')
                out['depth'] = depth

        return out

    def __len__(self):
        return len(self.annotation)


                rgb, indices, fps = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"])
                out['rgb'] = rgb.to(torch.float32)

            if modality == 'frame':
                indices = None
                clip = None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                frms, indices = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                assert len(rgb) == getattr(self, f"{modality}_processor").n_frms
                out['rgb'] = rgb

            if modality == 'depth':
                assert indices is not None
                ann[f"{modality}_path"] = getattr(self, f"get_{modality}_path")(ann)
                depth, _ = getattr(self, f"{modality}_processor")(ann[f"{modality}_path"], clip_proposal=clip, indices=indices, type='depth')
                out['depth'] = depth

        return out

    def __len__(self):
        return len(self.annotation)


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import json
import os
import torch
from collections import OrderedDict

from lavis.datasets.datasets.multimodal_classification_datasets import (
    MultimodalClassificationDataset,
)
import random

class __DisplMixin:
    def displ_item(self, index):
        ann = self.annotation[index]
        vname = ann["video"]
        vpath = os.path.join(self.vis_root, vname)

        return OrderedDict(
            {"file": vpath, "question": ann["question"], "answer": ann["answer"]}
        )

ANS_MAPPING = {0:'A',1:'B',2:'C',3:'D',4:'E'}

# NextQA
class MCVideoQADataset(MultimodalClassificationDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths, modality_type):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths, modality_type)

    def _load_auxiliary_mappings(self):
        pass

    def _get_answer_label(self, answer):
        if answer in self.class_labels:
            return self.class_labels[answer]
        else:
            return len(self.class_labels)

    def __getitem__(self, index):

        result, flow_flag = None, False
        out = {}

        while result is None:
            ann = self.annotation[index]
            qid = ann['qid']
            q = ann['question']

            if 'start' in ann:
                start, end = float(ann['start']), float(ann['end'])
                clip = [start, end]
            else:
                clip = None

            # for QA
            prompt = 'Question: ' + q
            for j in range(ann['num_option']):
                a = ann['a{}'.format(j)]
                prompt += ' Option {}: '.format(ANS_MAPPING[j])
                prompt += a
            qa_prompt = prompt + ' Considering the information presented in the frame, select the correct answer from the options.'
            # loc_prompt = 'Question: ' + q +  ' ' + hints + ' Does the information within the frame provide the necessary details to accurately answer the given question?'
            answers = 'Option ' + ANS_MAPPING[int(ann['answer'])]
            duration = 1

            # print(self.modality_type)
            indices = None

            if 'rgb' in self.modality_type:
                vpath = os.path.join(self.vis_root, str(ann['video']))
                frms, indices = self.vis_processor(vpath, clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                # print(indices)
                assert len(rgb) == self.vis_processor.n_frms
                out['rgb'] = rgb

            if 'depth' in self.modality_type:
                depth_root = self.vis_root[:-1] + '_depth/'
                # if 'nas-hdd' in depth_root:
                #     depth_root = depth_root.replace('nas-hdd','nas-ssd2')
                depth_path = os.path.join(depth_root, str(ann['video']))
                depth, indices_ = self.vis_processor(depth_path, clip_proposal=clip, indices=indices, type='depth')
                if indices is not None:
                    assert indices == indices_
                    indices = indices_

                out['depth'] = depth

            if 'flow' in self.modality_type:
                flow_root = self.vis_root[:-1] + '_flow/'
                # if 'nas-hdd' in flow_root:
                #     flow_root = flow_root.replace('nas-hdd','nas-ssd2')
                flow_path = os.path.join(flow_root, str(ann['video']))
                try:
                    flow, indices_ = self.vis_processor(flow_path, clip_proposal=clip, indices=indices, type='flow')
                    if indices is not None:
                        assert indices == indices_
                        indices = indices_
                    out['flow'] = flow
                    result = True
                    flow_flag = False
                except Exception as e:
                    print(f"Error while read flow file idx")
                    print("video is: {}".format(ann['video']))
                    index = random.randint(0, len(self.annotation) - 1)
                    flow_flag = True

            if 'norm' in self.modality_type:
                norm_root = self.vis_root[:-1] + '_norm/'
                norm_path = os.path.join(norm_root, str(ann['video']))
                norm, indices_ = self.vis_processor(norm_path, clip_proposal=clip, indices=indices, type='norm')
                if indices is not None:
                    assert indices == indices_
                    indices = indices_
                out['norm'] = norm

            if not flow_flag:
                result = True

            out['qa_input'] = qa_prompt
            out['qa_output'] = answers
            out['question_id'] = qid
            out['duration'] = duration

        return out


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import json
import os
import torch
from collections import OrderedDict

from lavis.datasets.datasets.multimodal_classification_datasets import (
    MultimodalClassificationDataset,
)
import random

class __DisplMixin:
    def displ_item(self, index):
        ann = self.annotation[index]
        vname = ann["video"]
        vpath = os.path.join(self.vis_root, vname)

        return OrderedDict(
            {"file": vpath, "question": ann["question"], "answer": ann["answer"]}
        )

ANS_MAPPING = {0:'A',1:'B',2:'C',3:'D',4:'E'}

# NextQA
class MCVideoQADataset(MultimodalClassificationDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths, modality_type):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths, modality_type)

    def _load_auxiliary_mappings(self):
        pass

    def _get_answer_label(self, answer):
        if answer in self.class_labels:
            return self.class_labels[answer]
        else:
            return len(self.class_labels)

    def __getitem__(self, index):

        result, flow_flag = None, False


MCVideoQADataset

from collections import OrderedDict

from lavis.datasets.datasets.multimodal_classification_datasets import (
    MultimodalClassificationDataset,
)
import random

class __DisplMixin:
    def displ_item(self, index):
        ann = self.annotation[index]
        vname = ann["video"]
        vpath = os.path.join(self.vis_root, vname)

        return OrderedDict(
            {"file": vpath, "question": ann["question"], "answer": ann["answer"]}
        )

ANS_MAPPING = {0:'A',1:'B',2:'C',3:'D',4:'E'}

# NextQA
class MCVideoQADataset(MultimodalClassificationDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths, modality_type):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths, modality_type)

    def _load_auxiliary_mappings(self):
        pass

    def _get_answer_label(self, answer):
        if answer in self.class_labels:
            return self.class_labels[answer]
        else:
            return len(self.class_labels)

    def __getitem__(self, index):

        result, flow_flag = None, False
        out = {}

        while result is None:
            ann = self.annotation[index]
            qid = ann['qid']
            q = ann['question']

            if 'start' in ann:
                start, end = float(ann['start']), float(ann['end'])
                clip = [start, end]
            else:
                clip = None

            # for QA
            prompt = 'Question: ' + q
            for j in range(ann['num_option']):
                a = ann['a{}'.format(j)]
                prompt += ' Option {}: '.format(ANS_MAPPING[j])
                prompt += a
            qa_prompt = prompt + ' Considering the information presented in the frame, select the correct answer from the options.'
            # loc_prompt = 'Question: ' + q +  ' ' + hints + ' Does the information within the frame provide the necessary details to accurately answer the given question?'
            answers = 'Option ' + ANS_MAPPING[int(ann['answer'])]
            duration = 1

            # print(self.modality_type)
            indices = None

            if 'rgb' in self.modality_type:
                vpath = os.path.join(self.vis_root, str(ann['video']))
                frms, indices = self.vis_processor(vpath, clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                # print(indices)
                assert len(rgb) == self.vis_processor.n_frms
                out['rgb'] = rgb

            if 'depth' in self.modality_type:
                depth_root = self.vis_root[:-1] + '_depth/'
                # if 'nas-hdd' in depth_root:
                #     depth_root = depth_root.replace('nas-hdd','nas-ssd2')
                depth_path = os.path.join(depth_root, str(ann['video']))
                depth, indices_ = self.vis_processor(depth_path, clip_proposal=clip, indices=indices, type='depth')
                if indices is not None:
                    assert indices == indices_
                    indices = indices_

                out['depth'] = depth

            if 'flow' in self.modality_type:
                flow_root = self.vis_root[:-1] + '_flow/'
                # if 'nas-hdd' in flow_root:
                #     flow_root = flow_root.replace('nas-hdd','nas-ssd2')
                flow_path = os.path.join(flow_root, str(ann['video']))
                try:
                    flow, indices_ = self.vis_processor(flow_path, clip_proposal=clip, indices=indices, type='flow')
                    if indices is not None:
                        assert indices == indices_
                        indices = indices_
                    out['flow'] = flow
                    result = True
                    flow_flag = False
                except Exception as e:
                    print(f"Error while read flow file idx")
                    print("video is: {}".format(ann['video']))
                    index = random.randint(0, len(self.annotation) - 1)
                    flow_flag = True

            if 'norm' in self.modality_type:
                norm_root = self.vis_root[:-1] + '_norm/'
                norm_path = os.path.join(norm_root, str(ann['video']))
                norm, indices_ = self.vis_processor(norm_path, clip_proposal=clip, indices=indices, type='norm')
                if indices is not None:
                    assert indices == indices_
                    indices = indices_
                out['norm'] = norm

            if not flow_flag:
                result = True

            out['qa_input'] = qa_prompt
            out['qa_output'] = answers
            out['question_id'] = qid
            out['duration'] = duration

        return out


from lavis.datasets.datasets.multimodal_classification_datasets import (
    MultimodalClassificationDataset,
)
import random

class __DisplMixin:
    def displ_item(self, index):
        ann = self.annotation[index]
        vname = ann["video"]
        vpath = os.path.join(self.vis_root, vname)

        return OrderedDict(
            {"file": vpath, "question": ann["question"], "answer": ann["answer"]}
        )

ANS_MAPPING = {0:'A',1:'B',2:'C',3:'D',4:'E'}

# NextQA
class MCVideoQADataset(MultimodalClassificationDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths, modality_type):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths, modality_type)

    def _load_auxiliary_mappings(self):
        pass

    def _get_answer_label(self, answer):
        if answer in self.class_labels:
            return self.class_labels[answer]
        else:
            return len(self.class_labels)

    def __getitem__(self, index):

        result, flow_flag = None, False
        out = {}

        while result is None:
            ann = self.annotation[index]
            qid = ann['qid']
            q = ann['question']


)
import random

class __DisplMixin:
    def displ_item(self, index):
        ann = self.annotation[index]
        vname = ann["video"]
        vpath = os.path.join(self.vis_root, vname)

        return OrderedDict(
            {"file": vpath, "question": ann["question"], "answer": ann["answer"]}
        )

ANS_MAPPING = {0:'A',1:'B',2:'C',3:'D',4:'E'}

# NextQA
class MCVideoQADataset(MultimodalClassificationDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths, modality_type):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths, modality_type)

    def _load_auxiliary_mappings(self):
        pass

    def _get_answer_label(self, answer):
        if answer in self.class_labels:
            return self.class_labels[answer]
        else:
            return len(self.class_labels)

    def __getitem__(self, index):

        result, flow_flag = None, False
        out = {}

        while result is None:
            ann = self.annotation[index]
            qid = ann['qid']
            q = ann['question']

            if 'start' in ann:
                start, end = float(ann['start']), float(ann['end'])
                clip = [start, end]


class __DisplMixin:
    def displ_item(self, index):
        ann = self.annotation[index]
        vname = ann["video"]
        vpath = os.path.join(self.vis_root, vname)

        return OrderedDict(
            {"file": vpath, "question": ann["question"], "answer": ann["answer"]}
        )

ANS_MAPPING = {0:'A',1:'B',2:'C',3:'D',4:'E'}

# NextQA
class MCVideoQADataset(MultimodalClassificationDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths, modality_type):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths, modality_type)

    def _load_auxiliary_mappings(self):
        pass

    def _get_answer_label(self, answer):
        if answer in self.class_labels:
            return self.class_labels[answer]
        else:
            return len(self.class_labels)

    def __getitem__(self, index):

        result, flow_flag = None, False
        out = {}

        while result is None:
            ann = self.annotation[index]
            qid = ann['qid']
            q = ann['question']

            if 'start' in ann:
                start, end = float(ann['start']), float(ann['end'])
                clip = [start, end]
            else:
                clip = None

            # for QA
            prompt = 'Question: ' + q
            for j in range(ann['num_option']):


        return OrderedDict(
            {"file": vpath, "question": ann["question"], "answer": ann["answer"]}
        )

ANS_MAPPING = {0:'A',1:'B',2:'C',3:'D',4:'E'}

# NextQA
class MCVideoQADataset(MultimodalClassificationDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths, modality_type):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths, modality_type)

    def _load_auxiliary_mappings(self):
        pass

    def _get_answer_label(self, answer):
        if answer in self.class_labels:
            return self.class_labels[answer]
        else:
            return len(self.class_labels)

    def __getitem__(self, index):

        result, flow_flag = None, False
        out = {}

        while result is None:
            ann = self.annotation[index]
            qid = ann['qid']
            q = ann['question']

            if 'start' in ann:
                start, end = float(ann['start']), float(ann['end'])
                clip = [start, end]
            else:
                clip = None

            # for QA
            prompt = 'Question: ' + q
            for j in range(ann['num_option']):
                a = ann['a{}'.format(j)]
                prompt += ' Option {}: '.format(ANS_MAPPING[j])
                prompt += a
            qa_prompt = prompt + ' Considering the information presented in the frame, select the correct answer from the options.'
            # loc_prompt = 'Question: ' + q +  ' ' + hints + ' Does the information within the frame provide the necessary details to accurately answer the given question?'
            answers = 'Option ' + ANS_MAPPING[int(ann['answer'])]
            duration = 1

            # print(self.modality_type)
            indices = None

            if 'rgb' in self.modality_type:
                vpath = os.path.join(self.vis_root, str(ann['video']))
                frms, indices = self.vis_processor(vpath, clip_proposal=clip, indices=indices)
                rgb = frms.permute(1, 0, 2, 3)
                # print(indices)
                assert len(rgb) == self.vis_processor.n_frms
                out['rgb'] = rgb

            if 'depth' in self.modality_type:
                depth_root = self.vis_root[:-1] + '_depth/'
                # if 'nas-hdd' in depth_root:
                #     depth_root = depth_root.replace('nas-hdd','nas-ssd2')
                depth_path = os.path.join(depth_root, str(ann['video']))
                depth, indices_ = self.vis_processor(depth_path, clip_proposal=clip, indices=indices, type='depth')
                if indices is not None:
                    assert indices == indices_
                    indices = indices_

                out['depth'] = depth

            if 'flow' in self.modality_type:
                flow_root = self.vis_root[:-1] + '_flow/'
                # if 'nas-hdd' in flow_root:
                #     flow_root = flow_root.replace('nas-hdd','nas-ssd2')
                flow_path = os.path.join(flow_root, str(ann['video']))
                try:
                    flow, indices_ = self.vis_processor(flow_path, clip_proposal=clip, indices=indices, type='flow')
                    if indices is not None:
                        assert indices == indices_
                        indices = indices_
                    out['flow'] = flow
                    result = True
                    flow_flag = False
                except Exception as e:
                    print(f"Error while read flow file idx")
                    print("video is: {}".format(ann['video']))
                    index = random.randint(0, len(self.annotation) - 1)
                    flow_flag = True

            if 'norm' in self.modality_type:
                norm_root = self.vis_root[:-1] + '_norm/'
                norm_path = os.path.join(norm_root, str(ann['video']))
                norm, indices_ = self.vis_processor(norm_path, clip_proposal=clip, indices=indices, type='norm')
                if indices is not None:
                    assert indices == indices_
                    indices = indices_
                out['norm'] = norm

            if not flow_flag:
                result = True

            out['qa_input'] = qa_prompt
            out['qa_output'] = answers
            out['question_id'] = qid
            out['duration'] = duration

        return out


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
import json

from PIL import Image

from lavis.datasets.datasets.vqa_datasets import VQADataset, VQAEvalDataset

from collections import OrderedDict

class __DisplMixin:
    def displ_item(self, index):
        sample, ann = self.__getitem__(index), self.annotation[index]

        return OrderedDict(
            {
                "file": ann["image"],
                "question": ann["question"],
                "question_id": ann["question_id"],
                "answers": "; ".join(ann["answer"]),
                "image": sample["image"],
            }
        )

class COCOVQADataset(VQADataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths)

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        answer_weight = {}
        for answer in ann["answer"]:
            if answer in answer_weight.keys():
                answer_weight[answer] += 1 / len(ann["answer"])
            else:
                answer_weight[answer] = 1 / len(ann["answer"])

        answers = list(answer_weight.keys())
        weights = list(answer_weight.values())

        return {
            "image": image,
            "text_input": question,
            "answers": answers,
            "weights": weights,
        }

class COCOVQAEvalDataset(VQAEvalDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        """
        vis_root (string): Root directory of images (e.g. coco/images/)
        ann_root (string): directory to store the annotation file
        """

        self.vis_root = vis_root

        self.annotation = json.load(open(ann_paths[0]))

        answer_list_path = ann_paths[1]
        if os.path.exists(answer_list_path):
            self.answer_list = json.load(open(answer_list_path))
        else:
            self.answer_list = None

        try:
            self.coco_fmt_qust_file = ann_paths[2]
            self.coco_fmt_anno_file = ann_paths[3]
        except IndexError:
            self.coco_fmt_qust_file = None
            self.coco_fmt_anno_file = None

        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        return {
            "image": image,
            "text_input": question,
            "question_id": ann["question_id"],
            "instance_id": ann["instance_id"],
        }


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
import json

from PIL import Image

from lavis.datasets.datasets.vqa_datasets import VQADataset, VQAEvalDataset

from collections import OrderedDict

class __DisplMixin:
    def displ_item(self, index):
        sample, ann = self.__getitem__(index), self.annotation[index]

        return OrderedDict(
            {
                "file": ann["image"],
                "question": ann["question"],
                "question_id": ann["question_id"],
                "answers": "; ".join(ann["answer"]),
                "image": sample["image"],
            }
        )

class COCOVQADataset(VQADataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths)

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        answer_weight = {}
        for answer in ann["answer"]:
            if answer in answer_weight.keys():
                answer_weight[answer] += 1 / len(ann["answer"])
            else:


COCOVQADataset

from lavis.datasets.datasets.vqa_datasets import VQADataset, VQAEvalDataset

from collections import OrderedDict

class __DisplMixin:
    def displ_item(self, index):
        sample, ann = self.__getitem__(index), self.annotation[index]

        return OrderedDict(
            {
                "file": ann["image"],
                "question": ann["question"],
                "question_id": ann["question_id"],
                "answers": "; ".join(ann["answer"]),
                "image": sample["image"],
            }
        )

class COCOVQADataset(VQADataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths)

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        answer_weight = {}
        for answer in ann["answer"]:
            if answer in answer_weight.keys():
                answer_weight[answer] += 1 / len(ann["answer"])
            else:
                answer_weight[answer] = 1 / len(ann["answer"])

        answers = list(answer_weight.keys())
        weights = list(answer_weight.values())

        return {
            "image": image,
            "text_input": question,
            "answers": answers,
            "weights": weights,
        }

class COCOVQAEvalDataset(VQAEvalDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        """
        vis_root (string): Root directory of images (e.g. coco/images/)
        ann_root (string): directory to store the annotation file
        """

        self.vis_root = vis_root

        self.annotation = json.load(open(ann_paths[0]))

        answer_list_path = ann_paths[1]
        if os.path.exists(answer_list_path):
            self.answer_list = json.load(open(answer_list_path))
        else:
            self.answer_list = None

        try:


COCOVQAEvalDataset

        question = self.text_processor(ann["question"])

        answer_weight = {}
        for answer in ann["answer"]:
            if answer in answer_weight.keys():
                answer_weight[answer] += 1 / len(ann["answer"])
            else:
                answer_weight[answer] = 1 / len(ann["answer"])

        answers = list(answer_weight.keys())
        weights = list(answer_weight.values())

        return {
            "image": image,
            "text_input": question,
            "answers": answers,
            "weights": weights,
        }

class COCOVQAEvalDataset(VQAEvalDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        """
        vis_root (string): Root directory of images (e.g. coco/images/)
        ann_root (string): directory to store the annotation file
        """

        self.vis_root = vis_root

        self.annotation = json.load(open(ann_paths[0]))

        answer_list_path = ann_paths[1]
        if os.path.exists(answer_list_path):
            self.answer_list = json.load(open(answer_list_path))
        else:
            self.answer_list = None

        try:
            self.coco_fmt_qust_file = ann_paths[2]
            self.coco_fmt_anno_file = ann_paths[3]
        except IndexError:
            self.coco_fmt_qust_file = None
            self.coco_fmt_anno_file = None

        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        return {
            "image": image,
            "text_input": question,
            "question_id": ann["question_id"],
            "instance_id": ann["instance_id"],
        }


        answer_weight = {}
        for answer in ann["answer"]:
            if answer in answer_weight.keys():
                answer_weight[answer] += 1 / len(ann["answer"])
            else:
                answer_weight[answer] = 1 / len(ann["answer"])

        answers = list(answer_weight.keys())
        weights = list(answer_weight.values())

        return {
            "image": image,
            "text_input": question,
            "answers": answers,
            "weights": weights,
        }

class COCOVQAEvalDataset(VQAEvalDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        """
        vis_root (string): Root directory of images (e.g. coco/images/)
        ann_root (string): directory to store the annotation file
        """

        self.vis_root = vis_root

        self.annotation = json.load(open(ann_paths[0]))

        answer_list_path = ann_paths[1]
        if os.path.exists(answer_list_path):
            self.answer_list = json.load(open(answer_list_path))
        else:
            self.answer_list = None

        try:
            self.coco_fmt_qust_file = ann_paths[2]
            self.coco_fmt_anno_file = ann_paths[3]
        except IndexError:
            self.coco_fmt_qust_file = None
            self.coco_fmt_anno_file = None

        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        return {
            "image": image,
            "text_input": question,
            "question_id": ann["question_id"],
            "instance_id": ann["instance_id"],
        }


        self.annotation = json.load(open(ann_paths[0]))

        answer_list_path = ann_paths[1]
        if os.path.exists(answer_list_path):
            self.answer_list = json.load(open(answer_list_path))
        else:
            self.answer_list = None

        try:
            self.coco_fmt_qust_file = ann_paths[2]
            self.coco_fmt_anno_file = ann_paths[3]
        except IndexError:
            self.coco_fmt_qust_file = None
            self.coco_fmt_anno_file = None

        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        return {
            "image": image,
            "text_input": question,
            "question_id": ann["question_id"],
            "instance_id": ann["instance_id"],
        }


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
import json

from PIL import Image

from lavis.datasets.datasets.vqa_datasets import VQADataset, VQAEvalDataset

from collections import OrderedDict

class __DisplMixin:
    def displ_item(self, index):
        sample, ann = self.__getitem__(index), self.annotation[index]

        return OrderedDict(
            {
                "file": ann["image"],
                "question": ann["question"],
                "question_id": ann["question_id"],
                "answers": "; ".join(ann["answer"]),
                "image": sample["image"],
            }
        )

class GQADataset(VQADataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths)

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        answers = [ann["answer"]]
        weights = [1]

        return {
            "image": image,
            "text_input": question,
            "answers": answers,
            "weights": weights,
        }

class GQAEvalDataset(VQAEvalDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        """
        vis_root (string): Root directory of images (e.g. gqa/images/)
        ann_root (string): directory to store the annotation file
        """

        self.vis_root = vis_root

        self.annotation = json.load(open(ann_paths[0]))

        ## TODO: support inference method == 'ranking'
        answer_list_path = ann_paths[1] if len(ann_paths) > 1 else ''
        if os.path.exists(answer_list_path):
            self.answer_list = json.load(open(answer_list_path))
        else:
            self.answer_list = None

        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        if "answer" in ann:
            # answer is a string
            answer = ann["answer"]
        else:
            answer = None

        return {
            "image": image,
            "text_input": question,
            "answer": answer,
            "question_id": ann["question_id"],
            "instance_id": ann["instance_id"],
        }


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import os
import json

from PIL import Image

from lavis.datasets.datasets.vqa_datasets import VQADataset, VQAEvalDataset

from collections import OrderedDict

class __DisplMixin:
    def displ_item(self, index):
        sample, ann = self.__getitem__(index), self.annotation[index]

        return OrderedDict(
            {
                "file": ann["image"],
                "question": ann["question"],
                "question_id": ann["question_id"],
                "answers": "; ".join(ann["answer"]),
                "image": sample["image"],
            }
        )

class GQADataset(VQADataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths)

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        answers = [ann["answer"]]
        weights = [1]

        return {
            "image": image,


GQADataset

from lavis.datasets.datasets.vqa_datasets import VQADataset, VQAEvalDataset

from collections import OrderedDict

class __DisplMixin:
    def displ_item(self, index):
        sample, ann = self.__getitem__(index), self.annotation[index]

        return OrderedDict(
            {
                "file": ann["image"],
                "question": ann["question"],
                "question_id": ann["question_id"],
                "answers": "; ".join(ann["answer"]),
                "image": sample["image"],
            }
        )

class GQADataset(VQADataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths)

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        answers = [ann["answer"]]
        weights = [1]

        return {
            "image": image,
            "text_input": question,
            "answers": answers,
            "weights": weights,
        }

class GQAEvalDataset(VQAEvalDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        """
        vis_root (string): Root directory of images (e.g. gqa/images/)
        ann_root (string): directory to store the annotation file
        """

        self.vis_root = vis_root

        self.annotation = json.load(open(ann_paths[0]))

        ## TODO: support inference method == 'ranking'
        answer_list_path = ann_paths[1] if len(ann_paths) > 1 else ''
        if os.path.exists(answer_list_path):
            self.answer_list = json.load(open(answer_list_path))
        else:
            self.answer_list = None


GQAEvalDataset

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        answers = [ann["answer"]]
        weights = [1]

        return {
            "image": image,
            "text_input": question,
            "answers": answers,
            "weights": weights,
        }

class GQAEvalDataset(VQAEvalDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        """
        vis_root (string): Root directory of images (e.g. gqa/images/)
        ann_root (string): directory to store the annotation file
        """

        self.vis_root = vis_root

        self.annotation = json.load(open(ann_paths[0]))

        ## TODO: support inference method == 'ranking'
        answer_list_path = ann_paths[1] if len(ann_paths) > 1 else ''
        if os.path.exists(answer_list_path):
            self.answer_list = json.load(open(answer_list_path))
        else:
            self.answer_list = None

        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        if "answer" in ann:
            # answer is a string
            answer = ann["answer"]
        else:
            answer = None

        return {
            "image": image,
            "text_input": question,
            "answer": answer,
            "question_id": ann["question_id"],
            "instance_id": ann["instance_id"],
        }


        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        answers = [ann["answer"]]
        weights = [1]

        return {
            "image": image,
            "text_input": question,
            "answers": answers,
            "weights": weights,
        }

class GQAEvalDataset(VQAEvalDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        """
        vis_root (string): Root directory of images (e.g. gqa/images/)
        ann_root (string): directory to store the annotation file
        """

        self.vis_root = vis_root

        self.annotation = json.load(open(ann_paths[0]))

        ## TODO: support inference method == 'ranking'
        answer_list_path = ann_paths[1] if len(ann_paths) > 1 else ''
        if os.path.exists(answer_list_path):
            self.answer_list = json.load(open(answer_list_path))
        else:
            self.answer_list = None

        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        if "answer" in ann:
            # answer is a string
            answer = ann["answer"]
        else:
            answer = None

        return {
            "image": image,
            "text_input": question,
            "answer": answer,


        vis_root (string): Root directory of images (e.g. gqa/images/)
        ann_root (string): directory to store the annotation file
        """

        self.vis_root = vis_root

        self.annotation = json.load(open(ann_paths[0]))

        ## TODO: support inference method == 'ranking'
        answer_list_path = ann_paths[1] if len(ann_paths) > 1 else ''
        if os.path.exists(answer_list_path):
            self.answer_list = json.load(open(answer_list_path))
        else:
            self.answer_list = None

        self.vis_processor = vis_processor
        self.text_processor = text_processor

        self._add_instance_ids()

    def __getitem__(self, index):
        ann = self.annotation[index]

        image_path = os.path.join(self.vis_root, ann["image"])
        image = Image.open(image_path).convert("RGB")

        image = self.vis_processor(image)
        question = self.text_processor(ann["question"])

        if "answer" in ann:
            # answer is a string
            answer = ann["answer"]
        else:
            answer = None

        return {
            "image": image,
            "text_input": question,
            "answer": answer,
            "question_id": ann["question_id"],
            "instance_id": ann["instance_id"],
        }


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import json
import os
from collections import OrderedDict

from lavis.datasets.datasets.multimodal_classification_datasets import (
    MultimodalClassificationDataset,
)

class __DisplMixin:
    def displ_item(self, index):
        ann = self.annotation[index]
        vname = ann["video"]
        vpath = os.path.join(self.vis_root, vname)

        return OrderedDict(
            {"file": vpath, "question": ann["question"], "answer": ann["answer"]}
        )

class VideoQADataset(MultimodalClassificationDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths)

    def _build_class_labels(self, ans_path):
        ans2label = json.load(open(ans_path))

        self.class_labels = ans2label

    def _get_answer_label(self, answer):
        if answer in self.class_labels:
            return self.class_labels[answer]
        else:
            return len(self.class_labels)

    def __getitem__(self, index):
        assert (
            self.class_labels
        ), f"class_labels of {__class__.__name__} is not built yet."

        ann = self.annotation[index]

        vname = ann["video"]
        vpath = os.path.join(self.vis_root, vname)

        frms = self.vis_processor(vpath)
        question = self.text_processor(ann["question"])

        return {
            "video": frms,
            "text_input": question,
            "answers": self._get_answer_label(ann["answer"]),
            "question_id": ann["question_id"],
            "instance_id": ann["instance_id"],
        }


"""
 Copyright (c) 2022, salesforce.com, inc.
 All rights reserved.
 SPDX-License-Identifier: BSD-3-Clause
 For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
"""

import json
import os
from collections import OrderedDict

from lavis.datasets.datasets.multimodal_classification_datasets import (
    MultimodalClassificationDataset,
)

class __DisplMixin:
    def displ_item(self, index):
        ann = self.annotation[index]
        vname = ann["video"]
        vpath = os.path.join(self.vis_root, vname)

        return OrderedDict(
            {"file": vpath, "question": ann["question"], "answer": ann["answer"]}
        )

class VideoQADataset(MultimodalClassificationDataset, __DisplMixin):
    def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
        super().__init__(vis_processor, text_processor, vis_root, ann_paths)

    def _build_class_labels(self, ans_path):
        ans2label = json.load(open(ans_path))

        self.class_labels = ans2label

    def _get_answer_label(self, answer):
        if answer in self.class_labels:
            return self.class_labels[answer]
        else:
            return len(self.class_labels)

    def __getitem__(self, index):
        assert (
            self.class_labels