diff --git a/.gitattributes b/.gitattributes index c6b5d2a8dafd72d1834a5b496d3a79bca8427af8..2ba0dbd16354e1d84010494d5ecd9e7a81a3a43d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text DFA-MoE/docs/intro.png filter=lfs diff=lfs merge=lfs -text +docs/intro.png filter=lfs diff=lfs merge=lfs -text diff --git a/MTIL_datasets/__init__.py b/MTIL_datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..15760204676b27d280b62826276a41652859057c --- /dev/null +++ b/MTIL_datasets/__init__.py @@ -0,0 +1,4 @@ +# This dataset is originally proposed by Zangwei Zheng et al. (https://arxiv.org/abs/2303.06628) +# Code here is based on CoOp and its following works. (https://github.com/KaiyangZhou/CoOp) +# Modified by Longxiang Tang (lloong.x@gmail.com) to release the dependence of Dassl lib. +# To prepare data, please refer to https://github.com/muzairkhattak/PromptSRC/blob/main/docs/DATASETS.md \ No newline at end of file diff --git a/MTIL_datasets/caltech101.py b/MTIL_datasets/caltech101.py new file mode 100644 index 0000000000000000000000000000000000000000..59ed25d6b30053e2a45a1ec6224cb3fe9cbbc644 --- /dev/null +++ b/MTIL_datasets/caltech101.py @@ -0,0 +1,92 @@ +import os +import pickle + +from .utils import * + +from .oxford_pets import OxfordPets +from .dtd import DescribableTextures as DTD + +IGNORED = ["BACKGROUND_Google", "Faces_easy"] +NEW_CNAMES = { + "airplanes": "airplane", + "Faces": "face", + "Leopards": "leopard", + "Motorbikes": "motorbike", +} + + +class Caltech101(DatasetBase): + + dataset_dir = "caltech-101" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = os.path.join(self.dataset_dir, "101_ObjectCategories") + self.split_path = os.path.join(self.dataset_dir, "split_zhou_Caltech101.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.image_dir) + else: + train, val, test = DTD.read_and_split_data(self.image_dir, ignored=IGNORED, new_cnames=NEW_CNAMES) + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c: f"a photo of a {c}.", + lambda c: f"a painting of a {c}.", + lambda c: f"a plastic {c}.", + lambda c: f"a sculpture of a {c}.", + lambda c: f"a sketch of a {c}.", + lambda c: f"a tattoo of a {c}.", + lambda c: f"a toy {c}.", + lambda c: f"a rendition of a {c}.", + lambda c: f"a embroidered {c}.", + lambda c: f"a cartoon {c}.", + lambda c: f"a {c} in a video game.", + lambda c: f"a plushie {c}.", + lambda c: f"a origami {c}.", + lambda c: f"art of a {c}.", + lambda c: f"graffiti of a {c}.", + lambda c: f"a drawing of a {c}.", + lambda c: f"a doodle of a {c}.", + lambda c: f"a photo of the {c}.", + lambda c: f"a painting of the {c}.", + lambda c: f"the plastic {c}.", + lambda c: f"a sculpture of the {c}.", + lambda c: f"a sketch of the {c}.", + lambda c: f"a tattoo of the {c}.", + lambda c: f"the toy {c}.", + lambda c: f"a rendition of the {c}.", + lambda c: f"the embroidered {c}.", + lambda c: f"the cartoon {c}.", + lambda c: f"the {c} in a video game.", + lambda c: f"the plushie {c}.", + lambda c: f"the origami {c}.", + lambda c: f"art of the {c}.", + lambda c: f"graffiti of the {c}.", + lambda c: f"a drawing of the {c}.", + lambda c: f"a doodle of the {c}.", + ] + + super().__init__(train_x=train, val=val, test=test) diff --git a/MTIL_datasets/cifar10.py b/MTIL_datasets/cifar10.py new file mode 100644 index 0000000000000000000000000000000000000000..d77c62dc9de14d1a0f9cfb55a345a247adb8555d --- /dev/null +++ b/MTIL_datasets/cifar10.py @@ -0,0 +1,94 @@ +import os +from typing import List + +from .utils import * # Datum, DatasetBase +from .oxford_pets import OxfordPets + +try: + from torchvision.datasets import CIFAR10 as TorchCIFAR10 +except Exception as e: + TorchCIFAR10 = None + print(f"Warning: torchvision not available for CIFAR10: {e}") + + +CIFAR10_CLASSES: List[str] = [ + 'airplane', + 'automobile', + 'bird', + 'cat', + 'deer', + 'dog', + 'frog', + 'horse', + 'ship', + 'truck', +] + +CIFAR10_TEMPLATES: List[str] = [ + 'a photo of a {}.', + 'a blurry photo of a {}.', + 'a black and white photo of a {}.', + 'a low contrast photo of a {}.', + 'a high contrast photo of a {}.', + 'a bad photo of a {}.', + 'a good photo of a {}.', + 'a photo of a small {}.', + 'a photo of a big {}.', + 'a photo of the {}.', + 'a blurry photo of the {}.', + 'a black and white photo of the {}.', + 'a low contrast photo of the {}.', + 'a high contrast photo of the {}.', + 'a bad photo of the {}.', + 'a good photo of the {}.', + 'a photo of the small {}.', + 'a photo of the big {}.', +] + + +class CIFAR10(DatasetBase): + + dataset_dir = "cifar10" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + if TorchCIFAR10 is None: + raise ImportError("torchvision is required for CIFAR10 dataset. Please install torchvision.") + + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + + # Use torchvision to download/load data to dataset_dir + train_ds = TorchCIFAR10(root=self.dataset_dir, train=True, download=True) + test_ds = TorchCIFAR10(root=self.dataset_dir, train=False, download=True) + + # Build Datum lists + trainval = [] + # train_ds.data: numpy array HWC, train_ds.targets: list[int] + for idx in range(len(train_ds.data)): + img = Image.fromarray(train_ds.data[idx]) + label = int(train_ds.targets[idx]) + classname = CIFAR10_CLASSES[label] + trainval.append(Datum(impath=img, label=label, classname=classname)) + + test = [] + for idx in range(len(test_ds.data)): + img = Image.fromarray(test_ds.data[idx]) + label = int(test_ds.targets[idx]) + classname = CIFAR10_CLASSES[label] + test.append(Datum(impath=img, label=label, classname=classname)) + + # Split train/val + train, val = OxfordPets.split_trainval(trainval) + + # Few-shot sampling if requested + if num_shots >= 1: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + + # Optional class subsampling (base/new) + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes) + + # Templates + self.templates = CIFAR10_TEMPLATES + + super().__init__(train_x=train, val=val, test=test) diff --git a/MTIL_datasets/cifar100.py b/MTIL_datasets/cifar100.py new file mode 100644 index 0000000000000000000000000000000000000000..3b8b353fc89ba68ad0a57b6336a5ecc2777d15ec --- /dev/null +++ b/MTIL_datasets/cifar100.py @@ -0,0 +1,89 @@ +import os +import pickle +import numpy as np + +from .utils import * + +from .oxford_pets import OxfordPets +from .dtd import DescribableTextures as DTD + + +class CIFAR100(DatasetBase): + + dataset_dir = "cifar100" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + + file_path = os.path.join(root, self.dataset_dir, 'train') + with open(file_path, "rb") as f: + entry = pickle.load(f, encoding="latin1") + trainval_data = entry["data"] + if "labels" in entry: + trainval_targets = entry["labels"] + else: + trainval_targets = entry["fine_labels"] + trainval_data = trainval_data.reshape(-1, 3, 32, 32) + trainval_data = trainval_data.transpose((0, 2, 3, 1)) + + file_path = os.path.join(root, self.dataset_dir, 'test') + with open(file_path, "rb") as f: + entry = pickle.load(f, encoding="latin1") + test_data = entry["data"] + if "labels" in entry: + test_targets = entry["labels"] + else: + test_targets = entry["fine_labels"] + test_data = test_data.reshape(-1, 3, 32, 32) + test_data = test_data.transpose((0, 2, 3, 1)) + + path = os.path.join(self.dataset_dir, "meta") + with open(path, "rb") as infile: + data = pickle.load(infile, encoding="latin1") + classes = data["fine_label_names"] + classes = [s.replace("_", " ") for s in classes] + + trainval = [] + for idx in range(trainval_data.shape[0]): + item = Datum(impath=Image.fromarray(trainval_data[idx]), + label=int(trainval_targets[idx]), classname=classes[trainval_targets[idx]]) + trainval.append(item) + + test = [] + for idx in range(test_data.shape[0]): + item = Datum(impath=Image.fromarray(test_data[idx]), + label=int(test_targets[idx]), classname=classes[test_targets[idx]]) + test.append(item) + + train, val = OxfordPets.split_trainval(trainval) + + if num_shots >= 1: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c : f'a bad photo of a {c}.', + lambda c : f'a blurry photo of a {c}.', + lambda c : f'a black and white photo of a {c}.', + lambda c : f'a low contrast photo of a {c}.', + lambda c : f'a high contrast photo of a {c}.', + lambda c : f'a photo of a {c}.', + lambda c : f'a good photo of a {c}.', + lambda c : f'a photo of a small {c}.', + lambda c : f'a photo of a big {c}.', + lambda c : f'a photo of the {c}.', + lambda c : f'a blurry photo of the {c}.', + lambda c : f'a black and white photo of the {c}.', + lambda c : f'a low contrast photo of the {c}.', + lambda c : f'a high contrast photo of the {c}.', + lambda c : f'a bad photo of the {c}.', + lambda c : f'a good photo of the {c}.', + lambda c : f'a photo of the small {c}.', + lambda c : f'a photo of the big {c}.', + ] + + super().__init__(train_x=train, val=val, test=test) diff --git a/MTIL_datasets/clevr_count.py b/MTIL_datasets/clevr_count.py new file mode 100644 index 0000000000000000000000000000000000000000..10bdfd7561eab8b7d75da74940d6c8f99b7f789d --- /dev/null +++ b/MTIL_datasets/clevr_count.py @@ -0,0 +1,118 @@ +import os +import pickle +from typing import List + +from .utils import Datum, DatasetBase, mkdir_if_missing +from .oxford_pets import OxfordPets +from .utils import read_json, write_json + +# Class list and templates as specified +CLEVR_COUNT_CLASSES: List[str] = [ + '10', '3', '4', '5', '6', '7', '8', '9' +] + +CLEVR_COUNT_TEMPLATES: List[str] = [ + 'a photo of {} objects.', +] + + +class CLEVRCount(DatasetBase): + + dataset_dir = 'clevr' + + def __init__(self, root, num_shots: int = 0, seed: int = 1, subsample_classes: str = 'all'): + # Root and directories + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.images_dir = os.path.join(self.dataset_dir, 'images') + self.scenes_dir = os.path.join(self.dataset_dir, 'scenes') + self.split_path = os.path.join(self.dataset_dir, 'split_custom_CLEVRCount.json') + self.split_fewshot_dir = os.path.join(self.dataset_dir, 'split_fewshot') + mkdir_if_missing(self.split_fewshot_dir) + + # Required files + train_scenes = os.path.join(self.scenes_dir, 'CLEVR_train_scenes.json') + val_scenes = os.path.join(self.scenes_dir, 'CLEVR_val_scenes.json') + if not os.path.isfile(train_scenes) or not os.path.isfile(val_scenes): + raise FileNotFoundError( + f"CLEVRCount expects scenes JSON at {train_scenes} and {val_scenes}" + ) + + # Load or build split + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir) + else: + trainval = self._read_scenes(train_scenes, split='train') + test = self._read_scenes(val_scenes, split='val') + train, val = OxfordPets.split_trainval(trainval) + OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir) + + # Few-shot + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, 'rb') as f: + data = pickle.load(f) + train, val = data['train'], data['val'] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {'train': train, 'val': val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, 'wb') as f: + pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL) + + # Optional class subsampling (base/new) + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes) + + # Templates + self.templates = CLEVR_COUNT_TEMPLATES + + # Debug stats + try: + def _hist(items: List[Datum]): + from collections import Counter + cnt = Counter([it.label for it in items]) + out = {i: int(cnt.get(i, 0)) for i in range(len(CLEVR_COUNT_CLASSES))} + return out + except Exception as e: + print(f"CLEVRCount stats printing failed: {e}") + + super().__init__(train_x=train, val=val, test=test) + # Ensure class metadata is stable and matches the provided list + self._classnames = CLEVR_COUNT_CLASSES + self._lab2cname = {i: c for i, c in enumerate(CLEVR_COUNT_CLASSES)} + self._num_classes = len(CLEVR_COUNT_CLASSES) + + def _read_scenes(self, json_path: str, split: str) -> List[Datum]: + """Read CLEVR scenes JSON and construct a list of Datum entries. + Only images whose object count appears in CLEVR_COUNT_CLASSES are kept. + """ + obj = read_json(json_path) + scenes = obj.get('scenes', []) + # Map count -> class index + lab2idx = {int(c): i for i, c in enumerate(CLEVR_COUNT_CLASSES)} + items: List[Datum] = [] + for sc in scenes: + # Some files use key 'split', some typos list 'spit'; be robust + image_filename = sc.get('image_filename', None) + if not image_filename: + continue + n_objects = sc.get('objects', []) + try: + num = int(len(n_objects)) + except Exception: + continue + if num not in lab2idx: + # skip counts not in the configured class list + continue + label_i = lab2idx[num] + # Build absolute path to the image based on split + if split not in ['train', 'val', 'test']: + split_dir = 'train' + else: + split_dir = split + impath = os.path.join(self.dataset_dir, 'images', split_dir, image_filename) + items.append(Datum(impath=impath, label=label_i, classname=str(num))) + return items diff --git a/MTIL_datasets/country211.py b/MTIL_datasets/country211.py new file mode 100644 index 0000000000000000000000000000000000000000..19634d3f372fd8bb7e33f4a1a5ff72d0f8dcf919 --- /dev/null +++ b/MTIL_datasets/country211.py @@ -0,0 +1,386 @@ +import os +import json +import pickle +import re + +from .utils import * +from .oxford_pets import OxfordPets +from .dtd import DescribableTextures as DTD + + +class Country211(DatasetBase): + """ + Country211 dataset loader for MTIL. + Expected structure: + /country211/ + ├─ train//*.jpg + ├─ valid//*.jpg + └─ test//*.jpg + + Where are ISO-3166 alpha-2 country codes (e.g., AD, US, CN). + Class names are mapped from ISO2 codes to provided human-readable names. + Optionally, a JSON mapping file can override defaults: + /country211/iso2_to_name.json => {"AD": "Andorra", ...} + """ + + dataset_dir = "country211" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + # Prefer explicit split directories if they exist + train_dir = os.path.join(self.dataset_dir, "train") + valid_dir = os.path.join(self.dataset_dir, "valid") + test_dir = os.path.join(self.dataset_dir, "test") + + use_folder_splits = os.path.isdir(train_dir) and os.path.isdir(valid_dir) and os.path.isdir(test_dir) + + if use_folder_splits: + # Build ISO2 -> name mapping (allow override via json) + iso_map = self._load_iso2_to_name() + # Create unified code set across splits to keep label mapping stable + all_codes = set() + for d in [train_dir, valid_dir, test_dir]: + if os.path.isdir(d): + for code in listdir_nohidden(d): + code_path = os.path.join(d, code) + if os.path.isdir(code_path): + all_codes.add(code) + # Validate code format and mapping coverage early + override_path = os.path.join(self.dataset_dir, 'iso2_to_name.json') + pat = re.compile(r'^[A-Z]{2}$') + invalid_codes = sorted([c for c in all_codes if not (pat.match(c) or c == 'XK')]) + if invalid_codes: + raise ValueError( + "Country211: Found invalid ISO2 code folder names: {}. " + "Codes must be two uppercase letters (e.g., 'US', 'CN') or 'XK'. " + "Please rename these folders accordingly.".format(invalid_codes) + ) + unknown_codes = sorted([c for c in all_codes if c not in iso_map]) + if unknown_codes: + raise ValueError( + "Country211: ISO2 codes missing from mapping: {}. " + "Add them to {} as a JSON dict, e.g., {\"XX\": \"Country Name\"}.".format( + unknown_codes, override_path + ) + ) + # Sort by human name (fallback to code) for stable labels + def _code_to_name(c): + return iso_map.get(c, c) + sorted_codes = sorted(list(all_codes), key=lambda c: _code_to_name(c)) + code_to_label = {c: i for i, c in enumerate(sorted_codes)} + + train = self._read_split_dir(train_dir, code_to_label, iso_map) + val = self._read_split_dir(valid_dir, code_to_label, iso_map) + test = self._read_split_dir(test_dir, code_to_label, iso_map) + else: + # Fallbacks: JSON split or naive folder split + image_dir = os.path.join(self.dataset_dir, "images") + self.image_dir = image_dir if os.path.isdir(image_dir) else self.dataset_dir + split_path = os.path.join(self.dataset_dir, "split_zhou_Country211.json") + if os.path.exists(split_path): + train, val, test = OxfordPets.read_split(split_path, self.image_dir) + else: + train, val, test = DTD.read_and_split_data(self.image_dir) + OxfordPets.save_split(train, val, test, split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + # Templates provided by user + self.templates = [ + lambda c: f'a photo i took in {c}.', + lambda c: f'a photo i took while visiting {c}.', + lambda c: f'a photo from my home country of {c}.', + lambda c: f'a photo from my visit to {c}.', + lambda c: f'a photo showing the country of {c}.', + ] + + super().__init__(train_x=train, val=val, test=test) + + def _read_split_dir(self, split_dir, code_to_label, iso_map): + items = [] + if not os.path.isdir(split_dir): + return items + codes = listdir_nohidden(split_dir) + for code in codes: + class_dir = os.path.join(split_dir, code) + if not os.path.isdir(class_dir): + continue + label = code_to_label.get(code) + if label is None: + raise ValueError( + f"Country211: Inconsistent label mapping for code '{code}' in split '{split_dir}'. " + f"This indicates an internal mismatch between discovered codes and label map." + ) + try: + cname = iso_map[code] + except KeyError: + raise ValueError( + f"Country211: Missing ISO mapping for code '{code}'. " + f"Please add it to '{os.path.join(self.dataset_dir, 'iso2_to_name.json')}'." + ) + for fname in listdir_nohidden(class_dir): + impath = os.path.join(class_dir, fname) + items.append(Datum(impath=impath, label=label, classname=cname)) + return items + + def _load_iso2_to_name(self): + """Load ISO2->country name mapping, allow local JSON override. + Fallback to built-in mapping; unknown codes map to themselves at usage time. + """ + override_path = os.path.join(self.dataset_dir, 'iso2_to_name.json') + if os.path.exists(override_path): + try: + data = read_json(override_path) + if not isinstance(data, dict): + raise ValueError( + f"Country211: Expected a JSON object in '{override_path}', got {type(data)}" + ) + # Validate keys as ISO2 (two uppercase letters) or 'XK', and values are non-empty strings + pat = re.compile(r'^[A-Z]{2}$') + bad_keys = [k for k in data.keys() if not (isinstance(k, str) and (pat.match(k) or k == 'XK'))] + bad_vals = [k for k, v in data.items() if not (isinstance(v, str) and v.strip())] + if bad_keys or bad_vals: + raise ValueError( + ( + "Country211: Invalid iso2_to_name.json entries. " + f"Bad keys (must be ISO2 like 'US'): {bad_keys}. " + f"Bad values (must be non-empty strings) for keys: {bad_vals}." + ) + ) + return data + except Exception as e: + raise ValueError(f"Country211: Failed to load '{override_path}': {e}") + # Built-in mapping aligned with provided classes + return { + 'AD': 'Andorra', + 'AE': 'United Arab Emirates', + 'AF': 'Afghanistan', + 'AG': 'Antigua and Barbuda', + 'AI': 'Anguilla', + 'AL': 'Albania', + 'AM': 'Armenia', + 'AO': 'Angola', + 'AQ': 'Antarctica', + 'AR': 'Argentina', + 'AT': 'Austria', + 'AU': 'Australia', + 'AW': 'Aruba', + 'AX': 'Aland Islands', + 'AZ': 'Azerbaijan', + 'BA': 'Bosnia and Herzegovina', + 'BB': 'Barbados', + 'BD': 'Bangladesh', + 'BE': 'Belgium', + 'BF': 'Burkina Faso', + 'BG': 'Bulgaria', + 'BH': 'Bahrain', + 'BJ': 'Benin', + 'BM': 'Bermuda', + 'BN': 'Brunei Darussalam', + 'BO': 'Bolivia', + 'BQ': 'Bonaire, Saint Eustatius and Saba', + 'BR': 'Brazil', + 'BS': 'Bahamas', + 'BT': 'Bhutan', + 'BW': 'Botswana', + 'BY': 'Belarus', + 'BZ': 'Belize', + 'CA': 'Canada', + 'CD': 'DR Congo', + 'CF': 'Central African Republic', + 'CH': 'Switzerland', + 'CI': "Cote d'Ivoire", + 'CK': 'Cook Islands', + 'CL': 'Chile', + 'CM': 'Cameroon', + 'CN': 'China', + 'CO': 'Colombia', + 'CR': 'Costa Rica', + 'CU': 'Cuba', + 'CV': 'Cabo Verde', + 'CW': 'Curacao', + 'CY': 'Cyprus', + 'CZ': 'Czech Republic', + 'DE': 'Germany', + 'DK': 'Denmark', + 'DM': 'Dominica', + 'DO': 'Dominican Republic', + 'DZ': 'Algeria', + 'EC': 'Ecuador', + 'EE': 'Estonia', + 'EG': 'Egypt', + 'ES': 'Spain', + 'ET': 'Ethiopia', + 'FI': 'Finland', + 'FJ': 'Fiji', + 'FK': 'Falkland Islands', + 'FO': 'Faeroe Islands', + 'FR': 'France', + 'GA': 'Gabon', + 'GB': 'United Kingdom', + 'GD': 'Grenada', + 'GE': 'Georgia', + 'GF': 'French Guiana', + 'GG': 'Guernsey', + 'GH': 'Ghana', + 'GI': 'Gibraltar', + 'GL': 'Greenland', + 'GM': 'Gambia', + 'GP': 'Guadeloupe', + 'GR': 'Greece', + 'GS': 'South Georgia and South Sandwich Is.', + 'GT': 'Guatemala', + 'GU': 'Guam', + 'GY': 'Guyana', + 'HK': 'Hong Kong', + 'HN': 'Honduras', + 'HR': 'Croatia', + 'HT': 'Haiti', + 'HU': 'Hungary', + 'ID': 'Indonesia', + 'IE': 'Ireland', + 'IL': 'Israel', + 'IM': 'Isle of Man', + 'IN': 'India', + 'IQ': 'Iraq', + 'IR': 'Iran', + 'IS': 'Iceland', + 'IT': 'Italy', + 'JE': 'Jersey', + 'JM': 'Jamaica', + 'JO': 'Jordan', + 'JP': 'Japan', + 'KE': 'Kenya', + 'KG': 'Kyrgyz Republic', + 'KH': 'Cambodia', + 'KN': 'St. Kitts and Nevis', + 'KP': 'North Korea', + 'KR': 'South Korea', + 'KW': 'Kuwait', + 'KY': 'Cayman Islands', + 'KZ': 'Kazakhstan', + 'LA': 'Laos', + 'LB': 'Lebanon', + 'LC': 'St. Lucia', + 'LI': 'Liechtenstein', + 'LK': 'Sri Lanka', + 'LR': 'Liberia', + 'LT': 'Lithuania', + 'LU': 'Luxembourg', + 'LV': 'Latvia', + 'LY': 'Libya', + 'MA': 'Morocco', + 'MC': 'Monaco', + 'MD': 'Moldova', + 'ME': 'Montenegro', + 'MF': 'Saint-Martin', + 'MG': 'Madagascar', + 'MK': 'Macedonia', + 'ML': 'Mali', + 'MM': 'Myanmar', + 'MN': 'Mongolia', + 'MO': 'Macau', + 'MQ': 'Martinique', + 'MR': 'Mauritania', + 'MT': 'Malta', + 'MU': 'Mauritius', + 'MV': 'Maldives', + 'MW': 'Malawi', + 'MX': 'Mexico', + 'MY': 'Malaysia', + 'MZ': 'Mozambique', + 'NA': 'Namibia', + 'NC': 'New Caledonia', + 'NG': 'Nigeria', + 'NI': 'Nicaragua', + 'NL': 'Netherlands', + 'NO': 'Norway', + 'NP': 'Nepal', + 'NZ': 'New Zealand', + 'OM': 'Oman', + 'PA': 'Panama', + 'PE': 'Peru', + 'PF': 'French Polynesia', + 'PG': 'Papua New Guinea', + 'PH': 'Philippines', + 'PK': 'Pakistan', + 'PL': 'Poland', + 'PR': 'Puerto Rico', + 'PS': 'Palestine', + 'PT': 'Portugal', + 'PW': 'Palau', + 'PY': 'Paraguay', + 'QA': 'Qatar', + 'RE': 'Reunion', + 'RO': 'Romania', + 'RS': 'Serbia', + 'RU': 'Russia', + 'RW': 'Rwanda', + 'SA': 'Saudi Arabia', + 'SB': 'Solomon Islands', + 'SC': 'Seychelles', + 'SD': 'Sudan', + 'SE': 'Sweden', + 'SG': 'Singapore', + 'SH': 'St. Helena', + 'SI': 'Slovenia', + 'SJ': 'Svalbard and Jan Mayen Islands', + 'SK': 'Slovakia', + 'SL': 'Sierra Leone', + 'SM': 'San Marino', + 'SN': 'Senegal', + 'SO': 'Somalia', + 'SS': 'South Sudan', + 'SV': 'El Salvador', + 'SX': 'Sint Maarten', + 'SY': 'Syria', + 'SZ': 'Eswatini', + 'TG': 'Togo', + 'TH': 'Thailand', + 'TJ': 'Tajikistan', + 'TL': 'Timor-Leste', + 'TM': 'Turkmenistan', + 'TN': 'Tunisia', + 'TO': 'Tonga', + 'TR': 'Turkey', + 'TT': 'Trinidad and Tobago', + 'TW': 'Taiwan', + 'TZ': 'Tanzania', + 'UA': 'Ukraine', + 'UG': 'Uganda', + 'US': 'United States', + 'UY': 'Uruguay', + 'UZ': 'Uzbekistan', + 'VA': 'Vatican', + 'VE': 'Venezuela', + 'VG': 'British Virgin Islands', + 'VI': 'United States Virgin Islands', + 'VN': 'Vietnam', + 'VU': 'Vanuatu', + 'WS': 'Samoa', + 'XK': 'Kosovo', + 'YE': 'Yemen', + 'ZA': 'South Africa', + 'ZM': 'Zambia', + 'ZW': 'Zimbabwe', + } diff --git a/MTIL_datasets/dtd.py b/MTIL_datasets/dtd.py new file mode 100644 index 0000000000000000000000000000000000000000..dad7b7957a48e6637b994fb2368687532dfe5765 --- /dev/null +++ b/MTIL_datasets/dtd.py @@ -0,0 +1,95 @@ +import os +import pickle +import random + +from .utils import * + +from .oxford_pets import OxfordPets + + +class DescribableTextures(DatasetBase): + + dataset_dir = "dtd" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = os.path.join(self.dataset_dir, "images") + self.split_path = os.path.join(self.dataset_dir, "split_zhou_DescribableTextures.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.image_dir) + else: + train, val, test = self.read_and_split_data(self.image_dir) + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c: f'a photo of a {c} texture.', + lambda c: f'a photo of a {c} pattern.', + lambda c: f'a photo of a {c} thing.', + lambda c: f'a photo of a {c} object.', + lambda c: f'a photo of the {c} texture.', + lambda c: f'a photo of the {c} pattern.', + lambda c: f'a photo of the {c} thing.', + lambda c: f'a photo of the {c} object.', + ] + + super().__init__(train_x=train, val=val, test=test) + + @staticmethod + def read_and_split_data(image_dir, p_trn=0.5, p_val=0.2, ignored=[], new_cnames=None): + categories = listdir_nohidden(image_dir) + categories = [c for c in categories if c not in ignored] + categories.sort() + + p_tst = 1 - p_trn - p_val + print(f"Splitting into {p_trn:.0%} train, {p_val:.0%} val, and {p_tst:.0%} test") + + def _collate(ims, y, c): + items = [] + for im in ims: + item = Datum(impath=im, label=y, classname=c) + items.append(item) + return items + + train, val, test = [], [], [] + for label, category in enumerate(categories): + category_dir = os.path.join(image_dir, category) + images = listdir_nohidden(category_dir) + images = [os.path.join(category_dir, im) for im in images] + random.shuffle(images) + n_total = len(images) + n_train = round(n_total * p_trn) + n_val = round(n_total * p_val) + n_test = n_total - n_train - n_val + assert n_train > 0 and n_val > 0 and n_test > 0 + + if new_cnames is not None and category in new_cnames: + category = new_cnames[category] + + train.extend(_collate(images[:n_train], label, category)) + val.extend(_collate(images[n_train : n_train + n_val], label, category)) + test.extend(_collate(images[n_train + n_val :], label, category)) + + return train, val, test diff --git a/MTIL_datasets/eurosat.py b/MTIL_datasets/eurosat.py new file mode 100644 index 0000000000000000000000000000000000000000..46be76d48ff89912a9a1a3ddbf2d0085052e47f7 --- /dev/null +++ b/MTIL_datasets/eurosat.py @@ -0,0 +1,75 @@ +import os +import pickle + +from .utils import * + +from .oxford_pets import OxfordPets +from .dtd import DescribableTextures as DTD + +NEW_CNAMES = { + "AnnualCrop": "Annual Crop Land", + "Forest": "Forest", + "HerbaceousVegetation": "Herbaceous Vegetation Land", + "Highway": "Highway or Road", + "Industrial": "Industrial Buildings", + "Pasture": "Pasture Land", + "PermanentCrop": "Permanent Crop Land", + "Residential": "Residential Buildings", + "River": "River", + "SeaLake": "Sea or Lake", +} + + +class EuroSAT(DatasetBase): + + dataset_dir = "eurosat" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = os.path.join(self.dataset_dir, "2750") + self.split_path = os.path.join(self.dataset_dir, "split_zhou_EuroSAT.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.image_dir) + else: + train, val, test = DTD.read_and_split_data(self.image_dir, new_cnames=NEW_CNAMES) + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c: f"a centered satellite photo of {c}.", + lambda c: f"a centered satellite photo of a {c}.", + lambda c: f"a centered satellite photo of the {c}.", + ] + + super().__init__(train_x=train, val=val, test=test) + + def update_classname(self, dataset_old): + dataset_new = [] + for item_old in dataset_old: + cname_old = item_old.classname + cname_new = NEW_CLASSNAMES[cname_old] + item_new = Datum(impath=item_old.impath, label=item_old.label, classname=cname_new) + dataset_new.append(item_new) + return dataset_new diff --git a/MTIL_datasets/fer2013.py b/MTIL_datasets/fer2013.py new file mode 100644 index 0000000000000000000000000000000000000000..d0be0c3f99cc86402a9e5247154c39c6218b7adb --- /dev/null +++ b/MTIL_datasets/fer2013.py @@ -0,0 +1,170 @@ +import os +import pickle +import random +import warnings +from collections import defaultdict + +from .utils import * +from .oxford_pets import OxfordPets + +# Synonym sets per class (label index is the list index) +FER2013_CLASSES_SYNONYMS = [ + ['angry'], + ['disgusted', 'disgust'], + ['fearful', 'fear'], + ['happy', 'smiling'], + ['sad', 'depressed'], + ['surprised', 'surprise', 'shocked', 'spooked'], + ['neutral', 'bored'], +] + +# Canonical class names are the first synonym in each list +FER2013_CANONICAL = [syns[0] for syns in FER2013_CLASSES_SYNONYMS] + +# Prompt templates +FER2013_TEMPLATES = [ + 'a photo of a {} looking face.', + 'a photo of a face showing the emotion: {}.', + 'a photo of a face looking {}.', + 'a face that looks {}.', + 'they look {}.', + 'look at how {} they are.', +] + +FER2013_DEBUG = os.environ.get("FER2013_DEBUG", "0") not in ("0", "false", "False", "") + +def _dbg(msg: str): + if FER2013_DEBUG: + print(f"[FER2013][DEBUG] {msg}") + + +def _norm(s: str) -> str: + s = s.lower().strip() + for ch in [" ", "_", "-", "."]: + s = s.replace(ch, "") + return s + + +def _build_syn_map(): + m = {} + for y, syns in enumerate(FER2013_CLASSES_SYNONYMS): + for s in syns: + m[_norm(s)] = y + # add common canonical variants for safety + aliases = { + 'disgust': 1, + 'fear': 2, + 'surprise': 5, + } + for k, v in aliases.items(): + m[_norm(k)] = v + return m + + +class FER2013(DatasetBase): + + dataset_dir = "fer2013" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.split_path = os.path.join(self.dataset_dir, "split_custom_FER2013.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + train_dir = os.path.join(self.dataset_dir, "train") + test_dir = os.path.join(self.dataset_dir, "test") + if not os.path.isdir(train_dir) or not os.path.isdir(test_dir): + raise ValueError( + f"FER2013: expected train/test folders under '{self.dataset_dir}'. Got train={os.path.isdir(train_dir)}, test={os.path.isdir(test_dir)}" + ) + + # try cache + if os.path.exists(self.split_path): + try: + train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir) + except Exception as e: + warnings.warn(f"FER2013: failed to read cached split; rebuilding. Error: {e}") + train, val, test = self._build_split(train_dir, test_dir) + try: + OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir) + except Exception as e2: + warnings.warn(f"FER2013: failed to save split: {e2}") + else: + train, val, test = self._build_split(train_dir, test_dir) + try: + OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir) + except Exception as e: + warnings.warn(f"FER2013: failed to save split: {e}") + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes) + self.templates = FER2013_TEMPLATES + super().__init__(train_x=train, val=val, test=test) + + def _build_split(self, train_dir, test_dir, p_val=0.2): + syn_map = _build_syn_map() + # read train per class + tr_items_by_label = defaultdict(list) + class_dirs = listdir_nohidden(train_dir, sort=True) + if not class_dirs: + warnings.warn(f"FER2013: no class folders found in {train_dir}") + for cls in class_dirs: + full = os.path.join(train_dir, cls) + if not os.path.isdir(full): + continue + key = _norm(cls) + y = syn_map.get(key) + if y is None: + warnings.warn(f"FER2013: unexpected class folder in train: '{cls}'") + continue + cname = FER2013_CANONICAL[y] + for fname in listdir_nohidden(full): + impath = os.path.join(full, fname) + tr_items_by_label[y].append(Datum(impath=impath, label=y, classname=cname)) + + # stratified split train->(train,val) + train, val = [], [] + for y, items in tr_items_by_label.items(): + random.shuffle(items) + n_val = max(1, round(len(items) * p_val)) if len(items) > 1 else 0 + val.extend(items[:n_val]) + train.extend(items[n_val:]) + + # read test + test = [] + class_dirs = listdir_nohidden(test_dir, sort=True) + if not class_dirs: + warnings.warn(f"FER2013: no class folders found in {test_dir}") + for cls in class_dirs: + full = os.path.join(test_dir, cls) + if not os.path.isdir(full): + continue + key = _norm(cls) + y = syn_map.get(key) + if y is None: + warnings.warn(f"FER2013: unexpected class folder in test: '{cls}'") + continue + cname = FER2013_CANONICAL[y] + for fname in listdir_nohidden(full): + impath = os.path.join(full, fname) + test.append(Datum(impath=impath, label=y, classname=cname)) + + # basic sanity + if not train or not val or not test: + warnings.warn(f"FER2013: split sizes train={len(train)} val={len(val)} test={len(test)}") + return train, val, test diff --git a/MTIL_datasets/fgvc_aircraft.py b/MTIL_datasets/fgvc_aircraft.py new file mode 100644 index 0000000000000000000000000000000000000000..52695dc556952dd0b3c08a6cfa3eb99205dfe23c --- /dev/null +++ b/MTIL_datasets/fgvc_aircraft.py @@ -0,0 +1,73 @@ +import os +import pickle + +from .utils import * + +from .oxford_pets import OxfordPets +import torchvision.datasets + + +class FGVCAircraft(DatasetBase): + + dataset_dir = "fgvc_aircraft" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = os.path.join(self.dataset_dir, "images") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + classnames = [] + with open(os.path.join(self.dataset_dir, "variants.txt"), "r") as f: + lines = f.readlines() + for line in lines: + classnames.append(line.strip()) + cname2lab = {c: i for i, c in enumerate(classnames)} + + train = self.read_data(cname2lab, "images_variant_train.txt") + val = self.read_data(cname2lab, "images_variant_val.txt") + test = self.read_data(cname2lab, "images_variant_test.txt") + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c: f"a photo of a {c}, a type of aircraft.", + lambda c: f"a photo of the {c}, a type of aircraft.", + ] + + super().__init__(train_x=train, val=val, test=test) + + def read_data(self, cname2lab, split_file): + filepath = os.path.join(self.dataset_dir, split_file) + items = [] + + with open(filepath, "r") as f: + lines = f.readlines() + for line in lines: + line = line.strip().split(" ") + imname = line[0] + ".jpg" + classname = " ".join(line[1:]) + impath = os.path.join(self.image_dir, imname) + label = cname2lab[classname] + item = Datum(impath=impath, label=label, classname=classname) + items.append(item) + + return items diff --git a/MTIL_datasets/food101.py b/MTIL_datasets/food101.py new file mode 100644 index 0000000000000000000000000000000000000000..822efc394c6e939b4b906c40dfe8c4d69b9bbc4a --- /dev/null +++ b/MTIL_datasets/food101.py @@ -0,0 +1,52 @@ +import os +import pickle + +from .utils import * + +from .oxford_pets import OxfordPets +from .dtd import DescribableTextures as DTD + + + +class Food101(DatasetBase): + + dataset_dir = "food-101" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = os.path.join(self.dataset_dir, "images") + self.split_path = os.path.join(self.dataset_dir, "split_zhou_Food101.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.image_dir) + else: + train, val, test = DTD.read_and_split_data(self.image_dir) + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c: f"a photo of a {c}, a type of food.", + ] + + super().__init__(train_x=train, val=val, test=test) diff --git a/MTIL_datasets/gtsrb.py b/MTIL_datasets/gtsrb.py new file mode 100644 index 0000000000000000000000000000000000000000..d79f3415b417027a71e80484b6a091a65eb97fd0 --- /dev/null +++ b/MTIL_datasets/gtsrb.py @@ -0,0 +1,202 @@ +import os +import pickle +import random +import warnings + +from .utils import * +from .oxford_pets import OxfordPets + +# 43 classes for GTSRB with human-readable names +classes = [ + 'red and white circle 20 kph speed limit', + 'red and white circle 30 kph speed limit', + 'red and white circle 50 kph speed limit', + 'red and white circle 60 kph speed limit', + 'red and white circle 70 kph speed limit', + 'red and white circle 80 kph speed limit', + 'end / de-restriction of 80 kph speed limit', + 'red and white circle 100 kph speed limit', + 'red and white circle 120 kph speed limit', + 'red and white circle red car and black car no passing', + 'red and white circle red truck and black car no passing', + 'red and white triangle road intersection warning', + 'white and yellow diamond priority road', + 'red and white upside down triangle yield right-of-way', + 'stop', + 'empty red and white circle', + 'red and white circle no truck entry', + 'red circle with white horizonal stripe no entry', + 'red and white triangle with exclamation mark warning', + 'red and white triangle with black left curve approaching warning', + 'red and white triangle with black right curve approaching warning', + 'red and white triangle with black double curve approaching warning', + 'red and white triangle rough / bumpy road warning', + 'red and white triangle car skidding / slipping warning', + 'red and white triangle with merging / narrow lanes warning', + 'red and white triangle with person digging / construction / road work warning', + 'red and white triangle with traffic light approaching warning', + 'red and white triangle with person walking warning', + 'red and white triangle with child and person walking warning', + 'red and white triangle with bicyle warning', + 'red and white triangle with snowflake / ice warning', + 'red and white triangle with deer warning', + 'white circle with gray strike bar no speed limit', + 'blue circle with white right turn arrow mandatory', + 'blue circle with white left turn arrow mandatory', + 'blue circle with white forward arrow mandatory', + 'blue circle with white forward or right turn arrow mandatory', + 'blue circle with white forward or left turn arrow mandatory', + 'blue circle with white keep right arrow mandatory', + 'blue circle with white keep left arrow mandatory', + 'blue circle with white arrows indicating a traffic circle', + 'white circle with gray strike bar indicating no passing for cars has ended', + 'white circle with gray strike bar indicating no passing for trucks has ended', +] + + +DEBUG = os.environ.get("GTSRB_DEBUG", "0") not in ("0", "false", "False", "") + + +def _dbg(msg: str): + if DEBUG: + print(f"[GTSRB][DEBUG] {msg}") + + +def _is_image_file(name): + name = name.lower() + return any(name.endswith(ext) for ext in ['.ppm', '.png', '.jpg', '.jpeg', '.bmp', '.webp']) + + +class GTSRB(DatasetBase): + """ + German Traffic Sign Recognition Benchmark (GTSRB) + + Expected structure: + /gtsrb/ + ├─ 00000/*.ppm + ├─ 00001/*.ppm + └─ ... up to 00042/ + + Note: The dataset has no official test split in this layout, so we will + randomly split each class into train/val/test (50%/20%/30%), cached to json. + """ + + dataset_dir = "gtsrb" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = self.dataset_dir + self.split_path = os.path.join(self.dataset_dir, "split_custom_GTSRB.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + _dbg(f"dataset_dir={self.dataset_dir}") + _dbg(f"image_dir={self.image_dir}") + _dbg(f"split_path exists? {os.path.exists(self.split_path)}") + + if os.path.exists(self.split_path): + try: + train, val, test = OxfordPets.read_split(self.split_path, self.image_dir) + except Exception as e: + warnings.warn(f"GTSRB: failed to read split file '{self.split_path}'; rebuilding split. Error: {e}") + train, val, test = self.read_and_split_data(self.image_dir) + try: + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + except Exception as e2: + warnings.warn(f"GTSRB: failed to save rebuilt split to '{self.split_path}': {e2}") + else: + train, val, test = self.read_and_split_data(self.image_dir) + try: + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + except Exception as e: + warnings.warn(f"GTSRB: failed to save split to '{self.split_path}': {e}") + + _dbg(f"loaded counts: train={len(train)}, val={len(val)}, test={len(test)}") + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + _dbg(f"few-shot applied: train={len(train)}, val={len(val)} (shots={num_shots})") + + # Ensure class subsampling behavior matches others + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes) + _dbg(f"after subsample='{subsample_classes}': train={len(train)}, val={len(val)}, test={len(test)})") + + # Prompt templates provided by user + self.templates = [ + lambda c: f'a zoomed in photo of a "{c}" traffic sign.', + lambda c: f'a centered photo of a "{c}" traffic sign.', + lambda c: f'a close up photo of a "{c}" traffic sign.', + ] + + super().__init__(train_x=train, val=val, test=test) + + @staticmethod + def read_and_split_data(image_root, p_trn=0.5, p_val=0.2): + # Discover class directories (e.g., 00000 .. 00042) + try: + all_entries = listdir_nohidden(image_root, sort=True) + except Exception as e: + warnings.warn(f"GTSRB: failed to list directory '{image_root}': {e}") + raise + class_dirs = [d for d in all_entries if os.path.isdir(os.path.join(image_root, d))] + class_dirs.sort() + assert len(class_dirs) > 0, f"GTSRB: no class folders found under {image_root}" + _dbg(f"found {len(class_dirs)} class folders; head={class_dirs[:5]}") + + # Map sorted class dirs to labels 0..N-1 and names from 'classes' + if len(class_dirs) != len(classes): + print(f"Warning: detected {len(class_dirs)} class folders but classes list has {len(classes)} entries. Proceeding with min overlap.") + num_labels = min(len(class_dirs), len(classes)) + + def _collate(paths, y, cname): + return [Datum(impath=p, label=y, classname=cname) for p in paths] + + train, val, test = [], [], [] + for y, cls_dir in enumerate(class_dirs[:num_labels]): + cname = classes[y] + cdir = os.path.join(image_root, cls_dir) + try: + files = listdir_nohidden(cdir, sort=False) + except Exception as e: + warnings.warn(f"GTSRB: failed to list class folder '{cdir}': {e}") + files = [] + imgs = [os.path.join(cdir, f) for f in files if _is_image_file(f)] + random.shuffle(imgs) + n_total = len(imgs) + if n_total == 0: + warnings.warn(f"GTSRB: empty or unreadable class folder: {cdir}; skipping this class") + continue + if n_total < 5: + warnings.warn(f"GTSRB: very few images in class '{cls_dir}' (n={n_total}); splits may be unstable") + n_train = round(n_total * p_trn) + n_val = round(n_total * p_val) + n_test = n_total - n_train - n_val + if not (n_train > 0 and n_val > 0 and n_test > 0): + warnings.warn(f"GTSRB: split would create empty split for class {cls_dir} (n={n_total}); adjusting strategy to keep at least 1 per split") + # Fallback: enforce at least 1 per split if possible + if n_total >= 3: + n_train, n_val, n_test = 1, 1, n_total - 2 + elif n_total == 2: + n_train, n_val, n_test = 1, 1, 0 + else: # n_total == 1 + n_train, n_val, n_test = 1, 0, 0 + + train.extend(_collate(imgs[:n_train], y, cname)) + val.extend(_collate(imgs[n_train:n_train + n_val], y, cname)) + test.extend(_collate(imgs[n_train + n_val:], y, cname)) + _dbg(f"class {cls_dir} -> label {y}: total={n_total}, train={n_train}, val={n_val}, test={n_test}") + + _dbg(f"aggregate sizes: train={len(train)}, val={len(val)}, test={len(test)}") + return train, val, test diff --git a/MTIL_datasets/hatefulmemes.py b/MTIL_datasets/hatefulmemes.py new file mode 100644 index 0000000000000000000000000000000000000000..6f0ef089f358eecd624f1754294f9ff0ea9daaea --- /dev/null +++ b/MTIL_datasets/hatefulmemes.py @@ -0,0 +1,117 @@ +import os +import json +import pickle + +from .utils import * +from .oxford_pets import OxfordPets + + +class HatefulMemes(DatasetBase): + """ + Hateful Memes dataset loader (image-only for MTIL). + + Expected structure: + /hatefulmemes/ + ├─ img/*.png|jpg + ├─ train.jsonl + ├─ dev.jsonl + └─ test.jsonl + + JSONL lines example: + {"id":85362, "img":"img/85362.png", "label":0, "text":"..."} + + We DO NOT use the 'text' field. Only image path and label are used. + + Classes: ['meme', 'hatespeech meme'] + Templates: ['a {}.'] + """ + + dataset_dir = "hatefulmemes" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + if not os.path.isdir(self.dataset_dir): + # allow alias without trailing 's' + alt = os.path.join(root, "hatefulmeme") + if os.path.isdir(alt): + self.dataset_dir = alt + else: + raise ValueError( + f"HatefulMemes: dataset folder not found at '{self.dataset_dir}' or '{alt}'" + ) + + split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(split_fewshot_dir) + + # paths + train_json = os.path.join(self.dataset_dir, "train.jsonl") + dev_json = os.path.join(self.dataset_dir, "dev.jsonl") + test_json = os.path.join(self.dataset_dir, "test.jsonl") + + if not os.path.isfile(train_json) or not os.path.isfile(dev_json) or not os.path.isfile(test_json): + raise ValueError( + f"HatefulMemes: missing jsonl files. Expected train/dev/test at '{self.dataset_dir}'." + ) + + # fixed classes + classes = ["meme", "hatespeech meme"] + lab_to_name = {0: classes[0], 1: classes[1]} + + # read splits + train = self._read_jsonl(train_json, lab_to_name) + val = self._read_jsonl(dev_json, lab_to_name) + test = self._read_jsonl(test_json, lab_to_name) + # Some public releases of Hateful Memes do not include labels for test.jsonl. + # In that case, fallback to use dev.jsonl for evaluation so zero-shot works. + if len(test) == 0 and len(val) > 0: + test = list(val) + + # few-shot + if num_shots >= 1: + preprocessed = os.path.join(split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + # subsample behavior consistent with others + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes) + + # templates per user spec + self.templates = [ + lambda c: f'a {c}.', + ] + + super().__init__(train_x=train, val=val, test=test) + + def _read_jsonl(self, filepath, lab_to_name): + items = [] + with open(filepath, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except Exception: + continue + # fields: id, img, label, text (ignored) + img_rel = obj.get('img', '') + label = obj.get('label', None) + if img_rel is None or label is None: + continue + if label not in (0, 1): + raise ValueError(f"HatefulMemes: unexpected label {label} in {filepath}") + impath = os.path.join(self.dataset_dir, img_rel.replace('/', os.sep)) + if not os.path.isfile(impath): + continue + cname = lab_to_name[label] + items.append(Datum(impath=impath, label=label, classname=cname)) + return items diff --git a/MTIL_datasets/imagenet_r.py b/MTIL_datasets/imagenet_r.py new file mode 100644 index 0000000000000000000000000000000000000000..b7823bf1ea3ae1a07bfbafd13d015302fe42973a --- /dev/null +++ b/MTIL_datasets/imagenet_r.py @@ -0,0 +1,231 @@ +import os +import random +from typing import List, Tuple, Dict + +from .utils import * # Datum, DatasetBase, listdir_nohidden, mkdir_if_missing +from .oxford_pets import OxfordPets + + +IMAGENETR_TEMPLATES: List[str] = [ + 'a bad photo of a {}.', + 'a photo of many {}.', + 'a sculpture of a {}.', + 'a photo of the hard to see {}.', + 'a low resolution photo of the {}.', + 'a rendering of a {}.', + 'graffiti of a {}.', + 'a bad photo of the {}.', + 'a cropped photo of the {}.', + 'a tattoo of a {}.', + 'the embroidered {}.', + 'a photo of a hard to see {}.', + 'a bright photo of a {}.', + 'a photo of a clean {}.', + 'a photo of a dirty {}.', + 'a dark photo of the {}.', + 'a drawing of a {}.', + 'a photo of my {}.', + 'the plastic {}.', + 'a photo of the cool {}.', + 'a close-up photo of a {}.', + 'a black and white photo of the {}.', + 'a painting of the {}.', + 'a painting of a {}.', + 'a pixelated photo of the {}.', + 'a sculpture of the {}.', + 'a bright photo of the {}.', + 'a cropped photo of a {}.', + 'a plastic {}.', + 'a photo of the dirty {}.', + 'a jpeg corrupted photo of a {}.', + 'a blurry photo of the {}.', + 'a photo of the {}.', + 'a good photo of the {}.', + 'a rendering of the {}.', + 'a {} in a video game.', + 'a photo of one {}.', + 'a doodle of a {}.', + 'a close-up photo of the {}.', + 'a photo of a {}.', + 'the origami {}.', + 'the {} in a video game.', + 'a sketch of a {}.', + 'a doodle of the {}.', + 'a origami {}.', + 'a low resolution photo of a {}.', + 'the toy {}.', + 'a rendition of the {}.', + 'a photo of the clean {}.', + 'a photo of a large {}.', + 'a rendition of a {}.', + 'a photo of a nice {}.', + 'a photo of a weird {}.', + 'a blurry photo of a {}.', + 'a cartoon {}.', + 'art of a {}.', + 'a sketch of the {}.', + 'a embroidered {}.', + 'a pixelated photo of a {}.', + 'itap of the {}.', + 'a jpeg corrupted photo of the {}.', + 'a good photo of a {}.', + 'a plushie {}.', + 'a photo of the nice {}.', + 'a photo of the small {}.', + 'a photo of the weird {}.', + 'the cartoon {}.', + 'art of the {}.', + 'a drawing of the {}.', + 'a photo of the large {}.', + 'a black and white photo of a {}.', + 'the plushie {}.', + 'a dark photo of a {}.', + 'itap of a {}.', + 'graffiti of the {}.', + 'a toy {}.', + 'itap of my {}.', + 'a photo of a cool {}.', + 'a photo of a small {}.', + 'a tattoo of the {}.', +] + + +class ImageNetR(DatasetBase): + + dataset_dir = "imagenet-r" + + def __init__(self, root: str, num_shots: int = 0, seed: int = 1, subsample_classes: str = 'all', test_ratio: float = 0.2): + """ + Expect directory structure: + {root}/imagenet-r//*.jpg|png|jpeg|bmp|webp + {root}/imagenet-r/classname.txt # lines: " " + There is no official test split; we perform a per-class split into train/test, then + split train into train/val using OxfordPets.split_trainval. + """ + rnd = random.Random(seed) + + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.split_path = os.path.join(self.dataset_dir, "split_custom_ImageNetR.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + # Read class mapping from classname.txt + class_map_path = os.path.join(self.dataset_dir, "classname.txt") + if not os.path.exists(class_map_path): + raise FileNotFoundError(f"Class mapping file not found: {class_map_path}") + wnids, classnames = self._read_class_map(class_map_path) + self._wnids = wnids + self._classnames_ref = classnames + + # Build or load split + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir) + else: + trainval, test = self._read_from_folders(self.dataset_dir, wnids, classnames, rnd=rnd, test_ratio=test_ratio) + train, val = self._split_trainval_safe(trainval) + OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + import pickle + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + import pickle + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes) + + # Expose templates + self.templates = IMAGENETR_TEMPLATES + + super().__init__(train_x=train, val=val, test=test) + + @staticmethod + def _read_class_map(filepath: str) -> Tuple[List[str], List[str]]: + wnids: List[str] = [] + cnames: List[str] = [] + with open(filepath, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + parts = line.split() + wnid = parts[0] + cname = ' '.join(parts[1:]) if len(parts) > 1 else wnid + wnids.append(wnid) + cnames.append(cname) + return wnids, cnames + + @staticmethod + def _split_trainval_safe(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]: + from collections import defaultdict + tracker: Dict[int, List[int]] = defaultdict(list) + for idx, item in enumerate(trainval): + tracker[item.label].append(idx) + + train, val = [], [] + for label, idxs in tracker.items(): + n = len(idxs) + if n <= 1: + for idx in idxs: + train.append(trainval[idx]) + continue + n_val = max(1, int(round(n * p_val))) + if n_val >= n: + n_val = n - 1 + random.shuffle(idxs) + for i, idx in enumerate(idxs): + if i < n_val: + val.append(trainval[idx]) + else: + train.append(trainval[idx]) + if len(val) == 0 and len(train) > 0: + val.append(train[-1]) + train = train[:-1] + return train, val + + def _read_from_folders(self, data_dir: str, wnids: List[str], classnames: List[str], rnd: random.Random, test_ratio: float) -> Tuple[List[Datum], List[Datum]]: + exts = {".png", ".jpg", ".jpeg", ".bmp", ".webp"} + class_to_label = {wnid: i for i, wnid in enumerate(wnids)} + label_to_cname = {i: classnames[i] for i in range(len(classnames))} + + items_by_label: Dict[int, List[Datum]] = {i: [] for i in range(len(wnids))} + for wnid in wnids: + cdir = os.path.join(data_dir, wnid) + if not os.path.isdir(cdir): + # If a class listed in mapping has no folder, skip gracefully + continue + label = class_to_label[wnid] + cname = label_to_cname[label] + for fname in listdir_nohidden(cdir, sort=True): + fext = os.path.splitext(fname)[1].lower() + if fext not in exts: + continue + impath = os.path.join(cdir, fname) + items_by_label[label].append(Datum(impath=impath, label=label, classname=cname)) + + trainval: List[Datum] = [] + test: List[Datum] = [] + for label, items in items_by_label.items(): + if not items: + continue + rnd.shuffle(items) + if len(items) == 1: + trainval.extend(items) + continue + n_test = max(1, int(round(len(items) * test_ratio))) + if n_test >= len(items): + n_test = len(items) - 1 + test.extend(items[:n_test]) + trainval.extend(items[n_test:]) + return trainval, test diff --git a/MTIL_datasets/kitti_distance.py b/MTIL_datasets/kitti_distance.py new file mode 100644 index 0000000000000000000000000000000000000000..e13259a3d42a3a105ac57cee99e33563b1dc0421 --- /dev/null +++ b/MTIL_datasets/kitti_distance.py @@ -0,0 +1,228 @@ +import os +import random +from typing import List, Tuple + +from .utils import * # Datum, DatasetBase, listdir_nohidden, mkdir_if_missing +from .oxford_pets import OxfordPets + +# Class names per user specification (ordered) +KITTI_DISTANCE_CLASSES: List[str] = [ + 'a photo i took of a car nearby', + 'a photo i took with a car in the middle distance', + 'a photo i took with a car faraway', + 'a photo i took with no car.', +] + +# Keep templates as strings with {} +KITTI_DISTANCE_TEMPLATES: List[str] = [ + '{}', +] + + +class KittiDistance(DatasetBase): + + dataset_dir = "kitti" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all', test_ratio: float = 0.2): + """ + Predict the distance category of the closest car from KITTI labels. + + Expected structure: + {root}/kitti/ + image/000004.png + label/000004.txt + + Label file format: KITTI object detection label lines. + We retain only lines with type == 'Car'. + For these, we read the 14th column (index 13, loc_z) as the camera-depth in meters. + We take the minimum positive loc_z across all 'Car' lines as z_min for the image. + If there is no valid positive loc_z for 'Car', we assign the 'no car' class. + + Discretization into classes: + 0: 0 < z_min < 10 -> 'nearby' + 1: 10 <= z_min < 30 -> 'middle distance' + 2: z_min >= 30 -> 'faraway' + 3: no car -> 'no car.' + + No official test set; we split randomly (per class) into train/test by test_ratio. + Then we split train into train/val using a safe per-class split. + Splits are saved to JSON for reproducibility. + """ + rnd = random.Random(seed) + + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + + # Image and label directories + image_dir = os.path.join(self.dataset_dir, 'image') + if not os.path.isdir(image_dir): + # Graceful fallback to standard KITTI naming variants if provided + alt1 = os.path.join(self.dataset_dir, 'image_2') + alt2 = os.path.join(self.dataset_dir, 'images') + if os.path.isdir(alt1): + image_dir = alt1 + elif os.path.isdir(alt2): + image_dir = alt2 + label_dir = os.path.join(self.dataset_dir, 'label') + if not os.path.isdir(label_dir): + alt_l1 = os.path.join(self.dataset_dir, 'label_2') + alt_l2 = os.path.join(self.dataset_dir, 'labels') + if os.path.isdir(alt_l1): + label_dir = alt_l1 + elif os.path.isdir(alt_l2): + label_dir = alt_l2 + + if not os.path.isdir(image_dir): + raise FileNotFoundError(f"KittiDistance: image dir not found: {image_dir}") + if not os.path.isdir(label_dir): + raise FileNotFoundError(f"KittiDistance: label dir not found: {label_dir}") + + self.image_dir = image_dir + self.label_dir = label_dir + + self.split_path = os.path.join(self.dataset_dir, "split_custom_KittiDistance.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.image_dir) + else: + trainval, test = self._read_and_split(self.image_dir, self.label_dir, rnd=rnd, test_ratio=test_ratio) + train, val = self._split_trainval_safe(trainval) + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + import pickle + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + import pickle + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + # Allow class subsampling if requested + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes) + + # Templates per user specification + self.templates = KITTI_DISTANCE_TEMPLATES + + super().__init__(train_x=train, val=val, test=test) + # Override inferred metadata to ensure stable 4-way classification regardless of train label coverage + self._classnames = KITTI_DISTANCE_CLASSES + self._lab2cname = {i: c for i, c in enumerate(KITTI_DISTANCE_CLASSES)} + self._num_classes = len(KITTI_DISTANCE_CLASSES) + + def _read_and_split(self, image_dir: str, label_dir: str, rnd: random.Random, test_ratio: float) -> Tuple[List[Datum], List[Datum]]: + exts = {".png", ".jpg", ".jpeg", ".bmp", ".webp"} + # Collect items with computed labels + items_by_label = {i: [] for i in range(len(KITTI_DISTANCE_CLASSES))} + + for fname in listdir_nohidden(image_dir, sort=True): + fext = os.path.splitext(fname)[1].lower() + if fext not in exts: + continue + impath = os.path.join(image_dir, fname) + stem = os.path.splitext(fname)[0] + label_path = os.path.join(label_dir, stem + '.txt') + label_idx = self._compute_label_from_kitti(label_path) + cname = KITTI_DISTANCE_CLASSES[label_idx] + items_by_label[label_idx].append(Datum(impath=impath, label=label_idx, classname=cname)) + + # Per-class balanced split into trainval/test + trainval, test = [], [] + for label, items in items_by_label.items(): + if not items: + continue + rnd.shuffle(items) + if len(items) == 1: + trainval.extend(items) + continue + n_test = max(1, int(round(len(items) * test_ratio))) + if n_test >= len(items): + n_test = len(items) - 1 + test.extend(items[:n_test]) + trainval.extend(items[n_test:]) + + return trainval, test + + @staticmethod + def _compute_label_from_kitti(label_path: str) -> int: + """ + Parse KITTI label file; focus only on lines with type == 'Car'. + Extract the 14th column (index 13, loc_z) as meters; consider only positive values. + If no positive loc_z found -> class 'no car' (index 3). + + Thresholds: + 0: 0 < z < 10 + 1: 10 <= z < 30 + 2: z >= 30 + """ + z_vals = [] + if os.path.isfile(label_path): + with open(label_path, 'r') as f: + for line in f: + line = line.strip() + if not line: + continue + parts = line.split() + obj_type = parts[0] + if obj_type != 'Car': + continue + if len(parts) < 14: + # Not a valid KITTI detection line; skip + continue + try: + # parts[13] is loc_z (0-based index), per KITTI format + z = float(parts[13]) + if z > 0: + z_vals.append(z) + except Exception: + continue + # Determine class index + if not z_vals: + return 3 # no car + z_min = min(z_vals) + if z_min < 10: + return 0 + elif z_min < 30: + return 1 + else: + return 2 + + @staticmethod + def _split_trainval_safe(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]: + from collections import defaultdict + tracker = defaultdict(list) + for idx, item in enumerate(trainval): + tracker[item.label].append(idx) + + train, val = [], [] + for label, idxs in tracker.items(): + n = len(idxs) + if n <= 1: + for idx in idxs: + train.append(trainval[idx]) + continue + n_val = max(1, int(round(n * p_val))) + if n_val >= n: + n_val = n - 1 + random.shuffle(idxs) + for i, idx in enumerate(idxs): + if i < n_val: + val.append(trainval[idx]) + else: + train.append(trainval[idx]) + + if len(val) == 0 and len(train) > 0: + val.append(train[-1]) + train = train[:-1] + + return train, val diff --git a/MTIL_datasets/mnist.py b/MTIL_datasets/mnist.py new file mode 100644 index 0000000000000000000000000000000000000000..075fde8796cef93f0e794fb1113c438635a16ef3 --- /dev/null +++ b/MTIL_datasets/mnist.py @@ -0,0 +1,132 @@ +import os +import pickle + +from .utils import * + +from .oxford_pets import OxfordPets +from .dtd import DescribableTextures as DTD +from .oxford_pets import OxfordPets + +import torch +import codecs +import numpy as np +import sys + +classes = [ + "0 - zero", + "1 - one", + "2 - two", + "3 - three", + "4 - four", + "5 - five", + "6 - six", + "7 - seven", + "8 - eight", + "9 - nine", + ] + +class MNIST(DatasetBase): + + dataset_dir = "mnist" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = self.dataset_dir + + trainval_image_file = "train-images-idx3-ubyte" + trainval_data = read_image_file(os.path.join(self.image_dir, trainval_image_file)) # Size([60000, 28, 28]) torch.uint8 + trainval_label_file = "train-labels-idx1-ubyte" + trainval_targets = read_label_file(os.path.join(self.image_dir, trainval_label_file)) # Size([60000]) torch.int64 + # trainval_names = + + test_image_file = "t10k-images-idx3-ubyte" + test_data = read_image_file(os.path.join(self.image_dir, test_image_file)) + test_label_file = "t10k-labels-idx1-ubyte" + test_targets = read_label_file(os.path.join(self.image_dir, test_label_file)) + + trainval = [] + for idx in range(trainval_data.size(0)): + item = Datum(impath=Image.fromarray(trainval_data[idx].numpy(), mode="L"), + label=int(trainval_targets[idx]), classname=classes[trainval_targets[idx]]) + trainval.append(item) + + test = [] + for idx in range(test_data.size(0)): + item = Datum(impath=Image.fromarray(test_data[idx].numpy(), mode="L"), + label=int(test_targets[idx]), classname=classes[test_targets[idx]]) + test.append(item) + + train, val = OxfordPets.split_trainval(trainval) + + if num_shots >= 1: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c: f'a photo of the number: "{c}".', + ] + + super().__init__(train_x=train, val=val, test=test) + + +def _flip_byte_order(t: torch.Tensor) -> torch.Tensor: + return ( + t.contiguous().view(torch.uint8).view(*t.shape, t.element_size()).flip(-1).view(*t.shape[:-1], -1).view(t.dtype) + ) + +def get_int(b: bytes) -> int: + return int(codecs.encode(b, "hex"), 16) + +SN3_PASCALVINCENT_TYPEMAP = { + 8: torch.uint8, + 9: torch.int8, + 11: torch.int16, + 12: torch.int32, + 13: torch.float32, + 14: torch.float64, +} + +def read_sn3_pascalvincent_tensor(path: str, strict: bool = True) -> torch.Tensor: + """Read a SN3 file in "Pascal Vincent" format (Lush file 'libidx/idx-io.lsh'). + Argument may be a filename, compressed filename, or file object. + """ + # read + with open(path, "rb") as f: + data = f.read() + # parse + magic = get_int(data[0:4]) + nd = magic % 256 + ty = magic // 256 + assert 1 <= nd <= 3 + assert 8 <= ty <= 14 + torch_type = SN3_PASCALVINCENT_TYPEMAP[ty] + s = [get_int(data[4 * (i + 1) : 4 * (i + 2)]) for i in range(nd)] + + parsed = torch.frombuffer(bytearray(data), dtype=torch_type, offset=(4 * (nd + 1))) + if sys.byteorder == "little" and parsed.element_size() > 1: + parsed = _flip_byte_order(parsed) + + assert parsed.shape[0] == np.prod(s) or not strict + return parsed.view(*s) + + +def read_label_file(path: str) -> torch.Tensor: + x = read_sn3_pascalvincent_tensor(path, strict=False) + if x.dtype != torch.uint8: + raise TypeError(f"x should be of dtype torch.uint8 instead of {x.dtype}") + if x.ndimension() != 1: + raise ValueError(f"x should have 1 dimension instead of {x.ndimension()}") + return x.long() + + +def read_image_file(path: str) -> torch.Tensor: + x = read_sn3_pascalvincent_tensor(path, strict=False) + if x.dtype != torch.uint8: + raise TypeError(f"x should be of dtype torch.uint8 instead of {x.dtype}") + if x.ndimension() != 3: + raise ValueError(f"x should have 3 dimension instead of {x.ndimension()}") + return x \ No newline at end of file diff --git a/MTIL_datasets/oxford_flowers.py b/MTIL_datasets/oxford_flowers.py new file mode 100644 index 0000000000000000000000000000000000000000..67dbd76915a76e9a25058106f2f3c535b7e097cf --- /dev/null +++ b/MTIL_datasets/oxford_flowers.py @@ -0,0 +1,89 @@ +import os +import pickle +import random +from scipy.io import loadmat +from collections import defaultdict + +from .utils import * + +from .oxford_pets import OxfordPets + + +class OxfordFlowers(DatasetBase): + + dataset_dir = "oxford_flowers" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = os.path.join(self.dataset_dir, "jpg") + self.label_file = os.path.join(self.dataset_dir, "imagelabels.mat") + self.lab2cname_file = os.path.join(self.dataset_dir, "cat_to_name.json") + self.split_path = os.path.join(self.dataset_dir, "split_zhou_OxfordFlowers.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.image_dir) + else: + train, val, test = self.read_data() + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c: f"a photo of a {c}, a type of flower.", + ] + + super().__init__(train_x=train, val=val, test=test) + + def read_data(self): + tracker = defaultdict(list) + label_file = loadmat(self.label_file)["labels"][0] + for i, label in enumerate(label_file): + imname = f"image_{str(i + 1).zfill(5)}.jpg" + impath = os.path.join(self.image_dir, imname) + label = int(label) + tracker[label].append(impath) + + print("Splitting data into 50% train, 20% val, and 30% test") + + def _collate(ims, y, c): + items = [] + for im in ims: + item = Datum(impath=im, label=y - 1, classname=c) + items.append(item) + return items + + lab2cname = read_json(self.lab2cname_file) + train, val, test = [], [], [] + for label, impaths in tracker.items(): + random.shuffle(impaths) + n_total = len(impaths) + n_train = round(n_total * 0.5) + n_val = round(n_total * 0.2) + n_test = n_total - n_train - n_val + assert n_train > 0 and n_val > 0 and n_test > 0 + cname = lab2cname[str(label)] + train.extend(_collate(impaths[:n_train], label, cname)) + val.extend(_collate(impaths[n_train : n_train + n_val], label, cname)) + test.extend(_collate(impaths[n_train + n_val :], label, cname)) + + return train, val, test diff --git a/MTIL_datasets/oxford_pets.py b/MTIL_datasets/oxford_pets.py new file mode 100644 index 0000000000000000000000000000000000000000..0ca7de5585c9e9288ebac3024968a049c788d542 --- /dev/null +++ b/MTIL_datasets/oxford_pets.py @@ -0,0 +1,176 @@ +import os +import pickle +import math +import random +from collections import defaultdict + +from .utils import * + + +class OxfordPets(DatasetBase): + + dataset_dir = "oxford_pets" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = os.path.join(self.dataset_dir, "images") + self.anno_dir = os.path.join(self.dataset_dir, "annotations") + self.split_path = os.path.join(self.dataset_dir, "split_zhou_OxfordPets.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + if os.path.exists(self.split_path): + train, val, test = self.read_split(self.split_path, self.image_dir) + else: + trainval = self.read_data(split_file="trainval.txt") + test = self.read_data(split_file="test.txt") + train, val = self.split_trainval(trainval) + self.save_split(train, val, test, self.split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = self.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c: f"a photo of a {c}, a type of pet.", + ] + + super().__init__(train_x=train, val=val, test=test) + + def read_data(self, split_file): + filepath = os.path.join(self.anno_dir, split_file) + items = [] + + with open(filepath, "r") as f: + lines = f.readlines() + for line in lines: + line = line.strip() + imname, label, species, _ = line.split(" ") + breed = imname.split("_")[:-1] + breed = "_".join(breed) + breed = breed.lower() + imname += ".jpg" + impath = os.path.join(self.image_dir, imname) + label = int(label) - 1 # convert to 0-based index + item = Datum(impath=impath, label=label, classname=breed) + items.append(item) + + return items + + @staticmethod + def split_trainval(trainval, p_val=0.2): + p_trn = 1 - p_val + tracker = defaultdict(list) + for idx, item in enumerate(trainval): + label = item.label + tracker[label].append(idx) + + train, val = [], [] + for label, idxs in tracker.items(): + n_val = round(len(idxs) * p_val) + assert n_val > 0 + random.shuffle(idxs) + for n, idx in enumerate(idxs): + item = trainval[idx] + if n < n_val: + val.append(item) + else: + train.append(item) + + return train, val + + @staticmethod + def save_split(train, val, test, filepath, path_prefix): + def _extract(items): + out = [] + for item in items: + impath = item.impath + label = item.label + classname = item.classname + impath = impath.replace(path_prefix, "") + if impath.startswith("/"): + impath = impath[1:] + out.append((impath, label, classname)) + return out + + train = _extract(train) + val = _extract(val) + test = _extract(test) + + split = {"train": train, "val": val, "test": test} + + write_json(split, filepath) + print(f"Saved split to {filepath}") + + @staticmethod + def read_split(filepath, path_prefix): + def _convert(items): + out = [] + for impath, label, classname in items: + impath = os.path.join(path_prefix, impath) + item = Datum(impath=impath, label=int(label), classname=classname) + out.append(item) + return out + + print(f"Reading split from {filepath}") + split = read_json(filepath) + train = _convert(split["train"]) + val = _convert(split["val"]) + test = _convert(split["test"]) + + return train, val, test + + @staticmethod + def subsample_classes(*args, subsample="all"): + assert subsample in ["all", "base", "new"] + + if subsample == "all": + return args + + dataset = args[0] + labels = set() + for item in dataset: + labels.add(item.label) + labels = list(labels) + labels.sort() + n = len(labels) + m = math.ceil(n / 2) + + print(f"SUBSAMPLE {subsample.upper()} CLASSES!") + if subsample == "base": + selected = labels[:m] + else: + selected = labels[m:] + relabeler = {y: y_new for y_new, y in enumerate(selected)} + + output = [] + for dataset in args: + dataset_new = [] + for item in dataset: + if item.label not in selected: + continue + item_new = Datum( + impath=item.impath, + label=relabeler[item.label], + classname=item.classname + ) + dataset_new.append(item_new) + output.append(dataset_new) + + return output diff --git a/MTIL_datasets/pcam.py b/MTIL_datasets/pcam.py new file mode 100644 index 0000000000000000000000000000000000000000..d46daa895b2349cfea994c49021cac5f67a7a73a --- /dev/null +++ b/MTIL_datasets/pcam.py @@ -0,0 +1,198 @@ +import os +import pickle +import random +from typing import List, Tuple + +import h5py +import numpy as np + +from .utils import Datum, DatasetBase, mkdir_if_missing, read_json, write_json +from .oxford_pets import OxfordPets + + +PCAM_CLASSES = [ + 'lymph node tissue without metastatic tumor', + 'metastatic tumor in lymph node tissue', +] + +# Multiple domain-specific templates for histopathology microscopy images +PCAM_TEMPLATES = [ + 'a microscopy image patch of {}', + 'a histopathology image of {}', + 'a hematoxylin and eosin stained image of {}', + 'a high-resolution histology patch of {}', + 'a digital pathology slide patch of {}', + 'this is a microscopy image of {}', + 'this is a histopathology image of {}', +] + + +class PCam(DatasetBase): + + dataset_dir = 'pcam' + + def __init__(self, root, num_shots: int = 0, seed: int = 1, subsample_classes: str = 'all', val_ratio: float = 0.2): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + mkdir_if_missing(self.dataset_dir) + + # HDF5 file paths + self.train_x_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_train_x.h5') + self.train_y_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_train_y.h5') + self.test_x_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_test_x.h5') + self.test_y_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_test_y.h5') + + for p in [self.train_x_path, self.train_y_path, self.test_x_path, self.test_y_path]: + if not os.path.isfile(p): + raise FileNotFoundError(f"PCam: expected file not found: {p}") + + self.split_path = os.path.join(self.dataset_dir, 'split_custom_PCam.json') + self.split_fewshot_dir = os.path.join(self.dataset_dir, 'split_fewshot') + mkdir_if_missing(self.split_fewshot_dir) + + random.seed(seed) + np.random.seed(seed) + + if os.path.exists(self.split_path): + train, val, test = self.read_split(self.split_path) + else: + trainval = self._read_train(self.train_x_path, self.train_y_path) + test = self._read_test(self.test_x_path, self.test_y_path) + train, val = self.split_trainval(trainval, p_val=val_ratio) + self.save_split(train, val, test, self.split_path) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, 'rb') as f: + data = pickle.load(f) + train, val = data['train'], data['val'] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {'train': train, 'val': val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, 'wb') as f: + pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL) + + # Optional class subsampling (kept for consistency with other datasets) + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes) + + # Templates per user specification + self.templates = PCAM_TEMPLATES + + # Debug: print split sizes and label histograms + def _hist(items): + from collections import Counter + cnt = Counter([it.label for it in items]) + # ensure keys present for both binary classes + out = {i: int(cnt.get(i, 0)) for i in range(len(PCAM_CLASSES))} + return out + + super().__init__(train_x=train, val=val, test=test) + # Ensure stable binary classification metadata + self._classnames = PCAM_CLASSES + self._lab2cname = {i: c for i, c in enumerate(PCAM_CLASSES)} + self._num_classes = len(PCAM_CLASSES) + + @staticmethod + def _first_key(h5_path: str) -> str: + with h5py.File(h5_path, 'r') as f: + keys = list(f.keys()) + if not keys: + raise RuntimeError(f"No datasets found in H5 file: {h5_path}") + return keys[0] + + @staticmethod + def _read_labels(h5_path: str) -> np.ndarray: + key = PCam._first_key(h5_path) + with h5py.File(h5_path, 'r') as f: + y = f[key][...] + y = np.asarray(y).squeeze() + y = y.astype(np.int64) + return y + + def _read_train(self, x_path: str, y_path: str) -> List[Datum]: + x_key = self._first_key(x_path) + y = self._read_labels(y_path) + items: List[Datum] = [] + for i, label in enumerate(y.tolist()): + label_i = int(label) + classname = PCAM_CLASSES[label_i] + # Lazy image reference: ('h5', abs_h5_path, dataset_key, index) + abs_path = os.path.abspath(x_path) + impath = ('h5', abs_path, x_key, i) + items.append(Datum(impath=impath, label=label_i, classname=classname)) + return items + + def _read_test(self, x_path: str, y_path: str) -> List[Datum]: + x_key = self._first_key(x_path) + y = self._read_labels(y_path) + items: List[Datum] = [] + for i, label in enumerate(y.tolist()): + label_i = int(label) + classname = PCAM_CLASSES[label_i] + abs_path = os.path.abspath(x_path) + impath = ('h5', abs_path, x_key, i) + items.append(Datum(impath=impath, label=label_i, classname=classname)) + return items + + @staticmethod + def split_trainval(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]: + from collections import defaultdict + p_trn = 1 - p_val + print(f"Splitting PCam train into {p_trn:.0%} train and {p_val:.0%} val") + tracker = defaultdict(list) + for idx, item in enumerate(trainval): + tracker[item.label].append(idx) + train, val = [], [] + for _, idxs in tracker.items(): + n_val = max(1, round(len(idxs) * p_val)) + random.shuffle(idxs) + for n, i in enumerate(idxs): + if n < n_val: + val.append(trainval[i]) + else: + train.append(trainval[i]) + return train, val + + def save_split(self, train: List[Datum], val: List[Datum], test: List[Datum], filepath: str): + def _ser(items: List[Datum]): + out = [] + for it in items: + impath = it.impath + if isinstance(impath, tuple) and len(impath) == 4 and impath[0] == 'h5': + tag, abs_path, key, idx = impath + # store relative path for portability + rel = os.path.relpath(abs_path, self.dataset_dir) + impath_ser = [tag, rel, key, int(idx)] + else: + raise ValueError('PCam expects H5 tuple paths') + out.append((impath_ser, int(it.label), it.classname)) + return out + split = { + 'train': _ser(train), + 'val': _ser(val), + 'test': _ser(test), + } + write_json(split, filepath) + print(f"Saved PCam split to {filepath}") + + def read_split(self, filepath: str): + def _deser(items): + out = [] + for impath_ser, label, classname in items: + if isinstance(impath_ser, (list, tuple)) and len(impath_ser) == 4 and impath_ser[0] == 'h5': + tag, rel, key, idx = impath_ser + fpath = os.path.join(self.dataset_dir, rel) + impath = (tag, fpath, key, int(idx)) + else: + raise ValueError('PCam split contains invalid path entries') + out.append(Datum(impath=impath, label=int(label), classname=classname)) + return out + split = read_json(filepath) + train = _deser(split['train']) + val = _deser(split['val']) + test = _deser(split['test']) + return train, val, test diff --git a/MTIL_datasets/resisc.py b/MTIL_datasets/resisc.py new file mode 100644 index 0000000000000000000000000000000000000000..21383b5f1432a5a512806bc079a7b6933c4aa784 --- /dev/null +++ b/MTIL_datasets/resisc.py @@ -0,0 +1,199 @@ +import os +import pickle +import random +import warnings + +from .utils import * +from .oxford_pets import OxfordPets + +# Canonical class list for RESISC45 (order defines label ids 0..44) +RESISC_CLASSES = [ + 'airplane', + 'airport', + 'baseball diamond', + 'basketball court', + 'beach', + 'bridge', + 'chaparral', + 'church', + 'circular farmland', + 'cloud', + 'commercial area', + 'dense residential', + 'desert', + 'forest', + 'freeway', + 'golf course', + 'ground track field', + 'harbor', + 'industrial area', + 'intersection', + 'island', + 'lake', + 'meadow', + 'medium residential', + 'mobile home park', + 'mountain', + 'overpass', + 'palace', + 'parking lot', + 'railway', + 'railway station', + 'rectangular farmland', + 'river', + 'roundabout', + 'runway', + 'sea ice', + 'ship', + 'snowberg', + 'sparse residential', + 'stadium', + 'storage tank', + 'tennis court', + 'terrace', + 'thermal power station', + 'wetland', +] + +# Prompt templates (strings) as provided +RESISC_TEMPLATES = [ + 'satellite imagery of {}.', + 'aerial imagery of {}.', + 'satellite photo of {}.', + 'aerial photo of {}.', + 'satellite view of {}.', + 'aerial view of {}.', + 'satellite imagery of a {}.', + 'aerial imagery of a {}.', + 'satellite photo of a {}.', + 'aerial photo of a {}.', + 'satellite view of a {}.', + 'aerial view of a {}.', + 'satellite imagery of the {}.', + 'aerial imagery of the {}.', + 'satellite photo of the {}.', + 'aerial photo of the {}.', + 'satellite view of the {}.', + 'aerial view of the {}.', +] + +RESISC_DEBUG = os.environ.get("RESISC_DEBUG", "0") not in ("0", "false", "False", "") + +def _dbg(msg: str): + if RESISC_DEBUG: + print(f"[RESISC45][DEBUG] {msg}") + + +def _norm_name(s: str) -> str: + s = s.lower().strip() + for ch in [" ", "_", "-", "."]: + s = s.replace(ch, "") + return s + + +class RESISC45(DatasetBase): + + dataset_dir = "resisc45" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = self.dataset_dir # images are under class subfolders directly + self.split_path = os.path.join(self.dataset_dir, "split_custom_RESISC45.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + _dbg(f"dataset_dir={self.dataset_dir}") + + if os.path.exists(self.split_path): + try: + train, val, test = OxfordPets.read_split(self.split_path, self.image_dir) + except Exception as e: + warnings.warn(f"RESISC45: failed to read split file '{self.split_path}'; rebuilding. Error: {e}") + train, val, test = self.read_and_split_data(self.image_dir) + try: + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + except Exception as e2: + warnings.warn(f"RESISC45: failed to save rebuilt split to '{self.split_path}': {e2}") + else: + train, val, test = self.read_and_split_data(self.image_dir) + try: + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + except Exception as e: + warnings.warn(f"RESISC45: failed to save split to '{self.split_path}': {e}") + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes) + self.templates = RESISC_TEMPLATES + super().__init__(train_x=train, val=val, test=test) + + @staticmethod + def read_and_split_data(image_dir, p_trn=0.5, p_val=0.2): + try: + categories = listdir_nohidden(image_dir, sort=True) + except Exception as e: + warnings.warn(f"RESISC45: failed to list directory '{image_dir}': {e}") + raise + categories = [c for c in categories if os.path.isdir(os.path.join(image_dir, c))] + cat_norm = {_norm_name(c): c for c in categories} + + missing = [] + class_to_dir = {} + for cname in RESISC_CLASSES: + key = _norm_name(cname) + if key in cat_norm: + class_to_dir[cname] = cat_norm[key] + else: + missing.append(cname) + if missing: + warnings.warn(f"RESISC45: missing class folders for {len(missing)} classes: {missing[:5]}{' ...' if len(missing)>5 else ''}") + + def _collate(paths, y, cname): + return [Datum(impath=p, label=y, classname=cname) for p in paths] + + train, val, test = [], [], [] + for y, cname in enumerate(RESISC_CLASSES): + if cname not in class_to_dir: + continue + cdir = os.path.join(image_dir, class_to_dir[cname]) + try: + images = listdir_nohidden(cdir, sort=False) + except Exception as e: + warnings.warn(f"RESISC45: failed to list class folder '{cdir}': {e}") + images = [] + images = [os.path.join(cdir, im) for im in images] + random.shuffle(images) + n_total = len(images) + if n_total == 0: + warnings.warn(f"RESISC45: empty class folder {cdir}; skipping") + continue + n_train = round(n_total * p_trn) + n_val = round(n_total * p_val) + n_test = n_total - n_train - n_val + if not (n_train > 0 and n_val > 0 and n_test > 0): + # Fallback to keep all splits non-empty where possible + if n_total >= 3: + n_train, n_val, n_test = 1, 1, n_total - 2 + elif n_total == 2: + n_train, n_val, n_test = 1, 1, 0 + else: # 1 + n_train, n_val, n_test = 1, 0, 0 + + train.extend(_collate(images[:n_train], y, cname)) + val.extend(_collate(images[n_train:n_train + n_val], y, cname)) + test.extend(_collate(images[n_train + n_val:], y, cname)) + + return train, val, test diff --git a/MTIL_datasets/sst2.py b/MTIL_datasets/sst2.py new file mode 100644 index 0000000000000000000000000000000000000000..a61368c0dd021e83e3e914a305ac9e7ae72b5ac7 --- /dev/null +++ b/MTIL_datasets/sst2.py @@ -0,0 +1,122 @@ +import os +import pickle + +from .utils import * +from .oxford_pets import OxfordPets +from .dtd import DescribableTextures as DTD + + +class SST2(DatasetBase): + """ + SST2 (rendered) dataset loader for MTIL. + + Expected structure (either of the following roots): + /sst2/ + ├─ train/{negative,positive}/*.png + ├─ valid/{negative,positive}/*.png + └─ test/{negative,positive}/*.png + + /rendered-sst2/ (alias supported) + ├─ train/{negative,positive}/*.png + ├─ valid/{negative,positive}/*.png + └─ test/{negative,positive}/*.png + + Classes: ['negative', 'positive'] + Templates: ['a {} review of a movie.'] + """ + + dataset_dir = "sst2" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + primary_dir = os.path.join(root, self.dataset_dir) + alt_dir = os.path.join(root, "rendered-sst2") + + if os.path.isdir(primary_dir): + self.dataset_dir = primary_dir + elif os.path.isdir(alt_dir): + self.dataset_dir = alt_dir + else: + raise ValueError( + "SST2: dataset folder not found. Expected one of: '{}' or '{}'".format(primary_dir, alt_dir) + ) + + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + train_dir = os.path.join(self.dataset_dir, "train") + valid_dir = os.path.join(self.dataset_dir, "valid") + test_dir = os.path.join(self.dataset_dir, "test") + + use_folder_splits = os.path.isdir(train_dir) and os.path.isdir(valid_dir) and os.path.isdir(test_dir) + + # fixed class order and validation + classes = ["negative", "positive"] + class_to_label = {c: i for i, c in enumerate(classes)} + + if use_folder_splits: + train = self._read_split_dir(train_dir, class_to_label) + val = self._read_split_dir(valid_dir, class_to_label) + test = self._read_split_dir(test_dir, class_to_label) + else: + # Fallbacks consistent with other datasets: look for a JSON split, else naive split + image_dir = os.path.join(self.dataset_dir, "images") + self.image_dir = image_dir if os.path.isdir(image_dir) else self.dataset_dir + split_path = os.path.join(self.dataset_dir, "split_zhou_SST2.json") + if os.path.exists(split_path): + train, val, test = OxfordPets.read_split(split_path, self.image_dir) + else: + train, val, test = DTD.read_and_split_data(self.image_dir) + OxfordPets.save_split(train, val, test, split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + # Template per user spec + self.templates = [ + lambda c: f'a {c} review of a movie.', + ] + + super().__init__(train_x=train, val=val, test=test) + + def _read_split_dir(self, split_dir, class_to_label): + items = [] + if not os.path.isdir(split_dir): + return items + codes = listdir_nohidden(split_dir) + # Validate folders: must be subset of expected classes and cover at least one class + unexpected = sorted([c for c in codes if c not in class_to_label]) + if unexpected: + raise ValueError( + f"SST2: Found unexpected class folders in '{split_dir}': {unexpected}. " + f"Expected only {list(class_to_label.keys())}." + ) + for cls in codes: + class_dir = os.path.join(split_dir, cls) + if not os.path.isdir(class_dir): + continue + label = class_to_label.get(cls) + if label is None: + raise ValueError( + f"SST2: Inconsistent label mapping for class '{cls}' in split '{split_dir}'." + ) + cname = cls + for fname in listdir_nohidden(class_dir): + impath = os.path.join(class_dir, fname) + items.append(Datum(impath=impath, label=label, classname=cname)) + return items diff --git a/MTIL_datasets/stanford_cars.py b/MTIL_datasets/stanford_cars.py new file mode 100644 index 0000000000000000000000000000000000000000..d22db094ad9794ba0070c6b266b533ade71d2f1b --- /dev/null +++ b/MTIL_datasets/stanford_cars.py @@ -0,0 +1,83 @@ +import os +import pickle +from scipy.io import loadmat + +from .utils import * + +from .oxford_pets import OxfordPets + + +class StanfordCars(DatasetBase): + + dataset_dir = "stanford_cars" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.split_path = os.path.join(self.dataset_dir, "split_zhou_StanfordCars.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir) + else: + trainval_file = os.path.join(self.dataset_dir, "devkit", "cars_train_annos.mat") + test_file = os.path.join(self.dataset_dir, "cars_test_annos_withlabels.mat") + meta_file = os.path.join(self.dataset_dir, "devkit", "cars_meta.mat") + trainval = self.read_data("cars_train", trainval_file, meta_file) + test = self.read_data("cars_test", test_file, meta_file) + train, val = OxfordPets.split_trainval(trainval) + OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c: f"a photo of a {c}, a type of car.", + lambda c: f"a photo of a {c}.", + lambda c: f"a photo of the {c}.", + lambda c: f"a photo of my {c}.", + lambda c: f"i love my {c}!", + lambda c: f"a photo of my dirty {c}.", + lambda c: f"a photo of my clean {c}.", + lambda c: f"a photo of my new {c}.", + lambda c: f"a photo of my old {c}.", + ] + + super().__init__(train_x=train, val=val, test=test) + + def read_data(self, image_dir, anno_file, meta_file): + anno_file = loadmat(anno_file)["annotations"][0] + meta_file = loadmat(meta_file)["class_names"][0] + items = [] + + for i in range(len(anno_file)): + imname = anno_file[i]["fname"][0] + impath = os.path.join(self.dataset_dir, image_dir, imname) + label = anno_file[i]["class"][0, 0] + label = int(label) - 1 + classname = meta_file[label][0] + names = classname.split(" ") + year = names.pop(-1) + names.insert(0, year) + classname = " ".join(names) + item = Datum(impath=impath, label=label, classname=classname) + items.append(item) + + return items diff --git a/MTIL_datasets/stl10.py b/MTIL_datasets/stl10.py new file mode 100644 index 0000000000000000000000000000000000000000..9113fac43dc4565852bb5d360d906bcb94e5f086 --- /dev/null +++ b/MTIL_datasets/stl10.py @@ -0,0 +1,143 @@ +import os +import math +import random +from typing import List, Tuple + +from .utils import * # Datum, DatasetBase, listdir_nohidden +from .oxford_pets import OxfordPets + +STL10_CLASSES: List[str] = [ + 'airplane', + 'bird', + 'car', + 'cat', + 'deer', + 'dog', + 'horse', + 'monkey', + 'ship', + 'truck', +] + +# keep templates as strings with {} +STL10_TEMPLATES: List[str] = [ + 'a photo of a {}.', + 'a photo of the {}.', +] + + +class STL10(DatasetBase): + + dataset_dir = "stl10" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all', test_ratio: float = 0.2): + """ + Expect directory structure: + {root}/stl10//*.png|jpg|jpeg|bmp|webp + No official test set provided; we split per-class into train/test. + Then we further split train into train/val using OxfordPets.split_trainval. + """ + rnd = random.Random(seed) + + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.split_path = os.path.join(self.dataset_dir, "split_custom_STL10.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + # If we have a saved split, reuse for reproducibility + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir) + else: + trainval, test = self._read_from_folders(self.dataset_dir, rnd=rnd, test_ratio=test_ratio) + train, val = self._split_trainval_safe(trainval) + OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + import pickle + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + import pickle + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes) + + self.templates = STL10_TEMPLATES + + super().__init__(train_x=train, val=val, test=test) + + def _read_from_folders(self, data_dir: str, rnd: random.Random, test_ratio: float) -> Tuple[List[Datum], List[Datum]]: + exts = {".png", ".jpg", ".jpeg", ".bmp", ".webp"} + class_to_label = {name: i for i, name in enumerate(STL10_CLASSES)} + + items_by_label = {i: [] for i in range(len(STL10_CLASSES))} + for cname in STL10_CLASSES: + cdir = os.path.join(data_dir, cname) + if not os.path.isdir(cdir): + raise FileNotFoundError(f"Class folder not found: {cdir}") + for fname in listdir_nohidden(cdir, sort=True): + fext = os.path.splitext(fname)[1].lower() + if fext not in exts: + continue + impath = os.path.join(cdir, fname) + label = class_to_label[cname] + items_by_label[label].append(Datum(impath=impath, label=label, classname=cname)) + + trainval, test = [], [] + for label, items in items_by_label.items(): + if not items: + continue + rnd.shuffle(items) + if len(items) == 1: + # keep the only sample for training to preserve class presence in train + trainval.extend(items) + continue + n_test = max(1, int(round(len(items) * test_ratio))) + if n_test >= len(items): + n_test = len(items) - 1 + test.extend(items[:n_test]) + trainval.extend(items[n_test:]) + + return trainval, test + + @staticmethod + def _split_trainval_safe(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]: + from collections import defaultdict + tracker = defaultdict(list) + for idx, item in enumerate(trainval): + tracker[item.label].append(idx) + + train, val = [], [] + for label, idxs in tracker.items(): + n = len(idxs) + if n <= 1: + # not enough to create a val sample for this class + for idx in idxs: + train.append(trainval[idx]) + continue + n_val = max(1, int(round(n * p_val))) + if n_val >= n: + n_val = n - 1 + random.shuffle(idxs) + for i, idx in enumerate(idxs): + if i < n_val: + val.append(trainval[idx]) + else: + train.append(trainval[idx]) + + # If val ended up empty (degenerate tiny dataset), move one from train to val + if len(val) == 0 and len(train) > 0: + val.append(train[-1]) + train = train[:-1] + + return train, val diff --git a/MTIL_datasets/sun397.py b/MTIL_datasets/sun397.py new file mode 100644 index 0000000000000000000000000000000000000000..d4631b94e4e4acc1bcc9e8e590812bb2ac46ff16 --- /dev/null +++ b/MTIL_datasets/sun397.py @@ -0,0 +1,81 @@ +import os +import pickle + +from .utils import * + +from .oxford_pets import OxfordPets + + +class SUN397(DatasetBase): + + dataset_dir = "sun397" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = os.path.join(self.dataset_dir, "SUN397") + self.split_path = os.path.join(self.dataset_dir, "split_zhou_SUN397.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + if os.path.exists(self.split_path): + train, val, test = OxfordPets.read_split(self.split_path, self.image_dir) + else: + classnames = [] + with open(os.path.join(self.dataset_dir, "ClassName.txt"), "r") as f: + lines = f.readlines() + for line in lines: + line = line.strip()[1:] + classnames.append(line) + cname2lab = {c: i for i, c in enumerate(classnames)} + trainval = self.read_data(cname2lab, "Training_01.txt") + test = self.read_data(cname2lab, "Testing_01.txt") + train, val = OxfordPets.split_trainval(trainval) + OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + subsample = subsample_classes + train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) + + self.templates = [ + lambda c: f"a photo of a {c}.", + lambda c: f"a photo of the {c}.", + ] + + super().__init__(train_x=train, val=val, test=test) + + def read_data(self, cname2lab, text_file): + text_file = os.path.join(self.dataset_dir, text_file) + items = [] + + with open(text_file, "r") as f: + lines = f.readlines() + for line in lines: + imname = line.strip()[1:] + classname = os.path.dirname(imname) + label = cname2lab[classname] + impath = os.path.join(self.image_dir, imname) + + names = classname.split("/")[1:] + names = names[::-1] + classname = " ".join(names) + + item = Datum(impath=impath, label=label, classname=classname) + items.append(item) + + return items diff --git a/MTIL_datasets/ucf101.py b/MTIL_datasets/ucf101.py new file mode 100644 index 0000000000000000000000000000000000000000..404d0354f9549ee8f5b129af8d13f79403152f27 --- /dev/null +++ b/MTIL_datasets/ucf101.py @@ -0,0 +1,360 @@ +import os +import pickle +import random +import math +from collections import defaultdict +from typing import List, Tuple + +from .utils import * # Datum, DatasetBase, mkdir_if_missing, read_json, write_json, listdir_nohidden + + +# Canonical class names as provided +UCF101_CANONICAL: List[str] = [ + 'Apply Eye Makeup', + 'Apply Lipstick', + 'Archery', + 'Baby Crawling', + 'Balance Beam', + 'Band Marching', + 'Baseball Pitch', + 'Basketball', + 'Basketball Dunk', + 'Bench Press', + 'Biking', + 'Billiards', + 'Blow Dry Hair', + 'Blowing Candles', + 'Body Weight Squats', + 'Bowling', + 'Boxing Punching Bag', + 'Boxing Speed Bag', + 'Breast Stroke', + 'Brushing Teeth', + 'Clean And Jerk', + 'Cliff Diving', + 'Cricket Bowling', + 'Cricket Shot', + 'Cutting In Kitchen', + 'Diving', + 'Drumming', + 'Fencing', + 'Field Hockey Penalty', + 'Floor Gymnastics', + 'Frisbee Catch', + 'Front Crawl', + 'Golf Swing', + 'Haircut', + 'Hammer Throw', + 'Hammering', + 'Hand Stand Pushups', + 'Handstand Walking', + 'Head Massage', + 'High Jump', + 'Horse Race', + 'Horse Riding', + 'Hula Hoop', + 'Ice Dancing', + 'Javelin Throw', + 'Juggling Balls', + 'Jump Rope', + 'Jumping Jack', + 'Kayaking', + 'Knitting', + 'Long Jump', + 'Lunges', + 'Military Parade', + 'Mixing', + 'Mopping Floor', + 'Nunchucks', + 'Parallel Bars', + 'Pizza Tossing', + 'Playing Cello', + 'Playing Daf', + 'Playing Dhol', + 'Playing Flute', + 'Playing Guitar', + 'Playing Piano', + 'Playing Sitar', + 'Playing Tabla', + 'Playing Violin', + 'Pole Vault', + 'Pommel Horse', + 'Pull Ups', + 'Punch', + 'Push Ups', + 'Rafting', + 'Rock Climbing Indoor', + 'Rope Climbing', + 'Rowing', + 'Salsa Spin', + 'Shaving Beard', + 'Shotput', + 'Skate Boarding', + 'Skiing', + 'Skijet', + 'Sky Diving', + 'Soccer Juggling', + 'Soccer Penalty', + 'Still Rings', + 'Sumo Wrestling', + 'Surfing', + 'Swing', + 'Table Tennis Shot', + 'Tai Chi', + 'Tennis Swing', + 'Throw Discus', + 'Trampoline Jumping', + 'Typing', + 'Uneven Bars', + 'Volleyball Spiking', + 'Walking With Dog', + 'Wall Pushups', + 'Writing On Board', + 'Yo Yo', +] + + +UCF101_TEMPLATES: List[str] = [ + 'a photo of a person {}.', + 'a video of a person {}.', + 'a example of a person {}.', + 'a demonstration of a person {}.', + 'a photo of the person {}.', + 'a video of the person {}.', + 'a example of the person {}.', + 'a demonstration of the person {}.', + 'a photo of a person using {}.', + 'a video of a person using {}.', + 'a example of a person using {}.', + 'a demonstration of a person using {}.', + 'a photo of the person using {}.', + 'a video of the person using {}.', + 'a example of the person using {}.', + 'a demonstration of the person using {}.', + 'a photo of a person doing {}.', + 'a video of a person doing {}.', + 'a example of a person doing {}.', + 'a demonstration of a person doing {}.', + 'a photo of the person doing {}.', + 'a video of the person doing {}.', + 'a example of the person doing {}.', + 'a demonstration of the person doing {}.', + 'a photo of a person during {}.', + 'a video of a person during {}.', + 'a example of a person during {}.', + 'a demonstration of a person during {}.', + 'a photo of the person during {}.', + 'a video of the person during {}.', + 'a example of the person during {}.', + 'a demonstration of the person during {}.', + 'a photo of a person performing {}.', + 'a video of a person performing {}.', + 'a example of a person performing {}.', + 'a demonstration of a person performing {}.', + 'a photo of the person performing {}.', + 'a video of the person performing {}.', + 'a example of the person performing {}.', + 'a demonstration of the person performing {}.', + 'a photo of a person practicing {}.', + 'a video of a person practicing {}.', + 'a example of a person practicing {}.', + 'a demonstration of a person practicing {}.', + 'a photo of the person practicing {}.', + 'a video of the person practicing {}.', + 'a example of the person practicing {}.', + 'a demonstration of the person practicing {}.', +] + + +class UCF101(DatasetBase): + """ + UCF101 midframes classification dataset adapter for MTIL. + Expected structure: + root/ucf101/ + UCF-101-midframes//.jpg + split_zhou_UCF101.json + """ + + dataset_dir = "ucf101" + + def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'): + root = os.path.abspath(os.path.expanduser(root)) + self.dataset_dir = os.path.join(root, self.dataset_dir) + self.image_dir = os.path.join(self.dataset_dir, "UCF-101-midframes") + self.split_path = os.path.join(self.dataset_dir, "split_zhou_UCF101.json") + self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") + mkdir_if_missing(self.split_fewshot_dir) + + if os.path.exists(self.split_path): + train, val, test = self.read_split(self.split_path, self.image_dir) + else: + # Fallback: build from directory and split train/val; use val as test too. + trainval = self._read_from_dir() + train, val = self.split_trainval(trainval) + test = list(val) + self.save_split(train, val, test, self.split_path, self.image_dir) + + if num_shots >= 1: + preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") + if os.path.exists(preprocessed): + print(f"Loading preprocessed few-shot data from {preprocessed}") + with open(preprocessed, "rb") as file: + data = pickle.load(file) + train, val = data["train"], data["val"] + else: + train = self.generate_fewshot_dataset(train, num_shots=num_shots) + val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) + data = {"train": train, "val": val} + print(f"Saving preprocessed few-shot data to {preprocessed}") + with open(preprocessed, "wb") as file: + pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) + + # Optionally subsample classes (keep interface consistent) + train, val, test = self.subsample_classes(train, val, test, subsample=subsample_classes) + + self.templates = UCF101_TEMPLATES + + super().__init__(train_x=train, val=val, test=test) + + @staticmethod + def subsample_classes(*args, subsample="all"): + assert subsample in ["all", "base", "new"] + + if subsample == "all": + return args + + dataset = args[0] + labels = set() + for item in dataset: + labels.add(item.label) + labels = list(labels) + labels.sort() + n = len(labels) + m = math.ceil(n / 2) + + print(f"SUBSAMPLE {subsample.upper()} CLASSES!") + if subsample == "base": + selected = labels[:m] + else: + selected = labels[m:] + relabeler = {y: y_new for y_new, y in enumerate(selected)} + + output = [] + for dataset in args: + dataset_new = [] + for item in dataset: + if item.label not in selected: + continue + item_new = Datum( + impath=item.impath, + label=relabeler[item.label], + classname=item.classname + ) + dataset_new.append(item_new) + output.append(dataset_new) + + return output + + # ---- IO helpers (compatible with OxfordPets) ---- + @staticmethod + def save_split(train, val, test, filepath, path_prefix): + def _extract(items): + out = [] + for item in items: + impath = item.impath + label = item.label + classname = item.classname + impath = impath.replace(path_prefix, "") + if impath.startswith("/"): + impath = impath[1:] + out.append((impath, label, classname)) + return out + + train = _extract(train) + val = _extract(val) + test = _extract(test) + split = {"train": train, "val": val, "test": test} + write_json(split, filepath) + print(f"Saved split to {filepath}") + + @staticmethod + def read_split(filepath, path_prefix): + def _convert(items): + out = [] + for impath, label, classname in items: + impath = os.path.join(path_prefix, impath) + item = Datum(impath=impath, label=int(label), classname=classname) + out.append(item) + return out + + print(f"Reading split from {filepath}") + split = read_json(filepath) + train = _convert(split["train"]) + val = _convert(split["val"]) + test = _convert(split["test"]) + return train, val, test + + @staticmethod + def split_trainval(trainval: List[Datum], p_val=0.2) -> Tuple[List[Datum], List[Datum]]: + p_trn = 1 - p_val + tracker = defaultdict(list) + for idx, item in enumerate(trainval): + tracker[item.label].append(idx) + train, val = [], [] + for label, idxs in tracker.items(): + n_val = max(1, round(len(idxs) * p_val)) + random.shuffle(idxs) + for n, idx in enumerate(idxs): + item = trainval[idx] + if n < n_val: + val.append(item) + else: + train.append(item) + return train, val + + # ---- Directory reader (fallback if JSON is missing) ---- + def _read_from_dir(self) -> List[Datum]: + if not os.path.isdir(self.image_dir): + raise FileNotFoundError(f"Image directory not found: {self.image_dir}") + + # Build canonical key mapping (lowercased, remove spaces/underscores) + def canon_key(s: str) -> str: + return ''.join(ch for ch in s.lower() if ch.isalnum()) + + canonical_to_label = {name: idx for idx, name in enumerate(UCF101_CANONICAL)} + key_to_canonical = {canon_key(name): name for name in UCF101_CANONICAL} + + items: List[Datum] = [] + class_dirs = listdir_nohidden(self.image_dir, sort=True) + for cls_dir in class_dirs: + cls_path = os.path.join(self.image_dir, cls_dir) + if not os.path.isdir(cls_path): + continue + # Try to map folder name to canonical class + k = canon_key(cls_dir.replace('_', ' ')) + cname = key_to_canonical.get(k, None) + if cname is None: + # Try removing underscores without space + k2 = canon_key(cls_dir.replace('_', '')) + cname = key_to_canonical.get(k2, None) + if cname is None: + # As a last resort, use the folder name with underscores replaced + cname = cls_dir.replace('_', ' ').strip() + if cname not in canonical_to_label: + # Unknown class; skip + continue + label = canonical_to_label[cname] + # Collect images + for vid in listdir_nohidden(cls_path, sort=False): + vpath = os.path.join(cls_path, vid) + if os.path.isdir(vpath): + # some datasets might nest frames under video folder; include all frames + for frame in listdir_nohidden(vpath, sort=False): + impath = os.path.join(vpath, frame) + if os.path.isfile(impath): + items.append(Datum(impath=impath, label=label, classname=cname)) + else: + # direct frames under class folder + if os.path.isfile(vpath): + items.append(Datum(impath=vpath, label=label, classname=cname)) + return items diff --git a/MTIL_datasets/utils.py b/MTIL_datasets/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..23fb044a1622f006642261770e375046557cb7af --- /dev/null +++ b/MTIL_datasets/utils.py @@ -0,0 +1,299 @@ +import os +import random +import os.path as osp +import tarfile +import zipfile +from collections import defaultdict +import gdown +import errno +import warnings +import json +import numpy as np +import h5py +from torch.utils.data import Dataset as TorchDataset +from PIL import Image + + +class Datum: + def __init__(self, impath="", label=0, domain=0, classname=""): + self._impath = impath + self._label = label + self._domain = domain + self._classname = classname + + @property + def impath(self): + return self._impath + + @property + def label(self): + return self._label + + @property + def domain(self): + return self._domain + + @property + def classname(self): + return self._classname + + +class DatasetBase: + dataset_dir = "" + domains = [] + + def __init__(self, train_x=None, train_u=None, val=None, test=None): + self._train_x = train_x + self._train_u = train_u + self._val = val + self._test = test + self._num_classes = self.get_num_classes(train_x) + self._lab2cname, self._classnames = self.get_lab2cname(train_x) + + @property + def train_x(self): + return self._train_x + + @property + def train_u(self): + return self._train_u + + @property + def val(self): + return self._val + + @property + def test(self): + return self._test + + @property + def lab2cname(self): + return self._lab2cname + + @property + def classnames(self): + return self._classnames + + @property + def num_classes(self): + return self._num_classes + + @staticmethod + def get_num_classes(data_source): + label_set = set() + for item in data_source: + label_set.add(item.label) + return max(label_set) + 1 + + @staticmethod + def get_lab2cname(data_source): + container = set() + for item in data_source: + container.add((item.label, item.classname)) + mapping = {label: classname for label, classname in container} + labels = list(mapping.keys()) + labels.sort() + classnames = [mapping[label] for label in labels] + return mapping, classnames + + @property + def template(self): + return self.templates[0] + + def check_input_domains(self, source_domains, target_domains): + assert len(source_domains) > 0, "source_domains (list) is empty" + assert len(target_domains) > 0, "target_domains (list) is empty" + self.is_input_domain_valid(source_domains) + self.is_input_domain_valid(target_domains) + + def is_input_domain_valid(self, input_domains): + for domain in input_domains: + if domain not in self.domains: + raise ValueError( + "Input domain must belong to {}, " + "but got [{}]".format(self.domains, domain) + ) + + def download_data(self, url, dst, from_gdrive=True): + if not osp.exists(osp.dirname(dst)): + os.makedirs(osp.dirname(dst)) + + if from_gdrive: + gdown.download(url, dst, quiet=False) + else: + raise NotImplementedError + + print("Extracting file ...") + + if dst.endswith(".zip"): + zip_ref = zipfile.ZipFile(dst, "r") + zip_ref.extractall(osp.dirname(dst)) + zip_ref.close() + + elif dst.endswith(".tar"): + tar = tarfile.open(dst, "r:") + tar.extractall(osp.dirname(dst)) + tar.close() + + elif dst.endswith(".tar.gz"): + tar = tarfile.open(dst, "r:gz") + tar.extractall(osp.dirname(dst)) + tar.close() + + else: + raise NotImplementedError + + print("File extracted to {}".format(osp.dirname(dst))) + + def generate_fewshot_dataset( + self, *data_sources, num_shots=-1, repeat=False + ): + if num_shots < 1: + if len(data_sources) == 1: + return data_sources[0] + return data_sources + + print(f"Creating a {num_shots}-shot dataset") + + output = [] + + for data_source in data_sources: + tracker = self.split_dataset_by_label(data_source) + dataset = [] + + for label, items in tracker.items(): + if len(items) >= num_shots: + sampled_items = random.sample(items, num_shots) + else: + if repeat: + sampled_items = random.choices(items, k=num_shots) + else: + sampled_items = items + dataset.extend(sampled_items) + + output.append(dataset) + + if len(output) == 1: + return output[0] + + return output + + def split_dataset_by_label(self, data_source): + output = defaultdict(list) + + for item in data_source: + output[item.label].append(item) + + return output + + def split_dataset_by_domain(self, data_source): + output = defaultdict(list) + + for item in data_source: + output[item.domain].append(item) + + return output + + +class DatasetWrapper(TorchDataset): + + def __init__(self, data_source, transform=None, is_train=False): + self.data_source = data_source + self.transform = transform + self.is_train = is_train + self._h5_cache = {} + self._h5_info_printed = set() + + def __len__(self): + return len(self.data_source) + + def __getitem__(self, idx): + item = self.data_source[idx] + + impath = item.impath + if isinstance(impath, str): + img0 = Image.open(impath).convert("RGB") + elif isinstance(impath, tuple) and len(impath) == 4 and impath[0] == 'h5': + _, fpath, key, index = impath + f = self._h5_cache.get(fpath) + if f is None: + f = h5py.File(fpath, 'r') + self._h5_cache[fpath] = f + # Print file info once + try: + keys = list(f.keys()) + print(f"H5 open: {os.path.basename(fpath)} keys={keys[:5]}{'...' if len(keys) > 5 else ''}") + except Exception as e: + print(f"H5 open (keys) failed for {fpath}: {e}") + # Print dataset info per file the first time we see this path + if fpath not in self._h5_info_printed: + try: + ds = f[key] + print(f"H5 dataset: {os.path.basename(fpath)}[{key}] shape={getattr(ds, 'shape', '?')} dtype={getattr(ds, 'dtype', '?')}") + except Exception as e: + print(f"H5 dataset info failed for {fpath}[{key}]: {e}") + self._h5_info_printed.add(fpath) + arr = f[key][int(index)] + arr = np.asarray(arr) + # Convert CHW -> HWC if needed + if arr.ndim == 3 and arr.shape[0] in (1, 3) and arr.shape[-1] not in (1, 3): + arr = np.transpose(arr, (1, 2, 0)) + # Ensure HWC and uint8 + if arr.ndim == 3 and arr.shape[-1] in (1, 3): + pass + else: + raise ValueError(f"Unexpected H5 image shape: {arr.shape}") + if arr.dtype != np.uint8: + arr = arr.astype(np.uint8) + if arr.shape[-1] == 1: + img0 = Image.fromarray(arr.squeeze(-1), mode='L').convert('RGB') + else: + img0 = Image.fromarray(arr, mode='RGB') + else: + # Fallback: if already PIL Image + if isinstance(impath, Image.Image): + img0 = impath + else: + raise ValueError("Unsupported impath type in DatasetWrapper") + + if self.transform: + img = self.transform(img0) + else: + img = img0 + + return img, item.label + + +def check_isfile(fpath): + isfile = osp.isfile(fpath) + if not isfile: + warnings.warn('No file found at "{}"'.format(fpath)) + return isfile + + +def mkdir_if_missing(dirname): + if not osp.exists(dirname): + try: + os.makedirs(dirname) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def listdir_nohidden(path, sort=False): + items = [f for f in os.listdir(path) if not f.startswith(".")] + if sort: + items.sort() + return items + + +def read_json(fpath): + with open(fpath, "r") as f: + obj = json.load(f) + return obj + + +def write_json(obj, fpath): + mkdir_if_missing(osp.dirname(fpath)) + with open(fpath, "w") as f: + json.dump(obj, f, indent=4, separators=(",", ": ")) \ No newline at end of file diff --git a/MTIL_datasets/voc2007.py b/MTIL_datasets/voc2007.py new file mode 100644 index 0000000000000000000000000000000000000000..8c19f6b1a002c401507adee0f5a90b3ac2ad5c42 --- /dev/null +++ b/MTIL_datasets/voc2007.py @@ -0,0 +1,173 @@ +import os +import os.path as osp +from typing import List, Dict, Tuple +from collections import defaultdict + +from .utils import Datum, DatasetBase, listdir_nohidden + + +VOC2007_CLASSES = [ + 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', + 'bus', 'car', 'cat', 'chair', 'cow', + 'dog', 'horse', 'motorbike', 'person', 'sheep', + 'sofa', 'diningtable', 'pottedplant', 'train', 'tvmonitor', +] + +VOC2007_TEMPLATES = [ + 'a photo of a {}.', +] + + +class VOC2007(DatasetBase): + """ + VOC2007 multi-label classification dataset adapter for MTIL evaluation. + + Expects directory structure: + data/VOC2007/ + JPEGImages/ + Main/ + _trainval.txt + _test.txt + + Each *_split.txt has lines: "