diff --git a/Tipsomaly/model/big_vision/datasets/ai2d/ai2d.py b/Tipsomaly/model/big_vision/datasets/ai2d/ai2d.py new file mode 100644 index 0000000000000000000000000000000000000000..1d9f940ea327acf7146b421b5b1d04c0ef9ec59c --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/ai2d/ai2d.py @@ -0,0 +1,209 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +r"""AI2D TFDS converter. + + +It's a small dataset, so can be built locally. Copy the data to local disk: + + mkdir -p /tmp/data/ai2d && cd /tmp/data/ai2d + wget https://ai2-public-datasets.s3.amazonaws.com/diagrams/ai2d-all.zip + wget https://s3-us-east-2.amazonaws.com/prior-datasets/ai2d_test_ids.csv + wget https://github.com/googlefonts/dm-fonts/raw/main/Sans/fonts/ttf/DMSans-Regular.ttf + unzip ai2d-all.zip + +Also download a font for rendering, set the location in the flag font_path. + +Then, run conversion locally (make sure to install tensorflow-datasets for the `tfds` util): + + cd third_party/py/big_vision/datasets + env TFDS_DATA_DIR=/tmp/tfds tfds build --datasets=ai2d + +Example to load: + + import tensorflow_datasets as tfds + dataset = tfds.load(ai2d', split='train', data_dir='/tmp/tfds') +""" + +import functools +import glob +import io +import json +import os +from typing import Any, Dict + +from absl import flags +import numpy as np +from PIL import Image +from PIL import ImageDraw +from PIL import ImageFont +import tensorflow_datasets as tfds + + +_DESCRIPTION = """AI2D dataset.""" + +# pylint: disable=line-too-long +_CITATION = """ +@inproceedings{kembhavi2016eccv, + author = {Aniruddha Kembhavi, Mike Salvato, Eric Kolve, Minjoon Seo, Hannaneh Hajishirzi, Ali Farhadi}, + title = {A Diagram Is Worth A Dozen Images}, + booktitle = {European Conference on Computer Vision (ECCV)}, + year = {2016} + url={https://api.semanticscholar.org/CorpusID:2682274} +} +""" +# pylint: enable=line-too-long + + +_INPUT_PATH = flags.DEFINE_string( + 'input_path', '/tmp/data/ai2d/', 'Downloaded AI2D data.' +) +_FONT_PATH = flags.DEFINE_string( + 'font_path', '/tmp/data/ai2d/DMSans-Regular.ttf', + 'Font for rendering annotations.' +) + + +class Ai2d(tfds.core.GeneratorBasedBuilder): + """DatasetBuilder for AI2D dataset.""" + + VERSION = tfds.core.Version('1.1.0') + RELEASE_NOTES = {'1.1.0': 'Re-create from scratch + more fields.'} + + def _info(self): + """Returns the metadata.""" + return tfds.core.DatasetInfo( + builder=self, + description=_DESCRIPTION, + features=tfds.features.FeaturesDict({ + 'id': tfds.features.Text(), + 'question': tfds.features.Text(), + 'label': tfds.features.Scalar(np.int32), + 'answer': tfds.features.Text(), + 'possible_answers': tfds.features.Sequence(tfds.features.Text()), + 'abc_label': tfds.features.Scalar(np.bool_), + 'image_name': tfds.features.Text(), + 'image': tfds.features.Image(encoding_format='png'), + }), + homepage='https://allenai.org/data/diagrams', + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + """Returns SplitGenerators.""" + return {split: self._generate_examples(split) + for split in ('test', 'train')} + + def _generate_examples(self, split: str): + """Yields (key, example) tuples.""" + with open( + os.path.join(_INPUT_PATH.value, 'ai2d_test_ids.csv'), 'r' + ) as f: + all_test_ids = f.readlines() + all_test_ids = [line.strip() for line in all_test_ids] + + all_annotation_paths = glob.glob( + os.path.join(_INPUT_PATH.value, 'ai2d/questions', '*.json')) + for annotation_path in all_annotation_paths: + basename = os.path.basename(annotation_path) + image_id = basename.split('.')[0] + if image_id in all_test_ids and split == 'train': + continue + elif image_id not in all_test_ids and split == 'test': + continue + + text_annotation_path = os.path.join( + _INPUT_PATH.value, 'ai2d/annotations', basename + ) + with open(annotation_path, 'r') as f: + with open(text_annotation_path, 'r') as g: + question_json = json.load(f) + text_annotation_json = json.load(g) + for question in question_json['questions']: + label_id = int( + question_json['questions'][question]['correctAnswer'] + ) + choices = question_json['questions'][question]['answerTexts'] + abc_label = question_json['questions'][question]['abcLabel'] + annotation = { + 'id': question_json['questions'][question]['questionId'], + 'question': question, + 'label': label_id, + 'answer': choices[label_id], + 'possible_answers': tuple(choices), + 'abc_label': abc_label, + 'image_name': question_json['imageName'], + } + annotation['image'] = _create_image( + annotation, text_annotation_json['text'] + ) + yield annotation['id'], annotation + + +@functools.cache +def Font( # pylint: disable=invalid-name + size: int, +) -> ImageFont.FreeTypeFont: + """Loads the font from in the specified style. + + Args: + size: The size of the returned font. + + Returns: + The loaded font. + """ + return ImageFont.truetype(_FONT_PATH.value, size=size) + + +def _create_image( + annotation: Dict[str, Any], text_annotation: Dict[str, Any] +) -> bytes: + """Adds image to one annotation.""" + img_path = os.path.join(_INPUT_PATH.value, 'ai2d/images', + annotation['image_name']) + with open(img_path, 'rb') as f: + if annotation['abc_label']: + raw_image = _draw_text(f, text_annotation) + else: + raw_image = f.read() + return raw_image + + +def _draw_text(image, text_annotations) -> bytes: + """Replaces text in image by the correct replacement letter from AI2D.""" + image = Image.open(image) + draw = ImageDraw.Draw(image) + for annotation in text_annotations: + current_annotation = text_annotations[annotation] + rectangle = current_annotation['rectangle'] + box = [tuple(rectangle[0]), tuple(rectangle[1]),] + text = current_annotation['replacementText'] + position = box[0] + draw.rectangle(box, fill='white') + font_size = 100 + x_diff = box[1][0] - box[0][0] + y_diff = box[1][1] - box[0][1] + font = Font(font_size) + size = font.getbbox(text) + while (size[2] > x_diff or size[3] > y_diff) and font_size > 0: + font = Font(font_size) + size = font.getbbox(text) + font_size -= 1 + delta = (x_diff - size[2]) // 2 + position = (position[0] + delta, position[1]) + draw.text(position, text, fill='black', font=font) + new_image_bytes = io.BytesIO() + image.save(new_image_bytes, format='PNG') + return new_image_bytes.getvalue() diff --git a/Tipsomaly/model/big_vision/datasets/aokvqa/aokvqa.py b/Tipsomaly/model/big_vision/datasets/aokvqa/aokvqa.py new file mode 100644 index 0000000000000000000000000000000000000000..bbff88f9bd7d9944d4022cd97799c372eebd0e1d --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/aokvqa/aokvqa.py @@ -0,0 +1,182 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +r"""Implements the OKVQA dataset for TFDS. + +Download the required files from https://aokvqa.allenai.org/download.html: + +mkdir -p /tmp/tfds +cd /tmp/tfds/ +wget http://images.cocodataset.org/zips/train2017.zip +wget http://images.cocodataset.org/zips/val2017.zip +wget http://images.cocodataset.org/zips/test2017.zip +wget https://prior-datasets.s3.us-east-2.amazonaws.com/aokvqa/aokvqa_v1p0.tar.gz +unzip val2017.zip +unzip train2017.zip +unzip test2017.zip +tar xzf aokvqa_v1p0.tar.gz + +Then, run conversion locally (make sure to install tensorflow-datasets for the `tfds` util): + + cd big_vision/datasets + env TFDS_DATA_DIR=/tmp/tfds tfds build --datasets=aokvqa + +Example to load: + + import tensorflow_datasets as tfds + dataset = tfds.load('aokvqa', split='val', data_dir='/tmp/tfds') +""" + +import json +import os +from typing import Any +import numpy as np +import tensorflow_datasets as tfds + +_DESCRIPTION = """ +A-OKVQA addresses the task of VQA with outside knowledge. +It is a follow-up dataset of OKVQA. + +This version of the dataset contains: +- Questions + Answers + Multiple Choice Answers + Rationales from A-OKVQA. +- Images from COCO. +""" + +_CITATION = """ +@article{AOKVQA, + title={A-OKVQA: A Benchmark for Visual Question Answering using World Knowledge}, + author={Dustin Schwenk and Apoorv Khandelwal and Christopher Clark and Kenneth Marino and Roozbeh Mottaghi}, + journal={arXiv}, + year={2022}, +} +""" + +ANNOTATION_FILES = { + 'train': 'aokvqa_v1p0_train.json', + 'val': 'aokvqa_v1p0_val.json', + 'test': 'aokvqa_v1p0_test.json', +} + + +# When running locally (recommended), copy files as above an use these: +_AOKVQA_PATH = '/tmp/tfds' + + +class AOkVqa(tfds.core.GeneratorBasedBuilder): + """AOKVQA dataset for TFDS.""" + + VERSION = tfds.core.Version('1.0.0') + RELEASE_NOTES = {'1.0.0': 'ArrayRecord version.'} + MANUAL_DOWNLOAD_INSTRUCTIONS = """ + In manual_dir/ you should have a directory a_ok_vqa which contains the + following files and directories: + From the A-OKVQA dataset: + - aokvqa_v1p0_train.json + - aokvqa_v1p0_val.json + - aokvqa_v1p0_test.json + It also requires the COCO data files. + """ + + def _info(self) -> tfds.core.DatasetInfo: + """Returns the dataset metadata.""" + features = tfds.features.FeaturesDict({ + 'image': tfds.features.Image(shape=(None, None, 3)), + 'image_id': tfds.features.Scalar(dtype=np.int64), + 'direct_answers': tfds.features.Sequence(tfds.features.Text()), + 'direct_answer_is_difficult': tfds.features.Scalar(dtype=np.bool_), + 'multiple_choice_possible_answers': # List of 4 possible answers. + tfds.features.Sequence(tfds.features.Text()), + 'multiple_choice_correct_idx': # Integer from 0-3. + tfds.features.Scalar(dtype=np.int32), + 'answer_rationales': tfds.features.Sequence(tfds.features.Text()), + 'question': tfds.features.Text(), + 'question_id': tfds.features.Text(), + }) + + return tfds.core.DatasetInfo( + builder=self, + features=features, + description=_DESCRIPTION, + supervised_keys=None, + homepage='https://okvqa.allenai.org/', + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager) -> ...: + """Call the function which defines the splits.""" + # data_dir = dl_manager.manual_dir + data_dir = _AOKVQA_PATH + return { + 'train': self._generate_examples(data_dir, 'train'), + 'val': self._generate_examples(data_dir, 'val'), + 'test': self._generate_examples(data_dir, 'test'), + } + + def _generate_examples(self, data_dir: str, split: str) -> ...: + annotations = get_annotations(data_dir, split) + + for question_id, feature_dict in annotations.items(): + image_id = feature_dict['image_id'] + + # Add image and GT segmentatio labels from total_transfer. + feature_dict['image'] = self.get_image_path(data_dir, split, image_id) + + # Add dummy features for several features in the test set. + if split not in ['train', 'val']: + assert split == 'test', f'Unknown split: {split}' + feature_dict['multiple_choice_correct_idx'] = -1 + feature_dict['direct_answers'] = [] + feature_dict['answer_rationales'] = [] + yield f'{question_id}', feature_dict + + def get_image_path(self, data_dir: str, split: str, image_id: int) -> str: + return f'{data_dir}/{split}2017/{image_id:012d}.jpg' + + +def get_annotations( + data_dir: str, split: str) -> dict[int, dict[str, Any]]: + """Return okvqa annotations (quesions and answers) as dictionary.""" + path = os.path.join(data_dir, ANNOTATION_FILES[split]) + with open(path) as f: + annotations = json.load(f) + + aokvqa_annotations = {} + for annotation in annotations: + # Sanity checks + assert len(annotation['choices']) == 4 + + question_id = annotation['question_id'] + + aokvqa_annotations[question_id] = { + 'image_id': annotation['image_id'], + 'direct_answer_is_difficult': annotation['difficult_direct_answer'], + 'multiple_choice_possible_answers': annotation['choices'], + 'question': annotation['question'], + 'question_id': annotation['question_id'], + } + + # Get answers and rationales for train and val only, not for test. + if split in ['train', 'val']: + assert len(annotation['direct_answers']) == 10 + assert len(annotation['rationales']) == 3 + + aokvqa_annotations[question_id]['direct_answers'] = annotation[ + 'direct_answers'] + aokvqa_annotations[question_id]['answer_rationales'] = annotation[ + 'rationales'] + aokvqa_annotations[question_id]['multiple_choice_correct_idx'] = ( + annotation['correct_choice_idx']) + + return aokvqa_annotations diff --git a/Tipsomaly/model/big_vision/datasets/chartqa/chartqa.py b/Tipsomaly/model/big_vision/datasets/chartqa/chartqa.py new file mode 100644 index 0000000000000000000000000000000000000000..13ea3e523530d104f241fb78bdbad2facc457325 --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/chartqa/chartqa.py @@ -0,0 +1,122 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +r"""Implements CharQA in TFDS structure. + +It's small data, so simple to run locally. First, copy the data to local disk: + + mkdir -p /tmp/data + wget -O /tmp/data/chartqa.zip https://huggingface.co/datasets/ahmed-masry/ChartQA/resolve/main/ChartQA%20Dataset.zip?download=true + unzip /tmp/data/chartqa.zip + +Then, run conversion locally (make sure to install tensorflow-datasets for the `tfds` util): + + cd big_vision/datasets + env TFDS_DATA_DIR=/tmp/tfds tfds build --datasets=chartqa + +Example to load: + + import tensorflow_datasets as tfds + dataset_augmented = tfds.load('chartqa/augmented', split='train', data_dir='/tmp/tfds') +""" +import json +import os + +import numpy as np +import tensorflow_datasets as tfds + + +_DESCRIPTION = """ChartQA dataset.""" + +# pylint: disable=line-too-long +_CITATION = """ +@inproceedings{masry-etal-2022-chartqa, + title = "{C}hart{QA}: A Benchmark for Question Answering about Charts with Visual and Logical Reasoning", + author = "Masry, Ahmed and + Do, Xuan Long and + Tan, Jia Qing and + Joty, Shafiq and + Hoque, Enamul", + editor = "Muresan, Smaranda and + Nakov, Preslav and + Villavicencio, Aline", + booktitle = "Findings of the Association for Computational Linguistics: ACL 2022", + month = may, + year = "2022", + address = "Dublin, Ireland", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.findings-acl.177", + doi = "10.18653/v1/2022.findings-acl.177", + pages = "2263--2279", + abstract = "Charts are very popular for analyzing data. When exploring charts, people often ask a variety of complex reasoning questions that involve several logical and arithmetic operations. They also commonly refer to visual features of a chart in their questions. However, most existing datasets do not focus on such complex reasoning questions as their questions are template-based and answers come from a fixed-vocabulary. In this work, we present a large-scale benchmark covering 9.6K human-written questions as well as 23.1K questions generated from human-written chart summaries. To address the unique challenges in our benchmark involving visual and logical reasoning over charts, we present two transformer-based models that combine visual features and the data table of the chart in a unified way to answer questions. While our models achieve the state-of-the-art results on the previous datasets as well as on our benchmark, the evaluation also reveals several challenges in answering complex reasoning questions.", +} +""" +# pylint: enable=line-too-long + +# When running locally (recommended), copy files as above an use these: +_CHARTQA_PATH = '/tmp/data/ChartQA Dataset/' + + +class ChartQAConfig(tfds.core.BuilderConfig): + """Configuration to build the dataset.""" + pass + + +class ChartQA(tfds.core.GeneratorBasedBuilder): + """DatasetBuilder for ChartQA dataset.""" + + VERSION = tfds.core.Version('1.0.0') + RELEASE_NOTES = {'1.0.0': 'First release.'} + BUILDER_CONFIGS = [ + ChartQAConfig(name='human', description='Human set'), + ChartQAConfig(name='augmented', description='Augmented set'), + ] + + def _info(self): + """Returns the metadata.""" + return tfds.core.DatasetInfo( + builder=self, + description=_DESCRIPTION, + features=tfds.features.FeaturesDict({ + 'question_id': tfds.features.Scalar(np.int32), + 'image/filename': tfds.features.Text(), + 'image': tfds.features.Image(encoding_format='png'), + 'question': tfds.features.Text(), + 'answer': tfds.features.Text(), + }), + homepage='https://github.com/vis-nlp/ChartQA', + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + """Returns SplitGenerators.""" + return {split: self._generate_examples(split, self.builder_config.name) + for split in ('val', 'train', 'test')} + + def _generate_examples(self, split: str, source: str): + """Yields (key, example) tuples from test set.""" + annot_fname = os.path.join(_CHARTQA_PATH, split, f'{split}_{source}.json') + + with open(annot_fname, 'r') as f: + data = json.loads(f.read()) + + for idx, v in enumerate(data): + yield idx, { + 'question_id': idx, + 'image/filename': v['imgname'], + 'image': os.path.join(_CHARTQA_PATH, split, 'png', v['imgname']), + 'question': v['query'], + 'answer': v['label'], + } diff --git a/Tipsomaly/model/big_vision/datasets/coco35l/coco35l.py b/Tipsomaly/model/big_vision/datasets/coco35l/coco35l.py new file mode 100644 index 0000000000000000000000000000000000000000..7615157a8d59101ecc4f5dd05fbc1ca487c3dd23 --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/coco35l/coco35l.py @@ -0,0 +1,154 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +r"""Generates COCO-35L in a TFDS-ready structure. + +First, download the captions from https://google.github.io/crossmodal-3600/ and the images from https://cocodataset.org/#download. +The coco Karpathy split is available at http://cs.stanford.edu/people/karpathy/deepimagesent/caption_datasets.zip: + mkdir -p /tmp/data/coco35l/images + wget https://storage.googleapis.com/crossmodal-3600/coco_mt_train.jsonl.bz2 -P /tmp/data/coco35l + wget https://storage.googleapis.com/crossmodal-3600/coco_mt_dev.jsonl.bz2 -P /tmp/data/coco35l + bzip2 -dk /tmp/data/coco35l/coco_mt_train.jsonl.bz2 /tmp/data/coco35l/coco_mt_dev.jsonl.bz2 + wget http://cs.stanford.edu/people/karpathy/deepimagesent/caption_datasets.zip -P /tmp/data/coco35l + unzip /tmp/data/coco35l/caption_datasets.zip -d /tmp/data/coco35l/ + wget http://images.cocodataset.org/zips/train2014.zip -P /tmp/data/coco35l/images + wget http://images.cocodataset.org/zips/val2014.zip -P /tmp/data/coco35l/images + unzip /tmp/data/coco35l/images/train2014.zip -d /tmp/data/coco35l/images/ + unzip /tmp/data/coco35l/images/val2014.zip -d /tmp/data/coco35l/images/ + +Then, run conversion locally (make sure to install tensorflow-datasets for the `tfds` util): + + cd big_vision/datasets + env TFDS_DATA_DIR=/tmp/tfds tfds build --datasets=coco35l + +Example to load: + + import tensorflow_datasets as tfds + dataset = tfds.load( + 'coco35l', split='dev_en', + data_dir='/tmp/tfds') +""" + +import json +import os.path + +import tensorflow_datasets as tfds + +_DESCRIPTION = """ +COCO image + captions, translated from English to 35 languages (English incl.). +""" + +# pylint: disable=line-too-long +_CITATION = """ +@inproceedings{thapliyal-etal-2022-crossmodal, + title = "Crossmodal-3600: A Massively Multilingual Multimodal Evaluation Dataset", + author = "Thapliyal, Ashish V. and + Pont Tuset, Jordi and + Chen, Xi and + Soricut, Radu", + editor = "Goldberg, Yoav and + Kozareva, Zornitsa and + Zhang, Yue", + booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing", + month = dec, + year = "2022", + address = "Abu Dhabi, United Arab Emirates", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.emnlp-main.45", + doi = "10.18653/v1/2022.emnlp-main.45", + pages = "715--729", +} +""" +# pylint: enable=line-too-long + + +_CAPTIONS_PATH = '/tmp/data/coco35l' +_IMAGES_PATH = '/tmp/data/mscoco/images' +_COCOCAPS_PATH = '/tmp/data/mscoco/dataset_coco.json' + +LANGUAGES = [ + 'ar', 'bn', 'cs', 'da', 'de', 'el', 'en', 'es', 'fa', 'fi', 'fil', 'fr', + 'he', 'hi', 'hr', 'hu', 'id', 'it', 'ja', 'ko', 'mi', 'nl', 'no', 'pl', + 'pt', 'ro', 'ru', 'sv', 'sw', 'te', 'th', 'tr', 'uk', 'vi', 'zh', +] + + +class Coco35l(tfds.core.GeneratorBasedBuilder): + """DatasetBuilder for COCO-35L dataset.""" + + VERSION = tfds.core.Version('1.0.0') + RELEASE_NOTES = {'1.0.0': 'First release.'} + + def _info(self): + """Returns the metadata.""" + + return tfds.core.DatasetInfo( + builder=self, + description=_DESCRIPTION, + features=tfds.features.FeaturesDict({ + 'image/id': tfds.features.Text(), + 'image': tfds.features.Image(encoding_format='jpeg'), + 'captions': tfds.features.Sequence(tfds.features.Text()), + 'language': tfds.features.Text(), + }), + supervised_keys=None, + homepage='https://google.github.io/crossmodal-3600/', + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + """Returns SplitGenerators.""" + splits = [] + for lang in LANGUAGES: + splits.extend([f'train_{lang}', f'dev_{lang}']) + return {split: self._generate_examples(split) for split in splits} + + def _generate_examples(self, split: str): + """Yields (key, example) tuples from dataset.""" + split, language = split.split('_') + + id_to_path = dict() + with open(_COCOCAPS_PATH, 'r') as f: + data = json.load(f)['images'] + for d in data: + id_to_path[d['cocoid']] = os.path.join( + _IMAGES_PATH, d['filepath'], d['filename'] + ) + + annot_fname = os.path.join(_CAPTIONS_PATH, f'coco_mt_{split}.jsonl') + data = {} + with open(annot_fname, 'r') as f: + for line in f: + j = json.loads(line) + image_id = f'{j["image_id"].split("_")[0]}_{language}' + if image_id not in data: + data[image_id] = [] + if language == 'en': + # COCO-35L was constructed from English into 35 other languages. + # To add English in our TFDS, we just select a language (eg. "de") to + # have each unique example, and add the corresponding source caption. + if j['trg_lang'] == 'de': + data[image_id].append(j['caption_tokenized']) + else: + if j['trg_lang'] == language: + data[image_id].append(j['translation_tokenized']) + + for image_id, captions in data.items(): + yield image_id, { + 'image/id': image_id, + 'image': id_to_path[int(image_id.split('_')[0])], + 'captions': captions, + 'language': language, + } diff --git a/Tipsomaly/model/big_vision/datasets/countbenchqa/countbenchqa.py b/Tipsomaly/model/big_vision/datasets/countbenchqa/countbenchqa.py new file mode 100644 index 0000000000000000000000000000000000000000..ba578ccec7f7951f27e7a5615e23eb76da885f38 --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/countbenchqa/countbenchqa.py @@ -0,0 +1,164 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +"""Import CountBenchQA dataset (CountBench dataset with added QA annotations). + +It's small data, so simple to run locally. First, download all the data: + + mkdir /tmp/data/ ; cd /tmp/data + wget https://huggingface.co/datasets/nielsr/countbench/resolve/main/data/train-00000-of-00001-cf54c241ba947306.parquet + wget https://raw.githubusercontent.com/teaching-clip-to-count/teaching-clip-to-count.github.io/main/CountBench.json + +Then, update the PATHs below and run conversion locally like so: + + cd big_vision/datasets + env TFDS_DATA_DIR=/tmp/tfds tfds build --datasets=countbenchqa + +The dataset contains 540 images so the dataset creation is very quick. + +There is a single split called huggingface to denote that the images come from +the hugginface parquet file. +""" + +import io +import json + +import numpy as np +import pandas as pd +import PIL +import tensorflow_datasets as tfds + + +# Huggingface dataset path; this is missing about 10% of the images. +_COUNTBENCH_PARQUET_PATH = '/tmp/data/train-00000-of-00001-cf54c241ba947306.parquet' +# Public path to the original CountBench JSON file. +_COUNTBENCH_JSON_PATH = '/tmp/data/CountBench.json' +# VQA annotations +_QA_JSON_PATH = 'countbenchqa/data/countbench_paired_questions.json' + +_DESCRIPTION = """ +CountBench: We introduce a new object counting benchmark called CountBench, + automatically curated (and manually verified) from the publicly available + LAION-400M image-text dataset. CountBench contains a total of 540 images + containing between two and ten instances of a particular object, where their + corresponding captions reflect this number. + +CountBenchQA: Each image is paired with a manually generated question about the + number of objects in the image to turn CountBench into a VQA task. +""" + +_CITATION = """ +@article{beyer2024paligemma, + title={{PaliGemma: A versatile 3B VLM for transfer}}, + author={Lucas Beyer and Andreas Steiner and André Susano Pinto and Alexander Kolesnikov and Xiao Wang and Daniel Salz and Maxim Neumann and Ibrahim Alabdulmohsin and Michael Tschannen and Emanuele Bugliarello and Thomas Unterthiner and Daniel Keysers and Skanda Koppula and Fangyu Liu and Adam Grycner and Alexey Gritsenko and Neil Houlsby and Manoj Kumar and Keran Rong and Julian Eisenschlos and Rishabh Kabra and Matthias Bauer and Matko Bošnjak and Xi Chen and Matthias Minderer and Paul Voigtlaender and Ioana Bica and Ivana Balazevic and Joan Puigcerver and Pinelopi Papalampidi and Olivier Henaff and Xi Xiong and Radu Soricut and Jeremiah Harmsen and Xiaohua Zhai}, + year={2024}, + journal={arXiv preprint arXiv:2407.07726} +} + +@article{paiss2023countclip, + title={{Teaching CLIP to Count to Ten}}, + author={Paiss, Roni and Ephrat, Ariel and Tov, Omer and Zada, Shiran and Mosseri, Inbar and Irani, Michal and Dekel, Tali}, + year={2023}, + journal={arXiv preprint arXiv:2302.12066} +} +""" + +_HOMEPAGE = 'https://teaching-clip-to-count.github.io/' + + +class CountbenchQA(tfds.core.GeneratorBasedBuilder): + """Create CountbenchQA dataset.""" + + VERSION = tfds.core.Version('1.2.0') + RELEASE_NOTES = {'1.1.0': 'Add `huggingface` split.', + '1.2.0': 'Fix image loading for `huggingface` split.'} + MANUAL_DOWNLOAD_INSTRUCTIONS = """ + There are two parts which should be downloaded: + * Countbench from Huggingface + * Questions found in `data/countbench_paired_questions.json` + """ + + def _info(self) -> tfds.core.DatasetInfo: + """Returns the dataset metadata.""" + features = tfds.features.FeaturesDict({ + 'image': tfds.features.Image(shape=(None, None, 3)), + 'image_id': tfds.features.Scalar(dtype=np.int32), + 'question': tfds.features.Text(), + 'text': tfds.features.Text(), + 'image_url': tfds.features.Text(), + 'number': tfds.features.Scalar(dtype=np.int32), + }) + + return tfds.core.DatasetInfo( + builder=self, + features=features, + description=_DESCRIPTION, + supervised_keys=None, + homepage=_HOMEPAGE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + """Call the function which defines the splits.""" + del dl_manager + return { + 'huggingface': self._generate_examples(split='huggingface'), + } + + def _generate_examples_hf(self): + """Generate examples from Huggingface parquet file. + + Note that the parquet file provided on Huggingface is missing about 10% + of the images as can be verified by running + ``` + import pyarrow.parquet as pq + with open(_COUNTBENCH_PARQUET_PATH, 'rb') as f: + x = pq.read_table(f) + sum([x['image'][i].is_valid for i in range(len(x['image']))]) # result: 491 + ``` + + Yields: + An index and a dictionary with features. + """ + with open(_COUNTBENCH_PARQUET_PATH, 'rb') as f: + df = pd.read_parquet(f) + + with open(_QA_JSON_PATH, 'r') as fq: + df_question = pd.read_json(fq) + + df['question'] = df_question + + for idx, row in df.iterrows(): + # Some entries have no image. + if row['image'] is None: + continue + image = np.array(PIL.Image.open(io.BytesIO(row['image']['bytes']))) + if len(image.shape) != 3: + continue # Filter out one bad image. + countbenchqa_dict = { + 'image': image, + 'image_id': idx, + 'question': row['question'], + 'text': row['text'], + 'image_url': row['image_url'], + 'number': row['number'], + } + yield idx, countbenchqa_dict + + def _generate_examples(self, split: str): + if split == 'huggingface': + yield from self._generate_examples_hf() + else: + raise ValueError(f'Unknown split: {split}') diff --git a/Tipsomaly/model/big_vision/datasets/docvqa/docvqa.py b/Tipsomaly/model/big_vision/datasets/docvqa/docvqa.py new file mode 100644 index 0000000000000000000000000000000000000000..b51e1e0c511ab0422809e693611cae708bfb723d --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/docvqa/docvqa.py @@ -0,0 +1,110 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +r"""Implements DocVQA in TFDS structure. + +It's small data, so simple to run locally. First, copy the data to local disk. +An account will be needed in https://rrc.cvc.uab.es/?ch=17&com=downloads and +from there the task annotations and images can be fetched separatedly. + + mkdir -p /tmp/data/docvqa + + +Then, run conversion locally (make sure to install tensorflow-datasets for the `tfds` util): + + cd big_vision/datasets + env TFDS_DATA_DIR=/tmp/tfds tfds build --datasets=docvqa + +Example to load: + + import tensorflow_datasets as tfds + dataset = tfds.load('docvqa', split='val', data_dir='/tmp/tfds') +""" +import json +import os + +import numpy as np +import tensorflow_datasets as tfds + + +_DESCRIPTION = """DocVQA dataset.""" + +# pylint: disable=line-too-long +_CITATION = """ +@article{DBLP:journals/corr/abs-2007-00398, + author = {Minesh Mathew and + Dimosthenis Karatzas and + R. Manmatha and + C. V. Jawahar}, + title = {DocVQA: {A} Dataset for {VQA} on Document Images}, + journal = {CoRR}, + volume = {abs/2007.00398}, + year = {2020}, + url = {https://arxiv.org/abs/2007.00398}, + eprinttype = {arXiv}, + eprint = {2007.00398}, + timestamp = {Mon, 06 Jul 2020 15:26:01 +0200}, + biburl = {https://dblp.org/rec/journals/corr/abs-2007-00398.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +""" +# pylint: enable=line-too-long + +# When running locally (recommended), copy files as above an use these: +_DOCVQA_PATH = '/tmp/data/docvqa/' + + +class DocVQA(tfds.core.GeneratorBasedBuilder): + """DatasetBuilder for DocVQA dataset.""" + + VERSION = tfds.core.Version('1.0.0') + RELEASE_NOTES = {'1.0.0': 'First release.'} + + def _info(self): + """Returns the metadata.""" + return tfds.core.DatasetInfo( + builder=self, + description=_DESCRIPTION, + features=tfds.features.FeaturesDict({ + 'question_id': tfds.features.Scalar(np.int32), + 'image/filename': tfds.features.Text(), + 'image': tfds.features.Image(encoding_format='png'), + 'question': tfds.features.Text(), + 'answers': tfds.features.Sequence(tfds.features.Text()), + }), + supervised_keys=None, + homepage='https://www.docvqa.org/', + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + """Returns SplitGenerators.""" + return {split: self._generate_examples(split) + for split in ('val', 'train', 'test')} + + def _generate_examples(self, split: str): + """Yields (key, example) tuples from split.""" + suffix = '' if split == 'test' else '_withQT' + with open(os.path.join(_DOCVQA_PATH, f'{split}_v1.0{suffix}.json')) as f: + data = json.load(f) + for v in data['data']: + question_id = v['questionId'] + yield question_id, { + 'question_id': question_id, + 'image/filename': v['image'], + 'image': os.path.join(_DOCVQA_PATH, split, v['image']), + 'question': v['question'], + 'answers': v.get('answers', []), + } diff --git a/Tipsomaly/model/big_vision/datasets/gqa/gqa.py b/Tipsomaly/model/big_vision/datasets/gqa/gqa.py new file mode 100644 index 0000000000000000000000000000000000000000..e22c75b356f7262fda53925f176583a534d51227 --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/gqa/gqa.py @@ -0,0 +1,167 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +r"""Generates GQA in a TFDS-ready structure, using Beam. + +Instructions below are to generate the dataset with a *local* Beam pipeline. +It's advisable to run the Beam job on Google Cloud Dataflow, see + https://www.tensorflow.org/datasets/beam_datasets. +for more details, which would significantly speed up generation. This would +involve uploading the locally downloaded data to a GCS bucket, and then +adding in the Beam pipeline options and your GCP/GCS bucket details +to the `tfds build` command below (as detailed in the link). + +First, copy the data to local disk: + + mkdir -p /tmp/data/gqa + wget -O /tmp/data/gqa/question1.2.zip https://downloads.cs.stanford.edu/nlp/data/gqa/questions1.2.zip?download=true + unzip /tmp/data/gqa/question1.2.zip + mv /tmp/data/gqa/question1.2/* /tmp/data/gqa/ + wget -O /tmp/data/gqa/images.zip https://downloads.cs.stanford.edu/nlp/data/gqa/images.zip?download=true + unzip /tmp/data/gqa/images.zip + +Then, run conversion (make sure to install tensorflow-datasets for the `tfds` util): + + cd big_vision/datasets + env TFDS_DATA_DIR=/tmp/tfds tfds build --datasets=gqa + +Example to load: + + import tensorflow_datasets as tfds + dataset = tfds.load('gqa', split='testdev_balanced', data_dir='/tmp/tfds') + +Some statistics: + train_all: 14305356 examples + train_balanced: 943000 examples + val_all: 2011853 examples + val_balanced: 132062 examples + testdev_all: 172174 examples + testdev_balanced: 12578 examples +""" +import glob +import json +import os + +import numpy as np +import tensorflow_datasets as tfds + + +_DESCRIPTION = """GQA: Visual Reasoning in the Real World.""" + +# pylint: disable=line-too-long +_CITATION = """ +@article{DBLP:journals/corr/abs-2306-14610, + author = {Drew Hudson and + Christopher Manning}, + title = {GQA: A New Dataset for Real-World Visual Reasoning and Compositional Question Answering}, + journal = {CVPR}, + volume = {abs/1902.09506}, + year = {2019}, + url = {https://doi.org/10.48550/arXiv.1902.09506}, + doi = {10.48550/arXiv.1902.09506}, + eprinttype = {arXiv}, + eprint = {1902.09506}, + timestamp = {Tue, 25 Jun 2019 00:00:00 +0100}, + biburl = {https://dblp.org/rec/journals/corr/abs-1902-09506}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +""" +# pylint: enable=line-too-long + + +_DATA_PATH = '/tmp/data/gqa/' + + +class GQA(tfds.core.GeneratorBasedBuilder): + """DatasetBuilder for GQA dataset.""" + + VERSION = tfds.core.Version('1.0.0') + RELEASE_NOTES = {'1.0.0': 'First release.'} + + def _info(self): + """Returns the metadata.""" + + return tfds.core.DatasetInfo( + builder=self, + description=_DESCRIPTION, + features=tfds.features.FeaturesDict({ + 'example_id': tfds.features.Scalar(np.int64), + 'image/id': tfds.features.Text(), + 'image': tfds.features.Image(encoding_format='jpeg'), + 'question': tfds.features.Text(), + 'answer': tfds.features.Text(), + 'full_answer': tfds.features.Text(), + 'is_balanced': tfds.features.Scalar(np.bool_), + }), + homepage='https://cs.stanford.edu/people/dorarad/gqa/', + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + """Returns SplitGenerators.""" + splits = [ + # 'debug', + 'train_all', + 'train_balanced', + 'testdev_all', + 'testdev_balanced', + 'val_all', + 'val_balanced', + 'challenge_all', + 'challenge_balanced', + ] + return {split: self._generate_examples(split) for split in splits} + + def _generate_examples(self, split: str): + """Yields (key, example) tuples from dataset.""" + if split == 'train_all': + train_json_dir = os.path.join(_DATA_PATH, 'train_all_questions', '*.json') + json_files = glob.glob(train_json_dir) + else: + json_files = [os.path.join(_DATA_PATH, f'{split}_questions.json')] + + def _prepare_data(json_path): + with open(os.path.join(json_path)) as f: + annotations = json.load(f) + return [(k, v) for k, v in annotations.items()] + + def _process_example(entry): + question_id, question_data = entry + image_id = question_data['imageId'] + image_path = os.path.join(_DATA_PATH, 'images', f'{image_id}.jpg') + answer = question_data['answer'] if 'answer' in question_data else '' + if 'fullAnswer' in question_data: + full_answer = question_data['fullAnswer'] + else: + full_answer = '' + + example = { + 'example_id': question_id, + 'image/id': image_id, + 'image': image_path, + 'question': question_data['question'], + 'answer': answer, + 'full_answer': full_answer, + 'is_balanced': question_data['isBalanced'], + } + return question_id, example + + beam = tfds.core.lazy_imports.apache_beam + return ( + beam.Create(json_files) + | beam.FlatMap(_prepare_data) + | beam.Reshuffle() + | beam.Map(_process_example) + ) diff --git a/Tipsomaly/model/big_vision/datasets/imagenet/class_names.py b/Tipsomaly/model/big_vision/datasets/imagenet/class_names.py new file mode 100644 index 0000000000000000000000000000000000000000..b6d238498a6870179e0de0ab9106ef014b41d6d7 --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/imagenet/class_names.py @@ -0,0 +1,4758 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Imagenet class names.""" + +# Copied from +# https://github.com/openai/CLIP/blob/main/notebooks/Prompt_Engineering_for_ImageNet.ipynb +CLIP_IMAGENET_CLASS_NAMES = [ + 'tench', 'goldfish', 'great white shark', 'tiger shark', 'hammerhead shark', + 'electric ray', 'stingray', 'rooster', 'hen', 'ostrich', 'brambling', + 'goldfinch', 'house finch', 'junco', 'indigo bunting', 'American robin', + 'bulbul', 'jay', 'magpie', 'chickadee', 'American dipper', + 'kite (bird of prey)', 'bald eagle', 'vulture', 'great grey owl', + 'fire salamander', 'smooth newt', 'newt', 'spotted salamander', 'axolotl', + 'American bullfrog', 'tree frog', 'tailed frog', 'loggerhead sea turtle', + 'leatherback sea turtle', 'mud turtle', 'terrapin', 'box turtle', + 'banded gecko', 'green iguana', 'Carolina anole', + 'desert grassland whiptail lizard', 'agama', 'frilled-necked lizard', + 'alligator lizard', 'Gila monster', 'European green lizard', 'chameleon', + 'Komodo dragon', 'Nile crocodile', 'American alligator', 'triceratops', + 'worm snake', 'ring-necked snake', 'eastern hog-nosed snake', + 'smooth green snake', 'kingsnake', 'garter snake', 'water snake', + 'vine snake', 'night snake', 'boa constrictor', 'African rock python', + 'Indian cobra', 'green mamba', 'sea snake', 'Saharan horned viper', + 'eastern diamondback rattlesnake', 'sidewinder rattlesnake', 'trilobite', + 'harvestman', 'scorpion', 'yellow garden spider', 'barn spider', + 'European garden spider', 'southern black widow', 'tarantula', + 'wolf spider', 'tick', 'centipede', 'black grouse', 'ptarmigan', + 'ruffed grouse', 'prairie grouse', 'peafowl', 'quail', 'partridge', + 'african grey parrot', 'macaw', 'sulphur-crested cockatoo', 'lorikeet', + 'coucal', 'bee eater', 'hornbill', 'hummingbird', 'jacamar', 'toucan', + 'duck', 'red-breasted merganser', 'goose', 'black swan', 'tusker', + 'echidna', 'platypus', 'wallaby', 'koala', 'wombat', 'jellyfish', + 'sea anemone', 'brain coral', 'flatworm', 'nematode', 'conch', 'snail', + 'slug', 'sea slug', 'chiton', 'chambered nautilus', 'Dungeness crab', + 'rock crab', 'fiddler crab', 'red king crab', 'American lobster', + 'spiny lobster', 'crayfish', 'hermit crab', 'isopod', 'white stork', + 'black stork', 'spoonbill', 'flamingo', 'little blue heron', 'great egret', + 'bittern bird', 'crane bird', 'limpkin', 'common gallinule', + 'American coot', 'bustard', 'ruddy turnstone', 'dunlin', 'common redshank', + 'dowitcher', 'oystercatcher', 'pelican', 'king penguin', 'albatross', + 'grey whale', 'killer whale', 'dugong', 'sea lion', 'Chihuahua', + 'Japanese Chin', 'Maltese', 'Pekingese', 'Shih Tzu', 'King Charles Spaniel', + 'Papillon', 'toy terrier', 'Rhodesian Ridgeback', 'Afghan Hound', + 'Basset Hound', 'Beagle', 'Bloodhound', 'Bluetick Coonhound', + 'Black and Tan Coonhound', 'Treeing Walker Coonhound', 'English foxhound', + 'Redbone Coonhound', 'borzoi', 'Irish Wolfhound', 'Italian Greyhound', + 'Whippet', 'Ibizan Hound', 'Norwegian Elkhound', 'Otterhound', 'Saluki', + 'Scottish Deerhound', 'Weimaraner', 'Staffordshire Bull Terrier', + 'American Staffordshire Terrier', 'Bedlington Terrier', 'Border Terrier', + 'Kerry Blue Terrier', 'Irish Terrier', 'Norfolk Terrier', 'Norwich Terrier', + 'Yorkshire Terrier', 'Wire Fox Terrier', 'Lakeland Terrier', + 'Sealyham Terrier', 'Airedale Terrier', 'Cairn Terrier', + 'Australian Terrier', 'Dandie Dinmont Terrier', 'Boston Terrier', + 'Miniature Schnauzer', 'Giant Schnauzer', 'Standard Schnauzer', + 'Scottish Terrier', 'Tibetan Terrier', 'Australian Silky Terrier', + 'Soft-coated Wheaten Terrier', 'West Highland White Terrier', 'Lhasa Apso', + 'Flat-Coated Retriever', 'Curly-coated Retriever', 'Golden Retriever', + 'Labrador Retriever', 'Chesapeake Bay Retriever', + 'German Shorthaired Pointer', 'Vizsla', 'English Setter', 'Irish Setter', + 'Gordon Setter', 'Brittany dog', 'Clumber Spaniel', + 'English Springer Spaniel', 'Welsh Springer Spaniel', 'Cocker Spaniel', + 'Sussex Spaniel', 'Irish Water Spaniel', 'Kuvasz', 'Schipperke', + 'Groenendael dog', 'Malinois', 'Briard', 'Australian Kelpie', 'Komondor', + 'Old English Sheepdog', 'Shetland Sheepdog', 'collie', 'Border Collie', + 'Bouvier des Flandres dog', 'Rottweiler', 'German Shepherd Dog', + 'Dobermann', 'Miniature Pinscher', 'Greater Swiss Mountain Dog', + 'Bernese Mountain Dog', 'Appenzeller Sennenhund', 'Entlebucher Sennenhund', + 'Boxer', 'Bullmastiff', 'Tibetan Mastiff', 'French Bulldog', 'Great Dane', + 'St. Bernard', 'husky', 'Alaskan Malamute', 'Siberian Husky', 'Dalmatian', + 'Affenpinscher', 'Basenji', 'pug', 'Leonberger', 'Newfoundland dog', + 'Great Pyrenees dog', 'Samoyed', 'Pomeranian', 'Chow Chow', 'Keeshond', + 'brussels griffon', 'Pembroke Welsh Corgi', 'Cardigan Welsh Corgi', + 'Toy Poodle', 'Miniature Poodle', 'Standard Poodle', + 'Mexican hairless dog (xoloitzcuintli)', 'grey wolf', 'Alaskan tundra wolf', + 'red wolf or maned wolf', 'coyote', 'dingo', 'dhole', 'African wild dog', + 'hyena', 'red fox', 'kit fox', 'Arctic fox', 'grey fox', 'tabby cat', + 'tiger cat', 'Persian cat', 'Siamese cat', 'Egyptian Mau', 'cougar', 'lynx', + 'leopard', 'snow leopard', 'jaguar', 'lion', 'tiger', 'cheetah', + 'brown bear', 'American black bear', 'polar bear', 'sloth bear', 'mongoose', + 'meerkat', 'tiger beetle', 'ladybug', 'ground beetle', 'longhorn beetle', + 'leaf beetle', 'dung beetle', 'rhinoceros beetle', 'weevil', 'fly', 'bee', + 'ant', 'grasshopper', 'cricket insect', 'stick insect', 'cockroach', + 'praying mantis', 'cicada', 'leafhopper', 'lacewing', 'dragonfly', + 'damselfly', 'red admiral butterfly', 'ringlet butterfly', + 'monarch butterfly', 'small white butterfly', 'sulphur butterfly', + 'gossamer-winged butterfly', 'starfish', 'sea urchin', 'sea cucumber', + 'cottontail rabbit', 'hare', 'Angora rabbit', 'hamster', 'porcupine', + 'fox squirrel', 'marmot', 'beaver', 'guinea pig', 'common sorrel horse', + 'zebra', 'pig', 'wild boar', 'warthog', 'hippopotamus', 'ox', + 'water buffalo', 'bison', 'ram (adult male sheep)', 'bighorn sheep', + 'Alpine ibex', 'hartebeest', 'impala (antelope)', 'gazelle', + 'arabian camel', 'llama', 'weasel', 'mink', 'European polecat', + 'black-footed ferret', 'otter', 'skunk', 'badger', 'armadillo', + 'three-toed sloth', 'orangutan', 'gorilla', 'chimpanzee', 'gibbon', + 'siamang', 'guenon', 'patas monkey', 'baboon', 'macaque', 'langur', + 'black-and-white colobus', 'proboscis monkey', 'marmoset', + 'white-headed capuchin', 'howler monkey', 'titi monkey', + 'Geoffroy\'s spider monkey', 'common squirrel monkey', 'ring-tailed lemur', + 'indri', 'Asian elephant', 'African bush elephant', 'red panda', + 'giant panda', 'snoek fish', 'eel', 'silver salmon', 'rock beauty fish', + 'clownfish', 'sturgeon', 'gar fish', 'lionfish', 'pufferfish', 'abacus', + 'abaya', 'academic gown', 'accordion', 'acoustic guitar', + 'aircraft carrier', 'airliner', 'airship', 'altar', 'ambulance', + 'amphibious vehicle', 'analog clock', 'apiary', 'apron', 'trash can', + 'assault rifle', 'backpack', 'bakery', 'balance beam', 'balloon', + 'ballpoint pen', 'Band-Aid', 'banjo', 'baluster / handrail', 'barbell', + 'barber chair', 'barbershop', 'barn', 'barometer', 'barrel', 'wheelbarrow', + 'baseball', 'basketball', 'bassinet', 'bassoon', 'swimming cap', + 'bath towel', 'bathtub', 'station wagon', 'lighthouse', 'beaker', + 'military hat (bearskin or shako)', 'beer bottle', 'beer glass', + 'bell tower', 'baby bib', 'tandem bicycle', 'bikini', 'ring binder', + 'binoculars', 'birdhouse', 'boathouse', 'bobsleigh', 'bolo tie', + 'poke bonnet', 'bookcase', 'bookstore', 'bottle cap', 'hunting bow', + 'bow tie', 'brass memorial plaque', 'bra', 'breakwater', 'breastplate', + 'broom', 'bucket', 'buckle', 'bulletproof vest', 'high-speed train', + 'butcher shop', 'taxicab', 'cauldron', 'candle', 'cannon', 'canoe', + 'can opener', 'cardigan', 'car mirror', 'carousel', 'tool kit', + 'cardboard box / carton', 'car wheel', 'automated teller machine', + 'cassette', 'cassette player', 'castle', 'catamaran', 'CD player', 'cello', + 'mobile phone', 'chain', 'chain-link fence', 'chain mail', 'chainsaw', + 'storage chest', 'chiffonier', 'bell or wind chime', 'china cabinet', + 'Christmas stocking', 'church', 'movie theater', 'cleaver', + 'cliff dwelling', 'cloak', 'clogs', 'cocktail shaker', 'coffee mug', + 'coffeemaker', 'spiral or coil', 'combination lock', 'computer keyboard', + 'candy store', 'container ship', 'convertible', 'corkscrew', 'cornet', + 'cowboy boot', 'cowboy hat', 'cradle', 'construction crane', 'crash helmet', + 'crate', 'infant bed', 'Crock Pot', 'croquet ball', 'crutch', 'cuirass', + 'dam', 'desk', 'desktop computer', 'rotary dial telephone', 'diaper', + 'digital clock', 'digital watch', 'dining table', 'dishcloth', 'dishwasher', + 'disc brake', 'dock', 'dog sled', 'dome', 'doormat', 'drilling rig', 'drum', + 'drumstick', 'dumbbell', 'Dutch oven', 'electric fan', 'electric guitar', + 'electric locomotive', 'entertainment center', 'envelope', + 'espresso machine', 'face powder', 'feather boa', 'filing cabinet', + 'fireboat', 'fire truck', 'fire screen', 'flagpole', 'flute', + 'folding chair', 'football helmet', 'forklift', 'fountain', 'fountain pen', + 'four-poster bed', 'freight car', 'French horn', 'frying pan', 'fur coat', + 'garbage truck', 'gas mask or respirator', 'gas pump', 'goblet', 'go-kart', + 'golf ball', 'golf cart', 'gondola', 'gong', 'gown', 'grand piano', + 'greenhouse', 'radiator grille', 'grocery store', 'guillotine', 'hair clip', + 'hair spray', 'half-track', 'hammer', 'hamper', 'hair dryer', + 'hand-held computer', 'handkerchief', 'hard disk drive', 'harmonica', + 'harp', 'combine harvester', 'hatchet', 'holster', 'home theater', + 'honeycomb', 'hook', 'hoop skirt', 'gymnastic horizontal bar', + 'horse-drawn vehicle', 'hourglass', 'iPod', 'clothes iron', + 'carved pumpkin', 'jeans', 'jeep', 'T-shirt', 'jigsaw puzzle', 'rickshaw', + 'joystick', 'kimono', 'knee pad', 'knot', 'lab coat', 'ladle', 'lampshade', + 'laptop computer', 'lawn mower', 'lens cap', 'letter opener', 'library', + 'lifeboat', 'lighter', 'limousine', 'ocean liner', 'lipstick', + 'slip-on shoe', 'lotion', 'music speaker', 'loupe magnifying glass', + 'sawmill', 'magnetic compass', 'messenger bag', 'mailbox', 'tights', + 'one-piece bathing suit', 'manhole cover', 'maraca', 'marimba', 'mask', + 'matchstick', 'maypole', 'maze', 'measuring cup', 'medicine cabinet', + 'megalith', 'microphone', 'microwave oven', 'military uniform', 'milk can', + 'minibus', 'miniskirt', 'minivan', 'missile', 'mitten', 'mixing bowl', + 'mobile home', 'ford model t', 'modem', 'monastery', 'monitor', 'moped', + 'mortar and pestle', 'graduation cap', 'mosque', 'mosquito net', 'vespa', + 'mountain bike', 'tent', 'computer mouse', 'mousetrap', 'moving van', + 'muzzle', 'metal nail', 'neck brace', 'necklace', 'baby pacifier', + 'notebook computer', 'obelisk', 'oboe', 'ocarina', 'odometer', 'oil filter', + 'pipe organ', 'oscilloscope', 'overskirt', 'bullock cart', 'oxygen mask', + 'product packet / packaging', 'paddle', 'paddle wheel', 'padlock', + 'paintbrush', 'pajamas', 'palace', 'pan flute', 'paper towel', 'parachute', + 'parallel bars', 'park bench', 'parking meter', 'railroad car', 'patio', + 'payphone', 'pedestal', 'pencil case', 'pencil sharpener', 'perfume', + 'Petri dish', 'photocopier', 'plectrum', 'Pickelhaube', 'picket fence', + 'pickup truck', 'pier', 'piggy bank', 'pill bottle', 'pillow', + 'ping-pong ball', 'pinwheel', 'pirate ship', 'drink pitcher', 'block plane', + 'planetarium', 'plastic bag', 'plate rack', 'farm plow', 'plunger', + 'Polaroid camera', 'pole', 'police van', 'poncho', 'pool table', + 'soda bottle', 'plant pot', 'potter\'s wheel', 'power drill', 'prayer rug', + 'printer', 'prison', 'missile', 'projector', 'hockey puck', 'punching bag', + 'purse', 'quill', 'quilt', 'race car', 'racket', 'radiator', 'radio', + 'radio telescope', 'rain barrel', 'recreational vehicle', + 'fishing casting reel', 'reflex camera', 'refrigerator', 'remote control', + 'restaurant', 'revolver', 'rifle', 'rocking chair', 'rotisserie', 'eraser', + 'rugby ball', 'ruler measuring stick', 'sneaker', 'safe', 'safety pin', + 'salt shaker', 'sandal', 'sarong', 'saxophone', 'scabbard', + 'weighing scale', 'school bus', 'schooner', 'scoreboard', 'CRT monitor', + 'screw', 'screwdriver', 'seat belt', 'sewing machine', 'shield', + 'shoe store', 'shoji screen / room divider', 'shopping basket', + 'shopping cart', 'shovel', 'shower cap', 'shower curtain', 'ski', + 'balaclava ski mask', 'sleeping bag', 'slide rule', 'sliding door', + 'slot machine', 'snorkel', 'snowmobile', 'snowplow', 'soap dispenser', + 'soccer ball', 'sock', 'solar thermal collector', 'sombrero', 'soup bowl', + 'keyboard space bar', 'space heater', 'space shuttle', 'spatula', + 'motorboat', 'spider web', 'spindle', 'sports car', 'spotlight', 'stage', + 'steam locomotive', 'through arch bridge', 'steel drum', 'stethoscope', + 'scarf', 'stone wall', 'stopwatch', 'stove', 'strainer', 'tram', + 'stretcher', 'couch', 'stupa', 'submarine', 'suit', 'sundial', 'sunglasses', + 'sunglasses', 'sunscreen', 'suspension bridge', 'mop', 'sweatshirt', + 'swim trunks / shorts', 'swing', 'electrical switch', 'syringe', + 'table lamp', 'tank', 'tape player', 'teapot', 'teddy bear', 'television', + 'tennis ball', 'thatched roof', 'front curtain', 'thimble', + 'threshing machine', 'throne', 'tile roof', 'toaster', 'tobacco shop', + 'toilet seat', 'torch', 'totem pole', 'tow truck', 'toy store', 'tractor', + 'semi-trailer truck', 'tray', 'trench coat', 'tricycle', 'trimaran', + 'tripod', 'triumphal arch', 'trolleybus', 'trombone', 'hot tub', + 'turnstile', 'typewriter keyboard', 'umbrella', 'unicycle', 'upright piano', + 'vacuum cleaner', 'vase', 'vaulted or arched ceiling', 'velvet fabric', + 'vending machine', 'vestment', 'viaduct', 'violin', 'volleyball', + 'waffle iron', 'wall clock', 'wallet', 'wardrobe', 'military aircraft', + 'sink', 'washing machine', 'water bottle', 'water jug', 'water tower', + 'whiskey jug', 'whistle', 'hair wig', 'window screen', 'window shade', + 'Windsor tie', 'wine bottle', 'airplane wing', 'wok', 'wooden spoon', + 'wool', 'split-rail fence', 'shipwreck', 'sailboat', 'yurt', 'website', + 'comic book', 'crossword', 'traffic or street sign', 'traffic light', + 'dust jacket', 'menu', 'plate', 'guacamole', 'consomme', 'hot pot', + 'trifle', 'ice cream', 'popsicle', 'baguette', 'bagel', 'pretzel', + 'cheeseburger', 'hot dog', 'mashed potatoes', 'cabbage', 'broccoli', + 'cauliflower', 'zucchini', 'spaghetti squash', 'acorn squash', + 'butternut squash', 'cucumber', 'artichoke', 'bell pepper', 'cardoon', + 'mushroom', 'Granny Smith apple', 'strawberry', 'orange', 'lemon', 'fig', + 'pineapple', 'banana', 'jackfruit', 'cherimoya (custard apple)', + 'pomegranate', 'hay', 'carbonara', 'chocolate syrup', 'dough', 'meatloaf', + 'pizza', 'pot pie', 'burrito', 'red wine', 'espresso', 'tea cup', 'eggnog', + 'mountain', 'bubble', 'cliff', 'coral reef', 'geyser', 'lakeshore', + 'promontory', 'sandbar', 'beach', 'valley', 'volcano', 'baseball player', + 'bridegroom', 'scuba diver', 'rapeseed', 'daisy', 'yellow lady\'s slipper', + 'corn', 'acorn', 'rose hip', 'horse chestnut seed', 'coral fungus', + 'agaric', 'gyromitra', 'stinkhorn mushroom', 'earth star fungus', + 'hen of the woods mushroom', 'bolete', 'corn cob', 'toilet paper' +] + +# ImageNet-A and ImageNet-R do not use the full label space of ImageNet. +# These were copied from third_party/py/robustness_metrics/datasets/tfds.py +# Kudos to mjlm@ who helped us notice this. +IMAGENET_A_LABELSET = [ + 6, 11, 13, 15, 17, 22, 23, 27, 30, 37, 39, 42, 47, 50, 57, 70, 71, 76, 79, + 89, 90, 94, 96, 97, 99, 105, 107, 108, 110, 113, 124, 125, 130, 132, 143, + 144, 150, 151, 207, 234, 235, 254, 277, 283, 287, 291, 295, 298, 301, 306, + 307, 308, 309, 310, 311, 313, 314, 315, 317, 319, 323, 324, 326, 327, 330, + 334, 335, 336, 347, 361, 363, 372, 378, 386, 397, 400, 401, 402, 404, 407, + 411, 416, 417, 420, 425, 428, 430, 437, 438, 445, 456, 457, 461, 462, 470, + 472, 483, 486, 488, 492, 496, 514, 516, 528, 530, 539, 542, 543, 549, 552, + 557, 561, 562, 569, 572, 573, 575, 579, 589, 606, 607, 609, 614, 626, 627, + 640, 641, 642, 643, 658, 668, 677, 682, 684, 687, 701, 704, 719, 736, 746, + 749, 752, 758, 763, 765, 768, 773, 774, 776, 779, 780, 786, 792, 797, 802, + 803, 804, 813, 815, 820, 823, 831, 833, 835, 839, 845, 847, 850, 859, 862, + 870, 879, 880, 888, 890, 897, 900, 907, 913, 924, 932, 933, 934, 937, 943, + 945, 947, 951, 954, 956, 957, 959, 971, 972, 980, 981, 984, 986, 987, 988, +] + +# Also check out https://github.com/hendrycks/imagenet-r/blob/master/eval.py +IMAGENET_R_LABELSET = [ + 1, 2, 4, 6, 8, 9, 11, 13, 22, 23, 26, 29, 31, 39, 47, 63, 71, 76, 79, 84, + 90, 94, 96, 97, 99, 100, 105, 107, 113, 122, 125, 130, 132, 144, 145, 147, + 148, 150, 151, 155, 160, 161, 162, 163, 171, 172, 178, 187, 195, 199, 203, + 207, 208, 219, 231, 232, 234, 235, 242, 245, 247, 250, 251, 254, 259, 260, + 263, 265, 267, 269, 276, 277, 281, 288, 289, 291, 292, 293, 296, 299, 301, + 308, 309, 310, 311, 314, 315, 319, 323, 327, 330, 334, 335, 337, 338, 340, + 341, 344, 347, 353, 355, 361, 362, 365, 366, 367, 368, 372, 388, 390, 393, + 397, 401, 407, 413, 414, 425, 428, 430, 435, 437, 441, 447, 448, 457, 462, + 463, 469, 470, 471, 472, 476, 483, 487, 515, 546, 555, 558, 570, 579, 583, + 587, 593, 594, 596, 609, 613, 617, 621, 629, 637, 657, 658, 701, 717, 724, + 763, 768, 774, 776, 779, 780, 787, 805, 812, 815, 820, 824, 833, 847, 852, + 866, 875, 883, 889, 895, 907, 928, 931, 932, 933, 934, 936, 937, 943, 945, + 947, 948, 949, 951, 953, 954, 957, 963, 965, 967, 980, 981, 983, 988, +] + +# The name of the imagenet21k classes. from nltk with post-processing +# (replace "_", no alias names), more see +# https://colab.research.google.com/drive/1I7oVTyT8nDB8K_-rGIa8GgeSA9BeWAk5#scrollTo=byaZcr50X-S6 +IMAGENET21k_CLASS_NAMES = [ + 'organism', 'benthos', 'heterotroph', 'cell', 'person', 'animal', 'plant', + 'food', 'artifact', 'hop', 'check-in', 'dressage', 'curvet', 'piaffe', + 'funambulism', 'rock climbing', 'contact sport', 'outdoor sport', + 'gymnastics', 'acrobatics', 'track and field', 'track', 'jumping', + 'broad jump', 'high jump', 'fosbury flop', 'skiing', 'cross-country skiing', + 'ski jumping', 'water sport', 'swimming', 'bathe', 'dip', 'dive', + 'floating', 'dead-man\'s float', 'belly flop', 'cliff diving', 'flip', + 'gainer', 'half gainer', 'jackknife', 'swan dive', 'skin diving', + 'scuba diving', 'snorkeling', 'surfing', 'water-skiing', 'rowing', + 'sculling', 'boxing', 'professional boxing', 'in-fighting', 'fight', + 'rope-a-dope', 'spar', 'archery', 'sledding', 'tobogganing', 'luging', + 'bobsledding', 'wrestling', 'greco-roman wrestling', + 'professional wrestling', 'sumo', 'skating', 'ice skating', + 'figure skating', 'rollerblading', 'roller skating', 'skateboarding', + 'speed skating', 'racing', 'auto racing', 'boat racing', + 'hydroplane racing', 'camel racing', 'greyhound racing', 'horse racing', + 'riding', 'equestrian sport', 'pony-trekking', 'showjumping', + 'cross-country riding', 'cycling', 'bicycling', 'motorcycling', + 'dune cycling', 'blood sport', 'bullfighting', 'cockfighting', 'hunt', + 'battue', 'beagling', 'coursing', 'deer hunting', 'ducking', 'fox hunting', + 'pigsticking', 'fishing', 'angling', 'fly-fishing', 'troll', 'casting', + 'bait casting', 'fly casting', 'overcast', 'surf casting', 'day game', + 'athletic game', 'ice hockey', 'tetherball', 'water polo', 'outdoor game', + 'golf', 'professional golf', 'round of golf', 'medal play', 'match play', + 'miniature golf', 'croquet', 'quoits', 'shuffleboard', 'field game', + 'field hockey', 'shinny', 'football', 'american football', + 'professional football', 'touch football', 'hurling', 'rugby', 'ball game', + 'baseball', 'ball', 'professional baseball', 'hardball', 'perfect game', + 'no-hit game', 'one-hitter', 'two-hitter', 'three-hitter', 'four-hitter', + 'five-hitter', 'softball', 'rounders', 'stickball', 'cricket', 'lacrosse', + 'polo', 'pushball', 'soccer', 'court game', 'handball', 'racquetball', + 'fives', 'squash', 'volleyball', 'jai alai', 'badminton', 'battledore', + 'basketball', 'professional basketball', 'deck tennis', 'netball', 'tennis', + 'professional tennis', 'singles', 'singles', 'doubles', 'doubles', + 'royal tennis', 'pallone', 'sport', 'clasp', 'judo', 'team sport', + 'last supper', 'seder', 'camping', 'pest', 'critter', 'creepy-crawly', + 'darter', 'peeper', 'homeotherm', 'poikilotherm', 'range animal', + 'scavenger', 'bottom-feeder', 'bottom-feeder', 'work animal', + 'beast of burden', 'draft animal', 'pack animal', 'domestic animal', + 'feeder', 'feeder', 'stocker', 'hatchling', 'head', 'migrator', 'molter', + 'pet', 'stayer', 'stunt', 'marine animal', 'by-catch', 'female', 'hen', + 'male', 'adult', 'young', 'orphan', 'young mammal', 'baby', 'pup', + 'wolf pup', 'puppy', 'cub', 'lion cub', 'bear cub', 'tiger cub', 'kit', + 'suckling', 'sire', 'dam', 'thoroughbred', 'giant', 'mutant', 'carnivore', + 'herbivore', 'insectivore', 'acrodont', 'pleurodont', 'microorganism', + 'monohybrid', 'arbovirus', 'adenovirus', 'arenavirus', 'marburg virus', + 'arenaviridae', 'vesiculovirus', 'reoviridae', 'variola major', 'viroid', + 'coliphage', 'paramyxovirus', 'poliovirus', 'herpes', 'herpes simplex 1', + 'herpes zoster', 'herpes varicella zoster', 'cytomegalovirus', + 'varicella zoster virus', 'polyoma', 'lyssavirus', 'reovirus', 'rotavirus', + 'moneran', 'archaebacteria', 'bacteroid', 'bacillus anthracis', + 'yersinia pestis', 'brucella', 'spirillum', 'botulinus', + 'clostridium perfringens', 'cyanobacteria', 'trichodesmium', + 'nitric bacteria', 'spirillum', 'francisella', 'gonococcus', + 'corynebacterium diphtheriae', 'enteric bacteria', 'klebsiella', + 'salmonella typhimurium', 'typhoid bacillus', 'nitrate bacterium', + 'nitrite bacterium', 'actinomycete', 'streptomyces', + 'streptomyces erythreus', 'streptomyces griseus', 'tubercle bacillus', + 'pus-forming bacteria', 'streptobacillus', 'myxobacteria', 'staphylococcus', + 'diplococcus', 'pneumococcus', 'streptococcus', 'spirochete', + 'planktonic algae', 'zooplankton', 'parasite', 'endoparasite', + 'ectoparasite', 'pathogen', 'commensal', 'myrmecophile', 'protoctist', + 'protozoan', 'sarcodinian', 'heliozoan', 'endameba', 'ameba', 'globigerina', + 'testacean', 'arcella', 'difflugia', 'ciliate', 'paramecium', 'stentor', + 'alga', 'arame', 'seagrass', 'golden algae', 'yellow-green algae', + 'brown algae', 'kelp', 'fucoid', 'fucoid', 'fucus', 'bladderwrack', + 'green algae', 'pond scum', 'chlorella', 'stonewort', 'desmid', 'sea moss', + 'eukaryote', 'prokaryote', 'zooid', 'leishmania', 'zoomastigote', + 'polymastigote', 'costia', 'giardia', 'cryptomonad', 'sporozoan', + 'sporozoite', 'trophozoite', 'merozoite', 'coccidium', 'gregarine', + 'plasmodium', 'leucocytozoan', 'microsporidian', 'ostariophysi', + 'cypriniform fish', 'loach', 'cyprinid', 'carp', 'domestic carp', + 'leather carp', 'mirror carp', 'european bream', 'tench', 'dace', 'chub', + 'shiner', 'common shiner', 'roach', 'rudd', 'minnow', 'gudgeon', 'goldfish', + 'crucian carp', 'electric eel', 'catostomid', 'buffalo fish', + 'black buffalo', 'hog sucker', 'redhorse', 'cyprinodont', 'killifish', + 'mummichog', 'striped killifish', 'rivulus', 'flagfish', 'swordtail', + 'guppy', 'topminnow', 'mosquitofish', 'platy', 'mollie', 'squirrelfish', + 'reef squirrelfish', 'deepwater squirrelfish', 'holocentrus ascensionis', + 'soldierfish', 'anomalops', 'flashlight fish', 'john dory', 'boarfish', + 'boarfish', 'cornetfish', 'stickleback', 'three-spined stickleback', + 'ten-spined stickleback', 'pipefish', 'dwarf pipefish', + 'deepwater pipefish', 'seahorse', 'snipefish', 'shrimpfish', 'trumpetfish', + 'pellicle', 'embryo', 'fetus', 'abortus', 'spawn', 'blastula', 'blastocyst', + 'gastrula', 'morula', 'yolk', 'chordate', 'cephalochordate', 'lancelet', + 'tunicate', 'ascidian', 'sea squirt', 'salp', 'doliolum', 'larvacean', + 'appendicularia', 'ascidian tadpole', 'vertebrate', 'amniota', 'amniote', + 'aquatic vertebrate', 'jawless vertebrate', 'ostracoderm', 'heterostracan', + 'anaspid', 'conodont', 'cyclostome', 'lamprey', 'sea lamprey', 'hagfish', + 'myxine glutinosa', 'eptatretus', 'gnathostome', 'placoderm', + 'cartilaginous fish', 'holocephalan', 'chimaera', 'rabbitfish', + 'elasmobranch', 'shark', 'cow shark', 'mackerel shark', 'porbeagle', 'mako', + 'shortfin mako', 'longfin mako', 'bonito shark', 'great white shark', + 'basking shark', 'thresher', 'carpet shark', 'nurse shark', 'sand tiger', + 'whale shark', 'requiem shark', 'bull shark', 'sandbar shark', + 'blacktip shark', 'whitetip shark', 'dusky shark', 'lemon shark', + 'blue shark', 'tiger shark', 'soupfin shark', 'dogfish', 'smooth dogfish', + 'smoothhound', 'american smooth dogfish', 'florida smoothhound', + 'whitetip shark', 'spiny dogfish', 'atlantic spiny dogfish', + 'pacific spiny dogfish', 'hammerhead', 'smooth hammerhead', + 'smalleye hammerhead', 'shovelhead', 'angel shark', 'ray', 'electric ray', + 'sawfish', 'smalltooth sawfish', 'guitarfish', 'stingray', + 'roughtail stingray', 'butterfly ray', 'eagle ray', 'spotted eagle ray', + 'cownose ray', 'manta', 'atlantic manta', 'devil ray', 'skate', + 'grey skate', 'little skate', 'thorny skate', 'barndoor skate', 'bird', + 'dickeybird', 'fledgling', 'nestling', 'cock', 'gamecock', 'hen', 'nester', + 'night bird', 'night raven', 'bird of passage', 'archaeopteryx', + 'archaeornis', 'ratite', 'carinate', 'ostrich', 'cassowary', 'emu', 'kiwi', + 'rhea', 'rhea', 'elephant bird', 'moa', 'passerine', 'nonpasserine bird', + 'oscine', 'songbird', 'honey eater', 'accentor', 'hedge sparrow', 'lark', + 'skylark', 'wagtail', 'pipit', 'meadow pipit', 'finch', 'chaffinch', + 'brambling', 'goldfinch', 'linnet', 'siskin', 'red siskin', 'redpoll', + 'redpoll', 'new world goldfinch', 'pine siskin', 'house finch', + 'purple finch', 'canary', 'common canary', 'serin', 'crossbill', + 'bullfinch', 'junco', 'dark-eyed junco', 'new world sparrow', + 'vesper sparrow', 'white-throated sparrow', 'white-crowned sparrow', + 'chipping sparrow', 'field sparrow', 'tree sparrow', 'song sparrow', + 'swamp sparrow', 'bunting', 'indigo bunting', 'ortolan', 'reed bunting', + 'yellowhammer', 'yellow-breasted bunting', 'snow bunting', 'honeycreeper', + 'banana quit', 'sparrow', 'english sparrow', 'tree sparrow', 'grosbeak', + 'evening grosbeak', 'hawfinch', 'pine grosbeak', 'cardinal', 'pyrrhuloxia', + 'towhee', 'chewink', 'green-tailed towhee', 'weaver', 'baya', 'whydah', + 'java sparrow', 'avadavat', 'grassfinch', 'zebra finch', 'honeycreeper', + 'lyrebird', 'scrubbird', 'broadbill', 'tyrannid', 'new world flycatcher', + 'kingbird', 'arkansas kingbird', 'cassin\'s kingbird', 'eastern kingbird', + 'grey kingbird', 'pewee', 'western wood pewee', 'phoebe', + 'vermillion flycatcher', 'cotinga', 'cock of the rock', 'cock of the rock', + 'manakin', 'bellbird', 'umbrella bird', 'ovenbird', 'antbird', 'ant thrush', + 'ant shrike', 'spotted antbird', 'woodhewer', 'pitta', 'scissortail', + 'old world flycatcher', 'spotted flycatcher', 'thickhead', 'thrush', + 'missel thrush', 'song thrush', 'fieldfare', 'redwing', 'blackbird', + 'ring ouzel', 'robin', 'clay-colored robin', 'hermit thrush', 'veery', + 'wood thrush', 'nightingale', 'thrush nightingale', 'bulbul', + 'old world chat', 'stonechat', 'whinchat', 'solitaire', 'redstart', + 'wheatear', 'bluebird', 'robin', 'bluethroat', 'warbler', 'gnatcatcher', + 'kinglet', 'goldcrest', 'gold-crowned kinglet', 'ruby-crowned kinglet', + 'old world warbler', 'blackcap', 'greater whitethroat', + 'lesser whitethroat', 'wood warbler', 'sedge warbler', 'wren warbler', + 'tailorbird', 'babbler', 'new world warbler', 'parula warbler', + 'wilson\'s warbler', 'flycatching warbler', 'american redstart', + 'cape may warbler', 'yellow warbler', 'blackburn', 'audubon\'s warbler', + 'myrtle warbler', 'blackpoll', 'new world chat', 'yellow-breasted chat', + 'ovenbird', 'water thrush', 'yellowthroat', 'common yellowthroat', + 'riflebird', 'new world oriole', 'northern oriole', 'baltimore oriole', + 'bullock\'s oriole', 'orchard oriole', 'meadowlark', 'eastern meadowlark', + 'western meadowlark', 'cacique', 'bobolink', 'new world blackbird', + 'grackle', 'purple grackle', 'rusty blackbird', 'cowbird', + 'red-winged blackbird', 'old world oriole', 'golden oriole', 'fig-bird', + 'starling', 'common starling', 'rose-colored starling', 'myna', + 'crested myna', 'hill myna', 'corvine bird', 'crow', 'american crow', + 'raven', 'rook', 'jackdaw', 'chough', 'jay', 'old world jay', + 'common european jay', 'new world jay', 'blue jay', 'canada jay', + 'rocky mountain jay', 'nutcracker', 'common nutcracker', + 'clark\'s nutcracker', 'magpie', 'european magpie', 'american magpie', + 'australian magpie', 'butcherbird', 'currawong', 'piping crow', 'wren', + 'winter wren', 'house wren', 'marsh wren', 'long-billed marsh wren', + 'sedge wren', 'rock wren', 'carolina wren', 'cactus wren', 'mockingbird', + 'blue mockingbird', 'catbird', 'thrasher', 'brown thrasher', + 'new zealand wren', 'rock wren', 'rifleman bird', 'creeper', + 'brown creeper', 'european creeper', 'wall creeper', 'european nuthatch', + 'red-breasted nuthatch', 'white-breasted nuthatch', 'titmouse', 'chickadee', + 'black-capped chickadee', 'tufted titmouse', 'carolina chickadee', + 'blue tit', 'bushtit', 'wren-tit', 'verdin', 'fairy bluebird', 'swallow', + 'barn swallow', 'cliff swallow', 'tree swallow', 'white-bellied swallow', + 'martin', 'house martin', 'bank martin', 'purple martin', 'wood swallow', + 'tanager', 'scarlet tanager', 'western tanager', 'summer tanager', + 'hepatic tanager', 'shrike', 'butcherbird', 'european shrike', + 'northern shrike', 'white-rumped shrike', 'loggerhead shrike', + 'migrant shrike', 'bush shrike', 'black-fronted bush shrike', 'bowerbird', + 'satin bowerbird', 'great bowerbird', 'water ouzel', 'european water ouzel', + 'american water ouzel', 'vireo', 'red-eyed vireo', 'solitary vireo', + 'blue-headed vireo', 'waxwing', 'cedar waxwing', 'bohemian waxwing', + 'bird of prey', 'accipitriformes', 'hawk', 'eyas', 'tiercel', 'goshawk', + 'sparrow hawk', 'cooper\'s hawk', 'chicken hawk', 'buteonine', 'redtail', + 'rough-legged hawk', 'red-shouldered hawk', 'buzzard', 'honey buzzard', + 'kite', 'black kite', 'swallow-tailed kite', 'white-tailed kite', 'harrier', + 'marsh harrier', 'montagu\'s harrier', 'marsh hawk', 'harrier eagle', + 'falcon', 'peregrine', 'falcon-gentle', 'gyrfalcon', 'kestrel', + 'sparrow hawk', 'pigeon hawk', 'hobby', 'caracara', 'audubon\'s caracara', + 'carancha', 'eagle', 'young bird', 'eaglet', 'harpy', 'golden eagle', + 'tawny eagle', 'bald eagle', 'sea eagle', 'kamchatkan sea eagle', 'ern', + 'fishing eagle', 'osprey', 'vulture', 'aegypiidae', 'old world vulture', + 'griffon vulture', 'bearded vulture', 'egyptian vulture', 'black vulture', + 'secretary bird', 'new world vulture', 'buzzard', 'condor', 'andean condor', + 'california condor', 'black vulture', 'king vulture', 'owl', 'owlet', + 'little owl', 'horned owl', 'great horned owl', 'great grey owl', + 'tawny owl', 'barred owl', 'screech owl', 'screech owl', 'scops owl', + 'spotted owl', 'old world scops owl', 'oriental scops owl', 'hoot owl', + 'hawk owl', 'long-eared owl', 'laughing owl', 'barn owl', 'amphibian', + 'ichyostega', 'urodele', 'salamander', 'european fire salamander', + 'spotted salamander', 'alpine salamander', 'newt', 'common newt', 'red eft', + 'pacific newt', 'rough-skinned newt', 'california newt', 'eft', + 'ambystomid', 'mole salamander', 'spotted salamander', 'tiger salamander', + 'axolotl', 'waterdog', 'hellbender', 'giant salamander', 'olm', 'mud puppy', + 'dicamptodon', 'pacific giant salamander', 'olympic salamander', + 'lungless salamander', 'eastern red-backed salamander', + 'western red-backed salamander', 'dusky salamander', 'climbing salamander', + 'arboreal salamander', 'slender salamander', 'web-toed salamander', + 'shasta salamander', 'limestone salamander', 'amphiuma', 'siren', 'frog', + 'true frog', 'wood-frog', 'leopard frog', 'bullfrog', 'green frog', + 'cascades frog', 'goliath frog', 'pickerel frog', 'tarahumara frog', + 'grass frog', 'leptodactylid frog', 'robber frog', 'barking frog', + 'crapaud', 'tree frog', 'tailed frog', 'liopelma hamiltoni', 'true toad', + 'bufo', 'agua', 'european toad', 'natterjack', 'american toad', + 'eurasian green toad', 'american green toad', 'yosemite toad', 'texas toad', + 'southwestern toad', 'western toad', 'obstetrical toad', 'midwife toad', + 'fire-bellied toad', 'spadefoot', 'western spadefoot', 'southern spadefoot', + 'plains spadefoot', 'tree toad', 'spring peeper', 'pacific tree toad', + 'canyon treefrog', 'chameleon tree frog', 'cricket frog', + 'northern cricket frog', 'eastern cricket frog', 'chorus frog', + 'lowland burrowing treefrog', 'western narrow-mouthed toad', + 'eastern narrow-mouthed toad', 'sheep frog', 'tongueless frog', + 'surinam toad', 'african clawed frog', 'south american poison toad', + 'caecilian', 'reptile', 'anapsid', 'diapsid', 'diapsida', 'chelonian', + 'turtle', 'sea turtle', 'green turtle', 'loggerhead', 'ridley', + 'atlantic ridley', 'pacific ridley', 'hawksbill turtle', + 'leatherback turtle', 'snapping turtle', 'common snapping turtle', + 'alligator snapping turtle', 'mud turtle', 'musk turtle', 'terrapin', + 'diamondback terrapin', 'red-bellied terrapin', 'slider', 'cooter', + 'box turtle', 'western box turtle', 'painted turtle', 'tortoise', + 'european tortoise', 'giant tortoise', 'gopher tortoise', 'desert tortoise', + 'texas tortoise', 'soft-shelled turtle', 'spiny softshell', + 'smooth softshell', 'tuatara', 'saurian', 'lizard', 'gecko', 'flying gecko', + 'banded gecko', 'iguanid', 'common iguana', 'marine iguana', + 'desert iguana', 'chuckwalla', 'zebra-tailed lizard', 'fringe-toed lizard', + 'earless lizard', 'collared lizard', 'leopard lizard', 'spiny lizard', + 'fence lizard', 'western fence lizard', 'eastern fence lizard', + 'sagebrush lizard', 'side-blotched lizard', 'tree lizard', 'horned lizard', + 'texas horned lizard', 'basilisk', 'american chameleon', 'worm lizard', + 'night lizard', 'skink', 'western skink', 'mountain skink', 'teiid lizard', + 'whiptail', 'racerunner', 'plateau striped whiptail', + 'chihuahuan spotted whiptail', 'western whiptail', 'checkered whiptail', + 'teju', 'caiman lizard', 'agamid', 'agama', 'frilled lizard', 'moloch', + 'mountain devil', 'anguid lizard', 'alligator lizard', 'blindworm', + 'glass lizard', 'legless lizard', 'lanthanotus borneensis', + 'venomous lizard', 'gila monster', 'beaded lizard', 'lacertid lizard', + 'sand lizard', 'green lizard', 'chameleon', 'african chameleon', + 'horned chameleon', 'monitor', 'african monitor', 'komodo dragon', + 'crocodilian reptile', 'crocodile', 'african crocodile', 'asian crocodile', + 'morlett\'s crocodile', 'false gavial', 'alligator', 'american alligator', + 'chinese alligator', 'caiman', 'spectacled caiman', 'gavial', + 'armored dinosaur', 'stegosaur', 'ankylosaur', 'edmontonia', + 'bone-headed dinosaur', 'pachycephalosaur', 'ceratopsian', 'protoceratops', + 'triceratops', 'styracosaur', 'psittacosaur', 'ornithopod', 'hadrosaur', + 'trachodon', 'saurischian', 'sauropod', 'apatosaur', 'barosaur', + 'diplodocus', 'argentinosaur', 'theropod', 'ceratosaur', 'coelophysis', + 'tyrannosaur', 'allosaur', 'ornithomimid', 'maniraptor', 'oviraptorid', + 'velociraptor', 'deinonychus', 'utahraptor', 'synapsid', 'dicynodont', + 'pelycosaur', 'dimetrodon', 'pterosaur', 'pterodactyl', 'ichthyosaur', + 'ichthyosaurus', 'stenopterygius', 'plesiosaur', 'nothosaur', 'snake', + 'colubrid snake', 'hoop snake', 'thunder snake', 'ringneck snake', + 'hognose snake', 'leaf-nosed snake', 'green snake', 'smooth green snake', + 'rough green snake', 'green snake', 'racer', 'blacksnake', 'blue racer', + 'horseshoe whipsnake', 'whip-snake', 'coachwhip', 'california whipsnake', + 'sonoran whipsnake', 'rat snake', 'corn snake', 'black rat snake', + 'chicken snake', 'indian rat snake', 'glossy snake', 'bull snake', + 'gopher snake', 'pine snake', 'king snake', 'common kingsnake', + 'milk snake', 'garter snake', 'common garter snake', 'ribbon snake', + 'western ribbon snake', 'lined snake', 'ground snake', + 'eastern ground snake', 'water snake', 'common water snake', + 'water moccasin', 'grass snake', 'viperine grass snake', + 'red-bellied snake', 'sand snake', 'banded sand snake', + 'black-headed snake', 'vine snake', 'lyre snake', 'sonoran lyre snake', + 'night snake', 'blind snake', 'western blind snake', 'indigo snake', + 'eastern indigo snake', 'constrictor', 'boa', 'boa constrictor', + 'rubber boa', 'rosy boa', 'anaconda', 'python', 'carpet snake', + 'reticulated python', 'indian python', 'rock python', 'amethystine python', + 'elapid', 'coral snake', 'eastern coral snake', 'western coral snake', + 'coral snake', 'african coral snake', 'australian coral snake', + 'copperhead', 'cobra', 'indian cobra', 'asp', 'black-necked cobra', + 'hamadryad', 'ringhals', 'mamba', 'black mamba', 'green mamba', + 'death adder', 'tiger snake', 'australian blacksnake', 'krait', + 'banded krait', 'taipan', 'sea snake', 'viper', 'adder', 'asp', + 'puff adder', 'gaboon viper', 'horned viper', 'pit viper', 'copperhead', + 'water moccasin', 'rattlesnake', 'diamondback', 'timber rattlesnake', + 'canebrake rattlesnake', 'prairie rattlesnake', 'sidewinder', + 'western diamondback', 'rock rattlesnake', 'tiger rattlesnake', + 'mojave rattlesnake', 'speckled rattlesnake', 'massasauga', + 'ground rattler', 'fer-de-lance', 'carcase', 'carrion', 'arthropod', + 'trilobite', 'arachnid', 'harvestman', 'scorpion', 'false scorpion', + 'book scorpion', 'whip-scorpion', 'vinegarroon', 'spider', + 'orb-weaving spider', 'black and gold garden spider', 'barn spider', + 'garden spider', 'comb-footed spider', 'black widow', 'tarantula', + 'wolf spider', 'european wolf spider', 'trap-door spider', 'acarine', + 'tick', 'hard tick', 'ixodes dammini', 'ixodes neotomae', + 'ixodes pacificus', 'ixodes scapularis', 'sheep-tick', 'ixodes persulcatus', + 'ixodes dentatus', 'ixodes spinipalpis', 'wood tick', 'soft tick', 'mite', + 'web-spinning mite', 'acarid', 'trombidiid', 'trombiculid', 'harvest mite', + 'acarus', 'itch mite', 'rust mite', 'spider mite', 'red spider', 'myriapod', + 'garden centipede', 'tardigrade', 'centipede', 'house centipede', + 'millipede', 'sea spider', 'merostomata', 'horseshoe crab', + 'asian horseshoe crab', 'eurypterid', 'tongue worm', 'gallinaceous bird', + 'domestic fowl', 'dorking', 'plymouth rock', 'cornish', 'rock cornish', + 'game fowl', 'cochin', 'jungle fowl', 'jungle cock', 'jungle hen', + 'red jungle fowl', 'chicken', 'bantam', 'chick', 'cock', 'cockerel', + 'capon', 'hen', 'cackler', 'brood hen', 'mother hen', 'layer', 'pullet', + 'spring chicken', 'rhode island red', 'dominique', 'orpington', 'turkey', + 'turkey cock', 'ocellated turkey', 'grouse', 'black grouse', + 'european black grouse', 'asian black grouse', 'blackcock', 'greyhen', + 'ptarmigan', 'red grouse', 'moorhen', 'capercaillie', 'spruce grouse', + 'sage grouse', 'ruffed grouse', 'sharp-tailed grouse', 'prairie chicken', + 'greater prairie chicken', 'lesser prairie chicken', 'heath hen', 'guan', + 'curassow', 'piping guan', 'chachalaca', 'texas chachalaca', 'megapode', + 'mallee fowl', 'mallee hen', 'brush turkey', 'maleo', 'phasianid', + 'pheasant', 'ring-necked pheasant', 'afropavo', 'argus', 'golden pheasant', + 'bobwhite', 'northern bobwhite', 'old world quail', 'migratory quail', + 'monal', 'peafowl', 'peachick', 'peacock', 'peahen', 'blue peafowl', + 'green peafowl', 'quail', 'california quail', 'tragopan', 'partridge', + 'hungarian partridge', 'red-legged partridge', 'greek partridge', + 'mountain quail', 'guinea fowl', 'guinea hen', 'hoatzin', 'tinamou', + 'columbiform bird', 'dodo', 'pigeon', 'pouter pigeon', 'dove', 'rock dove', + 'band-tailed pigeon', 'wood pigeon', 'turtledove', 'streptopelia turtur', + 'ringdove', 'australian turtledove', 'mourning dove', 'domestic pigeon', + 'squab', 'fairy swallow', 'roller', 'homing pigeon', 'carrier pigeon', + 'passenger pigeon', 'sandgrouse', 'painted sandgrouse', + 'pin-tailed sandgrouse', 'pallas\'s sandgrouse', 'parrot', 'popinjay', + 'poll', 'african grey', 'amazon', 'macaw', 'kea', 'cockatoo', + 'sulphur-crested cockatoo', 'pink cockatoo', 'cockateel', 'lovebird', + 'lory', 'lorikeet', 'varied lorikeet', 'rainbow lorikeet', 'parakeet', + 'carolina parakeet', 'budgerigar', 'ring-necked parakeet', + 'cuculiform bird', 'cuckoo', 'european cuckoo', 'black-billed cuckoo', + 'roadrunner', 'ani', 'coucal', 'crow pheasant', 'touraco', + 'coraciiform bird', 'roller', 'european roller', 'ground roller', + 'kingfisher', 'eurasian kingfisher', 'belted kingfisher', 'kookaburra', + 'bee eater', 'hornbill', 'hoopoe', 'euopean hoopoe', 'wood hoopoe', + 'motmot', 'tody', 'apodiform bird', 'swift', 'european swift', + 'chimney swift', 'swiftlet', 'tree swift', 'hummingbird', + 'archilochus colubris', 'thornbill', 'goatsucker', 'european goatsucker', + 'chuck-will\'s-widow', 'whippoorwill', 'poorwill', 'frogmouth', 'oilbird', + 'piciform bird', 'woodpecker', 'green woodpecker', 'downy woodpecker', + 'flicker', 'yellow-shafted flicker', 'gilded flicker', + 'red-shafted flicker', 'ivorybill', 'redheaded woodpecker', 'sapsucker', + 'yellow-bellied sapsucker', 'red-breasted sapsucker', 'wryneck', 'piculet', + 'barbet', 'puffbird', 'honey guide', 'jacamar', 'toucan', 'toucanet', + 'trogon', 'quetzal', 'resplendent quetzel', 'aquatic bird', 'waterfowl', + 'anseriform bird', 'duck', 'drake', 'quack-quack', 'duckling', + 'diving duck', 'dabbling duck', 'mallard', 'black duck', 'teal', + 'greenwing', 'bluewing', 'garganey', 'widgeon', 'american widgeon', + 'shoveler', 'pintail', 'sheldrake', 'shelduck', 'ruddy duck', 'bufflehead', + 'goldeneye', 'barrow\'s goldeneye', 'canvasback', 'pochard', 'redhead', + 'scaup', 'greater scaup', 'lesser scaup', 'wild duck', 'wood duck', + 'wood drake', 'mandarin duck', 'muscovy duck', 'sea duck', 'eider', + 'scoter', 'common scoter', 'old squaw', 'merganser', 'goosander', + 'american merganser', 'red-breasted merganser', 'smew', 'hooded merganser', + 'goose', 'gosling', 'gander', 'chinese goose', 'greylag', 'blue goose', + 'snow goose', 'brant', 'common brant goose', 'honker', 'barnacle goose', + 'coscoroba', 'swan', 'cob', 'pen', 'cygnet', 'mute swan', 'whooper', + 'tundra swan', 'whistling swan', 'bewick\'s swan', 'trumpeter', + 'black swan', 'screamer', 'horned screamer', 'crested screamer', 'chaja', + 'mammal', 'female mammal', 'tusker', 'prototherian', 'monotreme', 'echidna', + 'echidna', 'platypus', 'marsupial', 'opossum', 'common opossum', + 'crab-eating opossum', 'opossum rat', 'bandicoot', 'rabbit-eared bandicoot', + 'kangaroo', 'giant kangaroo', 'wallaby', 'common wallaby', 'hare wallaby', + 'nail-tailed wallaby', 'rock wallaby', 'pademelon', 'tree wallaby', + 'musk kangaroo', 'rat kangaroo', 'potoroo', 'bettong', 'jerboa kangaroo', + 'phalanger', 'cuscus', 'brush-tailed phalanger', 'flying phalanger', + 'koala', 'wombat', 'dasyurid marsupial', 'dasyure', 'eastern dasyure', + 'native cat', 'thylacine', 'tasmanian devil', 'pouched mouse', 'numbat', + 'pouched mole', 'placental', 'livestock', 'bull', 'cow', 'calf', 'calf', + 'yearling', 'buck', 'doe', 'insectivore', 'mole', 'starnose mole', + 'brewer\'s mole', 'golden mole', 'shrew mole', 'asiatic shrew mole', + 'american shrew mole', 'shrew', 'common shrew', 'masked shrew', + 'short-tailed shrew', 'water shrew', 'american water shrew', + 'european water shrew', 'mediterranean water shrew', 'least shrew', + 'hedgehog', 'tenrec', 'tailless tenrec', 'otter shrew', 'eiderdown', + 'aftershaft', 'sickle feather', 'contour feather', 'bastard wing', + 'saddle hackle', 'encolure', 'hair', 'squama', 'scute', 'sclerite', + 'plastron', 'scallop shell', 'oyster shell', 'theca', 'invertebrate', + 'sponge', 'choanocyte', 'glass sponge', 'venus\'s flower basket', + 'metazoan', 'coelenterate', 'planula', 'polyp', 'medusa', 'jellyfish', + 'scyphozoan', 'chrysaora quinquecirrha', 'hydrozoan', 'hydra', + 'siphonophore', 'nanomia', 'portuguese man-of-war', 'praya', 'apolemia', + 'anthozoan', 'sea anemone', 'actinia', 'sea pen', 'coral', 'gorgonian', + 'sea feather', 'sea fan', 'red coral', 'stony coral', 'brain coral', + 'staghorn coral', 'mushroom coral', 'ctenophore', 'beroe', 'platyctenean', + 'sea gooseberry', 'venus\'s girdle', 'worm', 'helminth', 'woodworm', + 'woodborer', 'acanthocephalan', 'arrowworm', 'bladder worm', 'flatworm', + 'planarian', 'fluke', 'cercaria', 'liver fluke', 'fasciolopsis buski', + 'schistosome', 'tapeworm', 'echinococcus', 'taenia', 'ribbon worm', + 'beard worm', 'rotifer', 'nematode', 'common roundworm', + 'chicken roundworm', 'pinworm', 'eelworm', 'vinegar eel', 'trichina', + 'hookworm', 'filaria', 'guinea worm', 'annelid', 'archiannelid', + 'oligochaete', 'earthworm', 'polychaete', 'lugworm', 'sea mouse', + 'bloodworm', 'leech', 'medicinal leech', 'horseleech', 'mollusk', + 'scaphopod', 'tooth shell', 'gastropod', 'abalone', 'ormer', + 'scorpion shell', 'conch', 'giant conch', 'snail', 'edible snail', + 'garden snail', 'brown snail', 'helix hortensis', 'slug', 'seasnail', + 'neritid', 'nerita', 'bleeding tooth', 'neritina', 'whelk', 'moon shell', + 'periwinkle', 'limpet', 'common limpet', 'keyhole limpet', 'river limpet', + 'sea slug', 'sea hare', 'hermissenda crassicornis', 'bubble shell', 'physa', + 'cowrie', 'money cowrie', 'tiger cowrie', 'solenogaster', 'chiton', + 'bivalve', 'spat', 'clam', 'seashell', 'soft-shell clam', 'quahog', + 'littleneck', 'cherrystone', 'geoduck', 'razor clam', 'giant clam', + 'cockle', 'edible cockle', 'oyster', 'japanese oyster', 'virginia oyster', + 'pearl oyster', 'saddle oyster', 'window oyster', 'ark shell', 'blood clam', + 'mussel', 'marine mussel', 'edible mussel', 'freshwater mussel', + 'pearly-shelled mussel', 'thin-shelled mussel', 'zebra mussel', 'scallop', + 'bay scallop', 'sea scallop', 'shipworm', 'teredo', 'piddock', 'cephalopod', + 'chambered nautilus', 'octopod', 'octopus', 'paper nautilus', 'decapod', + 'squid', 'loligo', 'ommastrephes', 'architeuthis', 'cuttlefish', 'spirula', + 'crustacean', 'malacostracan crustacean', 'decapod crustacean', + 'brachyuran', 'crab', 'stone crab', 'hard-shell crab', 'soft-shell crab', + 'dungeness crab', 'rock crab', 'jonah crab', 'swimming crab', + 'english lady crab', 'american lady crab', 'blue crab', 'fiddler crab', + 'pea crab', 'king crab', 'spider crab', 'european spider crab', + 'giant crab', 'lobster', 'true lobster', 'american lobster', + 'european lobster', 'cape lobster', 'norway lobster', 'spiny lobster', + 'crayfish', 'old world crayfish', 'american crayfish', 'hermit crab', + 'shrimp', 'snapping shrimp', 'prawn', 'long-clawed prawn', 'tropical prawn', + 'krill', 'euphausia pacifica', 'opossum shrimp', 'stomatopod', + 'mantis shrimp', 'squilla', 'isopod', 'woodlouse', 'pill bug', 'sow bug', + 'sea louse', 'amphipod', 'skeleton shrimp', 'whale louse', 'daphnia', + 'fairy shrimp', 'brine shrimp', 'tadpole shrimp', 'copepod', 'cyclops', + 'seed shrimp', 'barnacle', 'acorn barnacle', 'goose barnacle', + 'onychophoran', 'wading bird', 'stork', 'white stork', 'black stork', + 'adjutant bird', 'marabou', 'openbill', 'jabiru', 'saddlebill', + 'policeman bird', 'wood ibis', 'shoebill', 'ibis', 'wood ibis', + 'sacred ibis', 'spoonbill', 'common spoonbill', 'roseate spoonbill', + 'flamingo', 'heron', 'great blue heron', 'great white heron', 'egret', + 'little blue heron', 'snowy egret', 'little egret', 'great white heron', + 'american egret', 'cattle egret', 'night heron', + 'black-crowned night heron', 'yellow-crowned night heron', 'boatbill', + 'bittern', 'american bittern', 'european bittern', 'least bittern', 'crane', + 'whooping crane', 'courlan', 'limpkin', 'crested cariama', 'chunga', 'rail', + 'weka', 'crake', 'corncrake', 'spotted crake', 'gallinule', + 'florida gallinule', 'moorhen', 'purple gallinule', 'european gallinule', + 'american gallinule', 'notornis', 'coot', 'american coot', 'old world coot', + 'bustard', 'great bustard', 'plain turkey', 'button quail', + 'striped button quail', 'plain wanderer', 'trumpeter', + 'brazilian trumpeter', 'seabird', 'shorebird', 'plover', 'piping plover', + 'killdeer', 'dotterel', 'golden plover', 'lapwing', 'turnstone', + 'ruddy turnstone', 'black turnstone', 'sandpiper', 'surfbird', + 'european sandpiper', 'spotted sandpiper', 'least sandpiper', + 'red-backed sandpiper', 'greenshank', 'redshank', 'yellowlegs', + 'greater yellowlegs', 'lesser yellowlegs', 'pectoral sandpiper', 'knot', + 'curlew sandpiper', 'sanderling', 'upland sandpiper', 'ruff', 'reeve', + 'tattler', 'polynesian tattler', 'willet', 'woodcock', 'eurasian woodcock', + 'american woodcock', 'snipe', 'whole snipe', 'wilson\'s snipe', + 'great snipe', 'jacksnipe', 'dowitcher', 'greyback', 'red-breasted snipe', + 'curlew', 'european curlew', 'eskimo curlew', 'godwit', 'hudsonian godwit', + 'stilt', 'black-necked stilt', 'black-winged stilt', 'white-headed stilt', + 'kaki', 'stilt', 'banded stilt', 'avocet', 'oystercatcher', 'phalarope', + 'red phalarope', 'northern phalarope', 'wilson\'s phalarope', 'pratincole', + 'courser', 'cream-colored courser', 'crocodile bird', 'stone curlew', + 'coastal diving bird', 'larid', 'gull', 'mew', 'black-backed gull', + 'herring gull', 'laughing gull', 'ivory gull', 'kittiwake', 'tern', + 'sea swallow', 'skimmer', 'jaeger', 'parasitic jaeger', 'skua', + 'great skua', 'auk', 'auklet', 'razorbill', 'little auk', 'guillemot', + 'black guillemot', 'pigeon guillemot', 'murre', 'common murre', + 'thick-billed murre', 'puffin', 'atlantic puffin', 'horned puffin', + 'tufted puffin', 'gaviiform seabird', 'loon', 'podicipitiform seabird', + 'grebe', 'great crested grebe', 'red-necked grebe', 'black-necked grebe', + 'dabchick', 'pied-billed grebe', 'pelecaniform seabird', 'pelican', + 'white pelican', 'old world white pelican', 'frigate bird', 'gannet', + 'solan', 'booby', 'cormorant', 'snakebird', 'water turkey', 'tropic bird', + 'sphenisciform seabird', 'penguin', 'adelie', 'king penguin', + 'emperor penguin', 'jackass penguin', 'rock hopper', 'pelagic bird', + 'procellariiform seabird', 'albatross', 'wandering albatross', + 'black-footed albatross', 'petrel', 'white-chinned petrel', 'giant petrel', + 'fulmar', 'shearwater', 'manx shearwater', 'storm petrel', 'stormy petrel', + 'mother carey\'s chicken', 'diving petrel', 'aquatic mammal', 'cetacean', + 'whale', 'baleen whale', 'right whale', 'bowhead', 'rorqual', 'blue whale', + 'finback', 'sei whale', 'lesser rorqual', 'humpback', 'grey whale', + 'toothed whale', 'sperm whale', 'pygmy sperm whale', 'dwarf sperm whale', + 'beaked whale', 'bottle-nosed whale', 'dolphin', 'common dolphin', + 'bottlenose dolphin', 'atlantic bottlenose dolphin', + 'pacific bottlenose dolphin', 'porpoise', 'harbor porpoise', 'vaquita', + 'grampus', 'killer whale', 'pilot whale', 'river dolphin', 'narwhal', + 'white whale', 'sea cow', 'manatee', 'dugong', 'steller\'s sea cow', + 'carnivore', 'omnivore', 'pinniped mammal', 'seal', 'crabeater seal', + 'eared seal', 'fur seal', 'guadalupe fur seal', 'fur seal', + 'alaska fur seal', 'sea lion', 'south american sea lion', + 'california sea lion', 'australian sea lion', 'steller sea lion', + 'earless seal', 'harbor seal', 'harp seal', 'elephant seal', 'bearded seal', + 'hooded seal', 'walrus', 'atlantic walrus', 'pacific walrus', 'fissipedia', + 'fissiped mammal', 'aardvark', 'canine', 'bitch', 'brood bitch', 'dog', + 'pooch', 'cur', 'feist', 'pariah dog', 'lapdog', 'toy dog', 'chihuahua', + 'japanese spaniel', 'maltese dog', 'pekinese', 'shih-tzu', 'toy spaniel', + 'english toy spaniel', 'blenheim spaniel', 'king charles spaniel', + 'papillon', 'toy terrier', 'hunting dog', 'courser', 'rhodesian ridgeback', + 'hound', 'afghan hound', 'basset', 'beagle', 'bloodhound', 'bluetick', + 'boarhound', 'coonhound', 'coondog', 'black-and-tan coonhound', 'dachshund', + 'sausage dog', 'foxhound', 'american foxhound', 'walker hound', + 'english foxhound', 'harrier', 'plott hound', 'redbone', 'wolfhound', + 'borzoi', 'irish wolfhound', 'greyhound', 'italian greyhound', 'whippet', + 'ibizan hound', 'norwegian elkhound', 'otterhound', 'saluki', + 'scottish deerhound', 'staghound', 'weimaraner', 'terrier', 'bullterrier', + 'staffordshire bullterrier', 'american staffordshire terrier', + 'bedlington terrier', 'border terrier', 'kerry blue terrier', + 'irish terrier', 'norfolk terrier', 'norwich terrier', 'yorkshire terrier', + 'rat terrier', 'manchester terrier', 'toy manchester', 'fox terrier', + 'smooth-haired fox terrier', 'wire-haired fox terrier', 'wirehair', + 'lakeland terrier', 'welsh terrier', 'sealyham terrier', 'airedale', + 'cairn', 'australian terrier', 'dandie dinmont', 'boston bull', 'schnauzer', + 'miniature schnauzer', 'giant schnauzer', 'standard schnauzer', + 'scotch terrier', 'tibetan terrier', 'silky terrier', 'skye terrier', + 'clydesdale terrier', 'soft-coated wheaten terrier', + 'west highland white terrier', 'lhasa', 'sporting dog', 'bird dog', + 'water dog', 'retriever', 'flat-coated retriever', 'curly-coated retriever', + 'golden retriever', 'labrador retriever', 'chesapeake bay retriever', + 'pointer', 'german short-haired pointer', 'setter', 'vizsla', + 'english setter', 'irish setter', 'gordon setter', 'spaniel', + 'brittany spaniel', 'clumber', 'field spaniel', 'springer spaniel', + 'english springer', 'welsh springer spaniel', 'cocker spaniel', + 'sussex spaniel', 'water spaniel', 'american water spaniel', + 'irish water spaniel', 'griffon', 'working dog', 'watchdog', 'kuvasz', + 'attack dog', 'housedog', 'schipperke', 'shepherd dog', 'belgian sheepdog', + 'groenendael', 'malinois', 'briard', 'kelpie', 'komondor', + 'old english sheepdog', 'shetland sheepdog', 'collie', 'border collie', + 'bouvier des flandres', 'rottweiler', 'german shepherd', 'police dog', + 'pinscher', 'doberman', 'miniature pinscher', 'sennenhunde', + 'greater swiss mountain dog', 'bernese mountain dog', 'appenzeller', + 'entlebucher', 'boxer', 'mastiff', 'bull mastiff', 'tibetan mastiff', + 'bulldog', 'french bulldog', 'great dane', 'guide dog', 'seeing eye dog', + 'hearing dog', 'saint bernard', 'seizure-alert dog', 'sled dog', + 'eskimo dog', 'malamute', 'siberian husky', 'dalmatian', + 'liver-spotted dalmatian', 'affenpinscher', 'basenji', 'pug', 'leonberg', + 'newfoundland', 'great pyrenees', 'spitz', 'samoyed', 'pomeranian', 'chow', + 'keeshond', 'griffon', 'brabancon griffon', 'corgi', 'pembroke', 'cardigan', + 'poodle', 'toy poodle', 'miniature poodle', 'standard poodle', + 'large poodle', 'mexican hairless', 'wolf', 'timber wolf', 'white wolf', + 'red wolf', 'coyote', 'coydog', 'jackal', 'wild dog', 'dingo', 'dhole', + 'crab-eating dog', 'raccoon dog', 'african hunting dog', 'hyena', + 'striped hyena', 'brown hyena', 'spotted hyena', 'aardwolf', 'fox', 'vixen', + 'reynard', 'red fox', 'black fox', 'silver fox', 'red fox', 'kit fox', + 'kit fox', 'arctic fox', 'blue fox', 'grey fox', 'feline', 'cat', + 'domestic cat', 'kitty', 'mouser', 'alley cat', 'stray', 'tom', 'gib', + 'tabby', 'kitten', 'tabby', 'tiger cat', 'tortoiseshell', 'persian cat', + 'angora', 'siamese cat', 'blue point siamese', 'burmese cat', + 'egyptian cat', 'maltese', 'abyssinian', 'manx', 'wildcat', 'sand cat', + 'european wildcat', 'cougar', 'ocelot', 'jaguarundi', 'kaffir cat', + 'jungle cat', 'serval', 'leopard cat', 'margay', 'manul', 'lynx', + 'common lynx', 'canada lynx', 'bobcat', 'spotted lynx', 'caracal', + 'big cat', 'leopard', 'leopardess', 'panther', 'snow leopard', 'jaguar', + 'lion', 'lioness', 'lionet', 'tiger', 'bengal tiger', 'tigress', 'liger', + 'tiglon', 'cheetah', 'saber-toothed tiger', 'smiledon californicus', 'bear', + 'brown bear', 'bruin', 'syrian bear', 'grizzly', 'alaskan brown bear', + 'american black bear', 'cinnamon bear', 'asiatic black bear', 'ice bear', + 'sloth bear', 'viverrine', 'civet', 'large civet', 'small civet', + 'binturong', 'cryptoprocta', 'fossa', 'fanaloka', 'genet', + 'banded palm civet', 'mongoose', 'indian mongoose', 'ichneumon', 'palm cat', + 'meerkat', 'slender-tailed meerkat', 'suricate', 'bat', 'fruit bat', + 'flying fox', 'pteropus capestratus', 'pteropus hypomelanus', 'harpy', + 'cynopterus sphinx', 'carnivorous bat', 'mouse-eared bat', 'leafnose bat', + 'macrotus', 'spearnose bat', 'phyllostomus hastatus', 'hognose bat', + 'horseshoe bat', 'horseshoe bat', 'orange bat', 'false vampire', + 'big-eared bat', 'vespertilian bat', 'frosted bat', 'red bat', 'brown bat', + 'little brown bat', 'cave myotis', 'big brown bat', 'serotine', + 'pallid bat', 'pipistrelle', 'eastern pipistrel', 'jackass bat', + 'long-eared bat', 'western big-eared bat', 'freetail', 'guano bat', + 'pocketed bat', 'mastiff bat', 'vampire bat', 'desmodus rotundus', + 'hairy-legged vampire bat', 'predator', 'prey', 'game', 'big game', + 'game bird', 'fossorial mammal', 'tetrapod', 'quadruped', 'hexapod', + 'biped', 'insect', 'social insect', 'holometabola', 'defoliator', + 'pollinator', 'gallfly', 'scorpion fly', 'hanging fly', 'collembolan', + 'beetle', 'tiger beetle', 'ladybug', 'two-spotted ladybug', + 'mexican bean beetle', 'hippodamia convergens', 'vedalia', 'ground beetle', + 'bombardier beetle', 'calosoma', 'searcher', 'firefly', 'glowworm', + 'long-horned beetle', 'sawyer', 'pine sawyer', 'leaf beetle', 'flea beetle', + 'colorado potato beetle', 'carpet beetle', 'buffalo carpet beetle', + 'black carpet beetle', 'clerid beetle', 'bee beetle', 'lamellicorn beetle', + 'scarabaeid beetle', 'dung beetle', 'scarab', 'tumblebug', 'dorbeetle', + 'june beetle', 'green june beetle', 'japanese beetle', 'oriental beetle', + 'rhinoceros beetle', 'melolonthid beetle', 'cockchafer', 'rose chafer', + 'rose chafer', 'stag beetle', 'elaterid beetle', 'click beetle', 'firefly', + 'wireworm', 'water beetle', 'whirligig beetle', 'deathwatch beetle', + 'weevil', 'snout beetle', 'boll weevil', 'blister beetle', 'oil beetle', + 'spanish fly', 'dutch-elm beetle', 'bark beetle', 'spruce bark beetle', + 'rove beetle', 'darkling beetle', 'mealworm', 'flour beetle', 'seed beetle', + 'pea weevil', 'bean weevil', 'rice weevil', 'asian longhorned beetle', + 'web spinner', 'louse', 'common louse', 'head louse', 'body louse', + 'crab louse', 'bird louse', 'flea', 'pulex irritans', 'dog flea', + 'cat flea', 'chigoe', 'sticktight', 'dipterous insect', 'gall midge', + 'hessian fly', 'fly', 'housefly', 'tsetse fly', 'blowfly', 'bluebottle', + 'greenbottle', 'flesh fly', 'tachina fly', 'gadfly', 'botfly', + 'human botfly', 'sheep botfly', 'warble fly', 'horsefly', 'bee fly', + 'robber fly', 'fruit fly', 'apple maggot', 'mediterranean fruit fly', + 'drosophila', 'vinegar fly', 'leaf miner', 'louse fly', 'horse tick', + 'sheep ked', 'horn fly', 'mosquito', 'wiggler', 'gnat', + 'yellow-fever mosquito', 'asian tiger mosquito', 'anopheline', + 'malarial mosquito', 'common mosquito', 'culex quinquefasciatus', 'gnat', + 'punkie', 'midge', 'fungus gnat', 'psychodid', 'sand fly', 'fungus gnat', + 'armyworm', 'crane fly', 'blackfly', 'hymenopterous insect', 'bee', 'drone', + 'queen bee', 'worker', 'soldier', 'worker bee', 'honeybee', + 'africanized bee', 'black bee', 'carniolan bee', 'italian bee', + 'carpenter bee', 'bumblebee', 'cuckoo-bumblebee', 'andrena', + 'nomia melanderi', 'leaf-cutting bee', 'mason bee', 'potter bee', 'wasp', + 'vespid', 'paper wasp', 'hornet', 'giant hornet', 'common wasp', + 'bald-faced hornet', 'yellow jacket', 'polistes annularis', 'mason wasp', + 'potter wasp', 'mutillidae', 'velvet ant', 'sphecoid wasp', 'mason wasp', + 'digger wasp', 'cicada killer', 'mud dauber', 'gall wasp', 'chalcid fly', + 'strawworm', 'chalcis fly', 'ichneumon fly', 'sawfly', 'birch leaf miner', + 'ant', 'pharaoh ant', 'little black ant', 'army ant', 'carpenter ant', + 'fire ant', 'wood ant', 'slave ant', 'formica fusca', 'slave-making ant', + 'sanguinary ant', 'bulldog ant', 'amazon ant', 'termite', + 'dry-wood termite', 'reticulitermes lucifugus', 'mastotermes darwiniensis', + 'mastotermes electrodominicus', 'powder-post termite', + 'orthopterous insect', 'grasshopper', 'short-horned grasshopper', 'locust', + 'migratory locust', 'migratory grasshopper', 'long-horned grasshopper', + 'katydid', 'mormon cricket', 'sand cricket', 'cricket', 'mole cricket', + 'european house cricket', 'field cricket', 'tree cricket', + 'snowy tree cricket', 'phasmid', 'walking stick', 'diapheromera', + 'walking leaf', 'cockroach', 'oriental cockroach', 'american cockroach', + 'australian cockroach', 'german cockroach', 'giant cockroach', 'mantis', + 'praying mantis', 'bug', 'hemipterous insect', 'leaf bug', 'mirid bug', + 'four-lined plant bug', 'lygus bug', 'tarnished plant bug', 'lace bug', + 'lygaeid', 'chinch bug', 'coreid bug', 'squash bug', 'leaf-footed bug', + 'bedbug', 'backswimmer', 'true bug', 'heteropterous insect', 'water bug', + 'giant water bug', 'water scorpion', 'water boatman', 'water strider', + 'common pond-skater', 'assassin bug', 'conenose', 'wheel bug', 'firebug', + 'cotton stainer', 'homopterous insect', 'whitefly', 'citrus whitefly', + 'greenhouse whitefly', 'sweet-potato whitefly', 'superbug', 'cotton strain', + 'coccid insect', 'scale insect', 'soft scale', 'brown soft scale', + 'armored scale', 'san jose scale', 'cochineal insect', 'mealybug', + 'citrophilous mealybug', 'comstock mealybug', 'citrus mealybug', + 'plant louse', 'aphid', 'apple aphid', 'blackfly', 'greenfly', + 'green peach aphid', 'ant cow', 'woolly aphid', 'woolly apple aphid', + 'woolly alder aphid', 'adelgid', 'balsam woolly aphid', 'spruce gall aphid', + 'woolly adelgid', 'jumping plant louse', 'cicada', 'dog-day cicada', + 'seventeen-year locust', 'spittle insect', 'froghopper', + 'meadow spittlebug', 'pine spittlebug', 'saratoga spittlebug', 'leafhopper', + 'plant hopper', 'treehopper', 'lantern fly', 'psocopterous insect', + 'psocid', 'bark-louse', 'booklouse', 'common booklouse', 'ephemerid', + 'mayfly', 'stonefly', 'neuropteron', 'ant lion', 'doodlebug', 'lacewing', + 'aphid lion', 'green lacewing', 'brown lacewing', 'dobson', 'hellgrammiate', + 'fish fly', 'alderfly', 'snakefly', 'mantispid', 'odonate', 'dragonfly', + 'damselfly', 'trichopterous insect', 'caddis fly', 'caseworm', 'caddisworm', + 'thysanuran insect', 'bristletail', 'silverfish', 'firebrat', + 'jumping bristletail', 'thysanopter', 'thrips', 'tobacco thrips', + 'onion thrips', 'earwig', 'common european earwig', 'lepidopterous insect', + 'butterfly', 'nymphalid', 'mourning cloak', 'tortoiseshell', + 'painted beauty', 'admiral', 'red admiral', 'white admiral', + 'banded purple', 'red-spotted purple', 'viceroy', 'anglewing', 'ringlet', + 'comma', 'fritillary', 'silverspot', 'emperor butterfly', 'purple emperor', + 'peacock', 'danaid', 'monarch', 'pierid', 'cabbage butterfly', + 'small white', 'large white', 'southern cabbage butterfly', + 'sulphur butterfly', 'lycaenid', 'blue', 'copper', 'american copper', + 'hairstreak', 'strymon melinus', 'moth', 'moth miller', 'tortricid', + 'leaf roller', 'tea tortrix', 'orange tortrix', 'codling moth', + 'lymantriid', 'tussock caterpillar', 'gypsy moth', 'browntail', + 'gold-tail moth', 'geometrid', 'paleacrita vernata', 'alsophila pometaria', + 'cankerworm', 'spring cankerworm', 'fall cankerworm', 'measuring worm', + 'pyralid', 'bee moth', 'corn borer', 'mediterranean flour moth', + 'tobacco moth', 'almond moth', 'raisin moth', 'tineoid', 'tineid', + 'clothes moth', 'casemaking clothes moth', 'webbing clothes moth', + 'carpet moth', 'gelechiid', 'grain moth', 'angoumois moth', 'potato moth', + 'potato tuberworm', 'noctuid moth', 'cutworm', 'underwing', 'red underwing', + 'antler moth', 'heliothis moth', 'army cutworm', 'armyworm', 'armyworm', + 'spodoptera exigua', 'beet armyworm', 'spodoptera frugiperda', + 'fall armyworm', 'hawkmoth', 'manduca sexta', 'tobacco hornworm', + 'manduca quinquemaculata', 'tomato hornworm', 'death\'s-head moth', + 'bombycid', 'domestic silkworm moth', 'silkworm', 'saturniid', 'emperor', + 'imperial moth', 'giant silkworm moth', 'silkworm', 'luna moth', 'cecropia', + 'cynthia moth', 'ailanthus silkworm', 'io moth', 'polyphemus moth', + 'pernyi moth', 'tussah', 'atlas moth', 'arctiid', 'tiger moth', 'cinnabar', + 'lasiocampid', 'eggar', 'tent-caterpillar moth', 'tent caterpillar', + 'tent-caterpillar moth', 'forest tent caterpillar', 'lappet', + 'lappet caterpillar', 'webworm', 'webworm moth', 'hyphantria cunea', + 'fall webworm', 'garden webworm', 'instar', 'caterpillar', 'corn borer', + 'bollworm', 'pink bollworm', 'corn earworm', 'cabbageworm', 'woolly bear', + 'woolly bear moth', 'larva', 'nymph', 'leptocephalus', 'grub', 'maggot', + 'leatherjacket', 'pupa', 'chrysalis', 'imago', 'queen', 'phoronid', + 'bryozoan', 'brachiopod', 'peanut worm', 'echinoderm', 'starfish', + 'brittle star', 'basket star', 'astrophyton muricatum', 'sea urchin', + 'edible sea urchin', 'sand dollar', 'heart urchin', 'crinoid', 'sea lily', + 'feather star', 'sea cucumber', 'trepang', 'duplicidentata', 'lagomorph', + 'leporid', 'rabbit', 'rabbit ears', 'lapin', 'bunny', 'european rabbit', + 'wood rabbit', 'eastern cottontail', 'swamp rabbit', 'marsh hare', 'hare', + 'leveret', 'european hare', 'jackrabbit', 'white-tailed jackrabbit', + 'blacktail jackrabbit', 'polar hare', 'snowshoe hare', 'belgian hare', + 'angora', 'pika', 'little chief hare', 'collared pika', 'rodent', 'mouse', + 'rat', 'pocket rat', 'murine', 'house mouse', 'harvest mouse', + 'field mouse', 'nude mouse', 'european wood mouse', 'brown rat', + 'wharf rat', 'sewer rat', 'black rat', 'bandicoot rat', 'jerboa rat', + 'kangaroo mouse', 'water rat', 'beaver rat', 'new world mouse', + 'american harvest mouse', 'wood mouse', 'white-footed mouse', 'deer mouse', + 'cactus mouse', 'cotton mouse', 'pygmy mouse', 'grasshopper mouse', + 'muskrat', 'round-tailed muskrat', 'cotton rat', 'wood rat', + 'dusky-footed wood rat', 'vole', 'packrat', 'dusky-footed woodrat', + 'eastern woodrat', 'rice rat', 'pine vole', 'meadow vole', 'water vole', + 'prairie vole', 'water vole', 'red-backed mouse', 'phenacomys', 'hamster', + 'eurasian hamster', 'golden hamster', 'gerbil', 'jird', 'tamarisk gerbil', + 'sand rat', 'lemming', 'european lemming', 'brown lemming', 'grey lemming', + 'pied lemming', 'hudson bay collared lemming', 'southern bog lemming', + 'northern bog lemming', 'porcupine', 'old world porcupine', + 'brush-tailed porcupine', 'long-tailed porcupine', 'new world porcupine', + 'canada porcupine', 'pocket mouse', 'silky pocket mouse', + 'plains pocket mouse', 'hispid pocket mouse', 'mexican pocket mouse', + 'kangaroo rat', 'ord kangaroo rat', 'kangaroo mouse', 'jumping mouse', + 'meadow jumping mouse', 'jerboa', 'typical jerboa', 'jaculus jaculus', + 'dormouse', 'loir', 'hazel mouse', 'lerot', 'gopher', + 'plains pocket gopher', 'southeastern pocket gopher', + 'valley pocket gopher', 'northern pocket gopher', 'squirrel', + 'tree squirrel', 'eastern grey squirrel', 'western grey squirrel', + 'fox squirrel', 'black squirrel', 'red squirrel', 'american red squirrel', + 'chickeree', 'antelope squirrel', 'ground squirrel', + 'mantled ground squirrel', 'suslik', 'flickertail', 'rock squirrel', + 'arctic ground squirrel', 'prairie dog', 'blacktail prairie dog', + 'whitetail prairie dog', 'eastern chipmunk', 'chipmunk', 'baronduki', + 'american flying squirrel', 'southern flying squirrel', + 'northern flying squirrel', 'marmot', 'groundhog', 'hoary marmot', + 'yellowbelly marmot', 'asiatic flying squirrel', 'beaver', + 'old world beaver', 'new world beaver', 'mountain beaver', 'cavy', + 'guinea pig', 'aperea', 'mara', 'capybara', 'agouti', 'paca', + 'mountain paca', 'coypu', 'chinchilla', 'mountain chinchilla', 'viscacha', + 'abrocome', 'mole rat', 'mole rat', 'sand rat', 'naked mole rat', 'queen', + 'damaraland mole rat', 'ungulata', 'ungulate', 'unguiculate', 'dinoceras', + 'hyrax', 'rock hyrax', 'odd-toed ungulate', 'equine', 'horse', 'roan', + 'stablemate', 'gee-gee', 'eohippus', 'foal', 'filly', 'colt', 'male horse', + 'ridgeling', 'stallion', 'stud', 'gelding', 'mare', 'broodmare', + 'saddle horse', 'remount', 'palfrey', 'warhorse', 'cavalry horse', + 'charger', 'steed', 'prancer', 'hack', 'cow pony', 'quarter horse', + 'morgan', 'tennessee walker', 'american saddle horse', 'appaloosa', + 'arabian', 'lippizan', 'pony', 'polo pony', 'mustang', 'bronco', + 'bucking bronco', 'buckskin', 'crowbait', 'dun', 'grey', 'wild horse', + 'tarpan', 'przewalski\'s horse', 'cayuse', 'hack', 'hack', 'plow horse', + 'pony', 'shetland pony', 'welsh pony', 'exmoor', 'racehorse', + 'thoroughbred', 'steeplechaser', 'racer', 'finisher', 'pony', 'yearling', + 'dark horse', 'mudder', 'nonstarter', 'stalking-horse', 'harness horse', + 'cob', 'hackney', 'workhorse', 'draft horse', 'packhorse', 'carthorse', + 'clydesdale', 'percheron', 'farm horse', 'shire', 'pole horse', + 'post horse', 'coach horse', 'pacer', 'pacer', 'trotting horse', + 'pole horse', 'stepper', 'chestnut', 'liver chestnut', 'bay', 'sorrel', + 'palomino', 'pinto', 'ass', 'domestic ass', 'burro', 'moke', 'jack', + 'jennet', 'mule', 'hinny', 'wild ass', 'african wild ass', 'kiang', + 'onager', 'chigetai', 'zebra', 'common zebra', 'mountain zebra', + 'grevy\'s zebra', 'quagga', 'rhinoceros', 'indian rhinoceros', + 'woolly rhinoceros', 'white rhinoceros', 'black rhinoceros', 'tapir', + 'new world tapir', 'malayan tapir', 'even-toed ungulate', 'swine', 'hog', + 'piglet', 'sucking pig', 'porker', 'boar', 'sow', 'razorback', 'wild boar', + 'babirusa', 'warthog', 'peccary', 'collared peccary', + 'white-lipped peccary', 'hippopotamus', 'ruminant', 'bovid', 'bovine', 'ox', + 'cattle', 'ox', 'stirk', 'bullock', 'bull', 'cow', 'heifer', 'bullock', + 'dogie', 'maverick', 'beef', 'longhorn', 'brahman', 'zebu', 'aurochs', + 'yak', 'banteng', 'welsh', 'red poll', 'santa gertrudis', 'aberdeen angus', + 'africander', 'dairy cattle', 'ayrshire', 'brown swiss', 'charolais', + 'jersey', 'devon', 'grade', 'durham', 'milking shorthorn', 'galloway', + 'friesian', 'guernsey', 'hereford', 'cattalo', 'old world buffalo', + 'water buffalo', 'indian buffalo', 'carabao', 'anoa', 'tamarau', + 'cape buffalo', 'asian wild ox', 'gaur', 'gayal', 'bison', 'american bison', + 'wisent', 'musk ox', 'sheep', 'ewe', 'ram', 'wether', 'lamb', 'lambkin', + 'baa-lamb', 'hog', 'teg', 'persian lamb', 'black sheep', 'domestic sheep', + 'cotswold', 'hampshire', 'lincoln', 'exmoor', 'cheviot', 'broadtail', + 'longwool', 'merino', 'rambouillet', 'wild sheep', 'argali', + 'marco polo sheep', 'urial', 'dall sheep', 'mountain sheep', 'bighorn', + 'mouflon', 'aoudad', 'goat', 'kid', 'billy', 'nanny', 'domestic goat', + 'cashmere goat', 'angora', 'wild goat', 'bezoar goat', 'markhor', 'ibex', + 'goat antelope', 'mountain goat', 'goral', 'serow', 'chamois', 'takin', + 'antelope', 'blackbuck', 'gerenuk', 'addax', 'gnu', 'dik-dik', 'hartebeest', + 'sassaby', 'impala', 'gazelle', 'thomson\'s gazelle', + 'gazella subgutturosa', 'springbok', 'bongo', 'kudu', 'greater kudu', + 'lesser kudu', 'harnessed antelope', 'nyala', 'mountain nyala', 'bushbuck', + 'nilgai', 'sable antelope', 'saiga', 'steenbok', 'eland', 'common eland', + 'giant eland', 'kob', 'lechwe', 'waterbuck', 'puku', 'oryx', 'gemsbok', + 'forest goat', 'pronghorn', 'deer', 'stag', 'royal', 'pricket', 'fawn', + 'red deer', 'hart', 'hind', 'brocket', 'sambar', 'wapiti', 'japanese deer', + 'virginia deer', 'mule deer', 'black-tailed deer', 'elk', 'fallow deer', + 'roe deer', 'roebuck', 'caribou', 'woodland caribou', + 'barren ground caribou', 'brocket', 'muntjac', 'musk deer', + 'pere david\'s deer', 'chevrotain', 'kanchil', 'napu', 'water chevrotain', + 'camel', 'arabian camel', 'bactrian camel', 'llama', 'domestic llama', + 'guanaco', 'alpaca', 'vicuna', 'giraffe', 'okapi', 'musteline mammal', + 'weasel', 'ermine', 'stoat', 'new world least weasel', + 'old world least weasel', 'longtail weasel', 'mink', 'american mink', + 'polecat', 'ferret', 'black-footed ferret', 'muishond', 'snake muishond', + 'striped muishond', 'otter', 'river otter', 'eurasian otter', 'sea otter', + 'skunk', 'striped skunk', 'hooded skunk', 'hog-nosed skunk', + 'spotted skunk', 'badger', 'american badger', 'eurasian badger', 'ratel', + 'ferret badger', 'hog badger', 'wolverine', 'glutton', 'grison', 'marten', + 'pine marten', 'sable', 'american marten', 'stone marten', 'fisher', + 'yellow-throated marten', 'tayra', 'fictional animal', 'pachyderm', + 'edentate', 'armadillo', 'peba', 'apar', 'tatouay', 'peludo', + 'giant armadillo', 'pichiciago', 'sloth', 'three-toed sloth', + 'two-toed sloth', 'two-toed sloth', 'megatherian', 'mylodontid', 'anteater', + 'ant bear', 'silky anteater', 'tamandua', 'pangolin', 'coronet', 'scapular', + 'tadpole', 'primate', 'simian', 'ape', 'anthropoid', 'anthropoid ape', + 'hominoid', 'hominid', 'homo', 'world', 'homo erectus', 'pithecanthropus', + 'java man', 'peking man', 'sinanthropus', 'homo soloensis', 'javanthropus', + 'homo habilis', 'homo sapiens', 'neandertal man', 'cro-magnon', + 'homo sapiens sapiens', 'australopithecine', 'australopithecus afarensis', + 'australopithecus africanus', 'australopithecus boisei', 'zinjanthropus', + 'australopithecus robustus', 'paranthropus', 'sivapithecus', 'rudapithecus', + 'proconsul', 'aegyptopithecus', 'great ape', 'orangutan', 'gorilla', + 'western lowland gorilla', 'eastern lowland gorilla', 'mountain gorilla', + 'silverback', 'chimpanzee', 'western chimpanzee', 'eastern chimpanzee', + 'central chimpanzee', 'pygmy chimpanzee', 'lesser ape', 'gibbon', 'siamang', + 'monkey', 'old world monkey', 'guenon', 'talapoin', 'grivet', 'vervet', + 'green monkey', 'mangabey', 'patas', 'baboon', 'chacma', 'mandrill', + 'drill', 'macaque', 'rhesus', 'bonnet macaque', 'barbary ape', + 'crab-eating macaque', 'langur', 'entellus', 'colobus', 'guereza', + 'proboscis monkey', 'new world monkey', 'marmoset', 'true marmoset', + 'pygmy marmoset', 'tamarin', 'silky tamarin', 'pinche', 'capuchin', + 'douroucouli', 'howler monkey', 'saki', 'uakari', 'titi', 'spider monkey', + 'squirrel monkey', 'woolly monkey', 'tree shrew', 'prosimian', 'lemur', + 'madagascar cat', 'aye-aye', 'slender loris', 'slow loris', 'potto', + 'angwantibo', 'galago', 'indri', 'woolly indris', 'tarsier', + 'tarsius syrichta', 'tarsius glis', 'flying lemur', + 'cynocephalus variegatus', 'proboscidean', 'elephant', 'rogue elephant', + 'indian elephant', 'african elephant', 'mammoth', 'woolly mammoth', + 'columbian mammoth', 'imperial mammoth', 'mastodon', 'plantigrade mammal', + 'digitigrade mammal', 'procyonid', 'raccoon', 'common raccoon', + 'crab-eating raccoon', 'bassarisk', 'kinkajou', 'coati', 'lesser panda', + 'giant panda', 'twitterer', 'fish', 'fingerling', 'game fish', 'food fish', + 'rough fish', 'groundfish', 'young fish', 'parr', 'mouthbreeder', 'spawner', + 'barracouta', 'crossopterygian', 'coelacanth', 'lungfish', 'ceratodus', + 'catfish', 'silurid', 'european catfish', 'electric catfish', 'bullhead', + 'horned pout', 'brown bullhead', 'channel catfish', 'blue catfish', + 'flathead catfish', 'armored catfish', 'sea catfish', 'gadoid', 'cod', + 'codling', 'atlantic cod', 'pacific cod', 'whiting', 'burbot', 'haddock', + 'pollack', 'hake', 'silver hake', 'ling', 'cusk', 'grenadier', 'eel', + 'elver', 'common eel', 'tuna', 'moray', 'conger', 'teleost fish', + 'beaked salmon', 'clupeid fish', 'whitebait', 'brit', 'shad', + 'common american shad', 'river shad', 'allice shad', 'alewife', 'menhaden', + 'herring', 'atlantic herring', 'pacific herring', 'sardine', 'sild', + 'brisling', 'pilchard', 'pacific sardine', 'anchovy', + 'mediterranean anchovy', 'salmonid', 'salmon', 'parr', 'blackfish', + 'redfish', 'atlantic salmon', 'landlocked salmon', 'sockeye', 'chinook', + 'coho', 'trout', 'brown trout', 'rainbow trout', 'sea trout', 'lake trout', + 'brook trout', 'char', 'arctic char', 'whitefish', 'lake whitefish', + 'cisco', 'round whitefish', 'smelt', 'sparling', 'capelin', 'tarpon', + 'ladyfish', 'bonefish', 'argentine', 'lanternfish', 'lizardfish', + 'lancetfish', 'opah', 'new world opah', 'ribbonfish', 'dealfish', 'oarfish', + 'batfish', 'goosefish', 'toadfish', 'oyster fish', 'frogfish', + 'sargassum fish', 'needlefish', 'timucu', 'flying fish', + 'monoplane flying fish', 'halfbeak', 'saury', 'spiny-finned fish', + 'lingcod', 'percoid fish', 'perch', 'climbing perch', 'perch', + 'yellow perch', 'european perch', 'pike-perch', 'walleye', 'blue pike', + 'snail darter', 'cusk-eel', 'brotula', 'pearlfish', 'robalo', 'snook', + 'pike', 'northern pike', 'muskellunge', 'pickerel', 'chain pickerel', + 'redfin pickerel', 'sunfish', 'crappie', 'black crappie', 'white crappie', + 'freshwater bream', 'pumpkinseed', 'bluegill', 'spotted sunfish', + 'freshwater bass', 'rock bass', 'black bass', 'kentucky black bass', + 'smallmouth', 'largemouth', 'bass', 'serranid fish', 'white perch', + 'yellow bass', 'blackmouth bass', 'rock sea bass', 'striped bass', + 'stone bass', 'grouper', 'hind', 'rock hind', 'creole-fish', 'jewfish', + 'soapfish', 'surfperch', 'rainbow seaperch', 'bigeye', 'catalufa', + 'cardinalfish', 'flame fish', 'tilefish', 'bluefish', 'cobia', 'remora', + 'sharksucker', 'whale sucker', 'carangid fish', 'jack', 'crevalle jack', + 'yellow jack', 'runner', 'rainbow runner', 'leatherjacket', 'threadfish', + 'moonfish', 'lookdown', 'amberjack', 'yellowtail', 'kingfish', 'pompano', + 'florida pompano', 'permit', 'scad', 'horse mackerel', 'horse mackerel', + 'bigeye scad', 'mackerel scad', 'round scad', 'dolphinfish', + 'coryphaena hippurus', 'coryphaena equisetis', 'pomfret', 'characin', + 'tetra', 'cardinal tetra', 'piranha', 'cichlid', 'bolti', 'snapper', + 'red snapper', 'grey snapper', 'mutton snapper', 'schoolmaster', + 'yellowtail', 'grunt', 'margate', 'spanish grunt', 'tomtate', 'cottonwick', + 'sailor\'s-choice', 'porkfish', 'pompon', 'pigfish', 'sparid', 'sea bream', + 'porgy', 'red porgy', 'european sea bream', 'atlantic sea bream', + 'sheepshead', 'pinfish', 'sheepshead porgy', 'snapper', 'black bream', + 'scup', 'scup', 'sciaenid fish', 'striped drum', 'jackknife-fish', + 'silver perch', 'red drum', 'mulloway', 'maigre', 'croaker', + 'atlantic croaker', 'yellowfin croaker', 'whiting', 'kingfish', + 'king whiting', 'northern whiting', 'corbina', 'white croaker', + 'white croaker', 'sea trout', 'weakfish', 'spotted weakfish', 'mullet', + 'goatfish', 'red goatfish', 'yellow goatfish', 'mullet', 'striped mullet', + 'white mullet', 'liza', 'silversides', 'jacksmelt', 'barracuda', + 'great barracuda', 'sweeper', 'sea chub', 'bermuda chub', 'spadefish', + 'butterfly fish', 'chaetodon', 'angelfish', 'rock beauty', 'damselfish', + 'beaugregory', 'anemone fish', 'clown anemone fish', 'sergeant major', + 'wrasse', 'pigfish', 'hogfish', 'slippery dick', 'puddingwife', 'bluehead', + 'pearly razorfish', 'tautog', 'cunner', 'parrotfish', 'threadfin', + 'jawfish', 'stargazer', 'sand stargazer', 'blenny', 'shanny', + 'molly miller', 'clinid', 'pikeblenny', 'bluethroat pikeblenny', 'gunnel', + 'rock gunnel', 'eelblenny', 'wrymouth', 'wolffish', 'viviparous eelpout', + 'ocean pout', 'sand lance', 'dragonet', 'goby', 'mudskipper', 'sleeper', + 'flathead', 'archerfish', 'surgeonfish', 'gempylid', 'snake mackerel', + 'escolar', 'oilfish', 'cutlassfish', 'scombroid', 'mackerel', + 'common mackerel', 'spanish mackerel', 'chub mackerel', 'wahoo', + 'spanish mackerel', 'king mackerel', 'scomberomorus maculatus', 'cero', + 'sierra', 'tuna', 'albacore', 'bluefin', 'yellowfin', 'bonito', 'skipjack', + 'chile bonito', 'skipjack', 'bonito', 'swordfish', 'sailfish', + 'atlantic sailfish', 'billfish', 'marlin', 'blue marlin', 'black marlin', + 'striped marlin', 'white marlin', 'spearfish', 'louvar', 'dollarfish', + 'palometa', 'harvestfish', 'driftfish', 'barrelfish', 'clingfish', + 'tripletail', 'atlantic tripletail', 'pacific tripletail', 'mojarra', + 'yellowfin mojarra', 'silver jenny', 'whiting', 'ganoid', 'bowfin', + 'paddlefish', 'chinese paddlefish', 'sturgeon', 'pacific sturgeon', + 'beluga', 'gar', 'scorpaenoid', 'scorpaenid', 'scorpionfish', + 'plumed scorpionfish', 'lionfish', 'stonefish', 'rockfish', + 'copper rockfish', 'vermillion rockfish', 'red rockfish', 'rosefish', + 'bullhead', 'miller\'s-thumb', 'sea raven', 'lumpfish', 'lumpsucker', + 'pogge', 'greenling', 'kelp greenling', 'painted greenling', 'flathead', + 'gurnard', 'tub gurnard', 'sea robin', 'northern sea robin', + 'flying gurnard', 'plectognath', 'triggerfish', 'queen triggerfish', + 'filefish', 'leatherjacket', 'boxfish', 'cowfish', 'puffer', 'spiny puffer', + 'porcupinefish', 'balloonfish', 'burrfish', 'ocean sunfish', + 'sharptail mola', 'flatfish', 'flounder', 'righteye flounder', 'plaice', + 'european flatfish', 'yellowtail flounder', 'winter flounder', 'lemon sole', + 'american plaice', 'halibut', 'atlantic halibut', 'pacific halibut', + 'lefteye flounder', 'southern flounder', 'summer flounder', 'whiff', + 'horned whiff', 'sand dab', 'windowpane', 'brill', 'turbot', 'tonguefish', + 'sole', 'european sole', 'english sole', 'hogchoker', 'aba', 'abacus', + 'abandoned ship', 'a battery', 'abattoir', 'abaya', 'abbe condenser', + 'abbey', 'abbey', 'abbey', 'abney level', 'abrader', 'abrading stone', + 'abutment', 'abutment arch', 'academic costume', 'academic gown', + 'accelerator', 'accelerator', 'accelerator', 'accelerometer', 'accessory', + 'accommodating lens implant', 'accommodation', 'accordion', 'acetate disk', + 'acetate rayon', 'achromatic lens', 'acoustic delay line', + 'acoustic device', 'acoustic guitar', 'acoustic modem', 'acropolis', + 'acrylic', 'acrylic', 'actinometer', 'action', 'active matrix screen', + 'actuator', 'adapter', 'adder', 'adding machine', 'addressing machine', + 'adhesive bandage', 'adit', 'adjoining room', 'adjustable wrench', 'adobe', + 'adz', 'aeolian harp', 'aerator', 'aerial torpedo', 'aerosol', 'aertex', + 'afghan', 'afro-wig', 'afterburner', 'after-shave', 'agateware', + 'agglomerator', 'aglet', 'aglet', 'agora', 'aigrette', 'aileron', 'air bag', + 'airbrake', 'airbrush', 'airbus', 'air compressor', 'air conditioner', + 'aircraft', 'aircraft carrier', 'aircraft engine', 'air cushion', 'airdock', + 'airfield', 'air filter', 'airfoil', 'airframe', 'air gun', 'air hammer', + 'air horn', 'airing cupboard', 'airliner', 'airmailer', 'airplane', + 'airplane propeller', 'airport', 'air pump', 'air search radar', 'airship', + 'air terminal', 'air-to-air missile', 'air-to-ground missile', 'aisle', + 'aladdin\'s lamp', 'alarm', 'alarm clock', 'alb', 'alcazar', + 'alcohol thermometer', 'alehouse', 'alembic', 'algometer', 'alidade', + 'alidade', 'a-line', 'allen screw', 'allen wrench', 'alligator wrench', + 'alms dish', 'alpaca', 'alpenstock', 'altar', 'altar', 'altarpiece', + 'altazimuth', 'alternator', 'altimeter', 'amati', 'ambulance', + 'amen corner', 'american organ', 'ammeter', 'ammonia clock', 'ammunition', + 'amphibian', 'amphibian', 'amphitheater', 'amphitheater', 'amphora', + 'amplifier', 'ampulla', 'amusement arcade', 'analog clock', + 'analog computer', 'analog watch', 'analytical balance', 'analyzer', + 'anamorphosis', 'anastigmat', 'anchor', 'anchor chain', 'anchor light', + 'and circuit', 'andiron', 'android', 'anechoic chamber', 'anemometer', + 'aneroid barometer', 'angiocardiogram', 'angioscope', 'angle bracket', + 'angledozer', 'ankle brace', 'anklet', 'anklet', 'ankus', 'anode', 'anode', + 'answering machine', 'antenna', 'anteroom', 'antiaircraft', + 'antiballistic missile', 'antifouling paint', 'anti-g suit', 'antimacassar', + 'antiperspirant', 'anti-submarine rocket', 'anvil', 'ao dai', 'apadana', + 'apartment', 'apartment building', 'aperture', 'aperture', 'apiary', + 'apparatus', 'apparel', 'applecart', 'appliance', 'appliance', 'applicator', + 'appointment', 'apron', 'apron string', 'apse', 'aqualung', 'aquaplane', + 'aquarium', 'arabesque', 'arbor', 'arcade', 'arch', 'architecture', + 'architrave', 'arch support', 'arc lamp', 'arctic', 'area', 'areaway', + 'argyle', 'ark', 'arm', 'armament', 'armature', 'armband', 'armchair', + 'armet', 'arm guard', 'armhole', 'armilla', 'armlet', 'armoire', 'armor', + 'armored car', 'armored car', 'armored personnel carrier', + 'armored vehicle', 'armor plate', 'armory', 'armrest', 'arquebus', 'array', + 'array', 'arrester', 'arrow', 'arsenal', 'arterial road', 'arthrogram', + 'arthroscope', 'artificial heart', 'artificial horizon', 'artificial joint', + 'artificial kidney', 'artificial skin', 'artillery', 'artillery shell', + 'artist\'s loft', 'art school', 'ascot', 'ashcan', 'ash-pan', 'ashtray', + 'aspergill', 'aspersorium', 'aspirator', 'aspirin powder', 'assault gun', + 'assault rifle', 'assegai', 'assembly', 'assembly', 'assembly hall', + 'assembly plant', 'astatic coils', 'astatic galvanometer', 'astrodome', + 'astrolabe', 'astronomical telescope', 'astronomy satellite', 'athenaeum', + 'athletic sock', 'athletic supporter', 'atlas', 'atmometer', 'atom bomb', + 'atomic clock', 'atomic pile', 'atomizer', 'atrium', 'attache case', + 'attachment', 'attack submarine', 'attenuator', 'attic', 'attic fan', + 'attire', 'audio amplifier', 'audiocassette', 'audio cd', 'audiometer', + 'audio system', 'audiotape', 'audiotape', 'audiovisual', 'auditorium', + 'auger', 'autobahn', 'autoclave', 'autofocus', 'autogiro', 'autoinjector', + 'autoloader', 'automat', 'automat', 'automatic choke', 'automatic firearm', + 'automatic pistol', 'automatic rifle', 'automatic transmission', + 'automation', 'automaton', 'automobile engine', 'automobile factory', + 'automobile horn', 'autopilot', 'autoradiograph', 'autostrada', + 'auxiliary boiler', 'auxiliary engine', 'auxiliary pump', + 'auxiliary research submarine', 'auxiliary storage', 'aviary', 'awl', + 'awning', 'ax', 'ax handle', 'ax head', 'axis', 'axle', 'axle bar', + 'axletree', 'babushka', 'baby bed', 'baby buggy', 'baby grand', + 'baby powder', 'baby shoe', 'back', 'back', 'backbench', 'backboard', + 'backboard', 'backbone', 'back brace', 'backgammon board', 'background', + 'backhoe', 'backlighting', 'backpack', 'backpacking tent', 'backplate', + 'back porch', 'backsaw', 'backscratcher', 'backseat', 'backspace key', + 'backstairs', 'backstay', 'backstop', 'backsword', 'backup system', + 'badminton court', 'badminton equipment', 'badminton racket', 'bag', 'bag', + 'bag', 'baggage', 'baggage', 'baggage car', 'baggage claim', 'bagpipe', + 'bailey', 'bailey', 'bailey bridge', 'bain-marie', 'bait', 'baize', + 'bakery', 'balaclava', 'balalaika', 'balance', 'balance beam', + 'balance wheel', 'balbriggan', 'balcony', 'balcony', 'baldachin', 'baldric', + 'bale', 'baling wire', 'ball', 'ball', 'ball and chain', + 'ball-and-socket joint', 'ballast', 'ball bearing', 'ball cartridge', + 'ballcock', 'balldress', 'ballet skirt', 'ball gown', + 'ballistic galvanometer', 'ballistic missile', 'ballistic pendulum', + 'ballistocardiograph', 'balloon', 'balloon bomb', 'balloon sail', + 'ballot box', 'ballpark', 'ball-peen hammer', 'ballpoint', 'ballroom', + 'ball valve', 'balsa raft', 'baluster', 'banana boat', 'band', 'bandage', + 'band aid', 'bandanna', 'bandbox', 'banderilla', 'bandoleer', 'bandoneon', + 'bandsaw', 'bandwagon', 'bangalore torpedo', 'bangle', 'banjo', 'banner', + 'bannister', 'banquette', 'banyan', 'baptismal font', 'bar', 'bar', + 'barbecue', 'barbed wire', 'barbell', 'barber chair', 'barbershop', + 'barbette carriage', 'barbican', 'bar bit', 'bareboat', 'barge', + 'barge pole', 'baritone', 'bark', 'bar magnet', 'bar mask', 'barn', + 'barndoor', 'barn door', 'barnyard', 'barograph', 'barometer', 'barong', + 'barouche', 'bar printer', 'barrack', 'barrage balloon', 'barrel', 'barrel', + 'barrelhouse', 'barrel knot', 'barrel organ', 'barrel vault', 'barrette', + 'barricade', 'barrier', 'barroom', 'barrow', 'bascule', 'base', 'base', + 'baseball', 'baseball bat', 'baseball cap', 'baseball equipment', + 'baseball glove', 'basement', 'basement', + 'basic point defense missile system', 'basilica', 'basilica', 'basilisk', + 'basin', 'basinet', 'basket', 'basket', 'basketball', 'basketball court', + 'basketball equipment', 'basket weave', 'bass', 'bass clarinet', + 'bass drum', 'basset horn', 'bass fiddle', 'bass guitar', 'bass horn', + 'bassinet', 'bassinet', 'bassoon', 'baster', 'bastinado', 'bastion', + 'bastion', 'bat', 'bath', 'bath chair', 'bathhouse', 'bathhouse', + 'bathing cap', 'bath oil', 'bathrobe', 'bathroom', 'bath salts', + 'bath towel', 'bathtub', 'bathyscaphe', 'bathysphere', 'batik', 'batiste', + 'baton', 'baton', 'baton', 'baton', 'battering ram', 'batter\'s box', + 'battery', 'battery', 'batting cage', 'batting glove', 'batting helmet', + 'battle-ax', 'battle cruiser', 'battle dress', 'battlement', 'battleship', + 'battle sight', 'bay', 'bay', 'bayonet', 'bay rum', 'bay window', 'bazaar', + 'bazaar', 'bazooka', 'b battery', 'bb gun', 'beach house', 'beach towel', + 'beach wagon', 'beachwear', 'beacon', 'beading plane', 'beaker', 'beaker', + 'beam', 'beam balance', 'beanbag', 'beanie', 'bearing', 'bearing rein', + 'bearing wall', 'bearskin', 'beater', 'beating-reed instrument', 'beaver', + 'beaver', 'beckman thermometer', 'bed', 'bed', 'bed and breakfast', + 'bedclothes', 'bedford cord', 'bed jacket', 'bedpan', 'bedpost', 'bedroll', + 'bedroom', 'bedroom furniture', 'bedsitting room', 'bedspread', 'bedspring', + 'bedstead', 'beefcake', 'beehive', 'beeper', 'beer barrel', 'beer bottle', + 'beer can', 'beer garden', 'beer glass', 'beer hall', 'beer mat', + 'beer mug', 'belaying pin', 'belfry', 'bell', 'bell arch', 'bellarmine', + 'bellbottom trousers', 'bell cote', 'bell foundry', 'bell gable', + 'bell jar', 'bellows', 'bellpull', 'bell push', 'bell seat', 'bell tent', + 'bell tower', 'bellyband', 'belt', 'belt', 'belt buckle', 'belting', + 'bench', 'bench clamp', 'bench hook', 'bench lathe', 'bench press', + 'bender', 'beret', 'berlin', 'bermuda shorts', 'berth', 'besom', + 'bessemer converter', 'bethel', 'betting shop', 'bevatron', 'bevel', + 'bevel gear', 'b-flat clarinet', 'bib', 'bib-and-tucker', 'bicorn', + 'bicycle', 'bicycle-built-for-two', 'bicycle chain', 'bicycle clip', + 'bicycle pump', 'bicycle rack', 'bicycle seat', 'bicycle wheel', 'bidet', + 'bier', 'bier', 'bi-fold door', 'bifocals', 'big blue', 'big board', + 'bight', 'bikini', 'bikini pants', 'bilge', 'bilge keel', 'bilge pump', + 'bilge well', 'bill', 'bill', 'billboard', 'billiard ball', 'billiard room', + 'bin', 'binder', 'binder', 'bindery', 'binding', 'bin liner', 'binnacle', + 'binoculars', 'binocular microscope', 'biochip', 'biohazard suit', + 'bioscope', 'biplane', 'birch', 'birchbark canoe', 'birdbath', 'birdcage', + 'birdcall', 'bird feeder', 'birdhouse', 'bird shot', 'biretta', 'bishop', + 'bistro', 'bit', 'bit', 'bite plate', 'bitewing', 'bitumastic', 'black', + 'black', 'blackboard', 'blackboard eraser', 'black box', 'blackface', + 'blackjack', 'black tie', 'blackwash', 'bladder', 'blade', 'blade', 'blade', + 'blank', 'blanket', 'blast furnace', 'blasting cap', 'blazer', 'blender', + 'blimp', 'blind', 'blind curve', 'blindfold', 'bling', 'blinker', + 'blister pack', 'block', 'blockade', 'blockade-runner', 'block and tackle', + 'blockbuster', 'blockhouse', 'block plane', 'bloodmobile', 'bloomers', + 'blouse', 'blower', 'blowtorch', 'blucher', 'bludgeon', 'blue', 'blue chip', + 'blunderbuss', 'blunt file', 'boarding', 'boarding house', 'boardroom', + 'boards', 'boat', 'boater', 'boat hook', 'boathouse', 'boatswain\'s chair', + 'boat train', 'boatyard', 'bobbin', 'bobby pin', 'bobsled', 'bobsled', + 'bocce ball', 'bodega', 'bodice', 'bodkin', 'bodkin', 'bodkin', 'body', + 'body armor', 'body lotion', 'body stocking', 'body plethysmograph', + 'body pad', 'bodywork', 'bofors gun', 'bogy', 'boiler', + 'boiling water reactor', 'bolero', 'bollard', 'bolo', 'bolo tie', 'bolt', + 'bolt', 'bolt', 'bolt cutter', 'bomb', 'bombazine', 'bomb calorimeter', + 'bomber', 'bomber jacket', 'bomblet', 'bomb rack', 'bombshell', + 'bomb shelter', 'bone-ash cup', 'bone china', 'bones', 'boneshaker', + 'bongo', 'bonnet', 'book', 'book bag', 'bookbindery', 'bookcase', 'bookend', + 'bookmark', 'bookmobile', 'bookshelf', 'bookshop', 'boom', 'boom', + 'boomerang', 'booster', 'booster', 'boot', 'boot', 'boot camp', 'bootee', + 'booth', 'booth', 'booth', 'boothose', 'bootjack', 'bootlace', 'bootleg', + 'bootstrap', 'bore bit', 'boron chamber', 'borstal', 'bosom', + 'boston rocker', 'bota', 'bottle', 'bottle', 'bottle bank', 'bottlebrush', + 'bottlecap', 'bottle opener', 'bottling plant', 'bottom', 'boucle', + 'boudoir', 'boulle', 'bouncing betty', 'bouquet', 'boutique', 'boutonniere', + 'bow', 'bow', 'bow', 'bow and arrow', 'bowed stringed instrument', + 'bowie knife', 'bowl', 'bowl', 'bowl', 'bowler hat', 'bowline', + 'bowling alley', 'bowling ball', 'bowling equipment', 'bowling pin', + 'bowling shoe', 'bowsprit', 'bowstring', 'bow tie', 'box', 'box', 'box', + 'box beam', 'box camera', 'boxcar', 'box coat', 'boxing equipment', + 'boxing glove', 'box office', 'box spring', 'box wrench', 'brace', 'brace', + 'brace', 'brace', 'brace and bit', 'bracelet', 'bracer', 'brace wrench', + 'bracket', 'bradawl', 'brake', 'brake', 'brake band', 'brake cylinder', + 'brake disk', 'brake drum', 'brake lining', 'brake pad', 'brake pedal', + 'brake shoe', 'brake system', 'brass', 'brass', 'brass', 'brassard', + 'brasserie', 'brassie', 'brassiere', 'brass knucks', 'brattice', 'brazier', + 'breadbasket', 'bread-bin', 'bread knife', 'breakable', 'breakfast area', + 'breakfast table', 'breakwater', 'breast drill', 'breast implant', + 'breastplate', 'breast pocket', 'breathalyzer', 'breechblock', + 'breechcloth', 'breeches', 'breeches buoy', 'breechloader', + 'breeder reactor', 'bren', 'brewpub', 'brick', 'brickkiln', + 'bricklayer\'s hammer', 'brick trowel', 'brickwork', 'bridal gown', + 'bridge', 'bridge', 'bridle', 'bridle path', 'bridoon', 'briefcase', + 'briefcase bomb', 'briefcase computer', 'briefs', 'brig', 'brig', + 'brigandine', 'brigantine', 'brilliantine', 'brilliant pebble', 'brim', + 'bristle brush', 'britches', 'broad arrow', 'broadax', 'brochette', + 'broadcaster', 'broadcloth', 'broadcloth', 'broad hatchet', 'broadloom', + 'broadside', 'broadsword', 'brocade', 'brogan', 'broiler', 'broken arch', + 'bronchoscope', 'broom', 'broom closet', 'broomstick', 'brougham', + 'browning automatic rifle', 'browning machine gun', 'brownstone', + 'brunch coat', 'brush', 'brussels carpet', 'brussels lace', 'bubble', + 'bubble chamber', 'bubble jet printer', 'buckboard', 'bucket', + 'bucket seat', 'bucket shop', 'buckle', 'buckram', 'bucksaw', 'buckskins', + 'buff', 'buffer', 'buffer', 'buffet', 'buffing wheel', 'buggy', 'bugle', + 'building', 'building complex', 'bulldog clip', 'bulldog wrench', + 'bulldozer', 'bullet', 'bulletproof vest', 'bullet train', 'bullhorn', + 'bullion', 'bullnose', 'bullpen', 'bullpen', 'bullring', 'bulwark', + 'bumboat', 'bumper', 'bumper', 'bumper car', 'bumper guard', 'bumper jack', + 'bundle', 'bung', 'bungalow', 'bungee', 'bunghole', 'bunk', 'bunk', + 'bunk bed', 'bunker', 'bunker', 'bunker', 'bunsen burner', 'bunting', 'bur', + 'burberry', 'burette', 'burglar alarm', 'burial chamber', 'burial garment', + 'burial mound', 'burin', 'burqa', 'burlap', 'burn bag', 'burner', 'burnous', + 'burp gun', 'burr', 'bus', 'bushel basket', 'bushing', 'bush jacket', + 'business suit', 'buskin', 'bustier', 'bustle', 'butcher knife', + 'butcher shop', 'butter dish', 'butterfly valve', 'butter knife', + 'butt hinge', 'butt joint', 'button', 'buttonhook', 'buttress', + 'butt shaft', 'butt weld', 'buzz bomb', 'buzzer', 'bvd', 'bypass condenser', + 'byway', 'cab', 'cab', 'cab', 'cabana', 'cabaret', 'caber', 'cabin', + 'cabin', 'cabin car', 'cabin class', 'cabin cruiser', 'cabinet', 'cabinet', + 'cabinet', 'cabinetwork', 'cabin liner', 'cable', 'cable', 'cable car', + 'cache', 'caddy', 'caesium clock', 'cafe', 'cafeteria', 'cafeteria tray', + 'caff', 'caftan', 'caftan', 'cage', 'cage', 'cagoule', 'caisson', 'calash', + 'calceus', 'calcimine', 'calculator', 'caldron', 'calico', 'caliper', + 'call-board', 'call center', 'caller id', 'calliope', 'calorimeter', + 'calpac', 'camail', 'camber arch', 'cambric', 'camcorder', 'camel\'s hair', + 'camera', 'camera lens', 'camera lucida', 'camera obscura', 'camera tripod', + 'camise', 'camisole', 'camisole', 'camlet', 'camouflage', 'camouflage', + 'camp', 'camp', 'camp', 'campaign hat', 'campanile', 'camp chair', 'camper', + 'camper trailer', 'campstool', 'camshaft', 'can', 'canal', 'canal boat', + 'candelabrum', 'candid camera', 'candle', 'candlepin', 'candlesnuffer', + 'candlestick', 'candlewick', 'candy thermometer', 'cane', 'cane', 'cangue', + 'canister', 'cannery', 'cannikin', 'cannikin', 'cannon', 'cannon', 'cannon', + 'cannon', 'cannonball', 'canoe', 'can opener', 'canopic jar', 'canopy', + 'canopy', 'canopy', 'canteen', 'canteen', 'canteen', 'canteen', 'canteen', + 'cant hook', 'cantilever', 'cantilever bridge', 'cantle', 'canton crepe', + 'canvas', 'canvas', 'canvas tent', 'cap', 'cap', 'cap', 'capacitor', + 'caparison', 'cape', 'capital ship', 'capitol', 'cap opener', 'capote', + 'capote', 'cap screw', 'capstan', 'capstone', 'capsule', 'captain\'s chair', + 'car', 'car', 'car', 'carabiner', 'carafe', 'caravansary', 'car battery', + 'carbine', 'car bomb', 'carbon arc lamp', 'carboy', 'carburetor', + 'car carrier', 'cardcase', 'cardiac monitor', 'cardigan', 'card index', + 'cardiograph', 'cardioid microphone', 'car door', 'cardroom', 'card table', + 'card table', 'car-ferry', 'cargo area', 'cargo container', 'cargo door', + 'cargo hatch', 'cargo helicopter', 'cargo liner', 'cargo ship', 'carillon', + 'car mirror', 'caroche', 'carousel', 'carpenter\'s hammer', + 'carpenter\'s kit', 'carpenter\'s level', 'carpenter\'s mallet', + 'carpenter\'s rule', 'carpenter\'s square', 'carpetbag', 'carpet beater', + 'carpet loom', 'carpet pad', 'carpet sweeper', 'carpet tack', 'carport', + 'carrack', 'carrel', 'carriage', 'carriage', 'carriage bolt', 'carriageway', + 'carriage wrench', 'carrick bend', 'carrier', 'carryall', 'carrycot', + 'car seat', 'cart', 'car tire', 'carton', 'cartouche', 'car train', + 'cartridge', 'cartridge', 'cartridge belt', 'cartridge extractor', + 'cartridge fuse', 'cartridge holder', 'cartwheel', 'carving fork', + 'carving knife', 'car wheel', 'caryatid', 'cascade liquefier', + 'cascade transformer', 'case', 'case', 'case', 'casein paint', 'case knife', + 'case knife', 'casement', 'casement window', 'casern', 'case shot', + 'cash bar', 'cashbox', 'cash machine', 'cashmere', 'cash register', + 'casing', 'casino', 'casket', 'casque', 'casquet', + 'cassegrainian telescope', 'casserole', 'cassette', 'cassette deck', + 'cassette player', 'cassette recorder', 'cassette tape', 'cassock', 'cast', + 'caster', 'caster', 'castle', 'castle', 'catacomb', 'catafalque', + 'catalytic converter', 'catalytic cracker', 'catamaran', 'catapult', + 'catapult', 'catboat', 'cat box', 'catch', 'catchall', 'catcher\'s mask', + 'catchment', 'caterpillar', 'cathedra', 'cathedral', 'cathedral', + 'catheter', 'cathode', 'cathode-ray tube', 'cat-o\'-nine-tails', + 'cat\'s-paw', 'catsup bottle', 'cattle car', 'cattle guard', 'cattleship', + 'cautery', 'cavalier hat', 'cavalry sword', 'cavetto', 'cavity wall', + 'c battery', 'c-clamp', 'cd drive', 'cd player', 'cd-r', 'cd-rom', + 'cd-rom drive', 'cedar chest', 'ceiling', 'celesta', 'cell', 'cell', + 'cellar', 'cellblock', 'cello', 'cellophane', 'cellular telephone', + 'cellulose tape', 'cenotaph', 'censer', 'center', 'center punch', + 'centigrade thermometer', 'central processing unit', 'centrifugal pump', + 'centrifuge', 'ceramic', 'ceramic ware', 'cereal bowl', 'cereal box', + 'cerecloth', 'cesspool', 'chachka', 'chador', 'chafing dish', 'chain', + 'chain', 'chainlink fence', 'chain mail', 'chain printer', 'chain saw', + 'chain store', 'chain tongs', 'chain wrench', 'chair', 'chair', + 'chair of state', 'chairlift', 'chaise', 'chaise longue', 'chalet', + 'chalice', 'chalk', 'challis', 'chamberpot', 'chambray', 'chamfer bit', + 'chamfer plane', 'chamois cloth', 'chancel', 'chancellery', 'chancery', + 'chandelier', 'chandlery', 'chanfron', 'chanter', 'chantry', 'chap', + 'chapel', 'chapterhouse', 'chapterhouse', 'character printer', + 'charcuterie', 'charge-exchange accelerator', 'charger', 'chariot', + 'chariot', 'charnel house', 'chassis', 'chassis', 'chasuble', 'chateau', + 'chatelaine', 'checker', 'checkout', 'cheekpiece', 'cheeseboard', + 'cheesecloth', 'cheese cutter', 'cheese press', 'chemical bomb', + 'chemical plant', 'chemical reactor', 'chemise', 'chemise', 'chenille', + 'chessman', 'chest', 'chesterfield', 'chest of drawers', 'chest protector', + 'cheval-de-frise', 'cheval glass', 'chicane', 'chicken coop', + 'chicken wire', 'chicken yard', 'chiffon', 'chiffonier', 'child\'s room', + 'chime', 'chimney breast', 'chimney corner', 'china', 'china cabinet', + 'chinchilla', 'chinese lantern', 'chinese puzzle', 'chinning bar', 'chino', + 'chino', 'chin rest', 'chin strap', 'chintz', 'chip', 'chip', 'chisel', + 'chlamys', 'choir', 'choir loft', 'choke', 'choke', 'chokey', 'choo-choo', + 'chopine', 'chordophone', 'christmas stocking', 'chronograph', + 'chronometer', 'chronoscope', 'chuck', 'chuck wagon', 'chukka', 'church', + 'church bell', 'church hat', 'church key', 'church tower', 'churidars', + 'churn', 'ciderpress', 'cigar band', 'cigar box', 'cigar cutter', + 'cigarette butt', 'cigarette case', 'cigarette holder', 'cigar lighter', + 'cinch', 'cinema', 'cinquefoil', 'circle', 'circlet', 'circuit', + 'circuit board', 'circuit breaker', 'circuitry', 'circular plane', + 'circular saw', 'circus tent', 'cistern', 'cistern', 'cittern', 'city hall', + 'cityscape', 'city university', 'civies', 'civilian clothing', + 'clack valve', 'clamp', 'clamshell', 'clapper', 'clapperboard', 'clarence', + 'clarinet', 'clark cell', 'clasp', 'clasp knife', 'classroom', 'clavichord', + 'clavier', 'clay pigeon', 'claymore mine', 'claymore', 'cleaners', + 'cleaning implement', 'cleaning pad', 'clean room', 'clearway', 'cleat', + 'cleat', 'cleats', 'cleaver', 'clerestory', 'clevis', 'clews', + 'cliff dwelling', 'climbing frame', 'clinch', 'clinch', 'clincher', + 'clinic', 'clinical thermometer', 'clinker', 'clinometer', 'clip', + 'clip lead', 'clip-on', 'clipper', 'clipper', 'clipper', 'cloak', 'cloak', + 'cloakroom', 'cloche', 'cloche', 'clock', 'clock pendulum', 'clock radio', + 'clock tower', 'clockwork', 'clog', 'cloisonne', 'cloister', + 'closed circuit', 'closed-circuit television', 'closed loop', 'closet', + 'closeup lens', 'cloth cap', 'cloth covering', 'clothesbrush', + 'clothes closet', 'clothes dryer', 'clothes hamper', 'clotheshorse', + 'clothespin', 'clothes tree', 'clothing', 'clothing store', 'clout nail', + 'clove hitch', 'club car', 'clubroom', 'cluster bomb', 'clutch', 'clutch', + 'clutch bag', 'coach', 'coach house', 'coal car', 'coal chute', + 'coal house', 'coal shovel', 'coaming', 'coaster brake', 'coat', + 'coat button', 'coat closet', 'coatdress', 'coatee', 'coat hanger', + 'coating', 'coating', 'coat of paint', 'coatrack', 'coattail', + 'coaxial cable', 'cobweb', 'cobweb', 'cockcroft and walton accelerator', + 'cocked hat', 'cockhorse', 'cockleshell', 'cockpit', 'cockpit', 'cockpit', + 'cockscomb', 'cocktail dress', 'cocktail lounge', 'cocktail shaker', + 'cocotte', 'codpiece', 'coelostat', 'coffee can', 'coffee cup', + 'coffee filter', 'coffee maker', 'coffee mill', 'coffee mug', 'coffeepot', + 'coffee stall', 'coffee table', 'coffee urn', 'coffer', 'coffey still', + 'coffin', 'cog', 'coif', 'coil', 'coil', 'coil', 'coil spring', 'coin box', + 'colander', 'cold cathode', 'cold chisel', 'cold cream', 'cold frame', + 'collar', 'collar', 'college', 'collet', 'collider', 'colliery', + 'collimator', 'collimator', 'cologne', 'colonnade', 'colonoscope', + 'colorimeter', 'colors', 'color television', 'color tube', 'color wash', + 'colt', 'colter', 'columbarium', 'columbarium', 'column', 'column', 'comb', + 'comb', 'comber', 'combination lock', 'combination plane', 'combine', + 'comforter', 'command module', 'commissary', 'commissary', 'commodity', + 'common ax', 'common room', 'communications satellite', + 'communication system', 'community center', 'commutator', 'commuter', + 'compact', 'compact', 'compact disk', 'compact-disk burner', 'companionway', + 'compartment', 'compartment', 'compass', 'compass', 'compass card', + 'compass saw', 'compound', 'compound lens', 'compound lever', + 'compound microscope', 'compress', 'compression bandage', 'compressor', + 'computer', 'computer circuit', 'computerized axial tomography scanner', + 'computer keyboard', 'computer monitor', 'computer network', + 'computer screen', 'computer store', 'computer system', + 'concentration camp', 'concert grand', 'concert hall', 'concertina', + 'concertina', 'concrete mixer', 'condensation pump', 'condenser', + 'condenser', 'condenser', 'condenser microphone', 'condominium', + 'condominium', 'conductor', 'cone clutch', 'confectionery', + 'conference center', 'conference room', 'conference table', 'confessional', + 'conformal projection', 'congress boot', 'conic projection', + 'connecting rod', 'connecting room', 'connection', 'conning tower', + 'conning tower', 'conservatory', 'conservatory', 'console', 'console', + 'console table', 'consulate', 'contact', 'contact', 'container', + 'container ship', 'containment', 'contrabassoon', 'control', + 'control center', 'control circuit', 'control key', 'control panel', + 'control rod', 'control room', 'control system', 'control tower', + 'convector', 'convenience store', 'convent', 'conventicle', + 'converging lens', 'converter', 'convertible', 'convertible', 'conveyance', + 'conveyer belt', 'cooker', 'cookfire', 'cookhouse', 'cookie cutter', + 'cookie jar', 'cookie sheet', 'cooking utensil', 'cookstove', + 'coolant system', 'cooler', 'cooling system', 'cooling system', + 'cooling tower', 'coonskin cap', 'cope', 'coping saw', 'copperware', + 'copyholder', 'coquille', 'coracle', 'corbel', 'corbel arch', 'corbel step', + 'corbie gable', 'cord', 'cord', 'cordage', 'cords', 'core', 'core bit', + 'core drill', 'corer', 'cork', 'corker', 'corkscrew', 'corncrib', 'corner', + 'corner', 'corner post', 'cornet', 'cornice', 'cornice', 'cornice', + 'correctional institution', 'corrugated fastener', 'corselet', 'corset', + 'cosmetic', 'cosmotron', 'costume', 'costume', 'costume', 'costume', 'cosy', + 'cot', 'cottage tent', 'cotter', 'cotter pin', 'cotton', 'cotton flannel', + 'cotton mill', 'couch', 'couch', 'couchette', 'coude telescope', 'counter', + 'counter', 'counter', 'counterbore', 'counter tube', 'country house', + 'country store', 'coupe', 'coupling', 'court', 'court', 'court', 'court', + 'courtelle', 'courthouse', 'courthouse', 'coverall', 'covered bridge', + 'covered couch', 'covered wagon', 'covering', 'coverlet', 'cover plate', + 'cowbarn', 'cowbell', 'cowboy boot', 'cowboy hat', 'cowhide', 'cowl', + 'cow pen', 'cpu board', 'crackle', 'cradle', 'craft', 'cramp', 'crampon', + 'crampon', 'crane', 'craniometer', 'crank', 'crankcase', 'crankshaft', + 'crash barrier', 'crash helmet', 'crate', 'cravat', 'crayon', 'crazy quilt', + 'cream', 'cream pitcher', 'creche', 'creche', 'credenza', 'creel', + 'crematory', 'crematory', 'crepe', 'crepe de chine', 'crescent wrench', + 'cretonne', 'crib', 'crib', 'cricket ball', 'cricket bat', + 'cricket equipment', 'cringle', 'crinoline', 'crinoline', 'crochet needle', + 'crock', 'crock pot', 'crook', 'crookes radiometer', 'crookes tube', + 'croquet ball', 'croquet equipment', 'croquet mallet', 'cross', 'crossbar', + 'crossbar', 'crossbar', 'crossbench', 'cross bit', 'crossbow', + 'crosscut saw', 'crossjack', 'crosspiece', 'crotchet', 'croupier\'s rake', + 'crowbar', 'crown', 'crown', 'crown jewels', 'crown lens', 'crow\'s nest', + 'crucible', 'crucifix', 'cruet', 'cruet-stand', 'cruise control', + 'cruise missile', 'cruiser', 'cruiser', 'cruise ship', 'crupper', 'cruse', + 'crusher', 'crutch', 'cryometer', 'cryoscope', 'cryostat', 'crypt', + 'crystal', 'crystal detector', 'crystal microphone', 'crystal oscillator', + 'crystal set', 'cubitiere', 'cucking stool', 'cuckoo clock', 'cuddy', + 'cudgel', 'cue', 'cue ball', 'cuff', 'cuirass', 'cuisse', 'cul', + 'culdoscope', 'cullis', 'culotte', 'cultivator', 'culverin', 'culverin', + 'culvert', 'cup', 'cupboard', 'cup hook', 'cupola', 'cupola', 'curb', + 'curb roof', 'curbstone', 'curette', 'curler', 'curling iron', 'currycomb', + 'cursor', 'curtain', 'customhouse', 'cutaway', 'cutlas', 'cutoff', 'cutout', + 'cutter', 'cutter', 'cutting implement', 'cutting room', 'cutty stool', + 'cutwork', 'cybercafe', 'cyclopean masonry', 'cyclostyle', 'cyclotron', + 'cylinder', 'cylinder', 'cylinder lock', 'cymbal', 'dacha', 'dacron', + 'dado', 'dado plane', 'dagger', 'dairy', 'dais', 'daisy print wheel', + 'daisywheel printer', 'dam', 'damask', 'dampener', 'damper', 'damper block', + 'dark lantern', 'darkroom', 'darning needle', 'dart', 'dart', 'dashboard', + 'dashiki', 'dash-pot', 'data converter', 'data input device', + 'data multiplexer', 'data system', 'davenport', 'davenport', 'davit', + 'daybed', 'daybook', 'day nursery', 'day school', 'dead axle', 'deadeye', + 'deadhead', 'deanery', 'deathbed', 'death camp', 'death house', + 'death knell', 'death seat', 'deck', 'deck', 'deck chair', 'deck-house', + 'deckle', 'deckle edge', 'declinometer', 'decoder', 'decolletage', + 'decoupage', 'dedicated file server', 'deep-freeze', 'deerstalker', + 'defense system', 'defensive structure', 'defibrillator', 'defilade', + 'deflector', 'delayed action', 'delay line', 'delft', 'delicatessen', + 'delivery truck', 'delta wing', 'demijohn', 'demitasse', 'den', 'denim', + 'densimeter', 'densitometer', 'dental appliance', 'dental floss', + 'dental implant', 'dentist\'s drill', 'denture', 'deodorant', + 'department store', 'departure lounge', 'depilatory', 'depressor', + 'depth finder', 'depth gauge', 'derrick', 'derrick', 'derringer', 'desk', + 'desk phone', 'desktop computer', 'dessert spoon', 'destroyer', + 'destroyer escort', 'detached house', 'detector', 'detector', + 'detention home', 'detonating fuse', 'detonator', 'developer', 'device', + 'dewar flask', 'dhoti', 'dhow', 'dial', 'dial', 'dial', 'dialog box', + 'dial telephone', 'dialyzer', 'diamante', 'diaper', 'diaper', 'diaphone', + 'diaphragm', 'diaphragm', 'diathermy machine', 'dibble', 'dice cup', + 'dicer', 'dickey', 'dickey', 'dictaphone', 'die', 'diesel', + 'diesel-electric locomotive', 'diesel-hydraulic locomotive', + 'diesel locomotive', 'diestock', 'differential analyzer', + 'differential gear', 'diffuser', 'diffuser', 'digester', 'diggings', + 'digital-analog converter', 'digital audiotape', 'digital camera', + 'digital clock', 'digital computer', 'digital display', + 'digital subscriber line', 'digital voltmeter', 'digital watch', + 'digitizer', 'dilator', 'dildo', 'dimity', 'dimmer', 'diner', 'dinette', + 'dinghy', 'dining area', 'dining car', 'dining-hall', 'dining room', + 'dining-room furniture', 'dining-room table', 'dining table', 'dinner bell', + 'dinner dress', 'dinner jacket', 'dinner napkin', 'dinner pail', + 'dinner table', 'dinner theater', 'diode', 'diode', 'dip', + 'diplomatic building', 'dipole', 'dipper', 'dipstick', 'dip switch', + 'directional antenna', 'directional microphone', 'direction finder', 'dirk', + 'dirndl', 'dirndl', 'dirty bomb', 'discharge lamp', 'discharge pipe', + 'disco', 'discount house', 'discus', 'disguise', 'dish', 'dish', 'dishpan', + 'dish rack', 'dishrag', 'dishtowel', 'dishwasher', 'disk', 'disk brake', + 'disk clutch', 'disk controller', 'disk drive', 'diskette', 'disk harrow', + 'dispatch case', 'dispensary', 'dispenser', 'display', 'display adapter', + 'display panel', 'display window', 'disposal', 'disrupting explosive', + 'distaff', 'distillery', 'distributor', 'distributor cam', + 'distributor cap', 'distributor housing', 'distributor point', 'ditch', + 'ditch spade', 'ditty bag', 'divan', 'divan', 'dive bomber', + 'diverging lens', 'divided highway', 'divider', 'diving bell', + 'divining rod', 'diving suit', 'dixie', 'dixie cup', 'dock', 'doeskin', + 'dogcart', 'doggie bag', 'dogsled', 'dog wrench', 'doily', 'doll', + 'dollhouse', 'dolly', 'dolman', 'dolman', 'dolman sleeve', 'dolmen', 'dome', + 'dome', 'domino', 'dongle', 'donkey jacket', 'door', 'door', 'door', + 'doorbell', 'doorframe', 'doorjamb', 'doorlock', 'doormat', 'doornail', + 'doorplate', 'doorsill', 'doorstop', 'doppler radar', 'dormer', + 'dormer window', 'dormitory', 'dormitory', 'dosemeter', 'dossal', + 'dot matrix printer', 'double bed', 'double-bitted ax', 'double boiler', + 'double-breasted jacket', 'double-breasted suit', 'double door', + 'double glazing', 'double-hung window', 'double knit', 'doubler', + 'double reed', 'double-reed instrument', 'doublet', 'doubletree', 'douche', + 'dovecote', 'dover\'s powder', 'dovetail', 'dovetail plane', 'dowel', + 'downstage', 'drafting instrument', 'drafting table', 'dragunov', + 'drainage ditch', 'drainage system', 'drain basket', 'drainplug', 'drape', + 'drapery', 'drawbar', 'drawbridge', 'drawer', 'drawers', 'drawing chalk', + 'drawing room', 'drawing room', 'drawknife', 'drawstring bag', 'dray', + 'dreadnought', 'dredge', 'dredger', 'dredging bucket', 'dress', + 'dress blues', 'dresser', 'dress hat', 'dressing', 'dressing case', + 'dressing gown', 'dressing room', 'dressing sack', 'dressing table', + 'dress rack', 'dress shirt', 'dress suit', 'dress uniform', 'drift net', + 'drill', 'electric drill', 'drilling platform', 'drill press', 'drill rig', + 'drinking fountain', 'drinking vessel', 'drip loop', 'drip mat', 'drip pan', + 'dripping pan', 'drip pot', 'drive', 'drive', 'drive line', 'driver', + 'driveshaft', 'driveway', 'driving iron', 'driving wheel', 'drogue', + 'drogue parachute', 'drone', 'drone', 'drop arch', 'drop cloth', + 'drop curtain', 'drop forge', 'drop-leaf table', 'dropper', 'droshky', + 'drove', 'drugget', 'drugstore', 'drum', 'drum', 'drum brake', 'drumhead', + 'drum printer', 'drum sander', 'drumstick', 'dry battery', + 'dry-bulb thermometer', 'dry cell', 'dry dock', 'dryer', 'dry fly', + 'dry kiln', 'dry masonry', 'dry point', 'dry wall', 'dual scan display', + 'duck', 'duckboard', 'duckpin', 'dudeen', 'duffel', 'duffel bag', + 'duffel coat', 'dugout', 'dugout canoe', 'dulciana', 'dulcimer', 'dulcimer', + 'dumbbell', 'dumb bomb', 'dumbwaiter', 'dumdum', 'dumpcart', 'dumpster', + 'dump truck', 'dumpy level', 'dunce cap', 'dune buggy', 'dungeon', + 'duplex apartment', 'duplex house', 'duplicator', 'dust bag', 'dustcloth', + 'dust cover', 'dust cover', 'dustmop', 'dustpan', 'dutch oven', + 'dutch oven', 'dwelling', 'dye-works', 'dynamo', 'dynamometer', + 'eames chair', 'earflap', 'early warning radar', 'early warning system', + 'earmuff', 'earphone', 'earplug', 'earplug', 'earthenware', 'earthwork', + 'easel', 'easy chair', 'eaves', 'ecclesiastical attire', 'echinus', + 'echocardiograph', 'edger', 'edge tool', 'efficiency apartment', + 'egg-and-dart', 'eggbeater', 'egg timer', 'eiderdown', 'eight ball', + 'ejection seat', 'elastic', 'elastic bandage', 'elastoplast', 'elbow', + 'elbow pad', 'electric', 'electrical cable', 'electrical contact', + 'electrical converter', 'electrical device', 'electrical system', + 'electric bell', 'electric blanket', 'electric chair', 'electric clock', + 'electric-discharge lamp', 'electric fan', 'electric frying pan', + 'electric furnace', 'electric guitar', 'electric hammer', 'electric heater', + 'electric lamp', 'electric locomotive', 'electric meter', 'electric mixer', + 'electric motor', 'electric organ', 'electric range', + 'electric refrigerator', 'electric toothbrush', 'electric typewriter', + 'electro-acoustic transducer', 'electrode', 'electrodynamometer', + 'electroencephalograph', 'electrograph', 'electrolytic', + 'electrolytic cell', 'electromagnet', 'electrometer', 'electromyograph', + 'electron accelerator', 'electron gun', 'electronic balance', + 'electronic converter', 'electronic device', 'electronic equipment', + 'electronic fetal monitor', 'electronic instrument', 'electronic voltmeter', + 'electron microscope', 'electron multiplier', 'electrophorus', + 'electroscope', 'electrostatic generator', 'electrostatic printer', + 'elevator', 'elevator', 'elevator shaft', 'embankment', 'embassy', + 'embellishment', 'emergency room', 'emesis basin', 'emitter', 'empty', + 'emulsion', 'enamel', 'enamel', 'enamelware', 'encaustic', 'encephalogram', + 'enclosure', 'endoscope', 'energizer', 'engine', 'engine', 'engineering', + 'enginery', 'english horn', 'english saddle', 'enlarger', 'ensemble', + 'ensign', 'entablature', 'entertainment center', 'entrenching tool', + 'entrenchment', 'envelope', 'envelope', 'envelope', 'eolith', 'epauliere', + 'epee', 'epergne', 'epicyclic train', 'epidiascope', 'epilating wax', + 'equalizer', 'equatorial', 'equipment', + 'erasable programmable read-only memory', 'eraser', 'erecting prism', + 'erection', 'erlenmeyer flask', 'escape hatch', 'escapement', + 'escape wheel', 'escarpment', 'escutcheon', 'esophagoscope', 'espadrille', + 'espalier', 'espresso maker', 'espresso shop', 'establishment', 'estaminet', + 'estradiol patch', 'etagere', 'etamine', 'etching', 'ethernet', + 'ethernet cable', 'eton jacket', 'etui', 'eudiometer', 'euphonium', + 'evaporative cooler', 'evening bag', 'exercise bike', 'exercise device', + 'exhaust', 'exhaust fan', 'exhaust valve', 'exhibition hall', 'exocet', + 'expansion bit', 'expansion bolt', 'explosive detection system', + 'explosive device', 'explosive trace detection', 'express', 'extension', + 'extension cord', 'external-combustion engine', 'external drive', + 'extractor', 'eyebrow pencil', 'eyecup', 'eyeliner', 'eyepatch', 'eyepiece', + 'eyeshadow', 'fabric', 'facade', 'face guard', 'face mask', 'faceplate', + 'face powder', 'face veil', 'facing', 'facing', 'facing', 'facsimile', + 'factory', 'factory ship', 'fagot', 'fagot stitch', + 'fahrenheit thermometer', 'faience', 'faille', 'fairlead', 'fairy light', + 'falchion', 'fallboard', 'fallout shelter', 'false face', 'false teeth', + 'family room', 'fan', 'fan belt', 'fan blade', 'fancy dress', 'fanion', + 'fanlight', 'fanjet', 'fanjet', 'fanny pack', 'fan tracery', 'fan vaulting', + 'farm building', 'farmer\'s market', 'farmhouse', 'farm machine', + 'farmplace', 'farmyard', 'farthingale', 'fastener', 'fast reactor', + 'fat farm', 'fatigues', 'faucet', 'fauld', 'fauteuil', 'feather boa', + 'featheredge', 'fedora', 'feedback circuit', 'feedlot', 'fell', 'felloe', + 'felt', 'felt-tip pen', 'felucca', 'fence', 'fencing mask', 'fencing sword', + 'fender', 'fender', 'ferris wheel', 'ferrule', 'ferry', 'ferule', 'festoon', + 'fetoscope', 'fetter', 'fez', 'fiber', 'fiber optic cable', 'fiberscope', + 'fichu', 'fiddlestick', 'field artillery', 'field coil', + 'field-effect transistor', 'field-emission microscope', 'field glass', + 'field hockey ball', 'field hospital', 'field house', 'field lens', + 'field magnet', 'field-sequential color television', 'field tent', + 'fieldwork', 'fife', 'fifth wheel', 'fighter', 'fighting chair', 'fig leaf', + 'figure eight', 'figure loom', 'figure skate', 'filament', 'filature', + 'file', 'file', 'file folder', 'file server', 'filigree', 'filling', 'film', + 'film', 'film advance', 'filter', 'filter', 'finder', 'finery', + 'fine-tooth comb', 'finger', 'fingerboard', 'finger bowl', 'finger paint', + 'finger-painting', 'finger plate', 'fingerstall', 'finish coat', + 'finish coat', 'finisher', 'fin keel', 'fipple', 'fipple flute', 'fire', + 'fire alarm', 'firearm', 'fire bell', 'fireboat', 'firebox', 'firebrick', + 'fire control radar', 'fire control system', 'fire engine', + 'fire extinguisher', 'fire iron', 'fireman\'s ax', 'fireplace', + 'fire screen', 'fire tongs', 'fire tower', 'firewall', 'firing chamber', + 'firing pin', 'firkin', 'firmer chisel', 'first-aid kit', + 'first-aid station', 'first base', 'first class', 'fishbowl', + 'fisherman\'s bend', 'fisherman\'s knot', 'fisherman\'s lure', 'fishhook', + 'fishing boat', 'fishing gear', 'fishing rod', 'fish joint', 'fish knife', + 'fishnet', 'fish slice', 'fitment', 'fixative', 'fixer-upper', 'flag', + 'flageolet', 'flagon', 'flagpole', 'flagship', 'flail', 'flambeau', + 'flamethrower', 'flange', 'flannel', 'flannel', 'flannelette', 'flap', + 'flash', 'flash', 'flash camera', 'flasher', 'flashlight', + 'flashlight battery', 'flash memory', 'flask', 'flat arch', 'flatbed', + 'flatbed press', 'flat bench', 'flatcar', 'flat file', 'flatlet', + 'flat panel display', 'flats', 'flat tip screwdriver', 'fleece', + 'fleet ballistic missile submarine', 'fleur-de-lis', 'flight simulator', + 'flintlock', 'flintlock', 'flip-flop', 'flipper', 'float', 'floating dock', + 'floatplane', 'flood', 'floor', 'floor', 'floor', 'floorboard', + 'floor cover', 'floor joist', 'floor lamp', 'flophouse', 'florist', 'floss', + 'flotsam', 'flour bin', 'flour mill', 'flowerbed', 'flugelhorn', + 'fluid drive', 'fluid flywheel', 'flume', 'fluorescent lamp', 'fluoroscope', + 'flush toilet', 'flute', 'flute', 'flux applicator', 'fluxmeter', 'fly', + 'flying boat', 'flying buttress', 'flying carpet', 'flying jib', 'fly rod', + 'fly tent', 'flytrap', 'flywheel', 'fob', 'foghorn', 'foglamp', 'foil', + 'fold', 'folder', 'folding chair', 'folding door', 'folding saw', + 'food court', 'food processor', 'food hamper', 'foot', 'footage', + 'football', 'football helmet', 'football stadium', 'footbath', 'foot brake', + 'footbridge', 'foothold', 'footlocker', 'foot rule', 'footstool', + 'footwear', 'footwear', 'forceps', 'force pump', 'fore-and-after', + 'fore-and-aft sail', 'forecastle', 'forecourt', 'foredeck', 'fore edge', + 'foreground', 'foremast', 'fore plane', 'foresail', 'forestay', 'foretop', + 'fore-topmast', 'fore-topsail', 'forge', 'fork', 'forklift', 'formalwear', + 'formica', 'fortification', 'fortress', 'forty-five', 'foucault pendulum', + 'foulard', 'foul-weather gear', 'foundation garment', 'foundry', 'fountain', + 'fountain pen', 'four-in-hand', 'four-poster', 'four-pounder', + 'four-stroke engine', 'four-wheel drive', 'four-wheel drive', + 'four-wheeler', 'fowling piece', 'foxhole', 'fragmentation bomb', 'frail', + 'fraise', 'frame', 'frame', 'frame buffer', 'framework', 'francis turbine', + 'franking machine', 'free house', 'free-reed', 'free-reed instrument', + 'freewheel', 'freight car', 'freight elevator', 'freight liner', + 'freight train', 'french door', 'french horn', 'french polish', + 'french roof', 'french window', 'fresnel lens', 'fret', 'friary', + 'friction clutch', 'frieze', 'frieze', 'frigate', 'frigate', 'frill', + 'frisbee', 'frock', 'frock coat', 'frontlet', 'front porch', + 'front projector', 'fruit machine', 'frying pan', 'fuel filter', + 'fuel gauge', 'fuel injection', 'fuel system', 'full-dress uniform', + 'full metal jacket', 'full skirt', 'fumigator', 'funeral home', 'funnel', + 'funny wagon', 'fur', 'fur coat', 'fur hat', 'furnace', 'furnace lining', + 'furnace room', 'furnishing', 'furnishing', 'furniture', 'fur-piece', + 'furrow', 'fuse', 'fusee drive', 'fuselage', 'fusil', 'fustian', 'futon', + 'gabardine', 'gable', 'gable roof', 'gadgetry', 'gaff', 'gaff', 'gaff', + 'gaffsail', 'gaff topsail', 'gag', 'gaiter', 'gaiter', 'galilean telescope', + 'galleon', 'gallery', 'gallery', 'galley', 'galley', 'galley', 'gallows', + 'gallows tree', 'galvanometer', 'gambling house', 'gambrel', 'game', + 'gamebag', 'game equipment', 'gaming table', 'gamp', 'gangplank', 'gangsaw', + 'gangway', 'gantlet', 'gantry', 'garage', 'garage', 'garand rifle', + 'garbage', 'garbage truck', 'garboard', 'garden', 'garden', 'garden rake', + 'garden spade', 'garden tool', 'garden trowel', 'gargoyle', 'garibaldi', + 'garlic press', 'garment', 'garment bag', 'garrison cap', 'garrote', + 'garter', 'garter belt', 'garter stitch', 'gas guzzler', 'gas shell', + 'gas bracket', 'gas burner', 'gas-cooled reactor', 'gas-discharge tube', + 'gas engine', 'gas fixture', 'gas furnace', 'gas gun', 'gas heater', + 'gas holder', 'gasket', 'gas lamp', 'gas maser', 'gasmask', 'gas meter', + 'gasoline engine', 'gasoline gauge', 'gas oven', 'gas oven', 'gas pump', + 'gas range', 'gas ring', 'gas tank', 'gas thermometer', 'gastroscope', + 'gas turbine', 'gas-turbine ship', 'gat', 'gate', 'gatehouse', + 'gateleg table', 'gatepost', 'gathered skirt', 'gatling gun', 'gauge', + 'gauntlet', 'gauntlet', 'gauze', 'gauze', 'gavel', 'gazebo', 'gear', 'gear', + 'gear', 'gearbox', 'gearing', 'gearset', 'gearshift', 'geiger counter', + 'geiger tube', 'gene chip', 'general-purpose bomb', 'generator', + 'generator', 'generator', 'geneva gown', 'geodesic dome', 'georgette', + 'gharry', 'ghat', 'ghetto blaster', 'gift shop', 'gift wrapping', 'gig', + 'gig', 'gig', 'gig', 'gildhall', 'gill net', 'gilt', 'gimbal', 'gingham', + 'girandole', 'girder', 'girdle', 'glass', 'glass', 'glass cutter', + 'glasses case', 'glebe house', 'glengarry', 'glider', + 'global positioning system', 'glockenspiel', 'glory hole', 'glove', + 'glove compartment', 'glow lamp', 'glow tube', 'glyptic art', 'glyptics', + 'gnomon', 'goal', 'goalmouth', 'goalpost', 'goblet', 'godown', 'goggles', + 'go-kart', 'gold plate', 'golf bag', 'golf ball', 'golfcart', 'golf club', + 'golf-club head', 'golf equipment', 'golf glove', 'golliwog', 'gondola', + 'gong', 'goniometer', 'gordian knot', 'gorget', 'gossamer', 'gothic arch', + 'gouache', 'gouge', 'gourd', 'government building', 'government office', + 'gown', 'gown', 'gown', 'grab', 'grab bag', 'grab bar', 'grace cup', + 'grade separation', 'graduated cylinder', 'graffito', 'gramophone', + 'granary', 'grandfather clock', 'grand piano', 'graniteware', 'granny knot', + 'grape arbor', 'grapnel', 'grapnel', 'grass skirt', 'grate', 'grate', + 'grater', 'graver', 'gravestone', 'gravimeter', 'gravure', 'gravy boat', + 'grey', 'grease-gun', 'greasepaint', 'greasy spoon', 'greatcoat', + 'great hall', 'greave', 'greengrocery', 'greenhouse', 'grenade', 'grid', + 'griddle', 'grill', 'grille', 'grillroom', 'grinder', 'grinding wheel', + 'grindstone', 'gripsack', 'gristmill', 'grocery bag', 'grocery store', + 'grogram', 'groined vault', 'groover', 'grosgrain', 'gros point', 'ground', + 'ground bait', 'ground control', 'ground floor', 'groundsheet', 'g-string', + 'guard', 'guard boat', 'guardroom', 'guardroom', 'guard ship', + 'guard\'s van', 'gueridon', 'guarnerius', 'guesthouse', 'guestroom', + 'guidance system', 'guided missile', 'guided missile cruiser', + 'guided missile frigate', 'guildhall', 'guilloche', 'guillotine', 'guimpe', + 'guimpe', 'guitar', 'guitar pick', 'gulag', 'gun', 'gunboat', + 'gun carriage', 'gun case', 'gun emplacement', 'gun enclosure', 'gunlock', + 'gunnery', 'gunnysack', 'gun pendulum', 'gun room', 'gunsight', + 'gun trigger', 'gurney', 'gusher', 'gusset', 'gusset', 'guy', + 'gymnastic apparatus', 'gym shoe', 'gym suit', 'gymslip', 'gypsy cab', + 'gyrocompass', 'gyroscope', 'gyrostabilizer', 'habergeon', 'habit', 'habit', + 'hacienda', 'hacksaw', 'haft', 'hairbrush', 'haircloth', 'hairdressing', + 'hairnet', 'hairpiece', 'hairpin', 'hair shirt', 'hair slide', 'hair spray', + 'hairspring', 'hair trigger', 'halberd', 'half binding', 'half hatchet', + 'half hitch', 'half track', 'hall', 'hall', 'hall', 'hall of fame', + 'hall of residence', 'hallstand', 'halter', 'halter', 'hame', 'hammer', + 'hammer', 'hammer', 'hammerhead', 'hammock', 'hamper', 'hand', 'handball', + 'handbarrow', 'handbell', 'hand blower', 'handbow', 'hand brake', + 'hand calculator', 'handcar', 'handcart', 'hand cream', 'handcuff', + 'hand drill', 'hand glass', 'hand glass', 'hand grenade', + 'hand-held computer', 'handhold', 'handkerchief', 'handlebar', 'handloom', + 'hand lotion', 'hand luggage', 'hand-me-down', 'hand mower', 'hand pump', + 'handrest', 'handsaw', 'handset', 'hand shovel', 'handspike', 'handstamp', + 'hand throttle', 'hand tool', 'hand towel', 'hand truck', 'handwear', + 'handwheel', 'handwheel', 'hangar queen', 'hanger', 'hang glider', + 'hangman\'s rope', 'hank', 'hansom', 'harbor', 'hard disc', 'hard hat', + 'hardtop', 'hardware', 'hardware store', 'harmonica', 'harmonium', + 'harness', 'harness', 'harp', 'harp', 'harpoon', 'harpoon gun', + 'harpoon log', 'harpsichord', 'harris tweed', 'harrow', 'harvester', + 'hash house', 'hasp', 'hat', 'hatbox', 'hatch', 'hatchback', 'hatchback', + 'hatchel', 'hatchet', 'hatpin', 'hauberk', 'hawaiian guitar', 'hawse', + 'hawser', 'hawser bend', 'hay bale', 'hayfork', 'hayloft', 'haymaker', + 'hayrack', 'hayrack', 'hazard', 'head', 'head', 'head', 'headboard', + 'head covering', 'headdress', 'header', 'header', 'header', 'header', + 'headfast', 'head gasket', 'head gate', 'headgear', 'headlight', + 'headpiece', 'headpin', 'headquarters', 'headrace', 'headrest', 'headsail', + 'headscarf', 'headset', 'head shop', 'headstall', 'headstock', 'health spa', + 'hearing aid', 'hearing aid', 'hearse', 'hearth', 'hearthrug', + 'heart-lung machine', 'heat engine', 'heater', 'heat exchanger', + 'heating pad', 'heat lamp', 'heat pump', 'heat-seeking missile', + 'heat shield', 'heat sink', 'heaume', 'heaver', 'heavier-than-air craft', + 'heckelphone', 'hectograph', 'hedge', 'hedge trimmer', 'helicon', + 'helicopter', 'heliograph', 'heliometer', 'helm', 'helmet', 'helmet', + 'hematocrit', 'hemming-stitch', 'hemostat', 'hemstitch', 'henroost', + 'heraldry', 'hermitage', 'herringbone', 'herringbone', + 'herschelian telescope', 'hessian boot', 'heterodyne receiver', 'hibachi', + 'hideaway', 'hi-fi', 'high altar', 'high-angle gun', 'highball glass', + 'highboard', 'highboy', 'highchair', 'high gear', 'high-hat cymbal', + 'highlighter', 'highlighter', 'high-pass filter', 'high-rise', 'high table', + 'high-warp loom', 'hijab', 'hinge', 'hinging post', 'hip boot', 'hipflask', + 'hip pad', 'hip pocket', 'hippodrome', 'hip roof', 'hitch', 'hitch', + 'hitching post', 'hitchrack', 'hob', 'hobble skirt', 'hockey skate', + 'hockey stick', 'hod', 'hodoscope', 'hoe', 'hoe handle', 'hogshead', + 'hoist', 'hold', 'holder', 'holding cell', 'holding device', 'holding pen', + 'hollowware', 'holster', 'holster', 'holy of holies', 'home', + 'home appliance', 'home computer', 'home plate', 'home room', 'homespun', + 'homestead', 'home theater', 'homing torpedo', 'hone', 'honeycomb', 'hood', + 'hood', 'hood', 'hood', 'hood', 'hood latch', 'hook', 'hook', 'hook', + 'hookah', 'hook and eye', 'hookup', 'hookup', 'hook wrench', 'hoopskirt', + 'hoosegow', 'hoover', 'hope chest', 'hopper', 'hopsacking', + 'horizontal bar', 'horizontal stabilizer', 'horizontal tail', 'horn', + 'horn', 'horn', 'horn button', 'hornpipe', 'horse', 'horsebox', 'horsecar', + 'horse cart', 'horsecloth', 'horse-drawn vehicle', 'horsehair', + 'horsehair wig', 'horseless carriage', 'horse pistol', 'horseshoe', + 'horseshoe', 'horse-trail', 'horsewhip', 'hose', 'hosiery', 'hospice', + 'hospital', 'hospital bed', 'hospital room', 'hospital ship', + 'hospital train', 'hostel', 'hostel', 'hot-air balloon', 'hotel', + 'hotel-casino', 'hotel-casino', 'hotel room', 'hot line', 'hot pants', + 'hot plate', 'hot rod', 'hot spot', 'hot tub', 'hot-water bottle', + 'houndstooth check', 'hourglass', 'hour hand', 'house', 'house', + 'houseboat', 'houselights', 'house of cards', 'house of correction', + 'house paint', 'housetop', 'housing', 'hovel', 'hovercraft', 'howdah', + 'huarache', 'hub-and-spoke', 'hubcap', 'huck', 'hug-me-tight', 'hula-hoop', + 'hulk', 'hull', 'humeral veil', 'humvee', 'hunter', 'hunting knife', + 'hurdle', 'hurricane deck', 'hurricane lamp', 'hut', 'hutch', 'hutment', + 'hydraulic brake', 'hydraulic press', 'hydraulic pump', 'hydraulic system', + 'hydraulic transmission', 'hydroelectric turbine', 'hydrofoil', 'hydrofoil', + 'hydrogen bomb', 'hydrometer', 'hygrodeik', 'hygrometer', 'hygroscope', + 'hyperbaric chamber', 'hypercoaster', 'hypermarket', 'hypodermic needle', + 'hypodermic syringe', 'hypsometer', 'hysterosalpingogram', 'i-beam', + 'ice ax', 'iceboat', 'icebreaker', 'iced-tea spoon', 'ice hockey rink', + 'ice machine', 'ice maker', 'ice pack', 'icepick', 'ice rink', 'ice skate', + 'ice tongs', 'icetray', 'iconoscope', 'identikit', 'idle pulley', 'igloo', + 'ignition coil', 'ignition key', 'ignition switch', 'imaret', + 'immovable bandage', 'impact printer', 'impeller', 'implant', 'implement', + 'impression', 'imprint', 'improvised explosive device', 'impulse turbine', + 'in-basket', 'incendiary bomb', 'incinerator', 'inclined plane', + 'inclinometer', 'inclinometer', 'incrustation', 'incubator', + 'index register', 'indiaman', 'indian club', 'indicator', 'induction coil', + 'inductor', 'industrial watercourse', 'inertial guidance system', + 'inflater', 'inhaler', 'injector', 'ink bottle', 'ink eraser', + 'ink-jet printer', 'inkle', 'inkstand', 'inkwell', 'inlay', + 'inside caliper', 'insole', 'instep', 'instillator', 'institution', + 'instrument', 'instrument of punishment', 'instrument of torture', + 'intaglio', 'intake valve', 'integrated circuit', 'integrator', 'intelnet', + 'interceptor', 'interchange', 'intercommunication system', + 'intercontinental ballistic missile', 'interface', 'interferometer', + 'interior door', 'internal-combustion engine', 'internal drive', 'internet', + 'interphone', 'interrupter', 'intersection', 'interstice', + 'intraocular lens', 'intravenous pyelogram', 'inverter', 'ion engine', + 'ionization chamber', 'ipod', 'video ipod', 'iron', 'iron', 'iron', 'irons', + 'ironclad', 'iron foundry', 'iron horse', 'ironing', 'iron lung', + 'ironmongery', 'ironworks', 'irrigation ditch', 'izar', 'jabot', 'jack', + 'jack', 'jack', 'jack', 'jacket', 'jacket', 'jacket', 'jack-in-the-box', + 'jack-o\'-lantern', 'jack plane', 'jacob\'s ladder', 'jaconet', + 'jacquard loom', 'jacquard', 'jag', 'jail', 'jalousie', 'jamb', 'jammer', + 'jampot', 'japan', 'jar', 'jarvik heart', 'jaunting car', 'javelin', 'jaw', + 'jaws of life', 'jean', 'jeep', 'jellaba', 'jerkin', 'jeroboam', 'jersey', + 'jersey', 'jet', 'jet bridge', 'jet engine', 'jetliner', 'jeweler\'s glass', + 'jewelled headdress', 'jew\'s harp', 'jib', 'jibboom', 'jig', 'jig', + 'jiggermast', 'jigsaw', 'jigsaw puzzle', 'jinrikisha', 'jobcentre', + 'jodhpurs', 'jodhpur', 'joinery', 'joint', 'joint direct attack munition', + 'jointer', 'joist', 'jolly boat', 'jorum', 'joss house', 'journal bearing', + 'journal box', 'joystick', 'jungle gym', 'junk', 'jug', 'jukebox', + 'jumbojet', 'jumper', 'jumper', 'jumper', 'jumper', 'jumper cable', + 'jump seat', 'jump suit', 'jump suit', 'junction', 'junction', + 'junction barrier', 'junk shop', 'jury box', 'jury mast', 'kachina', + 'kaffiyeh', 'kalansuwa', 'kalashnikov', 'kameez', 'kanzu', 'katharometer', + 'kayak', 'kazoo', 'keel', 'keelboat', 'keelson', 'keep', 'keg', 'kennel', + 'kepi', 'keratoscope', 'kerchief', 'ketch', 'kettle', 'kettle', 'key', + 'key', 'keyboard', 'keyboard buffer', 'keyboard instrument', 'keyhole', + 'keyhole saw', 'khadi', 'khaki', 'khakis', 'khimar', 'khukuri', + 'kick pleat', 'kicksorter', 'kickstand', 'kick starter', 'kid glove', + 'kiln', 'kilt', 'kimono', 'kinescope', 'kinetoscope', 'king', 'king', + 'kingbolt', 'king post', 'kipp\'s apparatus', 'kirk', 'kirpan', 'kirtle', + 'kirtle', 'kit', 'kit', 'kitbag', 'kitchen', 'kitchen appliance', + 'kitchenette', 'kitchen table', 'kitchen utensil', 'kitchenware', + 'kite balloon', 'klaxon', 'klieg light', 'klystron', 'knee brace', + 'knee-high', 'knee pad', 'knee piece', 'knife', 'knife', 'knife blade', + 'knight', 'knit', 'knitting machine', 'knitting needle', 'knitwear', 'knob', + 'knob', 'knobble', 'knobkerrie', 'knocker', 'knot', 'knuckle joint', 'kohl', + 'koto', 'kraal', 'kremlin', 'kris', 'krummhorn', 'kundt\'s tube', + 'kurdistan', 'kurta', 'kylix', 'kymograph', 'lab bench', 'lab coat', 'lace', + 'lacquer', 'lacquerware', 'lacrosse ball', 'ladder-back', 'ladder-back', + 'ladder truck', 'ladies\' room', 'ladle', 'lady chapel', 'lagerphone', + 'lag screw', 'lake dwelling', 'lally', 'lamasery', 'lambrequin', 'lame', + 'laminar flow clean room', 'laminate', 'lamination', 'lamp', 'lamp', + 'lamp house', 'lamppost', 'lampshade', 'lanai', 'lancet arch', + 'lancet window', 'landau', 'lander', 'landing craft', 'landing flap', + 'landing gear', 'landing net', 'landing skid', 'land line', 'land mine', + 'land office', 'lanolin', 'lantern', 'lanyard', 'lap', 'laparoscope', + 'lapboard', 'lapel', 'lap joint', 'laptop', 'laryngoscope', 'laser', + 'laser-guided bomb', 'laser printer', 'lash', 'lashing', 'lasso', 'latch', + 'latch', 'latchet', 'latchkey', 'lateen', 'latex paint', 'lath', 'lathe', + 'latrine', 'lattice', 'launch', 'launcher', 'laundry', 'laundry cart', + 'laundry truck', 'lavalava', 'lavaliere', 'laver', 'lawn chair', + 'lawn furniture', 'lawn mower', 'layette', 'lead-acid battery', 'lead-in', + 'leading rein', 'lead pencil', 'leaf spring', 'lean-to', 'lean-to tent', + 'leash', 'leatherette', 'leather strip', 'leclanche cell', 'lectern', + 'lecture room', 'lederhosen', 'ledger board', 'leg', 'leg', 'legging', + 'leiden jar', 'leisure wear', 'lens', 'lens', 'lens cap', 'lens implant', + 'leotard', 'letter case', 'letter opener', 'levee', 'level', 'lever', + 'lever', 'lever', 'lever lock', 'levi\'s', 'liberty ship', 'library', + 'library', 'lid', 'liebig condenser', 'lie detector', 'lifeboat', + 'life buoy', 'life jacket', 'life office', 'life preserver', + 'life-support system', 'life-support system', 'lifting device', 'lift pump', + 'ligament', 'ligature', 'light', 'light arm', 'light bulb', 'light circuit', + 'light-emitting diode', 'lighter', 'lighter-than-air craft', 'light filter', + 'lighting', 'light machine gun', 'light meter', 'light microscope', + 'lightning rod', 'light pen', 'lightship', 'lilo', 'limber', 'limekiln', + 'limiter', 'limousine', 'linear accelerator', 'linen', 'line printer', + 'liner', 'liner', 'lingerie', 'lining', 'link', 'linkage', 'link trainer', + 'linocut', 'linoleum knife', 'linotype', 'linsey-woolsey', 'linstock', + 'lion-jaw forceps', 'lip-gloss', 'lipstick', 'liqueur glass', + 'liquid crystal display', 'liquid metal reactor', 'lisle', 'lister', + 'litterbin', 'little theater', 'live axle', 'living quarters', + 'living room', 'load', 'loafer', 'loaner', 'lobe', 'lobster pot', 'local', + 'local area network', 'local oscillator', 'lochaber ax', 'lock', 'lock', + 'lock', 'lock', 'lockage', 'locker', 'locker room', 'locket', 'lock-gate', + 'locking pliers', 'lockring', 'lockstitch', 'lockup', 'locomotive', 'lodge', + 'lodge', 'lodge', 'lodging house', 'loft', 'loft', 'loft', 'log cabin', + 'loggia', 'longbow', 'long iron', 'long johns', 'long sleeve', 'long tom', + 'long trousers', 'long underwear', 'looking glass', 'lookout', 'loom', + 'loop knot', 'lorgnette', 'lorraine cross', 'lorry', 'lota', 'lotion', + 'loudspeaker', 'lounge', 'lounger', 'lounging jacket', 'lounging pajama', + 'loungewear', 'loupe', 'louvered window', 'love knot', 'love seat', + 'loving cup', 'lowboy', 'low-pass filter', 'low-warp-loom', 'lp', 'l-plate', + 'lubber\'s hole', 'lubricating system', 'luff', 'lug', 'luge', 'luger', + 'luggage carrier', 'luggage compartment', 'luggage rack', 'lugger', + 'lugsail', 'lug wrench', 'lumberjack', 'lumbermill', + 'lunar excursion module', 'lunchroom', 'lunette', 'lungi', 'lunula', + 'lusterware', 'lute', 'luxury liner', 'lyceum', 'lychgate', 'lyre', + 'machete', 'machicolation', 'machine', 'machine', 'machine bolt', + 'machine gun', 'machinery', 'machine screw', 'machine tool', + 'machinist\'s vise', 'machmeter', 'mackinaw', 'mackinaw', 'mackinaw', + 'mackintosh', 'macrame', 'madras', 'mae west', 'magazine rack', + 'magic lantern', 'magnet', 'magnetic bottle', 'magnetic compass', + 'magnetic core memory', 'magnetic disk', 'magnetic head', 'magnetic mine', + 'magnetic needle', 'magnetic recorder', 'magnetic stripe', 'magnetic tape', + 'magneto', 'magnetometer', 'magnetron', 'magnifier', 'magnum', + 'magnus hitch', 'mail', 'mailbag', 'mailbag', 'mailboat', 'mailbox', + 'mail car', 'maildrop', 'mailer', 'maillot', 'maillot', 'mailsorter', + 'mail train', 'mainframe', 'mainmast', 'main rotor', 'mainsail', + 'mainspring', 'main-topmast', 'main-topsail', 'main yard', 'maisonette', + 'majolica', 'makeup', 'maksutov telescope', 'malacca', 'mallet', 'mallet', + 'mallet', 'mammogram', 'mandola', 'mandolin', 'manger', 'mangle', 'manhole', + 'manhole cover', 'man-of-war', 'manometer', 'manor', 'manor hall', 'manpad', + 'mansard', 'manse', 'mansion', 'mantel', 'mantelet', 'mantilla', + 'mao jacket', 'map', 'maquiladora', 'maraca', 'marble', 'marching order', + 'marimba', 'marina', 'marker', 'marketplace', 'marlinespike', 'marocain', + 'marquee', 'marquetry', 'marriage bed', 'martello tower', 'martingale', + 'mascara', 'maser', 'masher', 'mashie', 'mashie niblick', 'masjid', 'mask', + 'mask', 'masonite', 'mason jar', 'masonry', 'mason\'s level', + 'massage parlor', 'massage parlor', 'mass spectrograph', + 'mass spectrometer', 'mast', 'mast', 'mastaba', 'master bedroom', + 'masterpiece', 'mat', 'mat', 'match', 'match', 'matchboard', 'matchbook', + 'matchbox', 'matchlock', 'match plane', 'matchstick', 'material', + 'materiel', 'maternity hospital', 'maternity ward', 'matrix', + 'matthew walker', 'matting', 'mattock', 'mattress cover', 'maul', + 'maulstick', 'mauser', 'mausoleum', 'maxi', 'maxim gun', + 'maximum and minimum thermometer', 'maypole', 'maze', 'mazer', 'means', + 'measure', 'measuring cup', 'measuring instrument', 'measuring stick', + 'meat counter', 'meat grinder', 'meat hook', 'meat house', 'meat safe', + 'meat thermometer', 'mechanical device', 'mechanical piano', + 'mechanical system', 'mechanism', 'medical building', 'medical instrument', + 'medicine ball', 'medicine chest', 'medline', 'megalith', 'megaphone', + 'memorial', 'memory', 'memory chip', 'memory device', 'menagerie', + 'mending', 'menhir', 'menorah', 'menorah', 'man\'s clothing', 'men\'s room', + 'mercantile establishment', 'mercury barometer', 'mercury cell', + 'mercury thermometer', 'mercury-vapor lamp', 'mercy seat', 'merlon', 'mess', + 'mess jacket', 'mess kit', 'messuage', 'metal detector', 'metallic', + 'metal screw', 'metal wood', 'meteorological balloon', 'meter', + 'meterstick', 'metronome', 'mezzanine', 'mezzanine', 'microbalance', + 'microbrewery', 'microfiche', 'microfilm', 'micrometer', 'microphone', + 'microprocessor', 'microscope', 'microtome', 'microwave', + 'microwave diathermy machine', 'microwave linear accelerator', 'middy', + 'midiron', 'mihrab', 'mihrab', 'military hospital', 'military quarters', + 'military uniform', 'military vehicle', 'milk bar', 'milk can', + 'milk float', 'milking machine', 'milking stool', 'milk wagon', 'mill', + 'milldam', 'miller', 'milliammeter', 'millinery', 'millinery', 'milling', + 'millivoltmeter', 'millstone', 'millstone', 'millwheel', 'mimeograph', + 'minaret', 'mincer', 'mine', 'mine detector', 'minelayer', 'mineshaft', + 'minibar', 'minibike', 'minibus', 'minicar', 'minicomputer', 'ministry', + 'miniskirt', 'minisub', 'minivan', 'miniver', 'mink', 'minster', 'mint', + 'minute hand', 'minuteman', 'mirror', 'missile', 'missile defense system', + 'miter box', 'miter joint', 'mitten', 'mixer', 'mixer', 'mixing bowl', + 'mixing faucet', 'mizzen', 'mizzenmast', 'mobcap', 'mobile home', + 'moccasin', 'mock-up', 'mod con', 'model t', 'modem', 'modillion', 'module', + 'module', 'mohair', 'moire', 'mold', 'moldboard', 'moldboard plow', + 'moleskin', 'molotov cocktail', 'monastery', 'monastic habit', 'moneybag', + 'money belt', 'monitor', 'monitor', 'monitor', 'monkey-wrench', + 'monk\'s cloth', 'monochrome', 'monocle', 'monofocal lens implant', + 'monoplane', 'monotype', 'monstrance', 'mooring tower', 'moorish arch', + 'moped', 'mop handle', 'moquette', 'morgue', 'morion', 'morning dress', + 'morning dress', 'morning room', 'morris chair', 'mortar', 'mortar', + 'mortarboard', 'mortise joint', 'mosaic', 'mosque', 'mosquito net', 'motel', + 'motel room', 'mother hubbard', 'motion-picture camera', + 'motion-picture film', 'motley', 'motley', 'motor', 'motorboat', + 'motorcycle', 'motor hotel', 'motorized wheelchair', 'motor scooter', + 'motor vehicle', 'mound', 'mound', 'mount', 'mountain bike', + 'mountain tent', 'mouse', 'mouse button', 'mousetrap', 'mousse', + 'mouthpiece', 'mouthpiece', 'mouthpiece', 'movement', 'movie projector', + 'moving-coil galvanometer', 'moving van', 'mud brick', 'mudguard', 'mudhif', + 'muff', 'muffle', 'muffler', 'mufti', 'mug', 'mulch', 'mule', + 'multichannel recorder', 'multiengine airplane', 'multiplex', 'multiplexer', + 'multiprocessor', 'multistage rocket', 'munition', 'murphy bed', 'musette', + 'musette pipe', 'museum', 'mushroom anchor', 'musical instrument', + 'music box', 'music hall', 'music school', 'music stand', 'music stool', + 'musket', 'musket ball', 'muslin', 'mustache cup', 'mustard plaster', + 'mute', 'muzzle loader', 'muzzle', 'myelogram', 'nacelle', 'nail', + 'nailbrush', 'nailfile', 'nailhead', 'nailhead', 'nail polish', 'nainsook', + 'napier\'s bones', 'nard', 'narrowbody aircraft', 'narrow wale', 'narthex', + 'narthex', 'nasotracheal tube', 'national monument', 'nautilus', + 'navigational system', 'naval equipment', 'naval gun', 'naval missile', + 'naval radar', 'naval tactical data system', 'naval weaponry', 'nave', + 'navigational instrument', 'nebuchadnezzar', 'neckband', 'neck brace', + 'neckcloth', 'neckerchief', 'necklace', 'necklet', 'neckline', 'neckpiece', + 'necktie', 'neckwear', 'needle', 'needle', 'needlenose pliers', + 'needlework', 'negative', 'negative magnetic pole', 'negative pole', + 'negligee', 'neolith', 'neon lamp', 'nephoscope', 'nest', 'nest egg', 'net', + 'net', 'net', 'net', 'network', 'network', 'neutron bomb', 'newel', + 'newel post', 'newspaper', 'newsroom', 'newsroom', 'newsstand', + 'newtonian telescope', 'nib', 'niblick', 'nicad', 'nickel-iron battery', + 'nicol prism', 'night bell', 'nightcap', 'nightgown', 'night latch', + 'night-light', 'nightshirt', 'nightwear', 'ninepin', 'ninepin ball', + 'ninon', 'nipple', 'nipple shield', 'niqab', 'nissen hut', 'nogging', + 'noisemaker', 'nonsmoker', 'non-volatile storage', 'norfolk jacket', + 'noria', 'nosebag', 'noseband', 'nose flute', 'nosewheel', 'notebook', + 'nuclear-powered ship', 'nuclear reactor', 'nuclear rocket', + 'nuclear weapon', 'nude', 'numdah', 'nun\'s habit', 'nursery', + 'nut and bolt', 'nutcracker', 'nylon', 'nylons', 'oar', 'oast', + 'oast house', 'obelisk', 'object ball', 'objective', 'oblique bandage', + 'oboe', 'oboe da caccia', 'oboe d\'amore', 'observation dome', + 'observatory', 'obstacle', 'obturator', 'ocarina', 'octant', + 'odd-leg caliper', 'odometer', 'oeil de boeuf', 'office', 'office building', + 'office furniture', 'officer\'s mess', 'off-line equipment', 'ogee', + 'ogee arch', 'ohmmeter', 'oil', 'oilcan', 'oilcloth', 'oil filter', + 'oil heater', 'oil lamp', 'oil paint', 'oil pump', 'oil refinery', + 'oilskin', 'oil slick', 'oilstone', 'oil tanker', 'old school tie', + 'olive drab', 'olive drab', 'olympian zeus', 'omelet pan', + 'omnidirectional antenna', 'omnirange', 'onion dome', 'open-air market', + 'open circuit', 'open-end wrench', 'opener', 'open-hearth furnace', + 'openside plane', 'open sight', 'openwork', 'opera', 'opera cloak', + 'operating microscope', 'operating room', 'operating table', + 'ophthalmoscope', 'optical device', 'optical disk', 'optical instrument', + 'optical pyrometer', 'optical telescope', 'orchestra pit', 'ordinary', + 'organ', 'organdy', 'organic light-emitting diode', 'organ loft', + 'organ pipe', 'organza', 'oriel', 'oriflamme', 'o ring', 'orlon', + 'orlop deck', 'orphanage', 'orphrey', 'orrery', 'orthicon', + 'orthochromatic film', 'orthopter', 'orthoscope', 'oscillograph', + 'oscilloscope', 'ossuary', 'otoscope', 'ottoman', 'oubliette', 'out-basket', + 'outboard motor', 'outboard motorboat', 'outbuilding', 'outerwear', + 'outfall', 'outfit', 'outfitter', 'outhouse', 'output device', 'outrigger', + 'outrigger canoe', 'outside caliper', 'outside mirror', 'outwork', 'oven', + 'oven thermometer', 'overall', 'overall', 'overcoat', 'overdrive', + 'overgarment', 'overhand knot', 'overhang', 'overhead projector', + 'overmantel', 'overnighter', 'overpass', 'override', 'overshoe', + 'overskirt', 'oxbow', 'oxbridge', 'oxcart', 'oxeye', 'oxford', 'oximeter', + 'oxyacetylene torch', 'oxygen mask', 'oyster bar', 'oyster bed', 'pace car', + 'pacemaker', 'pack', 'pack', 'pack', 'package', 'package store', + 'packaging', 'packet', 'packing box', 'packinghouse', 'packinghouse', + 'packing needle', 'packsaddle', 'paddle', 'paddle', 'paddle', 'paddle box', + 'paddle steamer', 'paddlewheel', 'paddock', 'padlock', 'page printer', + 'paint', 'paintball', 'paintball gun', 'paintbox', 'paintbrush', 'paisley', + 'pajama', 'pajama', 'palace', 'palace', 'palace', 'palanquin', 'paleolith', + 'palestra', 'palette', 'palette knife', 'palisade', 'pallet', 'pallette', + 'pallium', 'pallium', 'pan', 'pan', 'pancake turner', 'panchromatic film', + 'panda car', 'paneling', 'panhandle', 'panic button', 'pannier', 'pannier', + 'pannikin', 'panopticon', 'panopticon', 'panpipe', 'pantaloon', + 'pantechnicon', 'pantheon', 'pantheon', 'pantie', 'panting', 'pant leg', + 'pantograph', 'pantry', 'pants suit', 'panty girdle', 'pantyhose', 'panzer', + 'paper chain', 'paper clip', 'paper cutter', 'paper fastener', 'paper feed', + 'paper mill', 'paper towel', 'parabolic mirror', 'parabolic reflector', + 'parachute', 'parallel bars', 'parallel circuit', 'parallel interface', + 'parang', 'parapet', 'parapet', 'parasail', 'parasol', 'parer', + 'parfait glass', 'pargeting', 'pari-mutuel machine', 'parka', 'park bench', + 'parking meter', 'parlor', 'parquet', 'parquetry', 'parsonage', + 'parsons table', 'partial denture', 'particle detector', 'partition', + 'parts bin', 'party line', 'party wall', 'parvis', 'passenger car', + 'passenger ship', 'passenger train', 'passenger van', 'passe-partout', + 'passive matrix display', 'passkey', 'pass-through', 'pastry cart', 'patch', + 'patchcord', 'patchouli', 'patch pocket', 'patchwork', 'patent log', + 'paternoster', 'patina', 'patio', 'patisserie', 'patka', 'patrol boat', + 'patty-pan', 'pave', 'pavilion', 'pavior', 'pavis', 'pawn', + 'pawnbroker\'s shop', 'pay-phone', 'pc board', 'peach orchard', + 'pea jacket', 'peavey', 'pectoral', 'pedal', 'pedal pusher', 'pedestal', + 'pedestal table', 'pedestrian crossing', 'pedicab', 'pediment', 'pedometer', + 'peeler', 'peep sight', 'peg', 'peg', 'peg', 'peg', 'pegboard', 'pelham', + 'pelican crossing', 'pelisse', 'pelvimeter', 'pen', 'penal colony', + 'penal institution', 'penalty box', 'pen-and-ink', 'pencil', 'pencil', + 'pencil box', 'pencil sharpener', 'pendant earring', 'pendulum', + 'pendulum clock', 'pendulum watch', 'penetration bomb', 'penile implant', + 'penitentiary', 'penknife', 'penlight', 'pennant', 'pennywhistle', + 'penthouse', 'pentode', 'peplos', 'peplum', 'pepper mill', 'pepper shaker', + 'pepper spray', 'percale', 'percolator', 'percussion cap', + 'percussion instrument', 'perforation', 'perfume', 'perfumery', 'perfumery', + 'perfumery', 'peripheral', 'periscope', 'peristyle', 'periwig', + 'permanent press', 'perpetual motion machine', 'personal computer', + 'personal digital assistant', 'personnel carrier', 'pestle', 'pestle', + 'petcock', 'petri dish', 'petrolatum gauze', 'pet shop', 'petticoat', 'pew', + 'phial', 'phillips screw', 'phillips screwdriver', 'phonograph needle', + 'phonograph record', 'photocathode', 'photocoagulator', 'photocopier', + 'photographic equipment', 'photographic paper', 'photometer', + 'photomicrograph', 'photostat', 'photostat', 'physical pendulum', 'piano', + 'piano action', 'piano keyboard', 'piano wire', 'piccolo', 'pick', 'pick', + 'pick', 'pickelhaube', 'picket boat', 'picket fence', 'picket ship', + 'pickle barrel', 'pickup', 'picture', 'picture frame', 'picture hat', + 'picture rail', 'picture window', 'piece of cloth', 'pied-a-terre', 'pier', + 'pier', 'pier arch', 'pier glass', 'pier table', 'pieta', 'piezometer', + 'pig bed', 'piggery', 'piggy bank', 'pilaster', 'pile', 'pile driver', + 'pill bottle', 'pillbox', 'pillion', 'pillory', 'pillow', 'pillow block', + 'pillow lace', 'pillow sham', 'pilot bit', 'pilot boat', 'pilot burner', + 'pilot cloth', 'pilot engine', 'pilothouse', 'pilot light', 'pin', 'pin', + 'pin', 'pinata', 'pinball machine', 'pince-nez', 'pincer', 'pinch bar', + 'pincurl clip', 'pinfold', 'ping-pong ball', 'pinhead', 'pinion', + 'pinnacle', 'pinprick', 'pinstripe', 'pinstripe', 'pinstripe', 'pintle', + 'pinwheel', 'pinwheel', 'tabor pipe', 'pipe', 'pipe bomb', 'pipe cleaner', + 'pipe cutter', 'pipefitting', 'pipet', 'pipe vise', 'pipe wrench', 'pique', + 'pirate', 'piste', 'pistol', 'pistol grip', 'piston', 'piston ring', + 'piston rod', 'pit', 'pitcher', 'pitchfork', 'pitching wedge', 'pitch pipe', + 'pith hat', 'piton', 'pitot-static tube', 'pitot tube', 'pitsaw', 'pivot', + 'pivoting window', 'pizzeria', 'place of business', 'place of worship', + 'placket', 'planchet', 'plane', 'plane', 'plane seat', 'planetarium', + 'planetarium', 'planetarium', 'planetary gear', 'plank-bed', 'planking', + 'planner', 'plant', 'planter', 'plaster', 'plasterboard', + 'plastering trowel', 'plastic bag', 'plastic bomb', 'plastic laminate', + 'plastic wrap', 'plastron', 'plastron', 'plastron', 'plate', 'plate', + 'plate', 'platen', 'platen', 'plate rack', 'plate rail', 'platform', + 'platform', 'platform', 'platform bed', 'platform rocker', 'plating', + 'platter', 'playback', 'playbox', 'playground', 'playpen', 'playsuit', + 'plaza', 'pleat', 'plenum', 'plethysmograph', 'pleximeter', 'plexor', + 'pliers', 'plimsoll', 'plotter', 'plow', 'plug', 'plug', 'plug fuse', + 'plughole', 'plumb bob', 'plumb level', 'plunger', 'plus fours', 'plush', + 'plywood', 'pneumatic drill', 'p-n junction', 'p-n-p transistor', 'poacher', + 'pocket', 'pocket battleship', 'pocketcomb', 'pocket flap', + 'pocket-handkerchief', 'pocketknife', 'pocket watch', 'pod', 'pogo stick', + 'point-and-shoot camera', 'pointed arch', 'pointing trowel', 'point lace', + 'poker', 'polarimeter', 'polaroid', 'polaroid camera', 'pole', 'pole', + 'poleax', 'poleax', 'police boat', 'police van', 'polling booth', + 'polo ball', 'polo mallet', 'polonaise', 'polo shirt', 'polyester', + 'polygraph', 'pomade', 'pommel horse', 'poncho', 'pongee', 'poniard', + 'pontifical', 'pontoon', 'pontoon bridge', 'pony cart', 'pool ball', + 'poolroom', 'pool table', 'poop deck', 'poor box', 'poorhouse', + 'pop bottle', 'popgun', 'poplin', 'popper', 'poppet', 'pop tent', + 'porcelain', 'porch', 'porkpie', 'porringer', 'portable', + 'portable computer', 'portable circular saw', 'portcullis', 'porte-cochere', + 'porte-cochere', 'portfolio', 'porthole', 'portico', 'portiere', + 'portmanteau', 'portrait camera', 'portrait lens', 'positive pole', + 'positive pole', 'positron emission tomography scanner', 'post', + 'postage meter', 'post and lintel', 'post chaise', 'postern', + 'post exchange', 'posthole digger', 'post horn', 'posthouse', 'pot', 'pot', + 'potbelly', 'potemkin village', 'potential divider', 'potentiometer', + 'potentiometer', 'potpourri', 'potsherd', 'potter\'s wheel', 'pottery', + 'pottle', 'potty seat', 'pouch', 'poultice', 'pound', 'pound net', 'powder', + 'powder and shot', 'powdered mustard', 'powder horn', 'powder keg', + 'power brake', 'power cord', 'power drill', 'power line', 'power loom', + 'power mower', 'power pack', 'power saw', 'power shovel', 'power steering', + 'power takeoff', 'power tool', 'praetorium', 'prayer rug', 'prayer shawl', + 'precipitator', 'prefab', 'presbytery', 'presence chamber', 'press', + 'press', 'press', 'press box', 'press gallery', 'press of sail', + 'pressure cabin', 'pressure cooker', 'pressure dome', 'pressure gauge', + 'pressurized water reactor', 'pressure suit', 'pricket', 'prie-dieu', + 'primary coil', 'primus stove', 'prince albert', 'print', 'print buffer', + 'printed circuit', 'printer', 'printer', 'printer cable', 'priory', + 'prison', 'prison camp', 'privateer', 'private line', 'privet hedge', + 'probe', 'proctoscope', 'prod', 'production line', 'projectile', + 'projector', 'projector', 'prolonge', 'prolonge knot', 'prompter', 'prong', + 'propeller', 'propeller plane', 'propjet', 'proportional counter tube', + 'propulsion system', 'proscenium', 'proscenium arch', 'prosthesis', + 'protective covering', 'protective garment', 'proton accelerator', + 'protractor', 'pruner', 'pruning knife', 'pruning saw', 'pruning shears', + 'psaltery', 'psychrometer', 'pt boat', 'public address system', + 'public house', 'public toilet', 'public transport', 'public works', 'puck', + 'pull', 'pullback', 'pull chain', 'pulley', 'pull-off', 'pullman', + 'pullover', 'pull-through', 'pulse counter', 'pulse generator', + 'pulse timing circuit', 'pump', 'pump', 'pump action', 'pump house', + 'pump room', 'pump-type pliers', 'pump well', 'punch', 'punchboard', + 'punch bowl', 'punching bag', 'punch pliers', 'punch press', 'punnet', + 'punt', 'pup tent', 'purdah', 'purifier', 'purl', 'purse', 'push-bike', + 'push broom', 'push button', 'push-button radio', 'pusher', 'put-put', + 'puttee', 'putter', 'putty knife', 'puzzle', 'pylon', 'pylon', + 'pyramidal tent', 'pyrograph', 'pyrometer', 'pyrometric cone', 'pyrostat', + 'pyx', 'pyx', 'pyxis', 'quad', 'quadrant', 'quadraphony', 'quartering', + 'quarterstaff', 'quartz battery', 'quartz lamp', 'queen', 'queen', + 'queen post', 'quern', 'quill', 'quilt', 'quilted bedspread', 'quilting', + 'quipu', 'quirk molding', 'quirt', 'quiver', 'quoin', 'quoit', + 'qwerty keyboard', 'rabbet', 'rabbet joint', 'rabbit ears', 'rabbit hutch', + 'raceabout', 'racer', 'raceway', 'racing boat', 'racing gig', + 'racing skiff', 'rack', 'rack', 'rack', 'rack and pinion', 'racket', + 'racquetball', 'radar', 'radial', 'radial engine', 'radiation pyrometer', + 'radiator', 'radiator', 'radiator cap', 'radiator hose', 'radio', + 'radio antenna', 'radio chassis', 'radio compass', 'radiogram', + 'radio interferometer', 'radio link', 'radiometer', 'radiomicrometer', + 'radio-phonograph', 'radio receiver', 'radiotelegraph', 'radiotelephone', + 'radio telescope', 'radiotherapy equipment', 'radio transmitter', 'radome', + 'raft', 'rafter', 'raft foundation', 'rag', 'ragbag', 'raglan', + 'raglan sleeve', 'rail', 'rail fence', 'railhead', 'railing', 'railing', + 'railroad bed', 'railroad tunnel', 'rain barrel', 'raincoat', 'rain gauge', + 'rain stick', 'rake', 'rake handle', 'ram disk', 'ramekin', 'ramjet', + 'rammer', 'ramp', 'rampant arch', 'rampart', 'ramrod', 'ramrod', 'ranch', + 'ranch house', 'random-access memory', 'rangefinder', 'range hood', + 'range pole', 'rapier', 'rariora', 'rasp', 'ratchet', 'ratchet wheel', + 'rathskeller', 'ratline', 'rat-tail file', 'rattan', 'rattrap', 'rayon', + 'razor', 'razorblade', 'reaction-propulsion engine', 'reaction turbine', + 'reactor', 'reading lamp', 'reading room', 'read-only memory', + 'read-only memory chip', 'readout', 'read/write head', 'ready-to-wear', + 'real storage', 'reamer', 'reamer', 'rearview mirror', + 'reaumur thermometer', 'rebozo', 'receiver', 'receptacle', 'reception desk', + 'reception room', 'recess', 'reciprocating engine', 'recliner', + 'reconnaissance plane', 'reconnaissance vehicle', 'record changer', + 'recorder', 'recording', 'recording system', 'record player', + 'record sleeve', 'recovery room', 'recreational vehicle', 'recreation room', + 'recycling bin', 'recycling plant', 'redbrick university', 'red carpet', + 'redoubt', 'redoubt', 'reduction gear', 'reed pipe', 'reed stop', + 'reef knot', 'reel', 'reel', 'refectory', 'refectory table', 'refinery', + 'reflecting telescope', 'reflectometer', 'reflector', 'reflex camera', + 'reflux condenser', 'reformatory', 'reformer', 'refracting telescope', + 'refractometer', 'refrigeration system', 'refrigerator', 'refrigerator car', + 'refuge', 'regalia', 'regimentals', 'regulator', 'rein', 'relay', 'release', + 'religious residence', 'reliquary', 'remote control', 'remote terminal', + 'removable disk', 'rendering', 'rep', 'repair shop', 'repeater', + 'repeating firearm', 'repository', 'reproducer', 'rerebrace', + 'rescue equipment', 'research center', 'reseau', 'reservoir', 'reset', + 'reset button', 'residence', 'resistance pyrometer', 'resistor', + 'resonator', 'resonator', 'resort hotel', 'respirator', 'restaurant', + 'rest house', 'restraint', 'resuscitator', 'retainer', 'retaining wall', + 'reticle', 'reticulation', 'reticule', 'retort', 'retractor', 'return key', + 'reverberatory furnace', 'revers', 'reverse', 'reversible', 'revetment', + 'revetment', 'revolver', 'revolving door', 'rheometer', 'rheostat', + 'rhinoscope', 'rib', 'riband', 'ribbed vault', 'ribbing', + 'ribbon development', 'rib joint pliers', 'ricer', 'riddle', 'ride', + 'ridge', 'ridge rope', 'riding boot', 'riding crop', 'riding mower', + 'rifle', 'rifle ball', 'rifle grenade', 'rig', 'rigger', 'rigger', + 'rigging', 'rigout', 'ringlet', 'rings', 'rink', 'riot gun', 'ripcord', + 'ripcord', 'ripping bar', 'ripping chisel', 'ripsaw', 'riser', 'riser', + 'ritz', 'river boat', 'rivet', 'riveting machine', 'roach clip', 'road', + 'roadbed', 'roadblock', 'roadhouse', 'roadster', 'roadway', 'roaster', + 'robe', 'robotics equipment', 'rochon prism', 'rock bit', 'rocker', + 'rocker', 'rocker arm', 'rocket', 'rocket', 'rocking chair', 'rod', 'rodeo', + 'roll', 'roller', 'roller', 'roller bandage', 'in-line skate', + 'rollerblade', 'roller blind', 'roller coaster', 'roller skate', + 'roller towel', 'roll film', 'rolling hitch', 'rolling mill', 'rolling pin', + 'rolling stock', 'roll-on', 'roll-on', 'roll-on roll-off', 'rolodex', + 'roman arch', 'roman building', 'romper', 'rood screen', 'roof', 'roof', + 'roofing', 'room', 'roomette', 'room light', 'roost', 'rope', 'rope bridge', + 'rope tow', 'rose water', 'rose window', 'rosin bag', 'rotary actuator', + 'rotary engine', 'rotary press', 'rotating mechanism', 'rotating shaft', + 'rotisserie', 'rotisserie', 'rotor', 'rotor', 'rotor', 'rotor blade', + 'rotor head', 'rotunda', 'rotunda', 'rouge', 'roughcast', 'rouleau', + 'roulette', 'roulette ball', 'roulette wheel', 'round', 'round arch', + 'round-bottom flask', 'roundel', 'round file', 'roundhouse', 'router', + 'router', 'router plane', 'rowel', 'row house', 'rowing boat', + 'rowlock arch', 'royal', 'royal mast', 'rubber band', 'rubber boot', + 'rubber bullet', 'rubber eraser', 'rudder', 'rudder', 'rudder blade', 'rug', + 'rugby ball', 'ruin', 'rule', 'rumble', 'rumble seat', 'rummer', + 'rumpus room', 'runcible spoon', 'rundle', 'running shoe', 'running suit', + 'runway', 'rushlight', 'russet', 'rya', 'saber', 'saber saw', 'sable', + 'sable', 'sable coat', 'sabot', 'sachet', 'sack', 'sack', 'sackbut', + 'sackcloth', 'sackcloth', 'sack coat', 'sacking', 'saddle', 'saddlebag', + 'saddle blanket', 'saddle oxford', 'saddlery', 'saddle seat', + 'saddle stitch', 'safe', 'safe', 'safe-deposit', 'safe house', + 'safety arch', 'safety belt', 'safety bicycle', 'safety bolt', + 'safety curtain', 'safety fuse', 'safety lamp', 'safety match', + 'safety net', 'safety pin', 'safety rail', 'safety razor', 'safety valve', + 'sail', 'sail', 'sailboat', 'sailcloth', 'sailing vessel', + 'sailing warship', 'sailor cap', 'sailor suit', 'salad bar', 'salad bowl', + 'salinometer', 'sallet', 'salon', 'salon', 'salon', 'saltbox', 'saltcellar', + 'saltshaker', 'saltworks', 'salver', 'salwar', 'sam browne belt', 'samisen', + 'samite', 'samovar', 'sampan', 'sandal', 'sandbag', 'sandblaster', + 'sandbox', 'sandglass', 'sand wedge', 'sandwich board', 'sanitary napkin', + 'cling film', 'sarcenet', 'sarcophagus', 'sari', 'sarong', 'sash', + 'sash fastener', 'sash window', 'satchel', 'sateen', 'satellite', + 'satellite receiver', 'satellite television', 'satellite transmitter', + 'satin', 'saturday night special', 'saucepan', 'saucepot', 'sauna', + 'savings bank', 'saw', 'sawed-off shotgun', 'sawhorse', 'sawmill', + 'saw set', 'sax', 'saxhorn', 'scabbard', 'scaffolding', 'scale', 'scale', + 'scaler', 'scaling ladder', 'scalpel', 'scanner', 'scanner', 'scanner', + 'scantling', 'scarf', 'scarf joint', 'scatter rug', 'scauper', + 'schmidt telescope', 'school', 'schoolbag', 'school bell', 'school bus', + 'school ship', 'school system', 'schooner', 'schooner', + 'scientific instrument', 'scimitar', 'scintillation counter', 'scissors', + 'sclerometer', 'scoinson arch', 'sconce', 'sconce', 'scoop', 'scooter', + 'scoreboard', 'scouring pad', 'scow', 'scow', 'scraper', 'scratcher', + 'screen', 'screen', 'screen', 'screen', 'screen door', 'screening', 'screw', + 'screw', 'screw', 'screwdriver', 'screw eye', 'screw key', 'screw thread', + 'screwtop', 'screw wrench', 'scriber', 'scrim', 'scrimshaw', 'scriptorium', + 'scrubber', 'scrub brush', 'scrub plane', 'scuffer', 'scuffle', 'scull', + 'scull', 'scullery', 'sculpture', 'scuttle', 'scyphus', 'scythe', 'seabag', + 'sea boat', 'sea chest', 'sealing wax', 'sealskin', 'seam', 'seaplane', + 'searchlight', 'searing iron', 'seat', 'seat', 'seat', 'seat belt', + 'secateurs', 'secondary coil', 'second balcony', 'second base', + 'second hand', 'secretary', 'sectional', 'security blanket', + 'security system', 'security system', 'sedan', 'sedan', 'seeder', 'seeker', + 'seersucker', 'segmental arch', 'segway', 'seidel', 'seine', 'seismograph', + 'selector', 'selenium cell', 'self-propelled vehicle', + 'self-registering thermometer', 'self-starter', 'selsyn', 'selvage', + 'semaphore', 'semiautomatic firearm', 'semiautomatic pistol', + 'semiconductor device', 'semi-detached house', 'semigloss', 'semitrailer', + 'sennit', 'sensitometer', 'sentry box', 'separate', 'septic tank', + 'sequence', 'sequencer', 'serape', 'serge', 'serger', 'serial port', + 'serpent', 'serration', 'server', 'server', 'service club', 'serving cart', + 'serving dish', 'servo', 'set', 'set gun', 'setscrew', 'setscrew', + 'set square', 'settee', 'settle', 'settlement house', 'seventy-eight', + 'seven wonders of the ancient world', 'sewage disposal plant', 'sewer', + 'sewing basket', 'sewing kit', 'sewing machine', 'sewing needle', + 'sewing room', 'sextant', 'sgraffito', 'shackle', 'shackle', 'shade', + 'shadow box', 'shaft', 'shag rug', 'shaker', 'shank', 'shank', 'shantung', + 'shaper', 'shaping tool', 'sharkskin', 'sharpener', 'sharpie', 'shaver', + 'shaving brush', 'shaving cream', 'shaving foam', 'shawl', 'shawm', + 'shears', 'sheath', 'sheathing', 'shed', 'sheep bell', 'sheepshank', + 'sheepskin coat', 'sheepwalk', 'sheet', 'sheet bend', 'sheeting', + 'sheet pile', 'sheetrock', 'shelf', 'shelf bracket', 'shell', 'shell', + 'shell', 'shellac', 'shelter', 'shelter', 'shelter', 'sheltered workshop', + 'sheraton', 'shield', 'shield', 'shielding', 'shift key', 'shillelagh', + 'shim', 'shingle', 'shin guard', 'ship', 'shipboard system', 'shipping', + 'shipping room', 'ship-towed long-range acoustic detection system', + 'shipwreck', 'shirt', 'shirt button', 'shirtdress', 'shirtfront', + 'shirting', 'shirtsleeve', 'shirttail', 'shirtwaist', 'shiv', + 'shock absorber', 'shoe', 'shoe', 'shoebox', 'shoehorn', 'shoe shop', + 'shoetree', 'shofar', 'shoji', 'shooting brake', 'shooting lodge', + 'shooting stick', 'shop', 'shop bell', 'shopping bag', 'shopping basket', + 'shopping cart', 'short circuit', 'short iron', 'short pants', + 'short sleeve', 'shortwave diathermy machine', 'shot', 'shot glass', + 'shotgun', 'shotgun shell', 'shot tower', 'shoulder', 'shoulder bag', + 'shouldered arch', 'shoulder holster', 'shoulder pad', 'shoulder patch', + 'shovel', 'shovel', 'shovel hat', 'showboat', 'shower', 'shower cap', + 'shower curtain', 'shower room', 'shower stall', 'showroom', 'shrapnel', + 'shredder', 'shrimper', 'shrine', 'shrink-wrap', 'shunt', 'shunt', + 'shunter', 'shutter', 'shutter', 'shuttle', 'shuttle', 'shuttle bus', + 'shuttlecock', 'shuttle helicopter', 'sibley tent', 'sickbay', 'sickbed', + 'sickle', 'sickroom', 'sideboard', 'sidecar', 'side chapel', 'sidelight', + 'sidesaddle', 'sidewalk', 'sidewall', 'side-wheeler', 'sidewinder', 'sieve', + 'sifter', 'sights', 'sigmoidoscope', 'signal box', 'signaling device', + 'signboard', 'silencer', 'silent butler', 'silex', 'silk', 'silks', 'silo', + 'silver plate', 'silverpoint', 'simple pendulum', 'simulator', 'single bed', + 'single-breasted jacket', 'single-breasted suit', 'single prop', + 'single-reed instrument', 'single-rotor helicopter', 'singlestick', + 'singlet', 'siren', 'sister ship', 'sitar', 'sitz bath', 'six-pack', + 'skate', 'skateboard', 'skeg', 'skein', 'skeleton', 'skeleton key', 'skep', + 'skep', 'sketch', 'sketcher', 'skew arch', 'skewer', 'ski', 'ski binding', + 'skibob', 'ski boot', 'ski cap', 'skidder', 'skid lid', 'skiff', 'ski jump', + 'ski lodge', 'ski mask', 'skimmer', 'ski parka', 'ski-plane', 'ski pole', + 'ski rack', 'skirt', 'skirt', 'ski tow', 'skivvies', 'skullcap', 'skybox', + 'skyhook', 'skylight', 'skysail', 'skyscraper', 'skywalk', 'slacks', + 'slack suit', 'slasher', 'slash pocket', 'slat', 'slate', 'slate pencil', + 'slate roof', 'sled', 'sleeper', 'sleeper', 'sleeping bag', 'sleeping car', + 'sleeve', 'sleeve', 'sleigh bed', 'sleigh bell', 'slice bar', 'slicer', + 'slicer', 'slide', 'slide fastener', 'slide projector', 'slide rule', + 'slide valve', 'sliding door', 'sliding seat', 'sliding window', 'sling', + 'sling', 'slingback', 'slinger ring', 'slip clutch', 'slipcover', + 'slip-joint pliers', 'slipknot', 'slip-on', 'slipper', 'slip ring', + 'slit lamp', 'slit trench', 'sloop', 'sloop of war', 'slop basin', + 'slop pail', 'slops', 'slopshop', 'slot', 'slot machine', 'sluice', 'smack', + 'small boat', 'small computer system interface', 'small ship', + 'small stores', 'smart bomb', 'smelling bottle', 'smocking', 'smoke bomb', + 'smokehouse', 'smoker', 'smoke screen', 'smoking room', 'smoothbore', + 'smooth plane', 'snack bar', 'snaffle', 'snap', 'snap brim', + 'snap-brim hat', 'snare', 'snare drum', 'snatch block', 'snifter', + 'sniper rifle', 'snips', 'sno-cat', 'snood', 'snorkel', 'snorkel', + 'snowbank', 'snowboard', 'snowmobile', 'snowplow', 'snowshoe', 'snowsuit', + 'snow thrower', 'snuffbox', 'snuffer', 'snuffers', 'soapbox', 'soap dish', + 'soap dispenser', 'soap pad', 'soccer ball', 'sock', 'socket', + 'socket wrench', 'socle', 'soda can', 'soda fountain', 'soda fountain', + 'sod house', 'sodium-vapor lamp', 'sofa', 'soffit', 'softball', + 'soft pedal', 'soil pipe', 'solar array', 'solar cell', 'solar dish', + 'solar heater', 'solar house', 'solar telescope', 'solar thermal system', + 'soldering iron', 'solenoid', 'solleret', 'sombrero', 'sonic depth finder', + 'sonogram', 'sonograph', 'sorter', 'souk', 'sound bow', 'soundbox', + 'sound camera', 'sounder', 'sound film', 'sounding board', + 'sounding rocket', 'sound recording', 'sound spectrograph', 'soup bowl', + 'soup ladle', 'soupspoon', 'source of illumination', 'sourdine', 'soutache', + 'soutane', 'sou\'wester', 'soybean future', 'space bar', 'space capsule', + 'spacecraft', 'space heater', 'space helmet', 'space rocket', + 'space shuttle', 'space station', 'spacesuit', 'spade', 'spade bit', + 'spaghetti junction', 'spandau', 'spandex', 'spandrel', 'spanker', 'spar', + 'sparge pipe', 'spark arrester', 'spark arrester', 'spark chamber', + 'spark coil', 'spark gap', 'spark lever', 'spark plug', 'sparkplug wrench', + 'spark transmitter', 'spat', 'spatula', 'spatula', 'speakerphone', + 'speaking trumpet', 'spear', 'spear', 'specialty store', 'specimen bottle', + 'spectacle', 'spectacles', 'spectator pump', 'spectrograph', + 'spectrophotometer', 'spectroscope', 'speculum', 'speedboat', 'speed bump', + 'speedometer', 'speed skate', 'spherometer', 'sphygmomanometer', + 'spicemill', 'spice rack', 'spider', 'spider web', 'spike', 'spike', + 'spindle', 'spindle', 'spindle', 'spin dryer', 'spinet', 'spinet', + 'spinnaker', 'spinner', 'spinning frame', 'spinning jenny', + 'spinning machine', 'spinning rod', 'spinning wheel', 'spiral bandage', + 'spiral ratchet screwdriver', 'spiral spring', 'spirit lamp', + 'spirit stove', 'spirometer', 'spit', 'spittoon', 'splashboard', 'splasher', + 'splice', 'splicer', 'splint', 'split rail', 'spode', 'spoiler', 'spoiler', + 'spoke', 'spokeshave', 'sponge cloth', 'sponge mop', 'spoon', 'spoon', + 'spork', 'sporran', 'sport kite', 'sports car', 'sports equipment', + 'sports implement', 'sportswear', 'sport utility', 'spot', 'spotlight', + 'spot weld', 'spouter', 'sprag', 'spray gun', 'spray paint', 'spreader', + 'sprig', 'spring', 'spring balance', 'springboard', 'sprinkler', + 'sprinkler system', 'sprit', 'spritsail', 'sprocket', 'sprocket', + 'spun yarn', 'spur', 'spur gear', 'sputnik', 'spy satellite', 'squad room', + 'square', 'square knot', 'square-rigger', 'square sail', 'squash ball', + 'squash racket', 'squawk box', 'squeegee', 'squeezer', 'squelch circuit', + 'squinch', 'stabilizer', 'stabilizer', 'stabilizer bar', 'stable', + 'stable gear', 'stabling', 'stacks', 'staddle', 'stadium', 'stage', + 'stagecoach', 'stained-glass window', 'stair-carpet', 'stair-rod', + 'stairwell', 'stake', 'stall', 'stall', 'stamp', 'stamp mill', + 'stamping machine', 'stanchion', 'stand', 'standard', 'standard cell', + 'standard transmission', 'standing press', 'stanhope', 'stanley steamer', + 'staple', 'staple', 'staple gun', 'stapler', 'starship', 'starter', + 'starting gate', 'stassano furnace', 'statehouse', 'stately home', + 'state prison', 'stateroom', 'static tube', 'station', 'stator', 'statue', + 'stay', 'staysail', 'steakhouse', 'steak knife', 'stealth aircraft', + 'stealth bomber', 'stealth fighter', 'steam bath', 'steamboat', + 'steam chest', 'steam engine', 'steamer', 'steamer', 'steam iron', + 'steam locomotive', 'steamroller', 'steam shovel', 'steam turbine', + 'steam whistle', 'steel', 'steel arch bridge', 'steel drum', 'steel mill', + 'steel-wool pad', 'steelyard', 'steeple', 'steerage', 'steering gear', + 'steering linkage', 'steering system', 'steering wheel', 'stele', + 'stem-winder', 'stencil', 'sten gun', 'stenograph', 'step', + 'step-down transformer', 'step stool', 'step-up transformer', 'stereo', + 'stereoscope', 'stern chaser', 'sternpost', 'sternwheeler', 'stethoscope', + 'stewing pan', 'stick', 'stick', 'stick', 'stick', 'stile', 'stiletto', + 'still', 'stillroom', 'stillson wrench', 'stilt', 'stinger', 'stink bomb', + 'stirrer', 'stirrup', 'stirrup pump', 'stob', 'stock', 'stockade', + 'stockcar', 'stock car', 'stockinet', 'stocking', 'stock-in-trade', + 'stockpot', 'stockroom', 'stocks', 'stock saddle', 'stockyard', 'stole', + 'stomacher', 'stomach pump', 'stone wall', 'stoneware', 'stonework', + 'stool', 'stoop', 'stop bath', 'stopcock', 'stopper knot', 'stopwatch', + 'storage battery', 'storage cell', 'storage ring', 'storage space', + 'storeroom', 'storm cellar', 'storm door', 'storm window', 'stoup', 'stoup', + 'stove', 'stove', 'stove bolt', 'stovepipe', 'stovepipe iron', + 'stradavarius', 'straight chair', 'straightedge', 'straightener', + 'straight flute', 'straight pin', 'straight razor', 'strainer', + 'straitjacket', 'strap', 'strap', 'strap hinge', 'strapless', + 'streamer fly', 'streamliner', 'street', 'street', 'streetcar', + 'street clothes', 'streetlight', 'stretcher', 'stretcher', 'stretch pants', + 'strickle', 'strickle', 'stringed instrument', 'stringer', 'stringer', + 'string tie', 'strip', 'strip lighting', 'strip mall', 'stroboscope', + 'strongbox', 'stronghold', 'strongroom', 'strop', 'structural member', + 'structure', 'student center', 'student lamp', 'student union', + 'stud finder', 'studio apartment', 'studio couch', 'study', 'study hall', + 'stuffing nut', 'stump', 'stun gun', 'stupa', 'sty', 'stylus', 'stylus', + 'sub-assembly', 'subcompact', 'submachine gun', 'submarine', + 'submarine torpedo', 'submersible', 'submersible', 'subtracter', + 'subway token', 'subway train', 'subwoofer', 'suction cup', 'suction pump', + 'sudatorium', 'suede cloth', 'sugar bowl', 'sugar refinery', 'sugar spoon', + 'suit', 'suite', 'suiting', 'sulky', 'summer house', 'sumo ring', 'sump', + 'sump pump', 'sunbonnet', 'sunday best', 'sun deck', 'sundial', 'sundress', + 'sundries', 'sun gear', 'sunglass', 'sunglasses', 'sunhat', 'sunlamp', + 'sun parlor', 'sunroof', 'sunscreen', 'sunsuit', 'supercharger', + 'supercomputer', 'superconducting supercollider', 'superhighway', + 'supermarket', 'superstructure', 'supertanker', 'supper club', 'supplejack', + 'supply chamber', 'supply closet', 'support', 'support', 'support column', + 'support hose', 'supporting structure', 'supporting tower', 'surcoat', + 'surface gauge', 'surface lift', 'surface search radar', 'surface ship', + 'surface-to-air missile', 'surface-to-air missile system', 'surfboat', + 'surcoat', 'surgeon\'s knot', 'surgery', 'surge suppressor', + 'surgical dressing', 'surgical instrument', 'surgical knife', 'surplice', + 'surrey', 'surtout', 'surveillance system', 'surveying instrument', + 'surveyor\'s level', 'sushi bar', 'suspension', 'suspension bridge', + 'suspensory', 'sustaining pedal', 'suture', 'swab', 'swab', + 'swaddling clothes', 'swag', 'swage block', 'swagger stick', + 'swallow-tailed coat', 'swamp buggy', 'swan\'s down', 'swathe', 'swatter', + 'sweat bag', 'sweatband', 'sweater', 'sweat pants', 'sweatshirt', + 'sweatshop', 'sweat suit', 'sweep', 'sweep hand', 'swimming trunks', + 'swimsuit', 'swing', 'swing door', 'switch', 'switchblade', 'switch engine', + 'swivel', 'swivel chair', 'swizzle stick', 'sword', 'sword cane', + 's wrench', 'synagogue', 'synchrocyclotron', 'synchroflash', 'synchromesh', + 'synchronous converter', 'synchronous motor', 'synchrotron', 'synchroscope', + 'synthesizer', 'syringe', 'system', 'tabard', 'tabernacle', 'tabi', + 'tab key', 'table', 'table', 'tablefork', 'table knife', 'table lamp', + 'table saw', 'tablespoon', 'tablet-armed chair', 'table-tennis table', + 'table-tennis racquet', 'tabletop', 'tableware', 'tabor', 'taboret', + 'tachistoscope', 'tachograph', 'tachometer', 'tachymeter', 'tack', + 'tack hammer', 'taffeta', 'taffrail', 'tailgate', 'taillight', + 'tailor-made', 'tailor\'s chalk', 'tailpipe', 'tail rotor', 'tailstock', + 'take-up', 'talaria', 'talcum', 'tam', 'tambour', 'tambour', 'tambourine', + 'tammy', 'tamp', 'tampax', 'tampion', 'tampon', 'tandoor', 'tangram', + 'tank', 'tank', 'tankard', 'tank car', 'tank destroyer', 'tank engine', + 'tanker plane', 'tank shell', 'tank top', 'tannoy', 'tap', 'tapa', 'tape', + 'tape', 'tape deck', 'tape drive', 'tape player', 'tape recorder', + 'taper file', 'tapestry', 'tappet', 'tap wrench', 'tare', 'target', + 'target acquisition system', 'tarmacadam', 'tarpaulin', 'tartan', 'tasset', + 'tattoo', 'tavern', 'tawse', 'taximeter', 't-bar lift', 'tea bag', + 'tea ball', 'tea cart', 'tea chest', 'teaching aid', 'teacup', 'tea gown', + 'teakettle', 'tea maker', 'teapot', 'teashop', 'teaspoon', 'tea-strainer', + 'tea table', 'tea tray', 'tea urn', 'tee', 'tee hinge', 'telecom hotel', + 'telecommunication system', 'telegraph', 'telegraph key', 'telemeter', + 'telephone', 'telephone bell', 'telephone booth', 'telephone cord', + 'telephone jack', 'telephone line', 'telephone plug', 'telephone pole', + 'telephone receiver', 'telephone system', 'telephone wire', + 'telephoto lens', 'teleprompter', 'telescope', 'telescopic sight', + 'telethermometer', 'teletypewriter', 'television', 'television antenna', + 'television camera', 'television equipment', 'television monitor', + 'television receiver', 'television room', 'television transmitter', + 'telpher', 'telpherage', 'tempera', 'temple', 'temple', 'temporary hookup', + 'tender', 'tender', 'tender', 'tenement', 'tennis ball', 'tennis camp', + 'tennis racket', 'tenon', 'tenor drum', 'tenoroon', 'tenpenny nail', + 'tenpin', 'tensimeter', 'tensiometer', 'tensiometer', 'tensiometer', 'tent', + 'tenter', 'tenterhook', 'tent-fly', 'tent peg', 'tepee', 'terminal', + 'terminal', 'terraced house', 'terra cotta', 'terrarium', 'terra sigillata', + 'terry', 'tesla coil', 'tessera', 'test equipment', 'test rocket', + 'test room', 'testudo', 'tetraskelion', 'tetrode', 'textile machine', + 'textile mill', 'thatch', 'theater', 'theater curtain', 'theater light', + 'theodolite', 'theremin', 'thermal printer', 'thermal reactor', + 'thermocouple', 'thermoelectric thermometer', 'thermograph', 'thermograph', + 'thermohydrometer', 'thermojunction', 'thermometer', + 'thermonuclear reactor', 'thermopile', 'thermos', 'thermostat', 'thigh pad', + 'thill', 'thimble', 'thinning shears', 'third base', 'third gear', + 'third rail', 'thong', 'thong', 'three-centered arch', 'three-decker', + 'three-dimensional radar', 'three-piece suit', 'three-quarter binding', + 'three-way switch', 'thresher', 'threshing floor', 'thriftshop', + 'throat protector', 'throne', 'thrust bearing', 'thruster', 'thumb', + 'thumbhole', 'thumbscrew', 'thumbstall', 'thumbtack', 'thunderer', 'thwart', + 'tiara', 'ticking', 'tickler coil', 'tie', 'tie', 'tie rack', 'tie rod', + 'tights', 'tile', 'tile cutter', 'tile roof', 'tiller', 'tilter', + 'tilt-top table', 'timber', 'timber', 'timber hitch', 'timbrel', + 'time bomb', 'time capsule', 'time clock', + 'time-delay measuring instrument', 'time-fuse', 'timepiece', 'timer', + 'timer', 'time-switch', 'tin', 'tinderbox', 'tine', 'tinfoil', 'tippet', + 'tire chain', 'tire iron', 'titfer', 'tithe barn', 'titrator', 'toaster', + 'toaster oven', 'toasting fork', 'toastrack', 'tobacco pouch', + 'tobacco shop', 'toboggan', 'toby', 'tocsin', 'toe', 'toecap', 'toehold', + 'toga', 'toga virilis', 'toggle', 'toggle bolt', 'toggle joint', + 'toggle switch', 'togs', 'toilet', 'toilet bag', 'toilet bowl', + 'toilet kit', 'toilet powder', 'toiletry', 'toilet seat', 'toilet water', + 'tokamak', 'token', 'tollbooth', 'toll bridge', 'tollgate', 'toll line', + 'tomahawk', 'tommy gun', 'tomograph', 'tone arm', 'toner', 'tongs', + 'tongue', 'tongue and groove joint', 'tongue depressor', 'tonometer', + 'tool', 'tool bag', 'toolbox', 'toolshed', 'tooth', 'tooth', 'toothbrush', + 'toothpick', 'top', 'top', 'topgallant', 'topgallant', 'topiary', 'topknot', + 'topmast', 'topper', 'topsail', 'toque', 'torch', 'torpedo', 'torpedo', + 'torpedo', 'torpedo boat', 'torpedo-boat destroyer', 'torpedo tube', + 'torque converter', 'torque wrench', 'torture chamber', 'totem pole', + 'touch screen', 'toupee', 'touring car', 'tourist class', 'towel', + 'toweling', 'towel rack', 'towel rail', 'tower', 'town hall', 'towpath', + 'tow truck', 'toy', 'toy box', 'toyshop', 'trace detector', 'track', + 'track', 'trackball', 'tracked vehicle', 'tract house', 'tract housing', + 'traction engine', 'tractor', 'tractor', 'trail bike', 'trailer', 'trailer', + 'trailer camp', 'trailer truck', 'trailing edge', 'train', 'tramline', + 'trammel', 'trampoline', 'tramp steamer', 'tramway', 'transdermal patch', + 'transept', 'transformer', 'transistor', 'transit instrument', + 'transmission', 'transmission shaft', 'transmitter', 'transom', 'transom', + 'transponder', 'transporter', 'transporter', 'transport ship', 'trap', + 'trap door', 'trapeze', 'trave', 'travel iron', 'trawl', 'trawl', 'trawler', + 'tray', 'tray cloth', 'tread', 'tread', 'treadmill', 'treadmill', + 'treasure chest', 'treasure ship', 'treenail', 'trefoil arch', 'trellis', + 'trench', 'trench coat', 'trench knife', 'trepan', 'trepan', 'trestle', + 'trestle', 'trestle bridge', 'trestle table', 'trestlework', 'trews', + 'trial balloon', 'triangle', 'triangle', 'triclinium', 'triclinium', + 'tricorn', 'tricot', 'tricycle', 'trident', 'trigger', 'trimaran', + 'trimmer', 'trimmer arch', 'triode', 'tripod', 'triptych', 'trip wire', + 'trireme', 'triskelion', 'triumphal arch', 'trivet', 'trivet', 'troika', + 'troll', 'trolleybus', 'trombone', 'troop carrier', 'troopship', + 'trophy case', 'trough', 'trouser', 'trouser cuff', 'trouser press', + 'trouser', 'trousseau', 'trowel', 'truck', 'trumpet arch', 'truncheon', + 'trundle bed', 'trunk', 'trunk hose', 'trunk lid', 'trunk line', 'truss', + 'truss bridge', 'try square', 't-square', 'tub', 'tube', 'tuck box', + 'tucker', 'tucker-bag', 'tuck shop', 'tudor arch', 'tudung', 'tugboat', + 'tulle', 'tumble-dryer', 'tumbler', 'tumbrel', 'tun', 'tunic', + 'tuning fork', 'tupik', 'turban', 'turbine', 'turbogenerator', 'tureen', + 'turkish bath', 'turkish towel', 'turk\'s head', 'turnbuckle', 'turner', + 'turnery', 'turnpike', 'turnspit', 'turnstile', 'turntable', 'turntable', + 'turret', 'turret clock', 'turtleneck', 'tweed', 'tweeter', 'twenty-two', + 'twenty-two pistol', 'twenty-two rifle', 'twill', 'twill', 'twin bed', + 'twinjet', 'twist bit', 'two-by-four', 'two-man tent', 'two-piece', + 'typesetting machine', 'typewriter', 'typewriter carriage', + 'typewriter keyboard', 'tyrolean', 'uke', 'ulster', 'ultracentrifuge', + 'ultramicroscope', 'ultrasuede', 'ultraviolet lamp', 'umbrella', + 'umbrella tent', 'undercarriage', 'undercoat', 'undergarment', 'underpants', + 'underwear', 'undies', 'uneven parallel bars', 'unicycle', 'uniform', + 'universal joint', 'university', 'upholstery', 'upholstery material', + 'upholstery needle', 'uplift', 'upper berth', 'upright', 'upset', + 'upstairs', 'urceole', 'urn', 'urn', 'used-car', 'utensil', 'uzi', + 'vacation home', 'vacuum', 'vacuum chamber', 'vacuum flask', 'vacuum gauge', + 'valenciennes', 'valise', 'valve', 'valve', 'valve-in-head engine', + 'vambrace', 'van', 'van', 'vane', 'vaporizer', 'variable-pitch propeller', + 'variometer', 'varnish', 'vase', 'vault', 'vault', 'vaulting horse', + 'vehicle', 'velcro', 'velocipede', 'velour', 'velvet', 'velveteen', + 'vending machine', 'veneer', 'venetian blind', 'venn diagram', + 'ventilation', 'ventilation shaft', 'ventilator', 'veranda', 'verdigris', + 'vernier caliper', 'vernier scale', 'vertical file', 'vertical stabilizer', + 'vertical tail', 'very pistol', 'vessel', 'vessel', 'vest', 'vestiture', + 'vestment', 'vest pocket', 'vestry', 'viaduct', 'vibraphone', 'vibrator', + 'vibrator', 'victrola', 'vicuna', 'videocassette', 'videocassette recorder', + 'videodisk', 'video recording', 'videotape', 'videotape', 'vigil light', + 'villa', 'villa', 'villa', 'viol', 'viola', 'viola da braccio', + 'viola da gamba', 'viola d\'amore', 'violin', 'virginal', 'viscometer', + 'viscose rayon', 'vise', 'visor', 'visual display unit', 'vivarium', + 'viyella', 'voile', 'volleyball', 'volleyball net', 'voltage regulator', + 'voltaic cell', 'voltaic pile', 'voltmeter', 'vomitory', + 'von neumann machine', 'voting booth', 'voting machine', 'voussoir', + 'vox angelica', 'vox humana', 'waders', 'wading pool', 'waffle iron', + 'wagon', 'wagon', 'wagon tire', 'wagon wheel', 'wain', 'wainscot', + 'wainscoting', 'waist pack', 'walker', 'walker', 'walker', 'walkie-talkie', + 'walk-in', 'walking shoe', 'walking stick', 'walkman', 'walk-up apartment', + 'wall', 'wall', 'wall clock', 'wallet', 'wall tent', 'wall unit', 'wand', + 'wankel engine', 'ward', 'wardrobe', 'wardroom', 'warehouse', 'warming pan', + 'war paint', 'warplane', 'war room', 'warship', 'wash', 'wash-and-wear', + 'washbasin', 'washboard', 'washboard', 'washer', 'washer', 'washhouse', + 'washroom', 'washstand', 'washtub', 'wastepaper basket', 'watch', + 'watch cap', 'watch case', 'watch glass', 'watchtower', 'water-base paint', + 'water bed', 'water bottle', 'water butt', 'water cart', 'water chute', + 'water closet', 'watercolor', 'water-cooled reactor', 'water cooler', + 'water faucet', 'water filter', 'water gauge', 'water glass', + 'water hazard', 'water heater', 'watering can', 'watering cart', + 'water jacket', 'water jug', 'water jump', 'water level', 'water meter', + 'water mill', 'waterproof', 'waterproofing', 'water pump', 'water scooter', + 'water ski', 'waterspout', 'water tower', 'water wagon', 'waterwheel', + 'waterwheel', 'water wings', 'waterworks', 'wattmeter', 'waxwork', 'ways', + 'weapon', 'weaponry', 'weapons carrier', 'weathercock', 'weatherglass', + 'weather satellite', 'weather ship', 'weathervane', 'web', 'web', 'webbing', + 'webcam', 'wedge', 'wedge', 'wedgie', 'wedgwood', 'weeder', 'weeds', + 'weekender', 'weighbridge', 'weight', 'weir', 'weir', 'welcome wagon', + 'weld', 'welder\'s mask', 'weldment', 'well', 'wellhead', 'welt', + 'weston cell', 'wet bar', 'wet-bulb thermometer', 'wet cell', 'wet fly', + 'wet suit', 'whaleboat', 'whaler', 'whaling gun', 'wheel', 'wheel', + 'wheel and axle', 'wheelchair', 'wheeled vehicle', 'wheelwork', 'wherry', + 'wherry', 'whetstone', 'whiffletree', 'whip', 'whipcord', 'whipping post', + 'whipstitch', 'whirler', 'whisk', 'whisk', 'whiskey bottle', 'whiskey jug', + 'whispering gallery', 'whistle', 'whistle', 'white', 'white goods', + 'whitewash', 'whorehouse', 'wick', 'wicker', 'wicker basket', 'wicket', + 'wicket', 'wickiup', 'wide-angle lens', 'widebody aircraft', 'wide wale', + 'widow\'s walk', 'wiffle', 'wig', 'wigwam', 'wilton', 'wimple', 'wincey', + 'winceyette', 'winch', 'winchester', 'windbreak', 'winder', + 'wind instrument', 'windjammer', 'windmill', 'windmill', 'window', 'window', + 'window blind', 'window box', 'window envelope', 'window frame', + 'window screen', 'window seat', 'window shade', 'windowsill', 'windshield', + 'windshield wiper', 'windsor chair', 'windsor knot', 'windsor tie', + 'wind tee', 'wind tunnel', 'wind turbine', 'wine bar', 'wine bottle', + 'wine bucket', 'wine cask', 'wineglass', 'winepress', 'winery', 'wineskin', + 'wing', 'wing chair', 'wing nut', 'wing tip', 'wing tip', 'winker', 'wiper', + 'wiper motor', 'wire', 'wire', 'wire cloth', 'wire cutter', 'wire gauge', + 'wireless local area network', 'wire matrix printer', 'wire recorder', + 'wire stripper', 'wirework', 'wiring', 'wishing cap', 'witness box', 'wok', + 'woman\'s clothing', 'wood', 'woodcarving', 'wood chisel', 'woodenware', + 'wooden spoon', 'woodscrew', 'woodshed', 'wood vise', 'woodwind', 'woof', + 'woofer', 'wool', 'workbasket', 'workbench', 'work-clothing', 'workhouse', + 'workhouse', 'workpiece', 'workroom', 'works', 'work-shirt', 'workstation', + 'worktable', 'workwear', 'world wide web', 'worm fence', 'worm gear', + 'worm wheel', 'worsted', 'worsted', 'wrap', 'wraparound', 'wrapping', + 'wreck', 'wrench', 'wrestling mat', 'wringer', 'wrist pad', 'wrist pin', + 'wristwatch', 'writing arm', 'writing desk', 'writing desk', + 'writing implement', 'xerographic printer', 'xerox', 'x-ray film', + 'x-ray machine', 'x-ray tube', 'yacht', 'yacht chair', 'yagi', 'yard', + 'yard', 'yardarm', 'yard marker', 'yardstick', 'yarmulke', 'yashmak', + 'yataghan', 'yawl', 'yawl', 'yoke', 'yoke', 'yoke', 'yurt', 'zamboni', + 'zero', 'ziggurat', 'zill', 'zip gun', 'zither', 'zoot suit', 'shading', + 'grain', 'wood grain', 'graining', 'marbleization', 'light', 'aura', + 'sunniness', 'glint', 'opalescence', 'polish', 'primary color for pigments', + 'primary color for light', 'colorlessness', 'mottle', 'achromia', 'shade', + 'chromatic color', 'black', 'coal black', 'alabaster', 'bone', 'gray', + 'ash grey', 'charcoal', 'sanguine', 'turkey red', 'crimson', 'dark red', + 'claret', 'fuschia', 'maroon', 'orange', 'reddish orange', 'yellow', + 'gamboge', 'pale yellow', 'green', 'greenishness', 'sea green', + 'sage green', 'bottle green', 'emerald', 'olive green', 'jade green', + 'blue', 'azure', 'steel blue', 'greenish blue', 'purplish blue', 'purple', + 'tyrian purple', 'indigo', 'lavender', 'reddish purple', 'pink', + 'carnation', 'rose', 'chestnut', 'chocolate', 'light brown', 'tan', 'beige', + 'reddish brown', 'brick red', 'copper', 'indian red', 'puce', 'olive', + 'ultramarine', 'complementary color', 'pigmentation', 'complexion', + 'ruddiness', 'nonsolid color', 'aposematic coloration', + 'cryptic coloration', 'ring', 'center of curvature', 'cadaver', + 'mandibular notch', 'rib', 'skin', 'skin graft', 'epidermal cell', + 'melanocyte', 'prickle cell', 'columnar cell', 'spongioblast', + 'squamous cell', 'amyloid plaque', 'dental plaque', 'macule', 'freckle', + 'bouffant', 'sausage curl', 'forelock', 'spit curl', 'pigtail', 'pageboy', + 'pompadour', 'thatch', 'soup-strainer', 'mustachio', 'walrus mustache', + 'stubble', 'vandyke beard', 'soul patch', 'esophageal smear', + 'paraduodenal smear', 'specimen', 'punctum', 'glenoid fossa', 'diastema', + 'marrow', 'mouth', 'canthus', 'milk', 'mother\'s milk', 'colostrum', 'vein', + 'ganglion cell', 'x chromosome', 'embryonic cell', 'myeloblast', + 'sideroblast', 'osteocyte', 'megalocyte', 'leukocyte', 'histiocyte', + 'fixed phagocyte', 'lymphocyte', 'monoblast', 'neutrophil', 'microphage', + 'sickle cell', 'siderocyte', 'spherocyte', 'ootid', 'oocyte', 'spermatid', + 'leydig cell', 'striated muscle cell', 'smooth muscle cell', + 'ranvier\'s nodes', 'neuroglia', 'astrocyte', 'protoplasmic astrocyte', + 'oligodendrocyte', 'proprioceptor', 'dendrite', 'sensory fiber', + 'subarachnoid space', 'cerebral cortex', 'renal cortex', 'prepuce', 'head', + 'scalp', 'frontal eminence', 'suture', 'foramen magnum', + 'esophagogastric junction', 'heel', 'cuticle', 'hangnail', 'exoskeleton', + 'abdominal wall', 'lemon', 'coordinate axis', 'landscape', 'medium', + 'vehicle', 'paper', 'channel', 'film', 'silver screen', 'free press', + 'press', 'print media', 'storage medium', 'magnetic storage medium', + 'journalism', 'fleet street', 'photojournalism', 'news photography', + 'rotogravure', 'newspaper', 'daily', 'gazette', 'school newspaper', + 'tabloid', 'yellow journalism', 'telecommunication', 'telephone', + 'voice mail', 'call', 'call-back', 'collect call', 'call forwarding', + 'call-in', 'call waiting', 'crank call', 'local call', 'long distance', + 'toll call', 'wake-up call', 'three-way calling', 'telegraphy', 'cable', + 'wireless', 'radiotelegraph', 'radiotelephone', 'broadcasting', + 'rediffusion', 'multiplex', 'radio', 'television', 'cable television', + 'high-definition television', 'reception', 'signal detection', 'hakham', + 'web site', 'chat room', 'portal site', 'jotter', 'breviary', 'wordbook', + 'desk dictionary', 'reckoner', 'document', 'album', 'concept album', + 'rock opera', 'tribute album', 'magazine', 'colour supplement', + 'comic book', 'news magazine', 'pulp', 'slick', 'trade magazine', 'movie', + 'outtake', 'shoot-\'em-up', 'spaghetti western', 'encyclical', + 'crossword puzzle', 'sign', 'street sign', 'traffic light', 'swastika', + 'concert', 'artwork', 'lobe', 'book jacket', 'cairn', 'three-day event', + 'comfort food', 'comestible', 'tuck', 'course', 'dainty', 'dish', + 'fast food', 'finger food', 'ingesta', 'kosher', 'fare', 'diet', 'diet', + 'dietary', 'balanced diet', 'bland diet', 'clear liquid diet', + 'diabetic diet', 'dietary supplement', 'carbohydrate loading', 'fad diet', + 'gluten-free diet', 'high-protein diet', 'high-vitamin diet', 'light diet', + 'liquid diet', 'low-calorie diet', 'low-fat diet', 'low-sodium diet', + 'macrobiotic diet', 'reducing diet', 'soft diet', 'vegetarianism', 'menu', + 'chow', 'board', 'mess', 'ration', 'field ration', 'k ration', 'c-ration', + 'foodstuff', 'starches', 'breadstuff', 'coloring', 'concentrate', + 'tomato concentrate', 'meal', 'kibble', 'cornmeal', 'farina', 'matzo meal', + 'oatmeal', 'pea flour', 'roughage', 'bran', 'flour', 'plain flour', + 'wheat flour', 'whole wheat flour', 'soybean meal', 'semolina', + 'corn gluten feed', 'nutriment', 'commissariat', 'larder', 'frozen food', + 'canned food', 'canned meat', 'spam', 'dehydrated food', 'square meal', + 'meal', 'potluck', 'refection', 'refreshment', 'breakfast', + 'continental breakfast', 'brunch', 'lunch', 'business lunch', 'high tea', + 'tea', 'dinner', 'supper', 'buffet', 'picnic', 'cookout', 'barbecue', + 'clambake', 'fish fry', 'bite', 'nosh', 'nosh-up', 'ploughman\'s lunch', + 'coffee break', 'banquet', 'entree', 'piece de resistance', 'plate', + 'adobo', 'side dish', 'special', 'casserole', 'chicken casserole', + 'chicken cacciatore', 'antipasto', 'appetizer', 'canape', 'cocktail', + 'fruit cocktail', 'crab cocktail', 'shrimp cocktail', 'hors d\'oeuvre', + 'relish', 'dip', 'bean dip', 'cheese dip', 'clam dip', 'guacamole', 'soup', + 'soup du jour', 'alphabet soup', 'consomme', 'madrilene', 'bisque', + 'borsch', 'broth', 'barley water', 'bouillon', 'beef broth', + 'chicken broth', 'broth', 'stock cube', 'chicken soup', 'cock-a-leekie', + 'gazpacho', 'gumbo', 'julienne', 'marmite', 'mock turtle soup', + 'mulligatawny', 'oxtail soup', 'pea soup', 'pepper pot', 'petite marmite', + 'potage', 'pottage', 'turtle soup', 'eggdrop soup', 'chowder', + 'corn chowder', 'clam chowder', 'manhattan clam chowder', + 'new england clam chowder', 'fish chowder', 'won ton', 'split-pea soup', + 'green pea soup', 'lentil soup', 'scotch broth', 'vichyssoise', 'stew', + 'bigos', 'brunswick stew', 'burgoo', 'burgoo', 'olla podrida', + 'mulligan stew', 'purloo', 'goulash', 'hotchpotch', 'hot pot', + 'beef goulash', 'pork-and-veal goulash', 'porkholt', 'irish stew', + 'oyster stew', 'lobster stew', 'lobscouse', 'fish stew', 'bouillabaisse', + 'matelote', 'paella', 'fricassee', 'chicken stew', 'turkey stew', + 'beef stew', 'ragout', 'ratatouille', 'salmi', 'pot-au-feu', 'slumgullion', + 'smorgasbord', 'viand', 'ready-mix', 'brownie mix', 'cake mix', + 'lemonade mix', 'self-rising flour', 'choice morsel', 'savory', + 'calf\'s-foot jelly', 'caramel', 'lump sugar', 'cane sugar', 'castor sugar', + 'powdered sugar', 'granulated sugar', 'icing sugar', 'corn sugar', + 'brown sugar', 'demerara', 'sweet', 'confectionery', 'confiture', + 'sweetmeat', 'candy', 'candy bar', 'carob bar', 'hardbake', 'hard candy', + 'barley-sugar', 'brandyball', 'jawbreaker', 'lemon drop', 'sourball', + 'patty', 'peppermint patty', 'bonbon', 'brittle', 'peanut brittle', + 'chewing gum', 'gum ball', 'bubble gum', 'butterscotch', 'candied fruit', + 'candied apple', 'crystallized ginger', 'grapefruit peel', 'lemon peel', + 'orange peel', 'candied citrus peel', 'candy cane', 'candy corn', 'caramel', + 'center', 'comfit', 'cotton candy', 'dragee', 'dragee', 'fondant', 'fudge', + 'chocolate fudge', 'divinity', 'penuche', 'gumdrop', 'jujube', + 'honey crisp', 'mint', 'horehound', 'peppermint', 'jelly bean', 'kiss', + 'molasses kiss', 'meringue kiss', 'chocolate kiss', 'licorice', + 'life saver', 'lollipop', 'lozenge', 'cachou', 'cough drop', 'marshmallow', + 'marzipan', 'nougat', 'nougat bar', 'nut bar', 'peanut bar', 'popcorn ball', + 'praline', 'rock candy', 'rock candy', 'sugar candy', 'sugarplum', 'taffy', + 'molasses taffy', 'truffle', 'turkish delight', 'dessert', 'ambrosia', + 'ambrosia', 'baked alaska', 'blancmange', 'charlotte', 'compote', + 'dumpling', 'flan', 'frozen dessert', 'junket', 'mousse', 'mousse', + 'pavlova', 'peach melba', 'whip', 'prune whip', 'pudding', 'pudding', + 'syllabub', 'tiramisu', 'trifle', 'tipsy cake', 'jello', 'apple dumpling', + 'ice', 'water ice', 'ice cream', 'ice-cream cone', 'chocolate ice cream', + 'neapolitan ice cream', 'peach ice cream', 'sherbert', + 'strawberry ice cream', 'tutti-frutti', 'vanilla ice cream', 'ice lolly', + 'ice milk', 'frozen yogurt', 'snowball', 'snowball', 'parfait', + 'ice-cream sundae', 'split', 'banana split', 'frozen pudding', + 'frozen custard', 'pudding', 'flummery', 'fish mousse', 'chicken mousse', + 'chocolate mousse', 'plum pudding', 'carrot pudding', 'corn pudding', + 'steamed pudding', 'duff', 'vanilla pudding', 'chocolate pudding', + 'brown betty', 'nesselrode', 'pease pudding', 'custard', 'creme caramel', + 'creme anglais', 'creme brulee', 'fruit custard', 'tapioca', + 'tapioca pudding', 'roly-poly', 'suet pudding', 'bavarian cream', + 'maraschino', 'nonpareil', 'zabaglione', 'garnish', 'pastry', 'turnover', + 'apple turnover', 'knish', 'pirogi', 'samosa', 'timbale', 'puff paste', + 'phyllo', 'puff batter', 'ice-cream cake', 'doughnut', 'fish cake', + 'fish stick', 'conserve', 'apple butter', 'chowchow', 'jam', 'lemon curd', + 'strawberry jam', 'jelly', 'apple jelly', 'crabapple jelly', 'grape jelly', + 'marmalade', 'orange marmalade', 'gelatin', 'gelatin dessert', + 'buffalo wing', 'barbecued wing', 'mess', 'mince', 'puree', 'barbecue', + 'biryani', 'escalope de veau orloff', 'saute', 'patty', 'veal parmesan', + 'veal cordon bleu', 'margarine', 'mincemeat', 'stuffing', 'turkey stuffing', + 'oyster stuffing', 'forcemeat', 'bread', 'anadama bread', 'bap', + 'barmbrack', 'breadstick', 'grissino', 'brown bread', 'bun', 'tea bread', + 'caraway seed bread', 'challah', 'cinnamon bread', 'cracked-wheat bread', + 'cracker', 'crouton', 'dark bread', 'english muffin', 'flatbread', + 'garlic bread', 'gluten bread', 'graham bread', 'host', 'flatbrod', + 'bannock', 'chapatti', 'pita', 'loaf of bread', 'french loaf', 'matzo', + 'nan', 'onion bread', 'raisin bread', 'quick bread', 'banana bread', + 'date bread', 'date-nut bread', 'nut bread', 'oatcake', 'irish soda bread', + 'skillet bread', 'rye bread', 'black bread', 'jewish rye bread', 'limpa', + 'swedish rye bread', 'salt-rising bread', 'simnel', 'sour bread', 'toast', + 'wafer', 'white bread', 'baguet', 'french bread', 'italian bread', + 'cornbread', 'corn cake', 'skillet corn bread', 'ashcake', 'hoecake', + 'cornpone', 'corn dab', 'hush puppy', 'johnnycake', 'shawnee cake', + 'spoon bread', 'cinnamon toast', 'orange toast', 'melba toast', 'zwieback', + 'frankfurter bun', 'hamburger bun', 'muffin', 'bran muffin', 'corn muffin', + 'yorkshire pudding', 'popover', 'scone', 'drop scone', 'cross bun', + 'brioche', 'crescent roll', 'hard roll', 'soft roll', 'kaiser roll', + 'parker house roll', 'clover-leaf roll', 'onion roll', 'bialy', + 'sweet roll', 'bear claw', 'cinnamon roll', 'honey bun', 'pinwheel roll', + 'danish', 'bagel', 'onion bagel', 'biscuit', 'rolled biscuit', + 'baking-powder biscuit', 'buttermilk biscuit', 'shortcake', 'hardtack', + 'saltine', 'soda cracker', 'oyster cracker', 'water biscuit', + 'graham cracker', 'pretzel', 'soft pretzel', 'sandwich', 'sandwich plate', + 'butty', 'ham sandwich', 'chicken sandwich', 'club sandwich', + 'open-face sandwich', 'hamburger', 'cheeseburger', 'tunaburger', 'hotdog', + 'sloppy joe', 'bomber', 'gyro', 'bacon-lettuce-tomato sandwich', 'reuben', + 'western', 'wrap', 'spaghetti', 'hasty pudding', 'gruel', 'congee', + 'skilly', 'edible fruit', 'vegetable', 'julienne', 'raw vegetable', + 'crudites', 'celery stick', 'legume', 'pulse', 'potherb', 'greens', + 'chop-suey greens', 'bean curd', 'solanaceous vegetable', 'root vegetable', + 'potato', 'baked potato', 'french fries', 'home fries', 'jacket potato', + 'mashed potato', 'potato skin', 'uruguay potato', 'yam', 'sweet potato', + 'yam', 'snack food', 'chip', 'corn chip', 'tortilla chip', 'nacho', + 'eggplant', 'pieplant', 'cruciferous vegetable', 'mustard', 'cabbage', + 'kale', 'collards', 'chinese cabbage', 'bok choy', 'head cabbage', + 'red cabbage', 'savoy cabbage', 'broccoli', 'cauliflower', + 'brussels sprouts', 'broccoli rabe', 'squash', 'summer squash', + 'yellow squash', 'crookneck', 'zucchini', 'marrow', 'cocozelle', + 'pattypan squash', 'spaghetti squash', 'winter squash', 'acorn squash', + 'butternut squash', 'hubbard squash', 'turban squash', 'buttercup squash', + 'cushaw', 'winter crookneck squash', 'cucumber', 'gherkin', 'artichoke', + 'artichoke heart', 'jerusalem artichoke', 'asparagus', 'bamboo shoot', + 'sprout', 'bean sprout', 'alfalfa sprout', 'beet', 'beet green', + 'sugar beet', 'mangel-wurzel', 'chard', 'pepper', 'sweet pepper', + 'bell pepper', 'green pepper', 'globe pepper', 'pimento', 'hot pepper', + 'chili', 'jalapeno', 'chipotle', 'cayenne', 'tabasco', 'onion', + 'bermuda onion', 'green onion', 'vidalia onion', 'spanish onion', + 'purple onion', 'leek', 'shallot', 'salad green', 'lettuce', + 'butterhead lettuce', 'buttercrunch', 'bibb lettuce', 'boston lettuce', + 'crisphead lettuce', 'cos', 'leaf lettuce', 'celtuce', 'bean', 'goa bean', + 'lentil', 'pea', 'green pea', 'marrowfat pea', 'snow pea', 'sugar snap pea', + 'split-pea', 'chickpea', 'cajan pea', 'field pea', 'mushy peas', + 'black-eyed pea', 'common bean', 'kidney bean', 'navy bean', 'pinto bean', + 'frijole', 'black bean', 'fresh bean', 'flageolet', 'green bean', + 'snap bean', 'string bean', 'kentucky wonder', 'scarlet runner', + 'haricot vert', 'wax bean', 'shell bean', 'lima bean', 'fordhooks', + 'sieva bean', 'fava bean', 'soy', 'green soybean', 'field soybean', + 'cardoon', 'carrot', 'carrot stick', 'celery', 'pascal celery', 'celeriac', + 'chicory', 'radicchio', 'coffee substitute', 'chicory', 'postum', + 'chicory escarole', 'belgian endive', 'corn', 'sweet corn', 'hominy', + 'lye hominy', 'pearl hominy', 'popcorn', 'cress', 'watercress', + 'garden cress', 'winter cress', 'dandelion green', 'gumbo', 'kohlrabi', + 'lamb\'s-quarter', 'wild spinach', 'tomato', 'beefsteak tomato', + 'cherry tomato', 'plum tomato', 'tomatillo', 'mushroom', 'stuffed mushroom', + 'salsify', 'oyster plant', 'scorzonera', 'parsnip', 'pumpkin', 'radish', + 'turnip', 'white turnip', 'rutabaga', 'turnip greens', 'sorrel', + 'french sorrel', 'spinach', 'taro', 'truffle', 'edible nut', 'bunya bunya', + 'peanut', 'freestone', 'cling', 'windfall', 'apple', 'crab apple', + 'eating apple', 'baldwin', 'cortland', 'cox\'s orange pippin', 'delicious', + 'golden delicious', 'red delicious', 'empire', 'grimes\' golden', + 'jonathan', 'mcintosh', 'macoun', 'northern spy', 'pearmain', 'pippin', + 'prima', 'stayman', 'winesap', 'stayman winesap', 'cooking apple', + 'bramley\'s seedling', 'granny smith', 'lane\'s prince albert', + 'newtown wonder', 'rome beauty', 'berry', 'bilberry', 'huckleberry', + 'blueberry', 'wintergreen', 'cranberry', 'lingonberry', 'currant', + 'gooseberry', 'black currant', 'red currant', 'blackberry', 'boysenberry', + 'dewberry', 'loganberry', 'raspberry', 'saskatoon', 'strawberry', + 'sugarberry', 'persimmon', 'acerola', 'carambola', 'ceriman', + 'carissa plum', 'citrus', 'orange', 'temple orange', 'mandarin', + 'clementine', 'satsuma', 'tangerine', 'tangelo', 'bitter orange', + 'sweet orange', 'jaffa orange', 'navel orange', 'valencia orange', + 'kumquat', 'lemon', 'lime', 'key lime', 'grapefruit', 'pomelo', 'citrange', + 'citron', 'almond', 'jordan almond', 'apricot', 'peach', 'nectarine', + 'pitahaya', 'plum', 'damson', 'greengage', 'beach plum', 'sloe', + 'victoria plum', 'dried fruit', 'dried apricot', 'prune', 'raisin', + 'seedless raisin', 'seeded raisin', 'currant', 'fig', 'pineapple', + 'anchovy pear', 'banana', 'passion fruit', 'granadilla', 'sweet calabash', + 'bell apple', 'breadfruit', 'jackfruit', 'cacao bean', 'cocoa', 'canistel', + 'melon', 'melon ball', 'muskmelon', 'cantaloup', 'winter melon', 'honeydew', + 'persian melon', 'net melon', 'casaba', 'watermelon', 'cherry', + 'sweet cherry', 'bing cherry', 'heart cherry', 'blackheart', 'capulin', + 'sour cherry', 'amarelle', 'morello', 'cocoa plum', 'gherkin', 'grape', + 'fox grape', 'concord grape', 'catawba', 'muscadine', 'scuppernong', + 'slipskin grape', 'vinifera grape', 'emperor', 'muscat', 'ribier', + 'sultana', 'tokay', 'flame tokay', 'thompson seedless', 'custard apple', + 'cherimoya', 'soursop', 'sweetsop', 'ilama', 'pond apple', 'papaw', + 'papaya', 'kai apple', 'ketembilla', 'ackee', 'durian', 'feijoa', 'genip', + 'genipap', 'kiwi', 'loquat', 'mangosteen', 'mango', 'sapodilla', 'sapote', + 'tamarind', 'avocado', 'date', 'elderberry', 'guava', 'mombin', 'hog plum', + 'hog plum', 'jaboticaba', 'jujube', 'litchi', 'longanberry', 'mamey', + 'marang', 'medlar', 'medlar', 'mulberry', 'olive', 'black olive', + 'green olive', 'pear', 'bosc', 'anjou', 'bartlett', 'seckel', 'plantain', + 'plumcot', 'pomegranate', 'prickly pear', 'barbados gooseberry', 'quandong', + 'quandong nut', 'quince', 'rambutan', 'pulasan', 'rose apple', 'sorb', + 'sour gourd', 'edible seed', 'pumpkin seed', 'betel nut', 'beechnut', + 'walnut', 'black walnut', 'english walnut', 'brazil nut', 'butternut', + 'souari nut', 'cashew', 'chestnut', 'chincapin', 'hazelnut', 'coconut', + 'coconut milk', 'grugru nut', 'hickory nut', 'cola extract', + 'macadamia nut', 'pecan', 'pine nut', 'pistachio', 'sunflower seed', + 'anchovy paste', 'rollmops', 'feed', 'cattle cake', 'creep feed', 'fodder', + 'feed grain', 'eatage', 'silage', 'oil cake', 'oil meal', 'alfalfa', + 'broad bean', 'hay', 'timothy', 'stover', 'grain', 'grist', 'groats', + 'millet', 'barley', 'pearl barley', 'buckwheat', 'bulgur', 'wheat', + 'cracked wheat', 'stodge', 'wheat germ', 'oat', 'rice', 'brown rice', + 'white rice', 'wild rice', 'paddy', 'slop', 'mash', 'chicken feed', 'cud', + 'bird feed', 'petfood', 'dog food', 'cat food', 'canary seed', 'salad', + 'tossed salad', 'green salad', 'caesar salad', 'salmagundi', + 'salad nicoise', 'combination salad', 'chef\'s salad', 'potato salad', + 'pasta salad', 'macaroni salad', 'fruit salad', 'waldorf salad', + 'crab louis', 'herring salad', 'tuna fish salad', 'chicken salad', + 'coleslaw', 'aspic', 'molded salad', 'tabbouleh', 'ingredient', 'flavorer', + 'bouillon cube', 'condiment', 'herb', 'fines herbes', 'spice', + 'spearmint oil', 'lemon oil', 'wintergreen oil', 'salt', 'celery salt', + 'onion salt', 'seasoned salt', 'sour salt', 'five spice powder', 'allspice', + 'cinnamon', 'stick cinnamon', 'clove', 'cumin', 'fennel', 'ginger', + 'ginger', 'mace', 'nutmeg', 'pepper', 'black pepper', 'white pepper', + 'sassafras', 'basil', 'bay leaf', 'borage', 'hyssop', 'caraway', 'chervil', + 'chives', 'comfrey', 'coriander', 'coriander', 'costmary', 'fennel', + 'fennel', 'fennel seed', 'fenugreek', 'garlic', 'clove', 'garlic chive', + 'lemon balm', 'lovage', 'marjoram', 'mint', 'mustard seed', 'mustard', + 'chinese mustard', 'nasturtium', 'parsley', 'salad burnet', 'rosemary', + 'rue', 'sage', 'clary sage', 'savory', 'summer savory', 'winter savory', + 'sweet woodruff', 'sweet cicely', 'tarragon', 'thyme', 'turmeric', 'caper', + 'catsup', 'cardamom', 'cayenne', 'chili powder', 'chili sauce', 'chutney', + 'steak sauce', 'taco sauce', 'salsa', 'mint sauce', 'cranberry sauce', + 'curry powder', 'curry', 'lamb curry', 'duck sauce', 'horseradish', + 'marinade', 'paprika', 'spanish paprika', 'pickle', 'dill pickle', + 'bread and butter pickle', 'pickle relish', 'piccalilli', 'sweet pickle', + 'applesauce', 'soy sauce', 'tabasco', 'tomato paste', 'angelica', + 'angelica', 'almond extract', 'anise', 'chinese anise', 'juniper berries', + 'saffron', 'sesame seed', 'caraway seed', 'poppy seed', 'dill', 'dill seed', + 'celery seed', 'lemon extract', 'monosodium glutamate', 'vanilla bean', + 'vinegar', 'cider vinegar', 'wine vinegar', 'sauce', 'anchovy sauce', + 'hot sauce', 'hard sauce', 'horseradish sauce', 'bolognese pasta sauce', + 'carbonara', 'tomato sauce', 'tartare sauce', 'wine sauce', + 'marchand de vin', 'bread sauce', 'plum sauce', 'peach sauce', + 'apricot sauce', 'pesto', 'ravigote', 'remoulade sauce', 'dressing', + 'sauce louis', 'bleu cheese dressing', 'blue cheese dressing', + 'french dressing', 'lorenzo dressing', 'anchovy dressing', + 'italian dressing', 'half-and-half dressing', 'mayonnaise', + 'green mayonnaise', 'aioli', 'russian dressing', 'salad cream', + 'thousand island dressing', 'barbecue sauce', 'hollandaise', 'bearnaise', + 'bercy', 'bordelaise', 'bourguignon', 'brown sauce', 'espagnole', + 'chinese brown sauce', 'blanc', 'cheese sauce', 'chocolate sauce', + 'hot-fudge sauce', 'cocktail sauce', 'colbert', 'white sauce', + 'cream sauce', 'mornay sauce', 'demiglace', 'gravy', 'gravy', + 'spaghetti sauce', 'marinara', 'mole', 'hunter\'s sauce', 'mushroom sauce', + 'mustard sauce', 'nantua', 'hungarian sauce', 'pepper sauce', 'roux', + 'smitane', 'soubise', 'lyonnaise sauce', 'veloute', 'allemande', + 'caper sauce', 'poulette', 'curry sauce', 'worcester sauce', 'coconut milk', + 'egg', 'egg white', 'egg yolk', 'boiled egg', 'hard-boiled egg', + 'easter egg', 'easter egg', 'chocolate egg', 'candy egg', 'poached egg', + 'scrambled eggs', 'deviled egg', 'shirred egg', 'omelet', 'firm omelet', + 'french omelet', 'fluffy omelet', 'western omelet', 'souffle', 'fried egg', + 'dairy product', 'milk', 'milk', 'sour milk', 'soya milk', 'formula', + 'pasteurized milk', 'cows\' milk', 'yak\'s milk', 'goats\' milk', + 'acidophilus milk', 'raw milk', 'scalded milk', 'homogenized milk', + 'certified milk', 'powdered milk', 'nonfat dry milk', 'evaporated milk', + 'condensed milk', 'skim milk', 'semi-skimmed milk', 'whole milk', + 'low-fat milk', 'buttermilk', 'cream', 'clotted cream', 'double creme', + 'half-and-half', 'heavy cream', 'light cream', 'sour cream', + 'whipping cream', 'butter', 'clarified butter', 'ghee', 'brown butter', + 'meuniere butter', 'yogurt', 'blueberry yogurt', 'raita', 'whey', 'curd', + 'curd', 'clabber', 'cheese', 'paring', 'cream cheese', 'double cream', + 'mascarpone', 'triple cream', 'cottage cheese', 'process cheese', 'bleu', + 'stilton', 'roquefort', 'gorgonzola', 'danish blue', 'bavarian blue', + 'brie', 'brick cheese', 'camembert', 'cheddar', 'rat cheese', + 'cheshire cheese', 'double gloucester', 'edam', 'goat cheese', 'gouda', + 'grated cheese', 'hand cheese', 'liederkranz', 'limburger', 'mozzarella', + 'muenster', 'parmesan', 'quark cheese', 'ricotta', 'string cheese', + 'swiss cheese', 'emmenthal', 'gruyere', 'sapsago', 'velveeta', 'nut butter', + 'peanut butter', 'marshmallow fluff', 'onion butter', 'pimento butter', + 'shrimp butter', 'lobster butter', 'yak butter', 'spread', 'cheese spread', + 'anchovy butter', 'fishpaste', 'garlic butter', 'miso', 'wasabi', + 'snail butter', 'hummus', 'pate', 'duck pate', 'foie gras', 'tapenade', + 'tahini', 'sweetening', 'aspartame', 'honey', 'saccharin', 'sugar', 'syrup', + 'sugar syrup', 'molasses', 'sorghum', 'treacle', 'grenadine', 'maple syrup', + 'corn syrup', 'miraculous food', 'batter', 'dough', 'bread dough', + 'pancake batter', 'fritter batter', 'coq au vin', 'chicken provencale', + 'chicken and rice', 'moo goo gai pan', 'arroz con pollo', 'bacon and eggs', + 'barbecued spareribs', 'beef bourguignonne', 'beef wellington', 'bitok', + 'boiled dinner', 'boston baked beans', 'bubble and squeak', 'pasta', + 'cannelloni', 'carbonnade flamande', 'cheese souffle', 'chicken marengo', + 'chicken cordon bleu', 'maryland chicken', 'chicken paprika', + 'chicken tetrazzini', 'tetrazzini', 'chicken kiev', 'chili', 'chili dog', + 'chop suey', 'chow mein', 'codfish ball', 'coquille', + 'coquilles saint-jacques', 'croquette', 'cottage pie', 'rissole', 'dolmas', + 'egg foo yong', 'egg roll', 'eggs benedict', 'enchilada', 'falafel', + 'fish and chips', 'fondue', 'cheese fondue', 'chocolate fondue', 'fondue', + 'beef fondue', 'french toast', 'fried rice', 'frittata', 'frog legs', + 'galantine', 'gefilte fish', 'haggis', 'ham and eggs', 'hash', + 'corned beef hash', 'jambalaya', 'kabob', 'kedgeree', 'souvlaki', 'lasagna', + 'seafood newburg', 'lobster newburg', 'shrimp newburg', 'newburg sauce', + 'lobster thermidor', 'lutefisk', 'macaroni and cheese', 'macedoine', + 'meatball', 'porcupine ball', 'swedish meatball', 'meat loaf', 'moussaka', + 'osso buco', 'marrow', 'pheasant under glass', 'pigs in blankets', 'pilaf', + 'bulgur pilaf', 'pizza', 'sausage pizza', 'pepperoni pizza', 'cheese pizza', + 'anchovy pizza', 'sicilian pizza', 'poi', 'pork and beans', 'porridge', + 'oatmeal', 'loblolly', 'potpie', 'rijsttaffel', 'risotto', 'roulade', + 'fish loaf', 'salmon loaf', 'salisbury steak', 'sauerbraten', 'sauerkraut', + 'scallopine', 'veal scallopini', 'scampi', 'scotch egg', 'scotch woodcock', + 'scrapple', 'spaghetti and meatballs', 'spanish rice', 'steak tartare', + 'pepper steak', 'steak au poivre', 'beef stroganoff', 'stuffed cabbage', + 'kishke', 'stuffed peppers', 'stuffed tomato', 'stuffed tomato', + 'succotash', 'sukiyaki', 'sashimi', 'sushi', 'swiss steak', 'tamale', + 'tamale pie', 'tempura', 'teriyaki', 'terrine', 'welsh rarebit', + 'schnitzel', 'taco', 'chicken taco', 'burrito', 'beef burrito', + 'quesadilla', 'tostada', 'bean tostada', 'refried beans', 'beverage', + 'wish-wash', 'concoction', 'mix', 'filling', 'lekvar', 'potion', 'elixir', + 'elixir of life', 'philter', 'alcohol', 'proof spirit', 'home brew', + 'hooch', 'kava', 'aperitif', 'brew', 'beer', 'draft beer', 'suds', + 'munich beer', 'bock', 'lager', 'light beer', 'oktoberfest', 'pilsner', + 'shebeen', 'weissbier', 'weizenbock', 'malt', 'wort', 'malt', 'ale', + 'bitter', 'burton', 'pale ale', 'porter', 'stout', 'guinness', 'kvass', + 'mead', 'metheglin', 'hydromel', 'oenomel', 'near beer', 'ginger beer', + 'sake', 'wine', 'vintage', 'red wine', 'white wine', 'blush wine', + 'altar wine', 'sparkling wine', 'champagne', 'cold duck', 'burgundy', + 'beaujolais', 'medoc', 'canary wine', 'chablis', 'montrachet', 'chardonnay', + 'pinot noir', 'pinot blanc', 'bordeaux', 'claret', 'chianti', 'cabernet', + 'merlot', 'sauvignon blanc', 'california wine', 'cotes de provence', + 'dessert wine', 'dubonnet', 'jug wine', 'macon', 'moselle', 'muscadet', + 'plonk', 'retsina', 'rhine wine', 'riesling', 'liebfraumilch', 'rhone wine', + 'rioja', 'sack', 'saint emilion', 'soave', 'zinfandel', 'sauterne', + 'straw wine', 'table wine', 'tokay', 'vin ordinaire', 'vermouth', + 'sweet vermouth', 'dry vermouth', 'chenin blanc', 'verdicchio', 'vouvray', + 'yquem', 'generic', 'varietal', 'fortified wine', 'madeira', 'malmsey', + 'port', 'sherry', 'marsala', 'muscat', 'liquor', 'neutral spirits', + 'aqua vitae', 'eau de vie', 'moonshine', 'bathtub gin', 'aquavit', 'arrack', + 'bitters', 'brandy', 'applejack', 'calvados', 'armagnac', 'cognac', + 'grappa', 'kirsch', 'slivovitz', 'gin', 'sloe gin', 'geneva', 'grog', + 'ouzo', 'rum', 'demerara', 'jamaica rum', 'schnapps', 'pulque', 'mescal', + 'tequila', 'vodka', 'whiskey', 'blended whiskey', 'bourbon', 'corn whiskey', + 'firewater', 'irish', 'poteen', 'rye', 'scotch', 'sour mash', 'liqueur', + 'absinth', 'amaretto', 'anisette', 'benedictine', 'chartreuse', + 'coffee liqueur', 'creme de cacao', 'creme de menthe', 'creme de fraise', + 'drambuie', 'galliano', 'orange liqueur', 'curacao', 'triple sec', + 'grand marnier', 'kummel', 'maraschino', 'pastis', 'pernod', 'pousse-cafe', + 'kahlua', 'ratafia', 'sambuca', 'mixed drink', 'cocktail', 'dom pedro', + 'highball', 'mixer', 'bishop', 'bloody mary', 'virgin mary', 'bullshot', + 'cobbler', 'collins', 'cooler', 'refresher', 'smoothie', 'daiquiri', + 'strawberry daiquiri', 'nada daiquiri', 'spritzer', 'flip', 'gimlet', + 'gin and tonic', 'grasshopper', 'harvey wallbanger', 'julep', 'manhattan', + 'rob roy', 'margarita', 'martini', 'gin and it', 'vodka martini', + 'old fashioned', 'pink lady', 'sazerac', 'screwdriver', 'sidecar', + 'scotch and soda', 'sling', 'brandy sling', 'gin sling', 'rum sling', + 'sour', 'whiskey sour', 'stinger', 'swizzle', 'hot toddy', 'zombie', 'fizz', + 'irish coffee', 'cafe au lait', 'cafe noir', 'decaffeinated coffee', + 'drip coffee', 'espresso', 'caffe latte', 'cappuccino', 'iced coffee', + 'instant coffee', 'mocha', 'mocha', 'cassareep', 'turkish coffee', + 'chocolate milk', 'cider', 'hard cider', 'scrumpy', 'sweet cider', + 'mulled cider', 'perry', 'rotgut', 'slug', 'cocoa', 'criollo', 'juice', + 'fruit juice', 'nectar', 'apple juice', 'cranberry juice', 'grape juice', + 'must', 'grapefruit juice', 'orange juice', 'frozen orange juice', + 'pineapple juice', 'lemon juice', 'lime juice', 'papaya juice', + 'tomato juice', 'carrot juice', 'v-8 juice', 'koumiss', 'fruit drink', + 'lemonade', 'limeade', 'orangeade', 'malted milk', 'mate', 'mulled wine', + 'negus', 'soft drink', 'pop', 'birch beer', 'bitter lemon', 'cola', + 'cream soda', 'egg cream', 'ginger ale', 'orange soda', 'phosphate', + 'coca cola', 'pepsi', 'root beer', 'sarsaparilla', 'tonic', 'coffee bean', + 'coffee', 'cafe royale', 'fruit punch', 'milk punch', 'mimosa', + 'pina colada', 'punch', 'cup', 'champagne cup', 'claret cup', 'wassail', + 'planter\'s punch', 'white russian', 'fish house punch', 'may wine', + 'eggnog', 'cassiri', 'spruce beer', 'rickey', 'gin rickey', 'tea', + 'tea bag', 'tea', 'tea-like drink', 'cambric tea', 'cuppa', 'herb tea', + 'tisane', 'camomile tea', 'ice tea', 'sun tea', 'black tea', 'congou', + 'darjeeling', 'orange pekoe', 'souchong', 'green tea', 'hyson', 'oolong', + 'water', 'bottled water', 'branch water', 'spring water', 'sugar water', + 'drinking water', 'ice water', 'soda water', 'mineral water', 'seltzer', + 'vichy water', 'perishable', 'couscous', 'ramekin', 'multivitamin', + 'vitamin pill', 'soul food', 'mold', 'people', 'collection', 'book', + 'library', 'baseball club', 'crowd', 'class', 'core', 'concert band', + 'dance', 'wedding', 'chain', 'power breakfast', 'aerie', 'agora', + 'amusement park', 'aphelion', 'apron', 'interplanetary space', + 'interstellar space', 'intergalactic space', 'bush', 'semidesert', + 'beam-ends', 'bridgehead', 'bus stop', 'campsite', 'detention basin', + 'cemetery', 'trichion', 'city', 'business district', 'outskirts', 'borough', + 'cow pasture', 'crest', 'eparchy', 'suburb', 'stockbroker belt', + 'crawlspace', 'sheikdom', 'residence', 'domicile', 'dude ranch', 'farmland', + 'midfield', 'firebreak', 'flea market', 'battlefront', 'garbage heap', + 'benthos', 'goldfield', 'grainfield', 'half-mast', 'hemline', 'heronry', + 'hipline', 'hipline', 'hole-in-the-wall', 'junkyard', 'isoclinic line', + 'littoral', 'magnetic pole', 'grassland', 'mecca', 'observer\'s meridian', + 'prime meridian', 'nombril', 'no-parking zone', 'outdoors', 'fairground', + 'pasture', 'perihelion', 'periselene', 'locus of infection', 'kasbah', + 'waterfront', 'resort', 'resort area', 'rough', 'ashram', 'harborage', + 'scrubland', 'weald', 'wold', 'schoolyard', 'showplace', 'bedside', + 'sideline', 'ski resort', 'soil horizon', 'geological horizon', 'coal seam', + 'coalface', 'field', 'oilfield', 'temperate zone', 'terreplein', + 'three-mile limit', 'desktop', 'top', 'kampong', 'subtropics', 'barrio', + 'veld', 'vertex', 'waterline', 'high-water mark', 'low-water mark', + 'continental divide', 'zodiac', 'aegean island', 'sultanate', + 'swiss canton', 'abyssal zone', 'aerie', 'air bubble', 'alluvial flat', + 'alp', 'alpine glacier', 'anthill', 'aquifer', 'archipelago', 'arete', + 'arroyo', 'ascent', 'asterism', 'asthenosphere', 'atoll', 'bank', 'bank', + 'bar', 'barbecue pit', 'barrier reef', 'baryon', 'basin', 'beach', + 'honeycomb', 'belay', 'ben', 'berm', 'bladder stone', 'bluff', 'borrow pit', + 'brae', 'bubble', 'burrow', 'butte', 'caldera', 'canyon', 'canyonside', + 'cave', 'cavern', 'chasm', 'cirque', 'cliff', 'cloud', 'coast', 'coastland', + 'col', 'collector', 'comet', 'continental glacier', 'coral reef', 'cove', + 'crag', 'crater', 'cultivated land', 'dale', 'defile', 'delta', 'descent', + 'diapir', 'divot', 'divot', 'down', 'downhill', 'draw', 'drey', 'drumlin', + 'dune', 'escarpment', 'esker', 'fireball', 'flare star', 'floor', 'fomite', + 'foothill', 'footwall', 'foreland', 'foreshore', 'gauge boson', + 'geological formation', 'geyser', 'glacier', 'glen', 'gopher hole', 'gorge', + 'grotto', 'growler', 'gulch', 'gully', 'hail', 'highland', 'hill', + 'hillside', 'hole', 'hollow', 'hot spring', 'iceberg', 'icecap', + 'ice field', 'ice floe', 'ice mass', 'inclined fault', 'ion', 'isthmus', + 'kidney stone', 'knoll', 'kopje', 'kuiper belt', 'lake bed', 'lakefront', + 'lakeside', 'landfall', 'landfill', 'lather', 'leak', 'ledge', 'lepton', + 'lithosphere', 'lowland', 'lunar crater', 'maar', 'massif', 'meander', + 'mesa', 'meteorite', 'microfossil', 'midstream', 'molehill', 'monocline', + 'mountain', 'mountainside', 'mouth', 'mull', 'natural depression', + 'natural elevation', 'nullah', 'ocean', 'ocean floor', 'oceanfront', + 'outcrop', 'oxbow', 'pallasite', 'perforation', 'photosphere', 'piedmont', + 'piedmont glacier', 'pinetum', 'plage', 'plain', 'point', 'polar glacier', + 'pothole', 'precipice', 'promontory', 'ptyalith', 'pulsar', 'quicksand', + 'rabbit burrow', 'radiator', 'rainbow', 'range', 'rangeland', 'ravine', + 'reef', 'ridge', 'ridge', 'rift valley', 'riparian forest', 'ripple mark', + 'riverbank', 'riverbed', 'rock', 'roof', 'saltpan', 'sandbank', 'sandbar', + 'sandpit', 'sanitary landfill', 'sawpit', 'scablands', 'seashore', + 'seaside', 'seif dune', 'shell', 'shiner', 'shoal', 'shore', 'shoreline', + 'sinkhole', 'ski slope', 'sky', 'slope', 'snowcap', 'snowdrift', + 'snowfield', 'soapsuds', 'spit', 'spoor', 'spume', 'star', 'steep', + 'steppe', 'strand', 'streambed', 'sun', 'supernova', 'swale', 'swamp', + 'swell', 'tableland', 'talus', 'tangle', 'tar pit', 'terrace', + 'tidal basin', 'tideland', 'tor', 'tor', 'trapezium', 'troposphere', + 'tundra', 'twinkler', 'uphill', 'urolith', 'valley', + 'vehicle-borne transmission', 'vein', 'volcanic crater', 'volcano', 'wadi', + 'wall', 'warren', 'wasp\'s nest', 'watercourse', 'waterside', 'water table', + 'whinstone', 'wormcast', 'xenolith', 'circe', 'gryphon', 'spiritual leader', + 'messiah', 'rhea silvia', 'number one', 'adventurer', 'anomaly', + 'appointee', 'argonaut', 'ashkenazi', 'benefactor', 'color-blind person', + 'commoner', 'conservator', 'contrarian', 'contadino', 'contestant', + 'cosigner', 'discussant', 'enologist', 'entertainer', 'eulogist', + 'ex-gambler', 'experimenter', 'experimenter', 'exponent', 'ex-president', + 'face', 'female', 'finisher', 'inhabitant', 'native', 'native', 'juvenile', + 'lover', 'male', 'mediator', 'mediatrix', 'national', 'peer', + 'prize winner', 'recipient', 'religionist', 'sensualist', 'traveler', + 'unwelcome person', 'unskilled person', 'worker', 'wrongdoer', + 'black african', 'afrikaner', 'aryan', 'black', 'black woman', 'mulatto', + 'white', 'circassian', 'semite', 'chaldean', 'elamite', 'white man', 'wasp', + 'gook', 'mongol', 'tatar', 'nahuatl', 'aztec', 'olmec', 'biloxi', + 'blackfoot', 'brule', 'caddo', 'cheyenne', 'chickasaw', 'cocopa', + 'comanche', 'creek', 'delaware', 'diegueno', 'esselen', 'eyeish', + 'havasupai', 'hunkpapa', 'iowa', 'kalapooia', 'kamia', 'kekchi', 'kichai', + 'kickapoo', 'kiliwa', 'malecite', 'maricopa', 'mohican', 'muskhogean', + 'navaho', 'nootka', 'oglala', 'osage', 'oneida', 'paiute', 'passamaquody', + 'penobscot', 'penutian', 'potawatomi', 'powhatan', 'kachina', 'salish', + 'shahaptian', 'shasta', 'shawnee', 'sihasapa', 'teton', 'taracahitian', + 'tarahumara', 'tuscarora', 'tutelo', 'yana', 'yavapai', 'yokuts', 'yuma', + 'gadaba', 'kolam', 'kui', 'toda', 'tulu', 'gujarati', 'kashmiri', 'punjabi', + 'slav', 'anabaptist', 'adventist', 'gentile', 'gentile', 'catholic', + 'old catholic', 'uniat', 'copt', 'jewess', 'jihadist', 'buddhist', + 'zen buddhist', 'mahayanist', 'swami', 'hare krishna', 'shintoist', + 'eurafrican', 'eurasian', 'gael', 'frank', 'afghan', 'albanian', 'algerian', + 'altaic', 'andorran', 'angolan', 'anguillan', 'austrian', 'bahamian', + 'bahraini', 'basotho', 'herero', 'luba', 'barbadian', 'bolivian', 'bornean', + 'carioca', 'tupi', 'bruneian', 'bulgarian', 'byelorussian', 'cameroonian', + 'canadian', 'french canadian', 'central american', 'chilean', 'congolese', + 'cypriot', 'dane', 'djiboutian', 'britisher', 'english person', + 'englishwoman', 'anglo-saxon', 'angle', 'west saxon', 'lombard', 'limey', + 'cantabrigian', 'cornishman', 'cornishwoman', 'lancastrian', 'lancastrian', + 'geordie', 'oxonian', 'ethiopian', 'amhara', 'eritrean', 'finn', 'komi', + 'livonian', 'lithuanian', 'selkup', 'parisian', 'parisienne', 'creole', + 'creole', 'gabonese', 'greek', 'dorian', 'athenian', 'laconian', 'guyanese', + 'haitian', 'malay', 'moro', 'netherlander', 'icelander', 'iraqi', + 'irishman', 'irishwoman', 'dubliner', 'italian', 'roman', 'sabine', + 'japanese', 'jordanian', 'korean', 'kenyan', 'lao', 'lapp', + 'latin american', 'lebanese', 'levantine', 'liberian', 'luxemburger', + 'macedonian', 'sabahan', 'mexican', 'chicano', 'mexican-american', + 'namibian', 'nauruan', 'gurkha', 'new zealander', 'nicaraguan', 'nigerian', + 'hausa', 'north american', 'nova scotian', 'omani', 'pakistani', 'brahui', + 'south american indian', 'carib', 'filipino', 'polynesian', 'qatari', + 'romanian', 'muscovite', 'georgian', 'sarawakian', 'scandinavian', + 'senegalese', 'slovene', 'south african', 'south american', 'sudanese', + 'syrian', 'tahitian', 'tanzanian', 'tibetan', 'togolese', 'tuareg', 'turki', + 'chuvash', 'turkoman', 'uzbek', 'ugandan', 'ukranian', 'yakut', 'tungus', + 'igbo', 'american', 'anglo-american', 'alaska native', 'arkansan', + 'carolinian', 'coloradan', 'connecticuter', 'delawarean', 'floridian', + 'german american', 'illinoisan', 'mainer', 'marylander', 'minnesotan', + 'nebraskan', 'new hampshirite', 'new jerseyan', 'new yorker', + 'north carolinian', 'oregonian', 'pennsylvanian', 'texan', 'utahan', + 'uruguayan', 'vietnamese', 'gambian', 'east german', 'berliner', 'prussian', + 'ghanian', 'guinean', 'papuan', 'walloon', 'yemeni', 'yugoslav', 'serbian', + 'xhosa', 'zairese', 'zimbabwean', 'zulu', 'gemini', 'sagittarius', 'pisces', + 'abbe', 'abbess', 'abnegator', 'abridger', 'abstractor', 'absconder', + 'absolver', 'abecedarian', 'aberrant', 'abettor', 'abhorrer', 'abomination', + 'abseiler', 'abstainer', 'academic administrator', 'academician', + 'accessory before the fact', 'companion', 'accompanist', 'accomplice', + 'account executive', 'accused', 'accuser', 'acid head', 'acquaintance', + 'acquirer', 'aerialist', 'action officer', 'active', 'active citizen', + 'actor', 'actor', 'addict', 'adducer', 'adjuster', 'adjutant', + 'adjutant general', 'admirer', 'adoptee', 'adulterer', 'adulteress', + 'advertiser', 'advisee', 'advocate', 'aeronautical engineer', 'affiliate', + 'affluent', 'aficionado', 'buck sergeant', 'agent-in-place', 'aggravator', + 'agitator', 'agnostic', 'agnostic', 'agonist', 'agony aunt', + 'agriculturist', 'air attache', 'air force officer', 'airhead', + 'air traveler', 'alarmist', 'albino', 'alcoholic', 'alderman', 'alexic', + 'alienee', 'alienor', 'aliterate', 'algebraist', 'allegorizer', + 'alliterator', 'almoner', 'alpinist', 'altar boy', 'alto', 'ambassador', + 'ambassador', 'ambusher', 'amicus curiae', 'amoralist', 'amputee', + 'analogist', 'analphabet', 'analyst', 'industry analyst', + 'market strategist', 'anarchist', 'anathema', 'ancestor', 'anchor', + 'ancient', 'anecdotist', 'angler', 'animator', 'animist', 'annotator', + 'announcer', 'announcer', 'anti', 'anti-american', 'anti-semite', 'anzac', + 'ape-man', 'aphakic', 'appellant', 'appointee', 'apprehender', 'april fool', + 'aspirant', 'appreciator', 'appropriator', 'arabist', 'archaist', + 'archbishop', 'archer', 'architect', 'archivist', 'archpriest', + 'aristotelian', 'armiger', 'army attache', 'army engineer', 'army officer', + 'arranger', 'arrival', 'arthritic', 'articulator', 'artilleryman', + 'artist\'s model', 'assayer', 'assemblyman', 'assemblywoman', 'assenter', + 'asserter', 'assignee', 'assistant', 'assistant professor', 'associate', + 'associate', 'associate professor', 'astronaut', 'cosmographer', 'atheist', + 'athlete', 'attendant', 'attorney general', 'auditor', 'augur', 'aunt', + 'au pair girl', 'authoritarian', 'authority', 'authorizer', + 'automobile mechanic', 'aviator', 'aviatrix', 'ayah', 'babu', 'baby', + 'baby', 'baby boomer', 'baby farmer', 'back', 'backbencher', 'backpacker', + 'backroom boy', 'backscratcher', 'bad person', 'baggage', 'bag lady', + 'bailee', 'bailiff', 'bailor', 'bairn', 'baker', 'balancer', 'balker', + 'ball-buster', 'ball carrier', 'ballet dancer', 'ballet master', + 'ballet mistress', 'balletomane', 'ball hawk', 'balloonist', 'ballplayer', + 'bullfighter', 'banderillero', 'matador', 'picador', 'bandsman', 'banker', + 'bank robber', 'bankrupt', 'bantamweight', 'barmaid', 'baron', 'baron', + 'baron', 'bartender', 'baseball coach', 'base runner', 'basketball player', + 'basketweaver', 'basket maker', 'bass', 'bastard', 'bat boy', 'bather', + 'batman', 'baton twirler', 'bavarian', 'beadsman', 'beard', 'beatnik', + 'beauty consultant', 'bedouin', 'bedwetter', 'beekeeper', 'beer drinker', + 'beggarman', 'beggarwoman', 'beldam', 'theist', 'believer', 'bell founder', + 'benedick', 'berserker', 'besieger', 'best', 'betrothed', 'big brother', + 'bigot', 'big shot', 'big sister', 'billiard player', 'biochemist', + 'biographer', 'bird fancier', 'birth', 'birth-control campaigner', + 'bisexual', 'black belt', 'blackmailer', 'black muslim', 'blacksmith', + 'blade', 'bleacher', 'blind date', 'bluecoat', 'bluestocking', + 'boatbuilder', 'boatman', 'boatswain', 'bobby', 'bodyguard', 'boffin', + 'bolshevik', 'bolshevik', 'bombshell', 'bondman', 'bondwoman', 'bondwoman', + 'bond servant', 'book agent', 'bookbinder', 'bookkeeper', 'bookmaker', + 'bookworm', 'booster', 'bootblack', 'bootlegger', 'bootmaker', 'borderer', + 'border patrolman', 'botanist', 'bottom feeder', 'boulevardier', + 'bounty hunter', 'bounty hunter', 'bourbon', 'bowler', 'slugger', 'cub', + 'boy scout', 'boy scout', 'boy wonder', 'bragger', 'brahman', 'brawler', + 'breadwinner', 'breaststroker', 'breeder', 'brick', 'bride', 'bridesmaid', + 'bridge agent', 'broadcast journalist', 'brother', 'brother-in-law', + 'browser', 'brummie', 'buddy', 'bull', 'bully', 'bunny', 'burglar', + 'bursar', 'busboy', 'business editor', 'business traveler', 'buster', + 'busybody', 'buttinsky', 'cabinetmaker', 'caddie', 'cadet', 'caller', + 'call girl', 'calligrapher', 'campaigner', 'camper', 'camp follower', + 'candidate', 'canonist', 'capitalist', 'captain', 'captain', 'captain', + 'captain', 'captive', 'captive', 'cardinal', 'cardiologist', 'card player', + 'cardsharp', 'careerist', 'career man', 'caregiver', 'caretaker', + 'caretaker', 'caricaturist', 'carillonneur', 'caroler', 'carpenter', + 'carper', 'cartesian', 'cashier', 'casualty', 'casualty', 'casuist', + 'catechist', 'catechumen', 'caterer', 'catholicos', 'cat fancier', + 'cavalier', 'cavalryman', 'caveman', 'celebrant', 'celebrant', 'celebrity', + 'cellist', 'censor', 'censor', 'centenarian', 'centrist', 'centurion', + 'certified public accountant', 'chachka', 'chambermaid', 'chameleon', + 'champion', 'chandler', 'prison chaplain', 'charcoal burner', + 'charge d\'affaires', 'charioteer', 'charmer', 'chartered accountant', + 'chartist', 'charwoman', 'male chauvinist', 'cheapskate', 'chechen', + 'checker', 'cheerer', 'cheerleader', 'cheerleader', 'cheops', + 'chess master', 'chief executive officer', 'chief of staff', + 'chief petty officer', 'chief secretary', 'child', 'child', 'child', + 'child prodigy', 'chimneysweeper', 'chiropractor', 'chit', 'choker', + 'choragus', 'choreographer', 'chorus girl', 'chosen', 'cicerone', + 'cigar smoker', 'cipher', 'circus acrobat', 'citizen', 'city editor', + 'city father', 'city man', 'city slicker', 'civic leader', + 'civil rights leader', 'cleaner', 'clergyman', 'cleric', 'clerk', + 'clever dick', 'climatologist', 'climber', 'clinician', 'closer', + 'closet queen', 'clown', 'clown', 'coach', 'coach', 'pitching coach', + 'coachman', 'coal miner', 'coastguardsman', 'cobber', 'cobbler', 'codger', + 'co-beneficiary', 'cog', 'cognitive neuroscientist', 'coiffeur', 'coiner', + 'collaborator', 'colleen', 'college student', 'collegian', 'colonial', + 'colonialist', 'colonizer', 'coloratura', 'color guard', 'colossus', + 'comedian', 'comedienne', 'comer', 'commander', 'commander in chief', + 'commanding officer', 'commissar', 'commissioned officer', + 'commissioned military officer', 'commissioner', 'commissioner', + 'committee member', 'committeewoman', 'commodore', 'communicant', + 'communist', 'communist', 'commuter', 'compere', 'complexifier', + 'compulsive', 'computational linguist', 'computer scientist', + 'computer user', 'comrade', 'concert-goer', 'conciliator', 'conductor', + 'confectioner', 'confederate', 'confessor', 'confidant', 'confucian', 'rep', + 'conqueror', 'conservative', 'nonconformist', 'anglican', 'consignee', + 'consigner', 'constable', 'constructivist', 'contractor', 'contralto', + 'contributor', 'control freak', 'convalescent', 'convener', 'convict', + 'copilot', 'copycat', 'coreligionist', 'cornerback', 'corporatist', + 'correspondent', 'cosmetician', 'cosmopolitan', 'cossack', + 'cost accountant', 'co-star', 'costumier', 'cotter', 'cotter', 'counselor', + 'counterterrorist', 'counterspy', 'countess', 'compromiser', 'countrywoman', + 'county agent', 'courtier', 'cousin', 'cover girl', 'cow', 'craftsman', + 'craftsman', 'crapshooter', 'crazy', 'creature', 'creditor', 'creep', + 'criminologist', 'critic', 'croesus', 'cross-examiner', 'crossover voter', + 'croupier', 'crown prince', 'crown princess', 'cryptanalyst', 'cub scout', + 'cuckold', 'cultist', 'curandera', 'curate', 'curator', 'customer agent', + 'cutter', 'cyberpunk', 'cyborg', 'cymbalist', 'cynic', 'cytogeneticist', + 'cytologist', 'czar', 'czar', 'dad', 'dairyman', 'dalai lama', 'dallier', + 'dancer', 'dancer', 'clog dancer', 'dancing-master', 'dark horse', + 'darling', 'date', 'daughter', 'dawdler', 'day boarder', 'day laborer', + 'deacon', 'deaconess', 'deadeye', 'deipnosophist', 'dropout', 'deadhead', + 'deaf person', 'debtor', 'deckhand', 'defamer', 'defense contractor', + 'deist', 'delegate', 'deliveryman', 'demagogue', 'demigod', 'demographer', + 'demonstrator', 'den mother', 'department head', 'depositor', 'deputy', + 'dermatologist', 'descender', 'designated hitter', 'designer', 'desk clerk', + 'desk officer', 'desk sergeant', 'detainee', 'detective', 'detective', + 'detractor', 'developer', 'deviationist', 'devisee', 'devisor', 'devourer', + 'dialectician', 'diarist', 'dietician', 'diocesan', 'director', 'director', + 'dirty old man', 'disbeliever', 'disk jockey', 'dispatcher', + 'distortionist', 'distributor', 'district attorney', 'district manager', + 'diver', 'divorcee', 'ex-wife', 'divorce lawyer', 'docent', 'doctor', + 'dodo', 'doge', 'dog in the manger', 'dogmatist', 'dolichocephalic', + 'domestic partner', 'dominican', 'dominus', 'don', 'donatist', 'donna', + 'dosser', 'double', 'double-crosser', 'down-and-out', 'doyenne', + 'draftsman', 'dramatist', 'dreamer', 'dressmaker', 'dressmaker\'s model', + 'dribbler', 'dribbler', 'drinker', 'drinker', 'drug addict', 'drug user', + 'druid', 'drum majorette', 'drummer', 'drunk', 'drunkard', 'druze', 'dry', + 'dry nurse', 'duchess', 'duke', 'duffer', 'dunker', 'dutch uncle', + 'dyspeptic', 'eager beaver', 'earl', 'earner', 'eavesdropper', 'eccentric', + 'eclectic', 'econometrician', 'economist', 'ectomorph', 'editor', + 'egocentric', 'egotist', 'ejaculator', 'elder', 'elder statesman', + 'elected official', 'electrician', 'elegist', 'elocutionist', 'emancipator', + 'embryologist', 'emeritus', 'emigrant', 'emissary', 'empress', 'employee', + 'employer', 'enchantress', 'enchantress', 'encyclopedist', 'endomorph', + 'enemy', 'energizer', 'end man', 'end man', 'endorser', 'enjoyer', + 'enlisted woman', 'enophile', 'entrant', 'entrant', 'entrepreneur', 'envoy', + 'enzymologist', 'eparch', 'epidemiologist', 'epigone', 'epileptic', + 'episcopalian', 'equerry', 'equerry', 'erotic', 'escapee', 'escapist', + 'eskimo', 'espionage agent', 'esthetician', 'etcher', 'ethnologist', + 'etonian', 'etymologist', 'evangelist', 'evangelist', 'event planner', + 'examiner', 'examiner', 'exarch', 'executant', 'executive secretary', + 'executive vice president', 'executrix', 'exegete', 'exhibitor', + 'exhibitionist', 'exile', 'existentialist', 'exorcist', 'ex-spouse', + 'extern', 'extremist', 'extrovert', 'eyewitness', 'facilitator', + 'fairy godmother', 'falangist', 'falconer', 'falsifier', 'familiar', 'fan', + 'fanatic', 'fancier', 'farm boy', 'farmer', 'farmhand', 'fascist', + 'fascista', 'fatalist', 'father', 'father', 'father-figure', + 'father-in-law', 'fauntleroy', 'fauve', 'favorite son', 'featherweight', + 'federalist', 'fellow traveler', 'female aristocrat', 'female offspring', + 'female child', 'fence', 'fiance', 'fielder', 'field judge', + 'fighter pilot', 'filer', 'film director', 'finder', 'fire chief', + 'fire-eater', 'fire-eater', 'fireman', 'fire marshall', 'fire walker', + 'first baseman', 'firstborn', 'first lady', 'first lieutenant', + 'first offender', 'first sergeant', 'fishmonger', 'flagellant', + 'flag officer', 'flak catcher', 'flanker back', 'flapper', 'flatmate', + 'flatterer', 'flibbertigibbet', 'flight surgeon', 'floorwalker', 'flop', + 'florentine', 'flower girl', 'flower girl', 'flutist', 'fly-by-night', + 'flyweight', 'flyweight', 'foe', 'folk dancer', 'folk poet', 'follower', + 'football hero', 'football player', 'footman', 'forefather', 'foremother', + 'foreign agent', 'foreigner', 'boss', 'foreman', 'forester', 'forewoman', + 'forger', 'forward', 'foster-brother', 'foster-father', 'foster-mother', + 'foster-sister', 'foster-son', 'founder', 'foundress', 'four-minute man', + 'framer', 'francophobe', 'freak', 'free agent', 'free agent', + 'freedom rider', 'free-liver', 'freeloader', 'free trader', 'freudian', + 'friar', 'monk', 'frontierswoman', 'front man', 'frotteur', 'fucker', + 'fucker', 'fuddy-duddy', 'fullback', 'funambulist', 'fundamentalist', + 'fundraiser', 'futurist', 'gadgeteer', 'gagman', 'gagman', 'gainer', 'gal', + 'galoot', 'gambist', 'gambler', 'gamine', 'garbage man', 'gardener', + 'garment cutter', 'garroter', 'gasman', 'gastroenterologist', 'gatherer', + 'gawker', 'gendarme', 'general', 'generator', 'geneticist', 'genitor', + 'gent', 'geologist', 'geophysicist', 'ghostwriter', 'gibson girl', 'girl', + 'girlfriend', 'girlfriend', 'girl wonder', 'girondist', 'gitano', + 'gladiator', 'glassblower', 'gleaner', 'goat herder', 'godchild', + 'godfather', 'godparent', 'godson', 'gofer', 'goffer', 'goldsmith', + 'golfer', 'gondolier', 'good guy', 'good old boy', 'good samaritan', + 'gossip columnist', 'gouger', 'governor general', 'grabber', 'grader', + 'graduate nurse', 'grammarian', 'granddaughter', 'grande dame', + 'grandfather', 'grand inquisitor', 'grandma', 'grandmaster', 'grandparent', + 'grantee', 'granter', 'grass widower', 'great-aunt', 'great grandchild', + 'great granddaughter', 'great grandmother', 'great grandparent', + 'great grandson', 'great-nephew', 'great-niece', 'green beret', 'grenadier', + 'greeter', 'gringo', 'grinner', 'grocer', 'groom', 'groom', 'grouch', + 'group captain', 'grunter', 'prison guard', 'guard', 'guesser', 'guest', + 'guest', 'guest of honor', 'guest worker', 'guide', 'guitarist', + 'gunnery sergeant', 'guru', 'guru', 'guvnor', 'guy', 'gymnast', 'gym rat', + 'gynecologist', 'gypsy', 'hack', 'hacker', 'haggler', 'hairdresser', + 'hakim', 'hakka', 'halberdier', 'halfback', 'half blood', 'hand', + 'animal trainer', 'handyman', 'hang glider', 'hardliner', 'harlequin', + 'harmonizer', 'hash head', 'hatchet man', 'hater', 'hatmaker', 'headman', + 'headmaster', 'head nurse', 'hearer', 'heartbreaker', 'heathen', + 'heavyweight', 'heavy', 'heckler', 'hedger', 'hedger', 'hedonist', 'heir', + 'heir apparent', 'heiress', 'heir presumptive', 'hellion', 'helmsman', + 'hire', 'hematologist', 'hemiplegic', 'herald', 'herbalist', 'herder', + 'hermaphrodite', 'heroine', 'heroin addict', 'hero worshiper', 'herr', + 'highbinder', 'highbrow', 'high commissioner', 'highflier', 'highlander', + 'high-muck-a-muck', 'high priest', 'highjacker', 'hireling', 'historian', + 'hitchhiker', 'hitter', 'hobbyist', 'holdout', 'holdover', 'holdup man', + 'homeboy', 'homeboy', 'home buyer', 'homegirl', 'homeless', 'homeopath', + 'honest woman', 'honor guard', 'hooker', 'hoper', 'hornist', 'horseman', + 'horse trader', 'horsewoman', 'horse wrangler', 'horticulturist', + 'hospital chaplain', 'host', 'host', 'hostess', 'hotelier', 'housekeeper', + 'housemaster', 'housemate', 'house physician', 'house sitter', + 'housing commissioner', 'huckster', 'hugger', 'humanist', 'humanitarian', + 'hunk', 'huntress', 'ex-husband', 'hydrologist', 'hyperope', 'hypertensive', + 'hypnotist', 'hypocrite', 'iceman', 'iconoclast', 'ideologist', 'idol', + 'idolizer', 'imam', 'imperialist', 'important person', 'inamorato', + 'incumbent', 'incurable', 'inductee', 'industrialist', 'infanticide', + 'inferior', 'infernal', 'infielder', 'infiltrator', 'informer', 'ingenue', + 'ingenue', 'polymath', 'in-law', 'inquiry agent', 'inspector', + 'inspector general', 'instigator', 'insurance broker', 'insurgent', + 'intelligence analyst', 'interior designer', 'interlocutor', 'interlocutor', + 'international grandmaster', 'internationalist', 'internist', 'interpreter', + 'interpreter', 'intervenor', 'introvert', 'invader', 'invalidator', + 'investigator', 'investor', 'invigilator', 'irreligionist', 'ivy leaguer', + 'jack of all trades', 'jacksonian', 'jane doe', 'janissary', 'jat', + 'javanese', 'jekyll and hyde', 'jester', 'jesuit', 'jezebel', 'jilt', + 'jobber', 'job candidate', 'job\'s comforter', 'jockey', 'john doe', + 'journalist', 'judge', 'judge advocate', 'juggler', 'jungian', 'junior', + 'junior', 'junior', 'junior lightweight', 'junior middleweight', 'jurist', + 'juror', 'justice of the peace', 'justiciar', 'kachina', 'keyboardist', + 'khedive', 'kingmaker', 'king', 'king\'s counsel', 'counsel to the crown', + 'kin', 'enate', 'kink', 'kinswoman', 'kisser', 'kitchen help', + 'kitchen police', 'klansman', 'kleptomaniac', 'kneeler', 'knight', + 'knocker', 'knower', 'know-it-all', 'kolkhoznik', 'kshatriya', + 'labor coach', 'laborer', 'labourite', 'lady', 'lady-in-waiting', + 'lady\'s maid', 'lama', 'lamb', 'lame duck', 'lamplighter', 'land agent', + 'landgrave', 'landlubber', 'landlubber', 'landowner', 'landscape architect', + 'langlaufer', 'languisher', 'lapidary', 'lass', 'latin', 'latin', + 'latitudinarian', 'jehovah\'s witness', 'law agent', 'lawgiver', 'lawman', + 'law student', 'lawyer', 'lay reader', 'lazybones', 'leaker', 'leaseholder', + 'lector', 'lector', 'lecturer', 'left-hander', 'legal representative', + 'legate', 'legatee', 'legionnaire', 'letterman', 'liberator', 'licenser', + 'licentiate', 'lieutenant', 'lieutenant colonel', 'lieutenant commander', + 'lieutenant junior grade', 'life', 'lifeguard', 'life tenant', + 'light flyweight', 'light heavyweight', 'light heavyweight', + 'light-o\'-love', 'lightweight', 'lightweight', 'lightweight', + 'lilliputian', 'limnologist', 'lineman', 'line officer', 'lion-hunter', + 'lisper', 'lister', 'literary critic', 'literate', 'litigant', 'litterer', + 'little brother', 'little sister', 'lobbyist', 'locksmith', 'locum tenens', + 'lord', 'loser', 'loser', 'failure', 'lothario', 'loudmouth', + 'lowerclassman', 'lowlander', 'loyalist', 'luddite', 'lumberman', 'lumper', + 'bedlamite', 'pyromaniac', 'lutist', 'lutheran', 'lyricist', 'macebearer', + 'machinist', 'madame', 'maenad', 'maestro', 'magdalen', 'magician', 'magus', + 'maharani', 'mahatma', 'maid', 'maid', 'major', 'major', 'major-domo', + 'maker', 'malahini', 'malcontent', 'malik', 'malingerer', 'malthusian', + 'adonis', 'man', 'man', 'manageress', 'mandarin', 'maneuverer', 'maniac', + 'manichaean', 'manicurist', 'manipulator', 'man-at-arms', 'man of action', + 'man of letters', 'manufacturer', 'marcher', 'marchioness', 'margrave', + 'margrave', 'marine', 'marquess', 'marquis', 'marshal', 'martinet', + 'mascot', 'masochist', 'mason', 'masquerader', 'masseur', 'masseuse', + 'master', 'master', 'master-at-arms', 'master of ceremonies', 'masturbator', + 'matchmaker', 'mate', 'mate', 'mate', 'mater', 'material', 'materialist', + 'matriarch', 'matriarch', 'matriculate', 'matron', 'mayor', 'mayoress', + 'mechanical engineer', 'medalist', 'medical officer', + 'medical practitioner', 'medical scientist', 'medium', 'megalomaniac', + 'melancholic', 'melkite', 'melter', 'nonmember', 'board member', 'clansman', + 'memorizer', 'mendelian', 'mender', 'mesoamerican', 'messmate', 'mestiza', + 'meteorologist', 'meter maid', 'methodist', 'metis', 'metropolitan', + 'mezzo-soprano', 'microeconomist', 'middle-aged man', 'middlebrow', + 'middleweight', 'midwife', 'mikado', 'milanese', 'miler', 'miles gloriosus', + 'military attache', 'military chaplain', 'military leader', + 'military officer', 'military policeman', 'mill agent', 'mill-hand', + 'millionairess', 'millwright', 'minder', 'mining engineer', 'minister', + 'ministrant', 'minor leaguer', 'minuteman', 'misanthrope', 'misfit', + 'mistress', 'mistress', 'mixed-blood', 'model', 'class act', 'modeler', + 'modifier', 'molecular biologist', 'monegasque', 'monetarist', + 'moneygrubber', 'moneymaker', 'mongoloid', 'monolingual', 'monologist', + 'moonlighter', 'moralist', 'morosoph', 'morris dancer', 'mortal enemy', + 'mortgagee', 'mortician', 'moss-trooper', 'mother', 'mother', 'mother', + 'mother figure', 'mother hen', 'mother-in-law', 'mother\'s boy', + 'mother\'s daughter', 'motorcycle cop', 'motorcyclist', 'mound builder', + 'mountebank', 'mourner', 'mouthpiece', 'mover', 'moviegoer', 'muffin man', + 'mugwump', 'mullah', 'muncher', 'murderess', 'murder suspect', 'musher', + 'musician', 'musicologist', 'music teacher', 'musketeer', 'muslimah', + 'mutilator', 'mutineer', 'mute', 'mutterer', 'muzzler', 'mycenaen', + 'mycologist', 'myope', 'myrmidon', 'mystic', 'mythologist', 'naif', + 'nailer', 'namby-pamby', 'name dropper', 'namer', 'nan', 'nanny', 'narc', + 'narcissist', 'nark', 'nationalist', 'nautch girl', 'naval commander', + 'navy seal', 'obstructionist', 'nazarene', 'nazarene', 'nazi', 'nebbish', + 'necker', 'neonate', 'nephew', 'neurobiologist', 'neurologist', + 'neurosurgeon', 'neutral', 'neutralist', 'newcomer', 'newcomer', + 'new dealer', 'newspaper editor', 'newsreader', 'newtonian', 'niece', + 'niggard', 'night porter', 'night rider', 'nimby', 'niqaabi', 'nitpicker', + 'nobelist', 'noc', 'noncandidate', 'noncommissioned officer', 'nondescript', + 'nondriver', 'nonparticipant', 'nonperson', 'nonresident', 'nonsmoker', + 'northern baptist', 'noticer', 'novelist', 'novitiate', 'nuclear chemist', + 'nudger', 'nullipara', 'number theorist', 'nurse', 'nursling', 'nymph', + 'nymphet', 'nympholept', 'nymphomaniac', 'oarswoman', 'oboist', + 'obscurantist', 'observer', 'obstetrician', 'occupier', 'occultist', + 'wine lover', 'offerer', 'office-bearer', 'office boy', 'officeholder', + 'officiant', 'federal', 'oilman', 'oil tycoon', 'old-age pensioner', + 'old boy', 'old lady', 'old man', 'oldster', 'old-timer', 'old woman', + 'oligarch', 'olympian', 'omnivore', 'oncologist', 'onlooker', 'onomancer', + 'operator', 'opportunist', 'optimist', 'orangeman', 'orator', 'orderly', + 'orderly', 'orderly sergeant', 'ordinand', 'ordinary', 'organ-grinder', + 'organist', 'organization man', 'organizer', 'organizer', 'originator', + 'ornithologist', 'orphan', 'orphan', 'osteopath', 'out-and-outer', + 'outdoorswoman', 'outfielder', 'outfielder', 'right fielder', + 'right-handed pitcher', 'outlier', 'owner-occupier', 'oyabun', 'packrat', + 'padrone', 'padrone', 'page', 'painter', 'paleo-american', 'paleontologist', + 'pallbearer', 'palmist', 'pamperer', 'panchen lama', 'panelist', + 'panhandler', 'paparazzo', 'paperboy', 'paperhanger', 'paperhanger', + 'papoose', 'pardoner', 'paretic', 'parishioner', 'park commissioner', + 'parliamentarian', 'parliamentary agent', 'parodist', 'parricide', 'parrot', + 'partaker', 'part-timer', 'party', 'party man', 'passenger', 'passer', + 'paster', 'pater', 'patient', 'patriarch', 'patriarch', 'patriarch', + 'patriot', 'patron', 'patternmaker', 'pawnbroker', 'payer', 'peacekeeper', + 'peasant', 'pedant', 'peddler', 'pederast', 'penologist', 'pentathlete', + 'pentecostal', 'percussionist', 'periodontist', 'peshmerga', 'personality', + 'personal representative', 'personage', 'persona grata', + 'persona non grata', 'personification', 'perspirer', 'pervert', 'pessimist', + 'pest', 'peter pan', 'petitioner', 'petit juror', 'pet sitter', 'petter', + 'pharaoh', 'pharmacist', 'philanthropist', 'philatelist', 'philosopher', + 'phonetician', 'phonologist', 'photojournalist', 'photometrist', + 'physical therapist', 'physicist', 'piano maker', 'picker', 'picnicker', + 'pilgrim', 'pill', 'pillar', 'pill head', 'pilot', 'piltdown man', 'pimp', + 'pipe smoker', 'pip-squeak', 'pisser', 'pitcher', 'pitchman', 'placeman', + 'placer miner', 'plagiarist', 'plainsman', 'planner', 'planter', + 'plasterer', 'platinum blond', 'platitudinarian', 'playboy', 'player', + 'playmate', 'pleaser', 'pledger', 'plenipotentiary', 'plier', 'plodder', + 'plodder', 'plotter', 'plumber', 'pluralist', 'pluralist', 'poet', + 'pointsman', 'point woman', 'policyholder', 'political prisoner', + 'political scientist', 'politician', 'politician', 'pollster', 'polluter', + 'pool player', 'portraitist', 'poseuse', 'positivist', 'postdoc', + 'poster girl', 'postulator', 'private citizen', 'problem solver', + 'pro-lifer', 'prosthetist', 'postulant', 'potboy', 'poultryman', + 'power user', 'power worker', 'practitioner', 'prayer', 'preceptor', + 'predecessor', 'preemptor', 'preemptor', 'premature baby', 'presbyter', + 'presenter', 'presentist', 'preserver', 'president', + 'president of the united states', 'president', 'press agent', + 'press photographer', 'priest', 'prima ballerina', 'prima donna', + 'prima donna', 'primigravida', 'primordial dwarf', 'prince charming', + 'prince consort', 'princeling', 'prince of wales', 'princess', + 'princess royal', 'principal', 'principal', 'print seller', 'prior', + 'private', 'probationer', 'processor', 'process-server', 'proconsul', + 'proconsul', 'proctologist', 'proctor', 'procurator', 'procurer', + 'profit taker', 'programmer', 'promiser', 'promoter', 'promulgator', + 'propagandist', 'propagator', 'property man', 'prophetess', 'prophet', + 'prosecutor', 'prospector', 'protectionist', 'protegee', 'protozoologist', + 'provost marshal', 'pruner', 'psalmist', 'psephologist', 'psychiatrist', + 'psychic', 'psycholinguist', 'psychophysicist', 'publican', 'pudge', + 'puerpera', 'punching bag', 'punter', 'punter', 'puppeteer', 'puppy', + 'purchasing agent', 'puritan', 'puritan', 'pursuer', 'pusher', 'pusher', + 'pusher', 'putz', 'pygmy', 'qadi', 'quadriplegic', 'quadruplet', 'quaker', + 'quarter', 'quarterback', 'quartermaster', 'quartermaster general', + 'quebecois', 'queen', 'queen of england', 'queen', 'queen', 'queen consort', + 'queen mother', 'queen\'s counsel', 'question master', 'quick study', + 'quietist', 'quitter', 'rabbi', 'racist', 'radiobiologist', + 'radiologic technologist', 'radiologist', 'rainmaker', 'raiser', 'raja', + 'rake', 'ramrod', 'ranch hand', 'ranker', 'ranter', 'rape suspect', + 'rapper', 'rapporteur', 'rare bird', 'ratepayer', 'raw recruit', 'reader', + 'reading teacher', 'realist', 'real estate broker', 'rear admiral', + 'receiver', 'reciter', 'recruit', 'recruit', 'recruiter', + 'recruiting-sergeant', 'redcap', 'redhead', 'redneck', 'reeler', + 'reenactor', 'referral', 'referee', 'refiner', 'reform jew', + 'registered nurse', 'registrar', 'regius professor', 'reliever', + 'anchorite', 'religious leader', 'remover', 'renaissance man', 'renegade', + 'rentier', 'repairman', 'reporter', 'newswoman', 'representative', + 'reprobate', 'rescuer', 'reservist', 'resident commissioner', 'respecter', + 'restaurateur', 'restrainer', 'retailer', 'retiree', 'returning officer', + 'revenant', 'revisionist', 'revolutionist', 'rheumatologist', + 'rhodesian man', 'rhymer', 'rich person', 'rider', 'riding master', + 'rifleman', 'right-hander', 'right-hand man', 'ringer', 'ringleader', + 'roadman', 'roarer', 'rocket engineer', 'rocket scientist', 'rock star', + 'romanov', 'romanticist', 'ropemaker', 'roper', 'roper', 'ropewalker', + 'rosebud', 'rosicrucian', 'mountie', 'rough rider', 'roundhead', + 'civil authority', 'runner', 'runner', 'runner', 'running back', 'rusher', + 'rustic', 'saboteur', 'sadist', 'sailing master', 'sailor', 'salesgirl', + 'salesman', 'salesperson', 'salvager', 'sandwichman', 'sangoma', 'sannup', + 'sapper', 'sassenach', 'satrap', 'saunterer', 'savoyard', 'sawyer', + 'scalper', 'scandalmonger', 'scapegrace', 'scene painter', 'schemer', + 'schizophrenic', 'schlemiel', 'schlockmeister', 'scholar', 'scholiast', + 'schoolchild', 'schoolfriend', 'schoolman', 'schoolmaster', 'schoolmate', + 'scientist', 'scion', 'scoffer', 'scofflaw', 'scorekeeper', 'scorer', + 'scourer', 'scout', 'scoutmaster', 'scrambler', 'scratcher', 'screen actor', + 'scrutineer', 'scuba diver', 'sculptor', 'sea scout', 'seasonal worker', + 'seasoner', 'second baseman', 'second cousin', 'seconder', 'second fiddle', + 'second-in-command', 'second lieutenant', 'second-rater', 'secretary', + 'secretary of agriculture', 'secretary of health and human services', + 'secretary of state', 'secretary of the interior', 'sectarian', + 'section hand', 'secularist', 'security consultant', 'seeded player', + 'seeder', 'seeker', 'segregate', 'segregator', 'selectman', 'selectwoman', + 'selfish person', 'self-starter', 'seller', 'selling agent', 'semanticist', + 'semifinalist', 'seminarian', 'senator', 'sendee', 'senior', + 'senior vice president', 'separatist', 'septuagenarian', 'serf', + 'spree killer', 'serjeant-at-law', 'server', 'serviceman', 'settler', + 'settler', 'sex symbol', 'sexton', 'shaheed', 'shakespearian', 'shanghaier', + 'sharecropper', 'shaver', 'shavian', 'sheep', 'sheik', 'shelver', + 'shepherd', 'ship-breaker', 'shipmate', 'shipowner', 'shipping agent', + 'shirtmaker', 'shogun', 'shopaholic', 'shop girl', 'shop steward', + 'shot putter', 'shrew', 'shuffler', 'shyster', 'sibling', 'sick person', + 'sightreader', 'signaler', 'signer', 'signor', 'signora', 'signore', + 'signorina', 'silent partner', 'addle-head', 'simperer', 'singer', + 'sinologist', 'sipper', 'sirrah', 'sister', 'sister', 'waverer', + 'sitar player', 'sixth-former', 'skateboarder', 'skeptic', 'sketcher', + 'skidder', 'skier', 'skinny-dipper', 'skin-diver', 'skinhead', 'slasher', + 'slattern', 'sleeper', 'sleeper', 'sleeping beauty', 'sleuth', 'slob', + 'sloganeer', 'slopseller', 'smasher', 'smirker', 'smith', 'smoothie', + 'smuggler', 'sneezer', 'snob', 'snoop', 'snorer', 'sob sister', + 'soccer player', 'social anthropologist', 'social climber', 'socialist', + 'socializer', 'social scientist', 'social secretary', 'socinian', + 'sociolinguist', 'sociologist', 'soda jerk', 'sodalist', 'sodomite', + 'soldier', 'son', 'songster', 'songstress', 'songwriter', 'sorcerer', + 'sorehead', 'soul mate', 'southern baptist', 'sovereign', 'spacewalker', + 'spanish american', 'sparring partner', 'spastic', 'speaker', + 'native speaker', 'speaker', 'speechwriter', 'specialist', 'specifier', + 'spectator', 'speech therapist', 'speedskater', 'spellbinder', 'sphinx', + 'spinster', 'split end', 'sport', 'sport', 'sporting man', + 'sports announcer', 'sports editor', 'sprog', 'square dancer', + 'square shooter', 'squatter', 'squire', 'squire', 'staff member', + 'staff sergeant', 'stage director', 'stainer', 'stakeholder', 'stalker', + 'stalking-horse', 'stammerer', 'stamper', 'standee', 'stand-in', 'star', + 'starlet', 'starter', 'statesman', 'state treasurer', 'stationer', + 'stenographer', 'stentor', 'stepbrother', 'stepmother', 'stepparent', + 'stevedore', 'steward', 'steward', 'steward', 'stickler', 'stiff', + 'stifler', 'stipendiary', 'stitcher', 'stockjobber', 'stock trader', + 'stockist', 'stoker', 'stooper', 'store detective', 'strafer', + 'straight man', 'stranger', 'stranger', 'strategist', 'straw boss', + 'streetwalker', 'stretcher-bearer', 'struggler', 'stud', 'student', + 'stumblebum', 'stylist', 'subaltern', 'subcontractor', 'subduer', 'subject', + 'subordinate', 'substitute', 'successor', 'successor', 'succorer', 'sufi', + 'suffragan', 'suffragette', 'sugar daddy', 'suicide bomber', 'suitor', + 'sumo wrestler', 'sunbather', 'sundowner', 'super heavyweight', 'superior', + 'supermom', 'supernumerary', 'supremo', 'surgeon', 'surgeon general', + 'surgeon general', 'surpriser', 'surveyor', 'surveyor', 'survivor', + 'sutler', 'sweeper', 'sweetheart', 'swinger', 'switcher', 'swot', + 'sycophant', 'sylph', 'sympathizer', 'symphonist', 'syncopator', 'syndic', + 'tactician', 'tagger', 'tailback', 'tallyman', 'tallyman', 'tanker', + 'tapper', 'tartuffe', 'tarzan', 'taster', 'tax assessor', 'taxer', + 'taxi dancer', 'taxonomist', 'teacher', 'teaching fellow', 'tearaway', + 'technical sergeant', 'technician', 'ted', 'teetotaler', + 'television reporter', 'temporizer', 'tempter', 'term infant', 'toiler', + 'tenant', 'tenant', 'tenderfoot', 'tennis player', 'tennis pro', + 'tenor saxophonist', 'termer', 'terror', 'tertigravida', 'testator', + 'testatrix', 'testee', 'test-tube baby', 'texas ranger', 'thane', + 'theatrical producer', 'theologian', 'theorist', 'theosophist', 'therapist', + 'thessalonian', 'thinker', 'thinker', 'thrower', 'thurifer', + 'ticket collector', 'tight end', 'tiler', 'timekeeper', 'timorese', + 'tinkerer', 'tinsmith', 'tinter', 'tippler', 'tipster', 't-man', + 'toastmaster', 'toast mistress', 'tobogganist', 'tomboy', 'toolmaker', + 'torchbearer', 'tory', 'tory', 'tosser', 'tosser', 'totalitarian', + 'tourist', 'tout', 'tout', 'tovarich', 'towhead', 'town clerk', + 'town crier', 'townsman', 'toxicologist', 'track star', 'trader', + 'trade unionist', 'traditionalist', 'traffic cop', 'tragedian', 'tragedian', + 'tragedienne', 'trail boss', 'trainer', 'traitor', 'traitress', + 'transactor', 'transcriber', 'transfer', 'transferee', 'translator', + 'transvestite', 'traveling salesman', 'traverser', 'trawler', 'treasury', + 'trencher', 'trend-setter', 'tribesman', 'trier', 'trifler', 'trooper', + 'trooper', 'trotskyite', 'truant', 'trumpeter', 'trusty', 'tudor', + 'tumbler', 'tutee', 'twin', 'two-timer', 'tyke', 'tympanist', 'typist', + 'tyrant', 'umpire', 'understudy', 'undesirable', 'unicyclist', + 'unilateralist', 'unitarian', 'arminian', 'universal donor', 'unix guru', + 'unknown soldier', 'upsetter', 'upstager', 'upstart', 'upstart', 'urchin', + 'urologist', 'usherette', 'usher', 'usurper', 'utility man', 'utilizer', + 'utopian', 'uxoricide', 'vacationer', 'valedictorian', 'valley girl', + 'vaulter', 'vegetarian', 'vegan', 'venerator', 'venture capitalist', + 'venturer', 'vermin', 'very important person', 'vibist', 'vicar', 'vicar', + 'vicar-general', 'vice chancellor', 'vicegerent', 'vice president', + 'vice-regent', 'victim', 'victorian', 'victualer', 'vigilante', 'villager', + 'vintager', 'vintner', 'violator', 'violator', 'violist', 'virago', + 'virologist', 'visayan', 'viscountess', 'viscount', 'visigoth', 'visionary', + 'visiting fireman', 'visiting professor', 'visualizer', 'vixen', 'vizier', + 'voicer', 'volunteer', 'volunteer', 'votary', 'votary', 'vouchee', 'vower', + 'voyager', 'voyeur', 'vulcanizer', 'waffler', 'wagnerian', 'waif', 'wailer', + 'waiter', 'waitress', 'walking delegate', 'walk-on', 'wallah', 'wally', + 'waltzer', 'wanderer', 'wandering jew', 'wanton', 'warrantee', 'warrantee', + 'washer', 'washerman', 'washwoman', 'wassailer', 'wastrel', 'wave', + 'weatherman', 'weekend warrior', 'weeder', 'welder', 'welfare case', + 'westerner', 'west-sider', 'wetter', 'whaler', 'whig', 'whiner', + 'whipper-in', 'whisperer', 'whiteface', 'carmelite', 'augustinian', + 'white hope', 'white supremacist', 'whoremaster', 'whoremaster', 'widow', + 'wife', 'wiggler', 'wimp', 'wing commander', 'winger', 'winner', 'winner', + 'window dresser', 'winker', 'wiper', 'wireman', 'wise guy', 'witch doctor', + 'withdrawer', 'withdrawer', 'woman', 'woman', 'wonder boy', 'wonderer', + 'working girl', 'workman', 'workmate', 'worldling', 'worshiper', 'worthy', + 'wrecker', 'wright', 'write-in candidate', 'writer', 'wykehamist', 'yakuza', + 'yard bird', 'yardie', 'yardman', 'yardmaster', 'yenta', 'yogi', + 'young buck', 'young turk', 'young turk', 'zionist', 'zoo keeper', 'genet', + 'kennan', 'munro', 'popper', 'stoker', 'townes', 'dust storm', 'parhelion', + 'snow', 'facula', 'wave', 'microflora', 'wilding', 'semi-climber', 'volva', + 'basidiocarp', 'domatium', 'apomict', 'aquatic', 'bryophyte', 'acrocarp', + 'sphagnum', 'liverwort', 'hepatica', 'pecopteris', 'pteridophyte', 'fern', + 'fern ally', 'spore', 'carpospore', 'chlamydospore', 'conidium', 'oospore', + 'tetraspore', 'zoospore', 'cryptogam', 'spermatophyte', 'seedling', + 'annual', 'biennial', 'perennial', 'hygrophyte', 'gymnosperm', 'gnetum', + 'catha edulis', 'ephedra', 'mahuang', 'welwitschia', 'cycad', 'sago palm', + 'false sago', 'zamia', 'coontie', 'ceratozamia', 'dioon', 'encephalartos', + 'kaffir bread', 'macrozamia', 'burrawong', 'pine', 'pinon', 'nut pine', + 'pinon pine', 'rocky mountain pinon', 'single-leaf', 'bishop pine', + 'california single-leaf pinyon', 'parry\'s pinyon', 'spruce pine', + 'black pine', 'pitch pine', 'pond pine', 'stone pine', 'swiss pine', + 'cembra nut', 'swiss mountain pine', 'ancient pine', 'white pine', + 'american white pine', 'western white pine', 'southwestern white pine', + 'limber pine', 'whitebark pine', 'yellow pine', 'ponderosa', 'jeffrey pine', + 'shore pine', 'sierra lodgepole pine', 'loblolly pine', 'jack pine', + 'swamp pine', 'longleaf pine', 'shortleaf pine', 'red pine', 'scotch pine', + 'scrub pine', 'monterey pine', 'bristlecone pine', 'table-mountain pine', + 'knobcone pine', 'japanese red pine', 'japanese black pine', 'torrey pine', + 'larch', 'american larch', 'western larch', 'subalpine larch', + 'european larch', 'siberian larch', 'golden larch', 'fir', 'silver fir', + 'amabilis fir', 'european silver fir', 'white fir', 'balsam fir', + 'fraser fir', 'lowland fir', 'alpine fir', 'santa lucia fir', 'cedar', + 'cedar of lebanon', 'deodar', 'atlas cedar', 'spruce', 'norway spruce', + 'weeping spruce', 'engelmann spruce', 'white spruce', 'black spruce', + 'siberian spruce', 'sitka spruce', 'oriental spruce', 'colorado spruce', + 'red spruce', 'hemlock', 'eastern hemlock', 'carolina hemlock', + 'mountain hemlock', 'western hemlock', 'douglas fir', 'green douglas fir', + 'big-cone spruce', 'cathaya', 'cedar', 'cypress', 'gowen cypress', + 'pygmy cypress', 'santa cruz cypress', 'arizona cypress', + 'guadalupe cypress', 'monterey cypress', 'mexican cypress', + 'italian cypress', 'king william pine', 'chilean cedar', 'incense cedar', + 'southern white cedar', 'oregon cedar', 'yellow cypress', 'japanese cedar', + 'juniper berry', 'incense cedar', 'kawaka', 'pahautea', 'metasequoia', + 'arborvitae', 'western red cedar', 'american arborvitae', + 'oriental arborvitae', 'hiba arborvitae', 'keteleeria', 'wollemi pine', + 'araucaria', 'monkey puzzle', 'norfolk island pine', 'new caledonian pine', + 'bunya bunya', 'hoop pine', 'kauri pine', 'kauri', 'amboina pine', + 'dundathu pine', 'red kauri', 'plum-yew', 'california nutmeg', + 'stinking cedar', 'celery pine', 'celery top pine', 'tanekaha', + 'alpine celery pine', 'yellowwood', 'gymnospermous yellowwood', 'podocarp', + 'yacca', 'brown pine', 'cape yellowwood', 'south-african yellowwood', + 'alpine totara', 'totara', 'common yellowwood', 'kahikatea', 'rimu', + 'tarwood', 'common sickle pine', 'yellow-leaf sickle pine', 'tarwood', + 'westland pine', 'huon pine', 'chilean rimu', 'mountain rimu', 'nagi', + 'miro', 'matai', 'plum-fruited yew', 'prince albert yew', + 'sundacarpus amara', 'japanese umbrella pine', 'yew', 'old world yew', + 'pacific yew', 'japanese yew', 'florida yew', 'new caledonian yew', + 'white-berry yew', 'ginkgo', 'angiosperm', 'dicot', 'monocot', 'floret', + 'flower', 'bloomer', 'wildflower', 'apetalous flower', 'inflorescence', + 'rosebud', 'gynostegium', 'pollinium', 'pistil', 'gynobase', 'gynophore', + 'stylopodium', 'carpophore', 'cornstalk', 'petiolule', 'mericarp', + 'micropyle', 'germ tube', 'pollen tube', 'gemma', 'galbulus', 'nectary', + 'pericarp', 'epicarp', 'mesocarp', 'pip', 'silique', 'cataphyll', + 'perisperm', 'monocarp', 'sporophyte', 'gametophyte', 'megasporangium', + 'microspore', 'microsporangium', 'microsporophyll', 'archespore', + 'bonduc nut', 'job\'s tears', 'oilseed', 'castor bean', 'cottonseed', + 'candlenut', 'peach pit', 'hypanthium', 'petal', 'corolla', 'lip', + 'perianth', 'thistledown', 'custard apple', 'cherimoya', 'ilama', 'soursop', + 'bullock\'s heart', 'sweetsop', 'pond apple', 'pawpaw', 'ilang-ilang', + 'lancewood', 'guinea pepper', 'barberry', 'american barberry', + 'common barberry', 'japanese barberry', 'oregon grape', 'oregon grape', + 'mayapple', 'may apple', 'allspice', 'carolina allspice', 'spicebush', + 'katsura tree', 'laurel', 'true laurel', 'camphor tree', 'cinnamon', + 'cassia', 'cassia bark', 'saigon cinnamon', 'cinnamon bark', 'spicebush', + 'avocado', 'laurel-tree', 'sassafras', 'california laurel', 'anise tree', + 'purple anise', 'star anise', 'star anise', 'magnolia', 'southern magnolia', + 'umbrella tree', 'earleaved umbrella tree', 'cucumber tree', + 'large-leaved magnolia', 'saucer magnolia', 'star magnolia', 'sweet bay', + 'manglietia', 'tulip tree', 'moonseed', 'common moonseed', + 'carolina moonseed', 'nutmeg', 'water nymph', 'european white lily', + 'southern spatterdock', 'lotus', 'water chinquapin', 'water-shield', + 'water-shield', 'peony', 'buttercup', 'meadow buttercup', 'water crowfoot', + 'lesser celandine', 'lesser spearwort', 'greater spearwort', + 'western buttercup', 'creeping buttercup', 'cursed crowfoot', 'aconite', + 'monkshood', 'wolfsbane', 'baneberry', 'baneberry', 'red baneberry', + 'pheasant\'s-eye', 'anemone', 'alpine anemone', 'canada anemone', + 'thimbleweed', 'wood anemone', 'wood anemone', 'longheaded thimbleweed', + 'snowdrop anemone', 'virginia thimbleweed', 'rue anemone', 'columbine', + 'meeting house', 'blue columbine', 'granny\'s bonnets', 'marsh marigold', + 'american bugbane', 'black cohosh', 'fetid bugbane', 'clematis', + 'pine hyacinth', 'blue jasmine', 'golden clematis', 'scarlet clematis', + 'leather flower', 'leather flower', 'virgin\'s bower', 'purple clematis', + 'goldthread', 'rocket larkspur', 'delphinium', 'larkspur', 'winter aconite', + 'lenten rose', 'green hellebore', 'hepatica', 'goldenseal', + 'false rue anemone', 'giant buttercup', 'nigella', 'love-in-a-mist', + 'fennel flower', 'black caraway', 'pasqueflower', 'meadow rue', + 'false bugbane', 'globeflower', 'winter\'s bark', 'pepper shrub', + 'sweet gale', 'wax myrtle', 'bay myrtle', 'bayberry', 'sweet fern', + 'corkwood', 'jointed rush', 'toad rush', 'slender rush', 'zebrawood', + 'connarus guianensis', 'legume', 'legume', 'peanut', 'granadilla tree', + 'arariba', 'tonka bean', 'courbaril', 'melilotus', 'darling pea', + 'smooth darling pea', 'clover', 'alpine clover', 'hop clover', + 'crimson clover', 'red clover', 'buffalo clover', 'white clover', 'mimosa', + 'acacia', 'shittah', 'wattle', 'black wattle', 'gidgee', 'catechu', + 'silver wattle', 'huisache', 'lightwood', 'golden wattle', 'fever tree', + 'coralwood', 'albizzia', 'silk tree', 'siris', 'rain tree', 'calliandra', + 'conacaste', 'inga', 'ice-cream bean', 'guama', 'lead tree', + 'wild tamarind', 'sabicu', 'nitta tree', 'parkia javanica', + 'manila tamarind', 'cat\'s-claw', 'honey mesquite', 'algarroba', + 'screw bean', 'screw bean', 'dogbane', 'indian hemp', 'bushman\'s poison', + 'impala lily', 'allamanda', 'common allamanda', 'dita', + 'nepal trumpet flower', 'carissa', 'hedge thorn', 'natal plum', + 'periwinkle', 'ivory tree', 'white dipladenia', 'chilean jasmine', + 'oleander', 'frangipani', 'west indian jasmine', 'rauwolfia', 'snakewood', + 'strophanthus kombe', 'yellow oleander', 'myrtle', 'large periwinkle', + 'arum', 'cuckoopint', 'black calla', 'calamus', 'alocasia', 'giant taro', + 'amorphophallus', 'pungapung', 'devil\'s tongue', 'anthurium', + 'flamingo flower', 'jack-in-the-pulpit', 'friar\'s-cowl', 'caladium', + 'caladium bicolor', 'wild calla', 'taro', 'taro', 'cryptocoryne', + 'dracontium', 'golden pothos', 'skunk cabbage', 'monstera', 'ceriman', + 'nephthytis', 'nephthytis afzelii', 'arrow arum', 'green arrow arum', + 'philodendron', 'pistia', 'pothos', 'spathiphyllum', 'skunk cabbage', + 'yautia', 'calla lily', 'pink calla', 'golden calla', 'duckweed', + 'common duckweed', 'star-duckweed', 'great duckweed', 'watermeal', + 'common wolffia', 'aralia', 'american angelica tree', 'american spikenard', + 'bristly sarsaparilla', 'japanese angelica tree', 'chinese angelica', 'ivy', + 'puka', 'ginseng', 'ginseng', 'umbrella tree', 'birthwort', + 'dutchman\'s-pipe', 'virginia snakeroot', 'canada ginger', 'heartleaf', + 'heartleaf', 'asarabacca', 'caryophyllaceous plant', 'corn cockle', + 'sandwort', 'mountain sandwort', 'pine-barren sandwort', + 'seabeach sandwort', 'rock sandwort', 'thyme-leaved sandwort', + 'mouse-ear chickweed', 'snow-in-summer', 'alpine mouse-ear', 'pink', + 'sweet william', 'carnation', 'china pink', 'japanese pink', 'maiden pink', + 'cheddar pink', 'button pink', 'cottage pink', 'fringed pink', 'drypis', + 'baby\'s breath', 'coral necklace', 'lychnis', 'ragged robin', + 'scarlet lychnis', 'mullein pink', 'sandwort', 'sandwort', 'soapwort', + 'knawel', 'silene', 'moss campion', 'wild pink', 'red campion', + 'white campion', 'fire pink', 'bladder campion', 'corn spurry', + 'sand spurry', 'chickweed', 'common chickweed', 'cowherb', 'hottentot fig', + 'livingstone daisy', 'fig marigold', 'ice plant', 'new zealand spinach', + 'amaranth', 'amaranth', 'tumbleweed', 'prince\'s-feather', 'pigweed', + 'thorny amaranth', 'alligator weed', 'cockscomb', 'cottonweed', + 'globe amaranth', 'bloodleaf', 'saltwort', 'lamb\'s-quarters', + 'good-king-henry', 'jerusalem oak', 'oak-leaved goosefoot', 'sowbane', + 'nettle-leaved goosefoot', 'red goosefoot', 'stinking goosefoot', 'orach', + 'saltbush', 'garden orache', 'desert holly', 'quail bush', 'beet', + 'beetroot', 'chard', 'mangel-wurzel', 'winged pigweed', 'halogeton', + 'glasswort', 'saltwort', 'russian thistle', 'greasewood', + 'scarlet musk flower', 'sand verbena', 'sweet sand verbena', + 'yellow sand verbena', 'beach pancake', 'beach sand verbena', + 'desert sand verbena', 'trailing four o\'clock', 'bougainvillea', + 'umbrellawort', 'four o\'clock', 'common four-o\'clock', + 'california four o\'clock', 'sweet four o\'clock', 'desert four o\'clock', + 'mountain four o\'clock', 'cockspur', 'rattail cactus', 'saguaro', + 'night-blooming cereus', 'echinocactus', 'hedgehog cactus', + 'golden barrel cactus', 'hedgehog cereus', 'rainbow cactus', 'epiphyllum', + 'barrel cactus', 'night-blooming cereus', 'chichipe', 'mescal', + 'mescal button', 'mammillaria', 'feather ball', 'garambulla', + 'knowlton\'s cactus', 'nopal', 'prickly pear', 'cholla', 'nopal', 'tuna', + 'barbados gooseberry', 'mistletoe cactus', 'christmas cactus', + 'night-blooming cereus', 'crab cactus', 'pokeweed', 'indian poke', 'poke', + 'ombu', 'bloodberry', 'portulaca', 'rose moss', 'common purslane', + 'rock purslane', 'red maids', 'carolina spring beauty', 'spring beauty', + 'virginia spring beauty', 'siskiyou lewisia', 'bitterroot', + 'broad-leaved montia', 'blinks', 'toad lily', 'winter purslane', + 'flame flower', 'pigmy talinum', 'jewels-of-opar', 'caper', + 'native pomegranate', 'caper tree', 'caper tree', 'common caper', + 'spiderflower', 'rocky mountain bee plant', 'clammyweed', 'crucifer', + 'cress', 'watercress', 'stonecress', 'garlic mustard', 'alyssum', + 'rose of jericho', 'arabidopsis thaliana', 'arabidopsis lyrata', + 'rock cress', 'sicklepod', 'tower mustard', 'horseradish', 'winter cress', + 'yellow rocket', 'hoary alison', 'buckler mustard', 'wild cabbage', + 'cabbage', 'head cabbage', 'savoy cabbage', 'brussels sprout', + 'cauliflower', 'broccoli', 'collard', 'kohlrabi', 'turnip plant', 'turnip', + 'rutabaga', 'broccoli raab', 'mustard', 'chinese mustard', 'bok choy', + 'rape', 'rapeseed', 'shepherd\'s purse', 'lady\'s smock', + 'coral-root bittercress', 'crinkleroot', 'american watercress', + 'spring cress', 'purple cress', 'wallflower', 'prairie rocket', + 'scurvy grass', 'sea kale', 'tansy mustard', 'draba', 'wallflower', + 'prairie rocket', 'siberian wall flower', 'western wall flower', + 'wormseed mustard', 'heliophila', 'damask violet', 'tansy-leaved rocket', + 'candytuft', 'woad', 'dyer\'s woad', 'bladderpod', 'sweet alyssum', + 'malcolm stock', 'virginian stock', 'stock', 'brompton stock', 'bladderpod', + 'chamois cress', 'radish plant', 'jointed charlock', 'radish', 'radish', + 'marsh cress', 'great yellowcress', 'schizopetalon', 'field mustard', + 'hedge mustard', 'desert plume', 'pennycress', 'field pennycress', + 'fringepod', 'bladderpod', 'wasabi', 'poppy', 'iceland poppy', + 'western poppy', 'prickly poppy', 'iceland poppy', 'oriental poppy', + 'corn poppy', 'opium poppy', 'prickly poppy', 'mexican poppy', 'bocconia', + 'celandine', 'corydalis', 'climbing corydalis', 'california poppy', + 'horn poppy', 'golden cup', 'plume poppy', 'blue poppy', 'welsh poppy', + 'creamcups', 'matilija poppy', 'wind poppy', 'celandine poppy', + 'climbing fumitory', 'bleeding heart', 'dutchman\'s breeches', + 'squirrel corn', 'composite', 'compass plant', 'everlasting', 'achillea', + 'yarrow', 'pink-and-white everlasting', 'white snakeroot', 'ageratum', + 'common ageratum', 'sweet sultan', 'ragweed', 'common ragweed', + 'great ragweed', 'western ragweed', 'ammobium', 'winged everlasting', + 'pellitory', 'pearly everlasting', 'andryala', 'plantain-leaved pussytoes', + 'field pussytoes', 'solitary pussytoes', 'mountain everlasting', 'mayweed', + 'yellow chamomile', 'corn chamomile', 'woolly daisy', 'burdock', + 'great burdock', 'african daisy', 'blue-eyed african daisy', 'marguerite', + 'silversword', 'arnica', 'heartleaf arnica', 'arnica montana', + 'lamb succory', 'artemisia', 'mugwort', 'sweet wormwood', 'field wormwood', + 'tarragon', 'sand sage', 'wormwood sage', 'western mugwort', + 'roman wormwood', 'bud brush', 'common mugwort', 'aster', 'wood aster', + 'whorled aster', 'heath aster', 'heart-leaved aster', 'white wood aster', + 'bushy aster', 'heath aster', 'white prairie aster', 'stiff aster', + 'goldilocks', 'large-leaved aster', 'new england aster', 'michaelmas daisy', + 'upland white aster', 'short\'s aster', 'sea aster', 'prairie aster', + 'annual salt-marsh aster', 'aromatic aster', 'arrow leaved aster', + 'azure aster', 'bog aster', 'crooked-stemmed aster', + 'eastern silvery aster', 'flat-topped white aster', 'late purple aster', + 'panicled aster', 'perennial salt marsh aster', 'purple-stemmed aster', + 'rough-leaved aster', 'rush aster', 'schreiber\'s aster', + 'small white aster', 'smooth aster', 'southern aster', 'starved aster', + 'tradescant\'s aster', 'wavy-leaved aster', 'western silvery aster', + 'willow aster', 'ayapana', 'mule fat', 'balsamroot', 'daisy', + 'common daisy', 'bur marigold', 'spanish needles', 'tickseed sunflower', + 'european beggar-ticks', 'slender knapweed', 'false chamomile', + 'swan river daisy', 'woodland oxeye', 'indian plantain', 'calendula', + 'common marigold', 'china aster', 'thistle', 'welted thistle', + 'musk thistle', 'carline thistle', 'stemless carline thistle', + 'common carline thistle', 'safflower', 'safflower seed', 'catananche', + 'blue succory', 'centaury', 'dusty miller', 'cornflower', 'star-thistle', + 'knapweed', 'sweet sultan', 'great knapweed', 'barnaby\'s thistle', + 'chamomile', 'chaenactis', 'chrysanthemum', 'corn marigold', 'crown daisy', + 'chop-suey greens', 'golden aster', 'maryland golden aster', 'goldenbush', + 'rabbit brush', 'chicory', 'endive', 'chicory', 'plume thistle', + 'canada thistle', 'field thistle', 'woolly thistle', + 'european woolly thistle', 'melancholy thistle', 'brook thistle', + 'bull thistle', 'blessed thistle', 'mistflower', 'horseweed', 'coreopsis', + 'giant coreopsis', 'sea dahlia', 'calliopsis', 'cosmos', 'brass buttons', + 'billy buttons', 'hawk\'s-beard', 'artichoke', 'cardoon', 'dahlia', + 'german ivy', 'florist\'s chrysanthemum', 'cape marigold', + 'leopard\'s-bane', 'coneflower', 'globe thistle', 'elephant\'s-foot', + 'tassel flower', 'brittlebush', 'sunray', 'engelmannia', 'fireweed', + 'fleabane', 'blue fleabane', 'daisy fleabane', 'orange daisy', + 'spreading fleabane', 'seaside daisy', 'philadelphia fleabane', + 'robin\'s plantain', 'showy daisy', 'woolly sunflower', 'golden yarrow', + 'dog fennel', 'joe-pye weed', 'boneset', 'joe-pye weed', 'blue daisy', + 'kingfisher daisy', 'cotton rose', 'herba impia', 'gaillardia', 'gazania', + 'treasure flower', 'african daisy', 'barberton daisy', 'desert sunflower', + 'cudweed', 'chafeweed', 'gumweed', 'grindelia robusta', 'curlycup gumweed', + 'little-head snakeweed', 'rabbitweed', 'broomweed', 'velvet plant', + 'goldenbush', 'camphor daisy', 'yellow spiny daisy', 'hoary golden bush', + 'sneezeweed', 'orange sneezeweed', 'rosilla', 'sunflower', + 'swamp sunflower', 'common sunflower', 'giant sunflower', 'showy sunflower', + 'maximilian\'s sunflower', 'prairie sunflower', 'jerusalem artichoke', + 'jerusalem artichoke', 'strawflower', 'heliopsis', 'strawflower', + 'hairy golden aster', 'hawkweed', 'rattlesnake weed', 'alpine coltsfoot', + 'alpine gold', 'dwarf hulsea', 'cat\'s-ear', 'inula', 'marsh elder', + 'burweed marsh elder', 'krigia', 'dwarf dandelion', 'garden lettuce', + 'cos lettuce', 'leaf lettuce', 'celtuce', 'prickly lettuce', 'goldfields', + 'tidytips', 'hawkbit', 'fall dandelion', 'edelweiss', 'oxeye daisy', + 'oxeye daisy', 'shasta daisy', 'pyrenees daisy', 'north island edelweiss', + 'blazing star', 'dotted gayfeather', 'dense blazing star', 'texas star', + 'african daisy', 'tahoka daisy', 'sticky aster', 'mojave aster', 'tarweed', + 'sweet false chamomile', 'pineapple weed', 'climbing hempweed', 'mutisia', + 'rattlesnake root', 'white lettuce', 'daisybush', 'new zealand daisybush', + 'cotton thistle', 'othonna', 'cascade everlasting', 'butterweed', + 'american feverfew', 'cineraria', 'florest\'s cineraria', 'butterbur', + 'winter heliotrope', 'sweet coltsfoot', 'oxtongue', 'hawkweed', + 'mouse-ear hawkweed', 'stevia', 'rattlesnake root', 'fleabane', + 'sheep plant', 'coneflower', 'mexican hat', 'long-head coneflower', + 'prairie coneflower', 'swan river everlasting', 'coneflower', + 'black-eyed susan', 'cutleaved coneflower', 'golden glow', + 'lavender cotton', 'creeping zinnia', 'golden thistle', + 'spanish oyster plant', 'nodding groundsel', 'dusty miller', 'butterweed', + 'ragwort', 'arrowleaf groundsel', 'black salsify', 'white-topped aster', + 'narrow-leaved white-topped aster', 'silver sage', 'sea wormwood', + 'sawwort', 'rosinweed', 'milk thistle', 'goldenrod', 'silverrod', + 'meadow goldenrod', 'missouri goldenrod', 'alpine goldenrod', + 'grey goldenrod', 'blue mountain tea', 'dyer\'s weed', 'seaside goldenrod', + 'narrow goldenrod', 'boott\'s goldenrod', 'elliott\'s goldenrod', + 'ohio goldenrod', 'rough-stemmed goldenrod', 'showy goldenrod', + 'tall goldenrod', 'zigzag goldenrod', 'sow thistle', 'milkweed', 'stevia', + 'stokes\' aster', 'marigold', 'african marigold', 'french marigold', + 'painted daisy', 'pyrethrum', 'northern dune tansy', 'feverfew', + 'dusty miller', 'tansy', 'dandelion', 'common dandelion', 'dandelion green', + 'russian dandelion', 'stemless hymenoxys', 'mexican sunflower', + 'easter daisy', 'yellow salsify', 'salsify', 'meadow salsify', + 'scentless camomile', 'turfing daisy', 'coltsfoot', 'ursinia', 'crownbeard', + 'wingstem', 'cowpen daisy', 'gravelweed', 'virginia crownbeard', 'ironweed', + 'mule\'s ears', 'white-rayed mule\'s ears', 'cocklebur', 'xeranthemum', + 'immortelle', 'zinnia', 'white zinnia', 'little golden zinnia', + 'blazing star', 'bartonia', 'achene', 'samara', 'campanula', + 'creeping bellflower', 'canterbury bell', 'tall bellflower', + 'marsh bellflower', 'clustered bellflower', 'peach bells', 'chimney plant', + 'rampion', 'tussock bellflower', 'orchid', 'orchis', 'male orchis', + 'butterfly orchid', 'showy orchis', 'aerides', 'angrecum', 'jewel orchid', + 'puttyroot', 'arethusa', 'bog rose', 'bletia', 'bletilla striata', + 'brassavola', 'spider orchid', 'spider orchid', 'caladenia', 'calanthe', + 'grass pink', 'calypso', 'cattleya', 'helleborine', 'red helleborine', + 'spreading pogonia', 'rosebud orchid', 'satyr orchid', 'frog orchid', + 'coelogyne', 'coral root', 'spotted coral root', 'striped coral root', + 'early coral root', 'swan orchid', 'cymbid', 'cypripedia', + 'lady\'s slipper', 'moccasin flower', 'common lady\'s-slipper', + 'ram\'s-head', 'yellow lady\'s slipper', 'large yellow lady\'s slipper', + 'california lady\'s slipper', 'clustered lady\'s slipper', + 'mountain lady\'s slipper', 'marsh orchid', 'common spotted orchid', + 'dendrobium', 'disa', 'phantom orchid', 'tulip orchid', 'butterfly orchid', + 'butterfly orchid', 'epidendron', 'helleborine', 'epipactis helleborine', + 'stream orchid', 'tongueflower', 'rattlesnake plantain', 'fragrant orchid', + 'short-spurred fragrant orchid', 'fringed orchis', 'frog orchid', + 'rein orchid', 'bog rein orchid', 'white fringed orchis', + 'elegant habenaria', 'purple-fringed orchid', 'coastal rein orchid', + 'hooker\'s orchid', 'ragged orchid', 'prairie orchid', 'snowy orchid', + 'round-leaved rein orchid', 'purple fringeless orchid', + 'purple-fringed orchid', 'alaska rein orchid', 'crested coral root', + 'texas purple spike', 'lizard orchid', 'laelia', 'liparis', 'twayblade', + 'fen orchid', 'broad-leaved twayblade', 'lesser twayblade', 'twayblade', + 'green adder\'s mouth', 'masdevallia', 'maxillaria', 'pansy orchid', + 'odontoglossum', 'oncidium', 'bee orchid', 'fly orchid', 'spider orchid', + 'early spider orchid', 'venus\' slipper', 'phaius', 'moth orchid', + 'butterfly plant', 'rattlesnake orchid', 'lesser butterfly orchid', + 'greater butterfly orchid', 'prairie white-fringed orchid', 'tangle orchid', + 'indian crocus', 'pleurothallis', 'pogonia', 'butterfly orchid', + 'psychopsis krameriana', 'psychopsis papilio', 'helmet orchid', + 'foxtail orchid', 'orange-blossom orchid', 'sobralia', 'ladies\' tresses', + 'screw augur', 'hooded ladies\' tresses', 'western ladies\' tresses', + 'european ladies\' tresses', 'stanhopea', 'stelis', 'fly orchid', 'vanda', + 'blue orchid', 'vanilla', 'vanilla orchid', 'yam', 'yam', 'white yam', + 'cinnamon vine', 'elephant\'s-foot', 'wild yam', 'cush-cush', + 'black bryony', 'primrose', 'english primrose', 'cowslip', 'oxlip', + 'chinese primrose', 'polyanthus', 'pimpernel', 'scarlet pimpernel', + 'bog pimpernel', 'chaffweed', 'cyclamen', 'sowbread', 'sea milkwort', + 'featherfoil', 'water gillyflower', 'water violet', 'loosestrife', + 'gooseneck loosestrife', 'yellow pimpernel', 'fringed loosestrife', + 'moneywort', 'swamp candles', 'whorled loosestrife', 'water pimpernel', + 'brookweed', 'brookweed', 'coralberry', 'marlberry', 'plumbago', 'leadwort', + 'thrift', 'sea lavender', 'barbasco', 'gramineous plant', 'grass', + 'midgrass', 'shortgrass', 'sword grass', 'tallgrass', 'herbage', + 'goat grass', 'wheatgrass', 'crested wheatgrass', 'bearded wheatgrass', + 'western wheatgrass', 'intermediate wheatgrass', 'slender wheatgrass', + 'velvet bent', 'cloud grass', 'meadow foxtail', 'foxtail', 'broom grass', + 'broom sedge', 'tall oat grass', 'toetoe', 'oat', 'cereal oat', 'wild oat', + 'slender wild oat', 'wild red oat', 'brome', 'chess', 'field brome', + 'grama', 'black grama', 'buffalo grass', 'reed grass', 'feather reed grass', + 'australian reed grass', 'burgrass', 'buffel grass', 'rhodes grass', + 'pampas grass', 'giant star grass', 'orchard grass', 'egyptian grass', + 'crabgrass', 'smooth crabgrass', 'large crabgrass', 'barnyard grass', + 'japanese millet', 'yardgrass', 'finger millet', 'lyme grass', 'wild rye', + 'giant ryegrass', 'sea lyme grass', 'canada wild rye', 'teff', + 'weeping love grass', 'plume grass', 'ravenna grass', 'fescue', + 'reed meadow grass', 'velvet grass', 'creeping soft grass', 'barleycorn', + 'barley grass', 'little barley', 'rye grass', 'perennial ryegrass', + 'italian ryegrass', 'darnel', 'nimblewill', 'cultivated rice', 'ricegrass', + 'smilo', 'switch grass', 'broomcorn millet', 'goose grass', 'dallisgrass', + 'bahia grass', 'knotgrass', 'fountain grass', 'reed canary grass', + 'canary grass', 'timothy', 'bluegrass', 'meadowgrass', 'wood meadowgrass', + 'noble cane', 'munj', 'broom beard grass', 'bluestem', 'rye', + 'bristlegrass', 'giant foxtail', 'yellow bristlegrass', + 'green bristlegrass', 'siberian millet', 'german millet', 'millet', + 'rattan', 'malacca', 'reed', 'sorghum', 'grain sorghum', 'durra', + 'feterita', 'hegari', 'kaoliang', 'milo', 'shallu', 'broomcorn', + 'cordgrass', 'salt reed grass', 'prairie cordgrass', 'smut grass', + 'sand dropseed', 'rush grass', 'st', 'grain', 'cereal', 'wheat', + 'wheat berry', 'durum', 'spelt', 'emmer', 'wild wheat', 'corn', 'mealie', + 'corn', 'dent corn', 'flint corn', 'popcorn', 'zoysia', 'manila grass', + 'korean lawn grass', 'bamboo', 'common bamboo', 'giant bamboo', + 'umbrella plant', 'chufa', 'galingale', 'nutgrass', 'sand sedge', + 'cypress sedge', 'cotton grass', 'common cotton grass', 'hardstem bulrush', + 'wool grass', 'spike rush', 'water chestnut', 'needle spike rush', + 'creeping spike rush', 'pandanus', 'textile screw pine', 'cattail', + 'cat\'s-tail', 'bur reed', 'grain', 'kernel', 'rye', 'gourd', 'gourd', + 'pumpkin', 'squash', 'summer squash', 'yellow squash', 'marrow', 'zucchini', + 'cocozelle', 'cymling', 'spaghetti squash', 'winter squash', 'acorn squash', + 'hubbard squash', 'turban squash', 'buttercup squash', 'butternut squash', + 'winter crookneck', 'cushaw', 'prairie gourd', 'prairie gourd', 'bryony', + 'white bryony', 'sweet melon', 'cantaloupe', 'winter melon', 'net melon', + 'cucumber', 'squirting cucumber', 'bottle gourd', 'luffa', 'loofah', + 'angled loofah', 'loofa', 'balsam apple', 'balsam pear', 'lobelia', + 'water lobelia', 'mallow', 'musk mallow', 'common mallow', 'okra', 'okra', + 'abelmosk', 'flowering maple', 'velvetleaf', 'hollyhock', 'rose mallow', + 'althea', 'marsh mallow', 'poppy mallow', 'fringed poppy mallow', + 'purple poppy mallow', 'clustered poppy mallow', 'sea island cotton', + 'levant cotton', 'upland cotton', 'peruvian cotton', 'wild cotton', 'kenaf', + 'sorrel tree', 'rose mallow', 'cotton rose', 'roselle', 'mahoe', + 'flower-of-an-hour', 'lacebark', 'wild hollyhock', 'mountain hollyhock', + 'seashore mallow', 'salt marsh mallow', 'chaparral mallow', 'malope', + 'false mallow', 'waxmallow', 'glade mallow', 'pavonia', 'ribbon tree', + 'bush hibiscus', 'virginia mallow', 'queensland hemp', 'indian mallow', + 'checkerbloom', 'globe mallow', 'prairie mallow', 'tulipwood tree', + 'portia tree', 'red silk-cotton tree', 'cream-of-tartar tree', 'baobab', + 'kapok', 'durian', 'montezuma', 'shaving-brush tree', 'quandong', + 'quandong', 'makomako', 'jamaican cherry', 'breakax', 'sterculia', + 'panama tree', 'kalumpang', 'bottle-tree', 'flame tree', 'flame tree', + 'kurrajong', 'queensland bottletree', 'kola', 'kola nut', + 'chinese parasol tree', 'flannelbush', 'screw tree', + 'nut-leaved screw tree', 'red beech', 'looking glass tree', + 'looking-glass plant', 'honey bell', 'mayeng', 'silver tree', 'cacao', + 'obeche', 'linden', 'american basswood', 'small-leaved linden', + 'white basswood', 'japanese linden', 'silver lime', 'corchorus', + 'african hemp', 'herb', 'protea', 'honeypot', 'honeyflower', 'banksia', + 'honeysuckle', 'smoke bush', 'chilean firebush', 'chilean nut', 'grevillea', + 'red-flowered silky oak', 'silky oak', 'beefwood', 'cushion flower', + 'rewa-rewa', 'honeyflower', 'silver tree', 'lomatia', 'macadamia', + 'macadamia integrifolia', 'macadamia nut', 'queensland nut', 'prickly ash', + 'geebung', 'wheel tree', 'scrub beefwood', 'waratah', 'waratah', + 'casuarina', 'she-oak', 'beefwood', 'australian pine', 'heath', + 'tree heath', 'briarroot', 'winter heath', 'bell heather', 'cornish heath', + 'spanish heath', 'prince-of-wales\'-heath', 'bog rosemary', + 'marsh andromeda', 'madrona', 'strawberry tree', 'bearberry', + 'alpine bearberry', 'heartleaf manzanita', 'parry manzanita', 'spike heath', + 'bryanthus', 'leatherleaf', 'connemara heath', 'trailing arbutus', + 'creeping snowberry', 'salal', 'huckleberry', 'black huckleberry', + 'dangleberry', 'box huckleberry', 'kalmia', 'mountain laurel', + 'swamp laurel', 'trapper\'s tea', 'wild rosemary', 'sand myrtle', + 'leucothoe', 'dog laurel', 'sweet bells', 'alpine azalea', 'staggerbush', + 'maleberry', 'fetterbush', 'false azalea', 'minniebush', 'sorrel tree', + 'mountain heath', 'purple heather', 'fetterbush', 'rhododendron', + 'coast rhododendron', 'rosebay', 'swamp azalea', 'azalea', 'cranberry', + 'american cranberry', 'european cranberry', 'blueberry', 'farkleberry', + 'low-bush blueberry', 'rabbiteye blueberry', 'dwarf bilberry', + 'evergreen blueberry', 'evergreen huckleberry', 'bilberry', 'bilberry', + 'bog bilberry', 'dryland blueberry', 'grouseberry', 'deerberry', 'cowberry', + 'diapensia', 'galax', 'pyxie', 'shortia', 'oconee bells', + 'australian heath', 'epacris', 'common heath', 'common heath', + 'port jackson heath', 'native cranberry', 'pink fivecorner', 'wintergreen', + 'false wintergreen', 'lesser wintergreen', 'wild lily of the valley', + 'wild lily of the valley', 'pipsissewa', 'love-in-winter', + 'one-flowered wintergreen', 'indian pipe', 'pinesap', 'beech', + 'common beech', 'copper beech', 'american beech', 'weeping beech', + 'japanese beech', 'chestnut', 'american chestnut', 'european chestnut', + 'chinese chestnut', 'japanese chestnut', 'allegheny chinkapin', + 'ozark chinkapin', 'oak chestnut', 'giant chinkapin', + 'dwarf golden chinkapin', 'tanbark oak', 'japanese oak', 'southern beech', + 'myrtle beech', 'coigue', 'new zealand beech', 'silver beech', + 'roble beech', 'rauli beech', 'black beech', 'hard beech', 'acorn', + 'cupule', 'oak', 'live oak', 'coast live oak', 'white oak', + 'american white oak', 'arizona white oak', 'swamp white oak', + 'european turkey oak', 'canyon oak', 'scarlet oak', 'jack oak', 'red oak', + 'southern red oak', 'oregon white oak', 'holm oak', 'bear oak', + 'shingle oak', 'bluejack oak', 'california black oak', + 'american turkey oak', 'laurel oak', 'california white oak', 'overcup oak', + 'bur oak', 'scrub oak', 'blackjack oak', 'swamp chestnut oak', + 'japanese oak', 'chestnut oak', 'chinquapin oak', 'myrtle oak', 'water oak', + 'nuttall oak', 'durmast', 'basket oak', 'pin oak', 'willow oak', + 'dwarf chinkapin oak', 'common oak', 'northern red oak', 'shumard oak', + 'post oak', 'cork oak', 'spanish oak', 'huckleberry oak', + 'chinese cork oak', 'black oak', 'southern live oak', 'interior live oak', + 'mast', 'birch', 'yellow birch', 'american white birch', 'grey birch', + 'silver birch', 'downy birch', 'black birch', 'sweet birch', + 'yukon white birch', 'swamp birch', 'newfoundland dwarf birch', 'alder', + 'common alder', 'grey alder', 'seaside alder', 'white alder', 'red alder', + 'speckled alder', 'smooth alder', 'green alder', 'green alder', 'hornbeam', + 'european hornbeam', 'american hornbeam', 'hop hornbeam', + 'old world hop hornbeam', 'eastern hop hornbeam', 'hazelnut', + 'american hazel', 'cobnut', 'beaked hazelnut', 'centaury', 'rosita', + 'lesser centaury', 'seaside centaury', 'slender centaury', + 'prairie gentian', 'persian violet', 'columbo', 'gentian', 'gentianella', + 'closed gentian', 'explorer\'s gentian', 'closed gentian', + 'great yellow gentian', 'marsh gentian', 'soapwort gentian', + 'striped gentian', 'agueweed', 'felwort', 'fringed gentian', + 'gentianopsis crinita', 'gentianopsis detonsa', 'gentianopsid procera', + 'gentianopsis thermalis', 'tufted gentian', 'spurred gentian', 'sabbatia', + 'toothbrush tree', 'olive tree', 'olive', 'olive', 'black maire', + 'white maire', 'fringe tree', 'fringe bush', 'forestiera', 'forsythia', + 'ash', 'white ash', 'swamp ash', 'flowering ash', 'european ash', + 'oregon ash', 'black ash', 'manna ash', 'red ash', 'green ash', 'blue ash', + 'mountain ash', 'pumpkin ash', 'arizona ash', 'jasmine', 'primrose jasmine', + 'winter jasmine', 'common jasmine', 'privet', 'amur privet', + 'japanese privet', 'ligustrum obtusifolium', 'common privet', 'devilwood', + 'mock privet', 'lilac', 'himalayan lilac', 'persian lilac', + 'japanese tree lilac', 'japanese lilac', 'common lilac', 'bloodwort', + 'kangaroo paw', 'virginian witch hazel', 'vernal witch hazel', + 'winter hazel', 'fothergilla', 'liquidambar', 'sweet gum', 'iron tree', + 'walnut', 'california black walnut', 'butternut', 'black walnut', + 'english walnut', 'hickory', 'water hickory', 'pignut', 'bitternut', + 'pecan', 'big shellbark', 'nutmeg hickory', 'shagbark', 'mockernut', + 'wing nut', 'caucasian walnut', 'dhawa', 'combretum', 'hiccup nut', + 'bush willow', 'bush willow', 'button tree', 'white mangrove', 'oleaster', + 'water milfoil', 'anchovy pear', 'brazil nut', 'loosestrife', + 'purple loosestrife', 'grass poly', 'crape myrtle', 'queen\'s crape myrtle', + 'myrtaceous tree', 'myrtle', 'common myrtle', 'bayberry', 'allspice', + 'allspice tree', 'sour cherry', 'nakedwood', 'surinam cherry', 'rose apple', + 'feijoa', 'jaboticaba', 'guava', 'guava', 'cattley guava', + 'brazilian guava', 'gum tree', 'eucalyptus', 'flooded gum', 'mallee', + 'stringybark', 'smoothbark', 'red gum', 'red gum', 'river red gum', + 'mountain swamp gum', 'snow gum', 'alpine ash', 'white mallee', + 'white stringybark', 'white mountain ash', 'blue gum', 'rose gum', + 'cider gum', 'swamp gum', 'spotted gum', 'lemon-scented gum', + 'black mallee', 'forest red gum', 'mountain ash', 'manna gum', 'clove', + 'clove', 'tupelo', 'water gum', 'sour gum', 'enchanter\'s nightshade', + 'circaea lutetiana', 'willowherb', 'fireweed', 'california fuchsia', + 'fuchsia', 'lady\'s-eardrop', 'evening primrose', 'common evening primrose', + 'sundrops', 'missouri primrose', 'pomegranate', 'mangrove', 'daphne', + 'garland flower', 'spurge laurel', 'mezereon', 'indian rhododendron', + 'medinilla magnifica', 'deer grass', 'canna', 'achira', 'arrowroot', + 'banana', 'dwarf banana', 'japanese banana', 'plantain', 'edible banana', + 'abaca', 'abyssinian banana', 'ginger', 'common ginger', 'turmeric', + 'galangal', 'shellflower', 'grains of paradise', 'cardamom', 'begonia', + 'fibrous-rooted begonia', 'tuberous begonia', 'rhizomatous begonia', + 'christmas begonia', 'angel-wing begonia', 'beefsteak begonia', + 'star begonia', 'rex begonia', 'wax begonia', 'socotra begonia', + 'hybrid tuberous begonia', 'dillenia', 'guinea gold vine', 'poon', 'calaba', + 'maria', 'laurelwood', 'alexandrian laurel', 'clusia', 'wild fig', + 'waxflower', 'pitch apple', 'mangosteen', 'gamboge tree', 'st john\'s wort', + 'common st john\'s wort', 'great st john\'s wort', + 'creeping st john\'s wort', 'low st andrew\'s cross', 'klammath weed', + 'shrubby st john\'s wort', 'st peter\'s wort', 'marsh st-john\'s wort', + 'mammee apple', 'rose chestnut', 'bower actinidia', 'chinese gooseberry', + 'silvervine', 'wild cinnamon', 'papaya', 'souari', 'rockrose', + 'white-leaved rockrose', 'common gum cistus', 'frostweed', 'dipterocarp', + 'red lauan', 'governor\'s plum', 'kei apple', 'ketembilla', 'chaulmoogra', + 'wild peach', 'candlewood', 'boojum tree', 'bird\'s-eye bush', 'granadilla', + 'granadilla', 'granadilla', 'maypop', 'jamaica honeysuckle', + 'banana passion fruit', 'sweet calabash', 'love-in-a-mist', 'reseda', + 'mignonette', 'dyer\'s rocket', 'false tamarisk', 'halophyte', 'viola', + 'violet', 'field pansy', 'american dog violet', 'dog violet', + 'horned violet', 'two-eyed violet', 'bird\'s-foot violet', + 'downy yellow violet', 'long-spurred violet', 'pale violet', 'hedge violet', + 'nettle', 'stinging nettle', 'roman nettle', 'ramie', 'wood nettle', + 'australian nettle', 'pellitory-of-the-wall', 'richweed', 'artillery plant', + 'friendship plant', 'queensland grass-cloth plant', 'pipturus albidus', + 'cannabis', 'indian hemp', 'mulberry', 'white mulberry', 'black mulberry', + 'red mulberry', 'osage orange', 'breadfruit', 'jackfruit', 'marang', + 'fig tree', 'fig', 'caprifig', 'golden fig', 'banyan', 'pipal', + 'india-rubber tree', 'mistletoe fig', 'port jackson fig', 'sycamore', + 'paper mulberry', 'trumpetwood', 'elm', 'winged elm', 'american elm', + 'smooth-leaved elm', 'cedar elm', 'witch elm', 'dutch elm', + 'huntingdon elm', 'water elm', 'chinese elm', 'english elm', 'siberian elm', + 'slippery elm', 'jersey elm', 'september elm', 'rock elm', 'hackberry', + 'european hackberry', 'american hackberry', 'sugarberry', + 'iridaceous plant', 'bearded iris', 'beardless iris', 'orrisroot', + 'dwarf iris', 'dutch iris', 'florentine iris', 'stinking iris', + 'german iris', 'japanese iris', 'german iris', 'dalmatian iris', + 'persian iris', 'dutch iris', 'dwarf iris', 'spanish iris', + 'blackberry-lily', 'crocus', 'saffron', 'corn lily', 'blue-eyed grass', + 'wandflower', 'amaryllis', 'salsilla', 'salsilla', 'blood lily', + 'cape tulip', 'hippeastrum', 'narcissus', 'daffodil', 'jonquil', 'jonquil', + 'jacobean lily', 'liliaceous plant', 'mountain lily', 'canada lily', + 'tiger lily', 'columbia tiger lily', 'tiger lily', 'easter lily', + 'coast lily', 'turk\'s-cap', 'michigan lily', 'leopard lily', 'turk\'s-cap', + 'african lily', 'colicroot', 'ague root', 'yellow colicroot', + 'alliaceous plant', 'hooker\'s onion', 'wild leek', 'canada garlic', + 'keeled garlic', 'onion', 'shallot', 'nodding onion', 'welsh onion', + 'red-skinned onion', 'daffodil garlic', 'few-flowered leek', 'garlic', + 'sand leek', 'chives', 'crow garlic', 'wild garlic', 'garlic chive', + 'round-headed leek', 'three-cornered leek', 'cape aloe', 'kniphofia', + 'poker plant', 'red-hot poker', 'fly poison', 'amber lily', 'asparagus', + 'asparagus fern', 'smilax', 'asphodel', 'jacob\'s rod', 'aspidistra', + 'coral drops', 'christmas bells', 'climbing onion', 'mariposa', + 'globe lily', 'cat\'s-ear', 'white globe lily', 'yellow globe lily', + 'rose globe lily', 'star tulip', 'desert mariposa tulip', + 'yellow mariposa tulip', 'sagebrush mariposa tulip', 'sego lily', 'camas', + 'common camas', 'leichtlin\'s camas', 'wild hyacinth', 'dogtooth violet', + 'white dogtooth violet', 'yellow adder\'s tongue', 'european dogtooth', + 'fawn lily', 'glacier lily', 'avalanche lily', 'fritillary', + 'mission bells', 'mission bells', 'stink bell', 'crown imperial', + 'white fritillary', 'snake\'s head fritillary', 'adobe lily', + 'scarlet fritillary', 'tulip', 'dwarf tulip', 'lady tulip', + 'tulipa gesneriana', 'cottage tulip', 'darwin tulip', 'gloriosa', + 'lemon lily', 'common hyacinth', 'roman hyacinth', 'summer hyacinth', + 'star-of-bethlehem', 'bath asparagus', 'grape hyacinth', + 'common grape hyacinth', 'tassel hyacinth', 'scilla', 'spring squill', + 'false asphodel', 'scotch asphodel', 'sea squill', 'squill', + 'butcher\'s broom', 'bog asphodel', 'european bog asphodel', + 'american bog asphodel', 'hellebore', 'white hellebore', 'squaw grass', + 'death camas', 'alkali grass', 'white camas', 'poison camas', + 'grassy death camas', 'prairie wake-robin', 'dwarf-white trillium', + 'herb paris', 'sarsaparilla', 'bullbrier', 'rough bindweed', 'clintonia', + 'false lily of the valley', 'false lily of the valley', 'solomon\'s-seal', + 'great solomon\'s-seal', 'bellwort', 'strawflower', 'pia', 'agave', + 'american agave', 'sisal', 'maguey', 'maguey', 'agave tequilana', + 'cabbage tree', 'dracaena', 'tuberose', 'sansevieria', + 'african bowstring hemp', 'ceylon bowstring hemp', + 'mother-in-law\'s tongue', 'spanish bayonet', 'spanish bayonet', + 'joshua tree', 'soapweed', 'adam\'s needle', 'bear grass', 'spanish dagger', + 'our lord\'s candle', 'water shamrock', 'butterfly bush', 'yellow jasmine', + 'flax', 'calabar bean', 'bonduc', 'divi-divi', 'mysore thorn', + 'brazilian ironwood', 'bird of paradise', 'shingle tree', 'mountain ebony', + 'msasa', 'cassia', 'golden shower tree', 'pink shower', 'rainbow shower', + 'horse cassia', 'carob', 'carob', 'paloverde', 'royal poinciana', + 'locust tree', 'water locust', 'honey locust', 'kentucky coffee tree', + 'logwood', 'jerusalem thorn', 'palo verde', 'dalmatian laburnum', 'senna', + 'avaram', 'alexandria senna', 'wild senna', 'sicklepod', 'coffee senna', + 'tamarind', 'false indigo', 'false indigo', 'hog peanut', 'angelim', + 'cabbage bark', 'kidney vetch', 'groundnut', 'rooibos', 'milk vetch', + 'alpine milk vetch', 'purple milk vetch', 'camwood', 'wild indigo', + 'blue false indigo', 'white false indigo', 'indigo broom', 'dhak', + 'pigeon pea', 'sword bean', 'pea tree', 'siberian pea tree', + 'chinese pea tree', 'moreton bay chestnut', 'butterfly pea', 'judas tree', + 'redbud', 'western redbud', 'tagasaste', 'weeping tree broom', 'flame pea', + 'chickpea', 'chickpea', 'kentucky yellowwood', 'glory pea', 'desert pea', + 'parrot\'s beak', 'butterfly pea', 'blue pea', 'telegraph plant', + 'bladder senna', 'axseed', 'crotalaria', 'guar', 'white broom', + 'common broom', 'rosewood', 'indian blackwood', 'sissoo', 'kingwood', + 'brazilian rosewood', 'cocobolo', 'blackwood', 'bitter pea', 'derris', + 'derris root', 'prairie mimosa', 'tick trefoil', 'beggarweed', + 'australian pea', 'coral tree', 'kaffir boom', 'coral bean tree', 'ceibo', + 'kaffir boom', 'indian coral tree', 'cork tree', 'goat\'s rue', + 'poison bush', 'spanish broom', 'woodwaxen', 'chanar', 'gliricidia', 'soy', + 'licorice', 'wild licorice', 'licorice root', 'western australia coral pea', + 'sweet vetch', 'french honeysuckle', 'anil', 'scarlet runner', + 'hyacinth bean', 'scotch laburnum', 'vetchling', 'wild pea', + 'everlasting pea', 'beach pea', 'grass vetch', 'marsh pea', + 'common vetchling', 'grass pea', 'tangier pea', 'heath pea', + 'bicolor lespediza', 'japanese clover', 'korean lespedeza', + 'sericea lespedeza', 'lentil', 'lentil', 'prairie bird\'s-foot trefoil', + 'bird\'s foot trefoil', 'winged pea', 'lupine', 'white lupine', + 'tree lupine', 'wild lupine', 'bluebonnet', 'texas bluebonnet', 'medic', + 'moon trefoil', 'sickle alfalfa', 'calvary clover', 'black medick', + 'alfalfa', 'millettia', 'mucuna', 'cowage', 'tolu tree', 'peruvian balsam', + 'sainfoin', 'restharrow', 'bead tree', 'jumby bead', 'locoweed', + 'purple locoweed', 'tumbleweed', 'yam bean', 'shamrock pea', 'pole bean', + 'kidney bean', 'haricot', 'wax bean', 'scarlet runner', 'lima bean', + 'sieva bean', 'tepary bean', 'chaparral pea', 'jamaica dogwood', 'pea', + 'garden pea', 'edible-pod pea', 'sugar snap pea', 'field pea', 'field pea', + 'common flat pea', 'quira', 'roble', 'panama redwood tree', 'indian beech', + 'winged bean', 'breadroot', 'bloodwood tree', 'kino', 'red sandalwood', + 'kudzu', 'bristly locust', 'black locust', 'clammy locust', 'carib wood', + 'colorado river hemp', 'scarlet wisteria tree', 'japanese pagoda tree', + 'mescal bean', 'kowhai', 'jade vine', 'hoary pea', 'bastard indigo', + 'catgut', 'bush pea', 'false lupine', 'carolina lupine', 'tipu', + 'bird\'s foot trefoil', 'fenugreek', 'gorse', 'vetch', 'tufted vetch', + 'broad bean', 'bitter betch', 'bush vetch', 'moth bean', 'snailflower', + 'mung', 'cowpea', 'cowpea', 'asparagus bean', 'swamp oak', 'keurboom', + 'keurboom', 'japanese wistaria', 'chinese wistaria', 'american wistaria', + 'silky wisteria', 'palm', 'sago palm', 'feather palm', 'fan palm', + 'palmetto', 'coyol', 'grugru', 'areca', 'betel palm', 'sugar palm', + 'piassava palm', 'coquilla nut', 'palmyra', 'calamus', 'rattan', + 'lawyer cane', 'fishtail palm', 'wine palm', 'wax palm', 'coconut', + 'carnauba', 'caranday', 'corozo', 'gebang palm', 'latanier', 'talipot', + 'oil palm', 'african oil palm', 'american oil palm', 'palm nut', + 'cabbage palm', 'cabbage palm', 'true sago palm', 'nipa palm', 'babassu', + 'babassu nut', 'cohune palm', 'cohune nut', 'date palm', 'ivory palm', + 'raffia palm', 'bamboo palm', 'lady palm', 'miniature fan palm', + 'reed rhapis', 'royal palm', 'cabbage palm', 'cabbage palmetto', + 'saw palmetto', 'thatch palm', 'key palm', 'english plantain', + 'broad-leaved plantain', 'hoary plantain', 'fleawort', 'rugel\'s plantain', + 'hoary plantain', 'buckwheat', 'prince\'s-feather', 'eriogonum', + 'umbrella plant', 'wild buckwheat', 'rhubarb', 'himalayan rhubarb', + 'pie plant', 'chinese rhubarb', 'sour dock', 'sheep sorrel', 'bitter dock', + 'french sorrel', 'yellow-eyed grass', 'commelina', 'spiderwort', + 'pineapple', 'pipewort', 'water hyacinth', 'water star grass', 'naiad', + 'water plantain', 'narrow-leaved water plantain', 'hydrilla', + 'american frogbit', 'waterweed', 'canadian pondweed', 'tape grass', + 'pondweed', 'curled leaf pondweed', 'loddon pondweed', 'frog\'s lettuce', + 'arrow grass', 'horned pondweed', 'eelgrass', 'rose', 'hip', 'banksia rose', + 'damask rose', 'sweetbrier', 'cherokee rose', 'musk rose', 'agrimonia', + 'harvest-lice', 'fragrant agrimony', 'alderleaf juneberry', + 'flowering quince', 'japonica', 'coco plum', 'cotoneaster', + 'cotoneaster dammeri', 'cotoneaster horizontalis', 'parsley haw', + 'scarlet haw', 'blackthorn', 'cockspur thorn', 'mayhaw', 'red haw', + 'red haw', 'quince', 'mountain avens', 'loquat', 'beach strawberry', + 'virginia strawberry', 'avens', 'yellow avens', 'yellow avens', + 'prairie smoke', 'bennet', 'toyon', 'apple tree', 'apple', 'wild apple', + 'crab apple', 'siberian crab', 'wild crab', 'american crab apple', + 'oregon crab apple', 'southern crab apple', 'iowa crab', 'bechtel crab', + 'medlar', 'cinquefoil', 'silverweed', 'salad burnet', 'plum', 'wild plum', + 'allegheny plum', 'american red plum', 'chickasaw plum', 'beach plum', + 'common plum', 'bullace', 'damson plum', 'big-tree plum', 'canada plum', + 'plumcot', 'apricot', 'japanese apricot', 'common apricot', + 'purple apricot', 'cherry', 'wild cherry', 'wild cherry', 'sweet cherry', + 'heart cherry', 'gean', 'capulin', 'cherry laurel', 'cherry plum', + 'sour cherry', 'amarelle', 'morello', 'marasca', 'almond tree', 'almond', + 'bitter almond', 'jordan almond', 'dwarf flowering almond', + 'holly-leaved cherry', 'fuji', 'flowering almond', 'cherry laurel', + 'catalina cherry', 'bird cherry', 'hagberry tree', 'hagberry', 'pin cherry', + 'peach', 'nectarine', 'sand cherry', 'japanese plum', 'black cherry', + 'flowering cherry', 'oriental cherry', 'japanese flowering cherry', + 'sierra plum', 'rosebud cherry', 'russian almond', 'flowering almond', + 'chokecherry', 'chokecherry', 'western chokecherry', 'pyracantha', 'pear', + 'fruit tree', 'bramble bush', 'lawyerbush', 'stone bramble', + 'sand blackberry', 'boysenberry', 'loganberry', 'american dewberry', + 'northern dewberry', 'southern dewberry', 'swamp dewberry', + 'european dewberry', 'raspberry', 'wild raspberry', 'american raspberry', + 'black raspberry', 'salmonberry', 'salmonberry', 'wineberry', + 'mountain ash', 'rowan', 'rowanberry', 'american mountain ash', + 'western mountain ash', 'service tree', 'wild service tree', 'spirea', + 'bridal wreath', 'madderwort', 'indian madder', 'madder', 'woodruff', + 'dagame', 'blolly', 'coffee', 'arabian coffee', 'liberian coffee', + 'robusta coffee', 'cinchona', 'cartagena bark', 'calisaya', 'cinchona tree', + 'cinchona', 'bedstraw', 'sweet woodruff', 'northern bedstraw', + 'yellow bedstraw', 'wild licorice', 'cleavers', 'wild madder', + 'cape jasmine', 'genipa', 'genipap fruit', 'hamelia', 'scarlet bush', + 'lemonwood', 'negro peach', 'wild medlar', 'spanish tamarind', 'abelia', + 'bush honeysuckle', 'american twinflower', 'honeysuckle', + 'american fly honeysuckle', 'italian honeysuckle', 'yellow honeysuckle', + 'hairy honeysuckle', 'japanese honeysuckle', 'hall\'s honeysuckle', + 'morrow\'s honeysuckle', 'woodbine', 'trumpet honeysuckle', + 'european fly honeysuckle', 'swamp fly honeysuckle', 'snowberry', + 'coralberry', 'blue elder', 'dwarf elder', 'american red elder', + 'european red elder', 'feverroot', 'cranberry bush', 'wayfaring tree', + 'guelder rose', 'arrow wood', 'black haw', 'weigela', 'teasel', + 'common teasel', 'fuller\'s teasel', 'wild teasel', 'scabious', + 'sweet scabious', 'field scabious', 'jewelweed', 'geranium', 'cranesbill', + 'wild geranium', 'meadow cranesbill', 'richardson\'s geranium', + 'herb robert', 'sticky geranium', 'dove\'s foot geranium', 'rose geranium', + 'fish geranium', 'ivy geranium', 'apple geranium', 'lemon geranium', + 'storksbill', 'musk clover', 'incense tree', 'elephant tree', 'gumbo-limbo', + 'boswellia carteri', 'salai', 'balm of gilead', 'myrrh tree', + 'protium heptaphyllum', 'protium guianense', 'water starwort', + 'barbados cherry', 'mahogany', 'chinaberry', 'neem', 'neem seed', + 'spanish cedar', 'satinwood', 'african scented mahogany', 'silver ash', + 'native beech', 'bunji-bunji', 'african mahogany', 'lanseh tree', + 'true mahogany', 'honduras mahogany', 'philippine mahogany', 'caracolito', + 'common wood sorrel', 'bermuda buttercup', 'creeping oxalis', 'goatsfoot', + 'violet wood sorrel', 'oca', 'carambola', 'bilimbi', 'milkwort', 'senega', + 'orange milkwort', 'flowering wintergreen', 'seneca snakeroot', + 'common milkwort', 'rue', 'citrus', 'orange', 'sour orange', 'bergamot', + 'pomelo', 'citron', 'grapefruit', 'mandarin', 'tangerine', 'clementine', + 'satsuma', 'sweet orange', 'temple orange', 'tangelo', 'rangpur', 'lemon', + 'sweet lemon', 'lime', 'citrange', 'fraxinella', 'kumquat', 'marumi', + 'nagami', 'cork tree', 'trifoliate orange', 'prickly ash', 'toothache tree', + 'hercules\'-club', 'bitterwood tree', 'marupa', 'paradise tree', + 'ailanthus', 'tree of heaven', 'wild mango', 'pepper tree', + 'jamaica quassia', 'quassia', 'nasturtium', 'garden nasturtium', + 'bush nasturtium', 'canarybird flower', 'bean caper', 'palo santo', + 'lignum vitae', 'creosote bush', 'caltrop', 'willow', 'osier', + 'white willow', 'silver willow', 'golden willow', 'cricket-bat willow', + 'arctic willow', 'weeping willow', 'wisconsin weeping willow', + 'pussy willow', 'sallow', 'goat willow', 'peachleaf willow', + 'almond willow', 'hoary willow', 'crack willow', 'prairie willow', + 'dwarf willow', 'grey willow', 'arroyo willow', 'shining willow', + 'swamp willow', 'bay willow', 'purple willow', 'balsam willow', + 'creeping willow', 'sitka willow', 'dwarf grey willow', 'bearberry willow', + 'common osier', 'poplar', 'balsam poplar', 'white poplar', 'grey poplar', + 'black poplar', 'lombardy poplar', 'cottonwood', 'eastern cottonwood', + 'black cottonwood', 'swamp cottonwood', 'aspen', 'quaking aspen', + 'american quaking aspen', 'canadian aspen', 'sandalwood tree', 'quandong', + 'rabbitwood', 'loranthaceae', 'mistletoe', 'american mistletoe', + 'mistletoe', 'american mistletoe', 'aalii', 'soapberry', 'wild china tree', + 'china tree', 'akee', 'soapberry vine', 'heartseed', 'balloon vine', + 'longan', 'harpullia', 'harpulla', 'moreton bay tulipwood', 'litchi', + 'spanish lime', 'rambutan', 'pulasan', 'pachysandra', 'allegheny spurge', + 'bittersweet', 'spindle tree', 'winged spindle tree', 'wahoo', + 'strawberry bush', 'evergreen bittersweet', 'cyrilla', 'titi', 'crowberry', + 'maple', 'silver maple', 'sugar maple', 'red maple', 'moosewood', + 'oregon maple', 'dwarf maple', 'mountain maple', 'vine maple', + 'hedge maple', 'norway maple', 'sycamore', 'box elder', + 'california box elder', 'pointed-leaf maple', 'japanese maple', + 'japanese maple', 'holly', 'chinese holly', 'bearberry', 'inkberry', 'mate', + 'american holly', 'low gallberry holly', 'tall gallberry holly', + 'yaupon holly', 'deciduous holly', 'juneberry holly', 'largeleaf holly', + 'geogia holly', 'common winterberry holly', 'smooth winterberry holly', + 'cashew', 'goncalo alves', 'venetian sumac', 'laurel sumac', 'mango', + 'pistachio', 'terebinth', 'mastic', 'australian sumac', 'sumac', + 'smooth sumac', 'sugar-bush', 'staghorn sumac', 'squawbush', + 'aroeira blanca', 'pepper tree', 'brazilian pepper tree', 'hog plum', + 'mombin', 'poison ash', 'poison ivy', 'western poison oak', + 'eastern poison oak', 'varnish tree', 'horse chestnut', 'buckeye', + 'sweet buckeye', 'ohio buckeye', 'dwarf buckeye', 'red buckeye', + 'particolored buckeye', 'ebony', 'marblewood', 'marblewood', 'persimmon', + 'japanese persimmon', 'american persimmon', 'date plum', 'buckthorn', + 'southern buckthorn', 'false buckthorn', 'star apple', 'satinleaf', + 'balata', 'sapodilla', 'gutta-percha tree', 'gutta-percha tree', 'canistel', + 'marmalade tree', 'sweetleaf', 'asiatic sweetleaf', 'styrax', 'snowbell', + 'japanese snowbell', 'texas snowbell', 'silver-bell tree', + 'carnivorous plant', 'pitcher plant', 'common pitcher plant', + 'hooded pitcher plant', 'huntsman\'s horn', 'tropical pitcher plant', + 'sundew', 'venus\'s flytrap', 'waterwheel plant', + 'drosophyllum lusitanicum', 'roridula', 'australian pitcher plant', 'sedum', + 'stonecrop', 'rose-root', 'orpine', 'pinwheel', 'christmas bush', + 'hortensia', 'fall-blooming hydrangea', 'carpenteria', 'decumary', + 'deutzia', 'philadelphus', 'mock orange', 'saxifrage', + 'yellow mountain saxifrage', 'meadow saxifrage', 'mossy saxifrage', + 'western saxifrage', 'purple saxifrage', 'star saxifrage', + 'strawberry geranium', 'astilbe', 'false goatsbeard', 'dwarf astilbe', + 'spirea', 'bergenia', 'coast boykinia', 'golden saxifrage', + 'umbrella plant', 'bridal wreath', 'alumroot', 'coralbells', + 'leatherleaf saxifrage', 'woodland star', 'prairie star', 'miterwort', + 'five-point bishop\'s cap', 'parnassia', 'bog star', + 'fringed grass of parnassus', 'false alumroot', 'foamflower', + 'false miterwort', 'pickaback plant', 'currant', 'black currant', + 'white currant', 'gooseberry', 'plane tree', 'london plane', + 'american sycamore', 'oriental plane', 'california sycamore', + 'arizona sycamore', 'greek valerian', 'northern jacob\'s ladder', + 'skunkweed', 'phlox', 'moss pink', 'evening-snow', 'acanthus', + 'bear\'s breech', 'caricature plant', 'black-eyed susan', 'catalpa', + 'catalpa bignioides', 'catalpa speciosa', 'desert willow', 'calabash', + 'calabash', 'borage', 'common amsinckia', 'anchusa', 'bugloss', + 'cape forget-me-not', 'cape forget-me-not', 'spanish elm', 'princewood', + 'chinese forget-me-not', 'hound\'s-tongue', 'hound\'s-tongue', 'blueweed', + 'beggar\'s lice', 'gromwell', 'puccoon', 'virginia bluebell', + 'garden forget-me-not', 'forget-me-not', 'false gromwell', 'comfrey', + 'common comfrey', 'convolvulus', 'bindweed', 'field bindweed', 'scammony', + 'silverweed', 'dodder', 'dichondra', 'cypress vine', 'moonflower', + 'wild potato vine', 'red morning-glory', 'man-of-the-earth', 'scammony', + 'japanese morning glory', 'imperial japanese morning glory', 'gesneriad', + 'gesneria', 'achimenes', 'aeschynanthus', 'lace-flower vine', 'columnea', + 'episcia', 'gloxinia', 'canterbury bell', 'kohleria', 'african violet', + 'streptocarpus', 'cape primrose', 'waterleaf', 'virginia waterleaf', + 'yellow bells', 'yerba santa', 'nemophila', 'baby blue-eyes', 'five-spot', + 'scorpionweed', 'california bluebell', 'california bluebell', 'fiddleneck', + 'fiesta flower', 'basil thyme', 'giant hyssop', 'yellow giant hyssop', + 'anise hyssop', 'mexican hyssop', 'bugle', 'creeping bugle', 'erect bugle', + 'pyramid bugle', 'wood mint', 'hairy wood mint', 'downy wood mint', + 'calamint', 'common calamint', 'large-flowered calamint', 'lesser calamint', + 'wild basil', 'horse balm', 'coleus', 'country borage', 'painted nettle', + 'apalachicola rosemary', 'dragonhead', 'elsholtzia', 'hemp nettle', + 'ground ivy', 'pennyroyal', 'hyssop', 'dead nettle', 'white dead nettle', + 'henbit', 'english lavender', 'french lavender', 'spike lavender', 'dagga', + 'lion\'s-ear', 'motherwort', 'pitcher sage', 'bugleweed', 'water horehound', + 'gipsywort', 'origanum', 'oregano', 'sweet marjoram', 'horehound', + 'common horehound', 'lemon balm', 'corn mint', 'water-mint', + 'bergamot mint', 'horsemint', 'peppermint', 'spearmint', 'apple mint', + 'pennyroyal', 'yerba buena', 'molucca balm', 'monarda', 'bee balm', + 'horsemint', 'bee balm', 'lemon mint', 'plains lemon monarda', 'basil balm', + 'mustang mint', 'catmint', 'basil', 'beefsteak plant', 'phlomis', + 'jerusalem sage', 'physostegia', 'plectranthus', 'patchouli', 'self-heal', + 'mountain mint', 'rosemary', 'clary sage', 'purple sage', 'cancerweed', + 'common sage', 'meadow clary', 'clary', 'pitcher sage', 'mexican mint', + 'wild sage', 'savory', 'summer savory', 'winter savory', 'skullcap', + 'blue pimpernel', 'hedge nettle', 'hedge nettle', 'germander', + 'american germander', 'cat thyme', 'wood sage', 'thyme', 'common thyme', + 'wild thyme', 'blue curls', 'turpentine camphor weed', 'bastard pennyroyal', + 'bladderwort', 'butterwort', 'genlisea', 'martynia', 'common unicorn plant', + 'sand devil\'s claw', 'sweet unicorn plant', 'figwort', 'snapdragon', + 'white snapdragon', 'yellow twining snapdragon', 'mediterranean snapdragon', + 'kitten-tails', 'alpine besseya', 'false foxglove', 'false foxglove', + 'calceolaria', 'indian paintbrush', 'desert paintbrush', + 'giant red paintbrush', 'great plains paintbrush', 'sulfur paintbrush', + 'shellflower', 'maiden blue-eyed mary', 'blue-eyed mary', 'foxglove', + 'common foxglove', 'yellow foxglove', 'gerardia', 'blue toadflax', + 'toadflax', 'golden-beard penstemon', 'scarlet bugler', + 'red shrubby penstemon', 'platte river penstemon', 'hot-rock penstemon', + 'jones\' penstemon', 'shrubby penstemon', 'narrow-leaf penstemon', + 'balloon flower', 'parry\'s penstemon', 'rock penstemon', + 'rydberg\'s penstemon', 'cascade penstemon', 'whipple\'s penstemon', + 'moth mullein', 'white mullein', 'purple mullein', 'common mullein', + 'veronica', 'field speedwell', 'brooklime', 'corn speedwell', 'brooklime', + 'germander speedwell', 'water speedwell', 'common speedwell', + 'purslane speedwell', 'thyme-leaved speedwell', 'nightshade', + 'horse nettle', 'african holly', 'potato vine', 'garden huckleberry', + 'naranjilla', 'potato vine', 'potato tree', 'belladonna', 'bush violet', + 'lady-of-the-night', 'angel\'s trumpet', 'angel\'s trumpet', + 'red angel\'s trumpet', 'cone pepper', 'bird pepper', 'day jessamine', + 'night jasmine', 'tree tomato', 'thorn apple', 'jimsonweed', 'pichi', + 'henbane', 'egyptian henbane', 'matrimony vine', 'common matrimony vine', + 'christmasberry', 'plum tomato', 'mandrake', 'mandrake root', + 'apple of peru', 'flowering tobacco', 'common tobacco', 'wild tobacco', + 'cupflower', 'whitecup', 'petunia', 'large white petunia', + 'violet-flowered petunia', 'hybrid petunia', 'cape gooseberry', + 'strawberry tomato', 'tomatillo', 'tomatillo', 'yellow henbane', + 'cock\'s eggs', 'salpiglossis', 'painted tongue', 'butterfly flower', + 'scopolia carniolica', 'chalice vine', 'verbena', 'lantana', + 'black mangrove', 'white mangrove', 'black mangrove', 'teak', 'spurge', + 'sun spurge', 'petty spurge', 'medusa\'s head', 'wild spurge', + 'snow-on-the-mountain', 'cypress spurge', 'leafy spurge', 'hairy spurge', + 'poinsettia', 'japanese poinsettia', 'fire-on-the-mountain', 'wood spurge', + 'dwarf spurge', 'scarlet plume', 'naboom', 'crown of thorns', + 'toothed spurge', 'three-seeded mercury', 'croton', 'cascarilla', + 'cascarilla bark', 'castor-oil plant', 'spurge nettle', 'physic nut', + 'para rubber tree', 'cassava', 'bitter cassava', 'cassava', 'sweet cassava', + 'candlenut', 'tung tree', 'slipper spurge', 'candelilla', 'jewbush', + 'jumping bean', 'camellia', 'japonica', 'umbellifer', 'wild parsley', + 'fool\'s parsley', 'dill', 'angelica', 'garden angelica', 'wild angelica', + 'chervil', 'cow parsley', 'wild celery', 'astrantia', 'greater masterwort', + 'caraway', 'whorled caraway', 'water hemlock', 'spotted cowbane', 'hemlock', + 'earthnut', 'cumin', 'wild carrot', 'eryngo', 'sea holly', + 'button snakeroot', 'rattlesnake master', 'fennel', 'common fennel', + 'florence fennel', 'cow parsnip', 'lovage', 'sweet cicely', 'water fennel', + 'parsnip', 'cultivated parsnip', 'wild parsnip', 'parsley', + 'italian parsley', 'hamburg parsley', 'anise', 'sanicle', 'purple sanicle', + 'european sanicle', 'water parsnip', 'greater water parsnip', 'skirret', + 'dogwood', 'common white dogwood', 'red osier', 'silky dogwood', + 'silky cornel', 'common european dogwood', 'bunchberry', 'cornelian cherry', + 'puka', 'kapuka', 'valerian', 'common valerian', 'common corn salad', + 'red valerian', 'filmy fern', 'bristle fern', 'hare\'s-foot bristle fern', + 'killarney fern', 'kidney fern', 'flowering fern', 'royal fern', + 'interrupted fern', 'crape fern', 'crepe fern', 'curly grass', 'pine fern', + 'climbing fern', 'creeping fern', 'climbing maidenhair', 'scented fern', + 'clover fern', 'nardoo', 'water clover', 'pillwort', 'regnellidium', + 'floating-moss', 'mosquito fern', 'adder\'s tongue', 'ribbon fern', + 'grape fern', 'daisyleaf grape fern', 'leathery grape fern', + 'rattlesnake fern', 'flowering fern', 'powdery mildew', 'dutch elm fungus', + 'ergot', 'rye ergot', 'black root rot fungus', 'dead-man\'s-fingers', + 'sclerotinia', 'brown cup', 'earthball', 'scleroderma citrinum', + 'scleroderma flavidium', 'scleroderma bovista', 'podaxaceae', + 'stalked puffball', 'stalked puffball', 'false truffle', + 'rhizopogon idahoensis', 'truncocolumella citrina', 'mucor', 'rhizopus', + 'bread mold', 'slime mold', 'true slime mold', 'cellular slime mold', + 'dictostylium', 'pond-scum parasite', 'potato wart fungus', 'white fungus', + 'water mold', 'downy mildew', 'blue mold fungus', 'onion mildew', + 'tobacco mildew', 'white rust', 'pythium', 'damping off fungus', + 'phytophthora citrophthora', 'phytophthora infestans', 'clubroot fungus', + 'geglossaceae', 'sarcosomataceae', 'rufous rubber cup', 'devil\'s cigar', + 'devil\'s urn', 'truffle', 'club fungus', 'coral fungus', 'tooth fungus', + 'lichen', 'ascolichen', 'basidiolichen', 'lecanora', 'manna lichen', + 'archil', 'roccella', 'beard lichen', 'horsehair lichen', 'reindeer moss', + 'crottle', 'iceland moss', 'fungus', 'promycelium', 'true fungus', + 'basidiomycete', 'mushroom', 'agaric', 'mushroom', 'mushroom', 'toadstool', + 'horse mushroom', 'meadow mushroom', 'shiitake', 'scaly lentinus', + 'royal agaric', 'false deathcap', 'fly agaric', 'death cap', + 'blushing mushroom', 'destroying angel', 'chanterelle', + 'floccose chanterelle', 'pig\'s ears', 'cinnabar chanterelle', + 'jack-o-lantern fungus', 'inky cap', 'shaggymane', 'milkcap', + 'fairy-ring mushroom', 'fairy ring', 'oyster mushroom', 'olive-tree agaric', + 'pholiota astragalina', 'pholiota aurea', 'pholiota destruens', + 'pholiota flammans', 'pholiota flavida', 'nameko', + 'pholiota squarrosa-adiposa', 'pholiota squarrosa', + 'pholiota squarrosoides', 'stropharia ambigua', 'stropharia hornemannii', + 'stropharia rugoso-annulata', 'gill fungus', 'entoloma lividum', + 'entoloma aprile', 'chlorophyllum molybdites', 'lepiota', + 'parasol mushroom', 'poisonous parasol', 'lepiota naucina', + 'lepiota rhacodes', 'american parasol', 'lepiota rubrotincta', + 'lepiota clypeolaria', 'onion stem', 'pink disease fungus', + 'bottom rot fungus', 'potato fungus', 'coffee fungus', 'blewits', + 'sandy mushroom', 'tricholoma pessundatum', 'tricholoma sejunctum', + 'man-on-a-horse', 'tricholoma venenata', 'tricholoma pardinum', + 'tricholoma vaccinum', 'tricholoma aurantium', 'volvaria bombycina', + 'pluteus aurantiorugosus', 'pluteus magnus', 'deer mushroom', + 'straw mushroom', 'volvariella bombycina', 'clitocybe clavipes', + 'clitocybe dealbata', 'clitocybe inornata', 'clitocybe robusta', + 'clitocybe irina', 'clitocybe subconnexa', 'winter mushroom', 'mycelium', + 'sclerotium', 'sac fungus', 'ascomycete', 'clavicipitaceae', 'grainy club', + 'yeast', 'baker\'s yeast', 'wine-maker\'s yeast', 'aspergillus fumigatus', + 'brown root rot fungus', 'discomycete', 'leotia lubrica', 'mitrula elegans', + 'sarcoscypha coccinea', 'caloscypha fulgens', 'aleuria aurantia', 'elf cup', + 'peziza domicilina', 'blood cup', 'urnula craterium', 'galiella rufa', + 'jafnea semitosta', 'morel', 'common morel', 'disciotis venosa', 'verpa', + 'verpa bohemica', 'verpa conica', 'black morel', 'morchella crassipes', + 'morchella semilibera', 'wynnea americana', 'wynnea sparassoides', + 'false morel', 'lorchel', 'helvella', 'helvella crispa', + 'helvella acetabulum', 'helvella sulcata', 'discina', 'gyromitra', + 'gyromitra californica', 'gyromitra sphaerospora', 'gyromitra esculenta', + 'gyromitra infula', 'gyromitra fastigiata', 'gyromitra gigas', + 'gasteromycete', 'stinkhorn', 'common stinkhorn', 'phallus ravenelii', + 'dog stinkhorn', 'calostoma lutescens', 'calostoma cinnabarina', + 'calostoma ravenelii', 'stinky squid', 'puffball', 'giant puffball', + 'earthstar', 'geastrum coronatum', 'radiigera fuscogleba', + 'astreus pteridis', 'astreus hygrometricus', 'bird\'s-nest fungus', + 'gastrocybe lateritia', 'macowanites americanus', 'polypore', + 'bracket fungus', 'albatrellus dispansus', 'albatrellus ovinus', + 'neolentinus ponderosus', 'oligoporus leucospongia', 'polyporus tenuiculus', + 'hen-of-the-woods', 'polyporus squamosus', 'beefsteak fungus', 'agaric', + 'bolete', 'boletus chrysenteron', 'boletus edulis', 'frost\'s bolete', + 'boletus luridus', 'boletus mirabilis', 'boletus pallidus', + 'boletus pulcherrimus', 'boletus pulverulentus', 'boletus roxanae', + 'boletus subvelutipes', 'boletus variipes', 'boletus zelleri', + 'fuscoboletinus paluster', 'fuscoboletinus serotinus', + 'leccinum fibrillosum', 'suillus albivelatus', 'old-man-of-the-woods', + 'boletellus russellii', 'jelly fungus', 'snow mushroom', 'witches\' butter', + 'tremella foliacea', 'tremella reticulata', 'jew\'s-ear', 'rust', 'aecium', + 'flax rust', 'blister rust', 'wheat rust', 'apple rust', 'smut', + 'covered smut', 'loose smut', 'cornsmut', 'boil smut', 'sphacelotheca', + 'head smut', 'bunt', 'bunt', 'onion smut', 'flag smut fungus', + 'wheat flag smut', 'felt fungus', 'waxycap', 'hygrocybe acutoconica', + 'hygrophorus borealis', 'hygrophorus caeruleus', + 'hygrophorus inocybiformis', 'hygrophorus kauffmanii', + 'hygrophorus marzuolus', 'hygrophorus purpurascens', 'hygrophorus russula', + 'hygrophorus sordidus', 'hygrophorus tennesseensis', 'hygrophorus turundus', + 'neohygrophorus angelesianus', 'cortinarius armillatus', + 'cortinarius atkinsonianus', 'cortinarius corrugatus', + 'cortinarius gentilis', 'cortinarius mutabilis', + 'cortinarius semisanguineus', 'cortinarius subfoetidus', + 'cortinarius violaceus', 'gymnopilus spectabilis', 'gymnopilus validipes', + 'gymnopilus ventricosus', 'mold', 'mildew', 'verticillium', 'monilia', + 'candida', 'candida albicans', 'blastomycete', 'yellow spot fungus', + 'green smut fungus', 'dry rot', 'rhizoctinia', 'houseplant', 'bedder', + 'succulent', 'cultivar', 'weed', 'wort', 'brier', 'aril', 'sporophyll', + 'sporangium', 'sporangiophore', 'ascus', 'ascospore', 'arthrospore', + 'eusporangium', 'tetrasporangium', 'gametangium', 'sorus', 'sorus', + 'partial veil', 'lignum', 'vascular ray', 'phloem', 'evergreen', + 'deciduous plant', 'poisonous plant', 'vine', 'creeper', 'tendril', + 'root climber', 'lignosae', 'arborescent plant', 'snag', 'tree', + 'timber tree', 'treelet', 'arbor', 'bean tree', 'pollard', 'sapling', + 'shade tree', 'gymnospermous tree', 'conifer', 'angiospermous tree', + 'nut tree', 'spice tree', 'fever tree', 'stump', 'bonsai', 'ming tree', + 'ming tree', 'undershrub', 'subshrub', 'bramble', 'liana', 'geophyte', + 'desert plant', 'mesophyte', 'marsh plant', 'hemiepiphyte', 'strangler', + 'lithophyte', 'saprobe', 'autophyte', 'root', 'taproot', 'prop root', + 'prophyll', 'rootstock', 'quickset', 'stolon', 'tuberous plant', 'rhizome', + 'rachis', 'caudex', 'cladode', 'receptacle', 'scape', 'umbel', 'petiole', + 'peduncle', 'pedicel', 'flower cluster', 'raceme', 'panicle', 'thyrse', + 'cyme', 'cymule', 'glomerule', 'scorpioid cyme', 'ear', 'spadix', + 'bulbous plant', 'bulbil', 'cormous plant', 'fruit', 'fruitlet', 'seed', + 'bean', 'nut', 'nutlet', 'kernel', 'syconium', 'berry', 'aggregate fruit', + 'simple fruit', 'acinus', 'drupe', 'drupelet', 'pome', 'pod', 'loment', + 'pyxidium', 'husk', 'cornhusk', 'pod', 'accessory fruit', 'buckthorn', + 'buckthorn berry', 'cascara buckthorn', 'cascara', 'carolina buckthorn', + 'coffeeberry', 'redberry', 'nakedwood', 'jujube', 'christ\'s-thorn', + 'hazel', 'fox grape', 'muscadine', 'vinifera', 'pinot blanc', + 'sauvignon grape', 'sauvignon blanc', 'muscadet', 'riesling', 'zinfandel', + 'chenin blanc', 'malvasia', 'verdicchio', 'boston ivy', 'virginia creeper', + 'true pepper', 'betel', 'cubeb', 'schizocarp', 'peperomia', + 'watermelon begonia', 'yerba mansa', 'pinna', 'frond', 'bract', 'bracteole', + 'involucre', 'glume', 'palmate leaf', 'pinnate leaf', 'bijugate leaf', + 'decompound leaf', 'acuminate leaf', 'deltoid leaf', 'ensiform leaf', + 'linear leaf', 'lyrate leaf', 'obtuse leaf', 'oblanceolate leaf', + 'pandurate leaf', 'reniform leaf', 'spatulate leaf', 'even-pinnate leaf', + 'odd-pinnate leaf', 'pedate leaf', 'crenate leaf', 'dentate leaf', + 'denticulate leaf', 'erose leaf', 'runcinate leaf', 'prickly-edged leaf', + 'deadwood', 'haulm', 'branchlet', 'osier', 'giant scrambling fern', + 'umbrella fern', 'floating fern', 'polypody', 'licorice fern', + 'grey polypody', 'leatherleaf', 'rock polypody', 'common polypody', + 'bear\'s-paw fern', 'strap fern', 'florida strap fern', 'basket fern', + 'snake polypody', 'climbing bird\'s nest fern', 'golden polypody', + 'staghorn fern', 'south american staghorn', 'common staghorn fern', + 'felt fern', 'potato fern', 'myrmecophyte', 'grass fern', 'spleenwort', + 'black spleenwort', 'bird\'s nest fern', 'ebony spleenwort', + 'black-stem spleenwort', 'walking fern', 'green spleenwort', + 'mountain spleenwort', 'lobed spleenwort', 'lanceolate spleenwort', + 'hart\'s-tongue', 'scale fern', 'scolopendrium', 'deer fern', 'doodia', + 'chain fern', 'virginia chain fern', 'silver tree fern', 'davallia', + 'hare\'s-foot fern', 'canary island hare\'s foot fern', + 'squirrel\'s-foot fern', 'bracken', 'soft tree fern', 'scythian lamb', + 'false bracken', 'thyrsopteris', 'shield fern', 'broad buckler-fern', + 'fragrant cliff fern', 'goldie\'s fern', 'wood fern', 'male fern', + 'marginal wood fern', 'mountain male fern', 'lady fern', 'alpine lady fern', + 'silvery spleenwort', 'holly fern', 'bladder fern', 'brittle bladder fern', + 'mountain bladder fern', 'bulblet fern', 'silvery spleenwort', 'oak fern', + 'limestone fern', 'ostrich fern', 'hart\'s-tongue', 'sensitive fern', + 'christmas fern', 'holly fern', 'braun\'s holly fern', 'western holly fern', + 'soft shield fern', 'leather fern', 'button fern', 'indian button fern', + 'woodsia', 'rusty woodsia', 'alpine woodsia', 'smooth woodsia', + 'boston fern', 'basket fern', 'golden fern', 'maidenhair', + 'common maidenhair', 'american maidenhair fern', 'bermuda maidenhair', + 'brittle maidenhair', 'farley maidenhair', 'annual fern', 'lip fern', + 'smooth lip fern', 'lace fern', 'wooly lip fern', 'southwestern lip fern', + 'bamboo fern', 'american rock brake', 'european parsley fern', 'hand fern', + 'cliff brake', 'coffee fern', 'purple rock brake', 'bird\'s-foot fern', + 'button fern', 'silver fern', 'golden fern', 'gold fern', 'pteris cretica', + 'spider brake', 'ribbon fern', 'potato fern', 'angiopteris', + 'skeleton fork fern', 'horsetail', 'common horsetail', 'swamp horsetail', + 'scouring rush', 'marsh horsetail', 'wood horsetail', + 'variegated horsetail', 'club moss', 'shining clubmoss', 'alpine clubmoss', + 'fir clubmoss', 'ground cedar', 'ground fir', 'foxtail grass', 'spikemoss', + 'meadow spikemoss', 'desert selaginella', 'resurrection plant', + 'florida selaginella', 'quillwort', 'earthtongue', 'snuffbox fern', + 'christella', 'mountain fern', 'new york fern', 'massachusetts fern', + 'beech fern', 'broad beech fern', 'long beech fern', 'shoestring fungus', + 'armillaria caligata', 'armillaria ponderosa', 'armillaria zelleri', + 'honey mushroom', 'milkweed', 'white milkweed', 'poke milkweed', + 'swamp milkweed', 'mead\'s milkweed', 'purple silkweed', 'showy milkweed', + 'poison milkweed', 'butterfly weed', 'whorled milkweed', 'cruel plant', + 'wax plant', 'silk vine', 'stapelia', 'stapelias asterias', 'stephanotis', + 'madagascar jasmine', 'negro vine', 'zygospore', 'tree of knowledge', + 'orangery', 'pocketbook', 'shit', 'cordage', 'yard', 'extremum', + 'leaf shape', 'equilateral', 'figure', 'pencil', 'plane figure', + 'solid figure', 'line', 'bulb', 'convex shape', 'concave shape', 'cylinder', + 'round shape', 'heart', 'polygon', 'convex polygon', 'concave polygon', + 'reentrant polygon', 'amorphous shape', 'closed curve', + 'simple closed curve', 's-shape', 'wave', 'extrados', 'hook', 'envelope', + 'bight', 'diameter', 'cone', 'funnel', 'oblong', 'circle', 'circle', + 'equator', 'scallop', 'ring', 'loop', 'bight', 'helix', 'element of a cone', + 'element of a cylinder', 'ellipse', 'quadrate', 'triangle', + 'acute triangle', 'isosceles triangle', 'obtuse triangle', 'right triangle', + 'scalene triangle', 'parallel', 'trapezoid', 'star', 'pentagon', 'hexagon', + 'heptagon', 'octagon', 'nonagon', 'decagon', 'rhombus', 'spherical polygon', + 'spherical triangle', 'convex polyhedron', 'concave polyhedron', 'cuboid', + 'quadrangular prism', 'bell', 'angular distance', 'true anomaly', + 'spherical angle', 'angle of refraction', 'acute angle', 'groove', 'rut', + 'bulge', 'belly', 'bow', 'crescent', 'ellipsoid', 'hypotenuse', 'balance', + 'conformation', 'symmetry', 'spheroid', 'spherule', 'toroid', 'column', + 'barrel', 'pipe', 'pellet', 'bolus', 'dewdrop', 'ridge', 'rim', 'taper', + 'boundary', 'incisure', 'notch', 'wrinkle', 'dermatoglyphic', 'frown line', + 'line of life', 'line of heart', 'crevice', 'cleft', 'roulette', 'node', + 'tree', 'stemma', 'brachium', 'fork', 'block', 'ovoid', 'tetrahedron', + 'pentahedron', 'hexahedron', 'regular polyhedron', 'polyhedral angle', + 'cube', 'truncated pyramid', 'truncated cone', 'tail', 'tongue', + 'trapezohedron', 'wedge', 'keel', 'place', 'herpes', 'chlamydia', 'wall', + 'micronutrient', 'chyme', 'ragweed pollen', 'pina cloth', + 'chlorobenzylidenemalononitrile', 'carbon', 'charcoal', 'rock', 'gravel', + 'aflatoxin', 'alpha-tocopheral', 'leopard', 'bricks and mortar', 'lagging', + 'hydraulic cement', 'choline', 'concrete', 'glass wool', 'soil', + 'high explosive', 'litter', 'fish meal', 'greek fire', 'culture medium', + 'agar', 'blood agar', 'hip tile', 'hyacinth', 'hydroxide ion', 'ice', + 'inositol', 'linoleum', 'lithia water', 'lodestone', 'pantothenic acid', + 'paper', 'papyrus', 'pantile', 'blacktop', 'tarmacadam', 'paving', + 'plaster', 'poison gas', 'ridge tile', 'roughcast', 'sand', 'spackle', + 'render', 'wattle and daub', 'stucco', 'tear gas', 'toilet tissue', + 'linseed', 'vitamin', 'fat-soluble vitamin', 'water-soluble vitamin', + 'vitamin a', 'vitamin a1', 'vitamin a2', 'b-complex vitamin', 'vitamin b1', + 'vitamin b12', 'vitamin b2', 'vitamin b6', 'vitamin bc', 'niacin', + 'vitamin d', 'vitamin e', 'biotin', 'vitamin k', 'vitamin k1', 'vitamin k3', + 'vitamin p', 'vitamin c', 'planking', 'chipboard', 'knothole', +] diff --git a/Tipsomaly/model/big_vision/datasets/infovqa/infovqa.py b/Tipsomaly/model/big_vision/datasets/infovqa/infovqa.py new file mode 100644 index 0000000000000000000000000000000000000000..3d231333dd8db15b7be24d1dea9885005b18aab1 --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/infovqa/infovqa.py @@ -0,0 +1,141 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +r"""Implements InfoVqa in TFDS structure. + +First, download and unzip the dataset from https://rrc.cvc.uab.es/?ch=17 +and place it in /tmp/data/infovqa. + +Then, run conversion locally (make sure to install tensorflow-datasets for the `tfds` util): + + cd third_party/py/big_vision/datasets + env TFDS_DATA_DIR=/tmp/tfds tfds build --datasets=infovqa + +Example to load: + + import tensorflow_datasets as tfds + dataset = tfds.load('infovqa', split='train', data_dir='/tmp/tfds') + +Dataset splits: + train: 23946 examples/questions (4406 images) + val: 2801 examples/questions (500 images) + test: 3288 examples/questions (579 images) (no answers) + +Recommended training splits: + train: train[:95%] (22749 examples/questions) + minitrain: train[:5%] (1197 examples/questions) + minival: train[95%:] (1197 examples/questions) + eval: val (2801 examples/questions) + +Note that according to task description in +https://rrc.cvc.uab.es/?ch=17&com=tasks: + - Order of items in a multi span answer does not matter. Therefore, we include + all permutations of the answer in the val split. + - Answers are not case sensitive. We leave it to the user to lower case + answers if they want to. +""" +import itertools +import json +import os + +import numpy as np +import tensorflow_datasets as tfds + + +_DESCRIPTION = """InfographicVQA dataset.""" + +# pylint: disable=line-too-long +_CITATION = """ +@inproceedings{Mathew_2022, + title={InfographicVQA}, + url={http://dx.doi.org/10.1109/WACV51458.2022.00264}, + DOI={10.1109/wacv51458.2022.00264}, + booktitle={2022 IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)}, + publisher={IEEE}, + author={Mathew, Minesh and Bagal, Viraj and Tito, Ruben and Karatzas, Dimosthenis and Valveny, Ernest and Jawahar, C. V.}, + year={2022}, + month=jan } +""" +# pylint: enable=line-too-long + +# When running locally (recommended), copy files as above an use these: +_INFOVQA_PATH = '/tmp/data/infovqa/' +_ANNOTATIONS = { + 'train': 'infographicsVQA_train_v1.0.json', + 'val': 'infographicsVQA_val_v1.0_withQT.json', + 'test': 'infographicsVQA_test_v1.0.json', + } + + +class Infovqa(tfds.core.GeneratorBasedBuilder): + """DatasetBuilder for infovqa dataset.""" + + VERSION = tfds.core.Version('1.1.0') + RELEASE_NOTES = { + '1.0.0': 'First release.', + '1.1.0': 'Add multi-span permutations to the val split answers.', + } + + def _info(self): + """Returns the metadata.""" + + return tfds.core.DatasetInfo( + builder=self, + description=_DESCRIPTION, + features=tfds.features.FeaturesDict({ + 'question_id': tfds.features.Scalar(np.int32), + 'filename': tfds.features.Text(), + 'image': tfds.features.Image(encoding_format='jpeg'), + 'question': tfds.features.Text(), + 'answers': tfds.features.Sequence(tfds.features.Text()), + }), + supervised_keys=None, + homepage='https://www.docvqa.org/datasets/infographicvqa', + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + """Returns SplitGenerators.""" + return {split: self._generate_examples(split) + for split in ('train', 'val', 'test')} + + def _generate_examples(self, split): + """Yields (key, example) tuples from test set.""" + annot_fname = os.path.join(_INFOVQA_PATH, _ANNOTATIONS[split]) + with open(annot_fname, 'r') as f: + data = json.loads(f.read()) + + for x in data['data']: + yield x['questionId'], { + 'question_id': x['questionId'], + 'filename': x['image_local_name'], + 'image': os.path.join(_INFOVQA_PATH, 'images', x['image_local_name']), + 'question': x['question'], + 'answers': maybe_permute(x.get('answers', []), split), + } + + +def maybe_permute(answers, split): + if split != 'val': + return answers + new_answers = [] + for x in answers: + if ', ' in x: # Create all permutations. + # The first element remains the same. + new_answers.extend([', '.join(y) + for y in itertools.permutations(x.split(', '))]) + else: + new_answers.append(x) + return new_answers diff --git a/Tipsomaly/model/big_vision/datasets/okvqa/okvqa.py b/Tipsomaly/model/big_vision/datasets/okvqa/okvqa.py new file mode 100644 index 0000000000000000000000000000000000000000..d4cb31afcafb04278b82af6b8b3d204d2580508c --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/okvqa/okvqa.py @@ -0,0 +1,213 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +r"""Implements the OKVQA dataset for TFDS. + +Download the required files from https://okvqa.allenai.org/download.html: + +mkdir -p /tmp/tfds +cd /tmp/tfds/ +wget http://images.cocodataset.org/zips/train2014.zip +wget http://images.cocodataset.org/zips/val2014.zip +wget https://okvqa.allenai.org/static/data/mscoco_train2014_annotations.json.zip +wget https://okvqa.allenai.org/static/data/mscoco_val2014_annotations.json.zip +wget https://okvqa.allenai.org/static/data/OpenEnded_mscoco_train2014_questions.json.zip +wget https://okvqa.allenai.org/static/data/OpenEnded_mscoco_val2014_questions.json.zip +unzip val2014.zip +unzip train2014.zip +unzip OpenEnded_mscoco_train2014_questions.json.zip +unzip OpenEnded_mscoco_val2014_questions.json.zip +unzip mscoco_train2014_annotations.json.zip +unzip mscoco_val2014_annotations.json.zip + +Then, run conversion locally (make sure to install tensorflow-datasets for the +`tfds` util): + + cd big_vision/datasets + env TFDS_DATA_DIR=/tmp/tfds tfds build --datasets=okvqa + +Example to load: + + import tensorflow_datasets as tfds + dataset = tfds.load('okvqa', split='val', data_dir='/tmp/tfds') +""" + +import json +import os +from typing import Any +import numpy as np +import tensorflow_datasets as tfds + +_DESCRIPTION = """ +OKVQA addresses the task of VQA with outside knowledge. +This version of the dataset contains: +- Questions + Answers from OKVQA. +- Images from COCO. +""" + +_CITATION = """ +@InProceedings{okvqa, +author = {Kenneth Marino and Mohammad Rastegari and Ali Farhadi and Roozbeh Mottaghi}, +title = {OK-VQA: A Visual Question Answering Benchmark Requiring External Knowledge}, +booktitle = {Conference on Computer Vision and Pattern Recognition (CVPR)}, +year = {2019}, +} +""" + +ANNOTATION_FILE = { + 'train': 'mscoco_train2014_annotations.json', + 'val': 'mscoco_val2014_annotations.json', +} +QUESTIONS_FILE = { + 'train': 'OpenEnded_mscoco_train2014_questions.json', + 'val': 'OpenEnded_mscoco_val2014_questions.json', +} +QUESTION_TYPES = { + 'one': 'Vehicles and Transportation', + 'two': 'Brands, Companies and Products', + 'three': 'Objects, Material and Clothing', + 'four': 'Sports and Recreation', + 'five': 'Cooking and Food', + 'six': 'Geography, History, Language and Culture', + 'seven': 'People and Everyday life', + 'eight': 'Plants and Animals', + 'nine': 'Science and Technology', + 'ten': 'Weather and Climate', + 'other': 'Other', +} + + +# When running locally (recommended), copy files as above an use these: +_OKVQA_PATH = '/media/scratch/okvqa' + + +class OkVqa(tfds.core.GeneratorBasedBuilder): + """Import COCO dataset for OKVQA with KAT features.""" + + VERSION = tfds.core.Version('1.0.0') + RELEASE_NOTES = {'1.0.0': 'Changed to array record format.'} + MANUAL_DOWNLOAD_INSTRUCTIONS = """ + In manual_dir/ you should have a directory okvqa which contains the + following files and directories: + From the OKVQA dataset: + - mscoco_train2014_annotations.json + - mscoco_val2014_annotations.json + - OpenEnded_mscoco_train2014_questions.json + - OpenEnded_mscoco_val2014_questions.json + - train2014.zip + - val2014.zip + """ + + def _info(self) -> tfds.core.DatasetInfo: + """Returns the dataset metadata.""" + features = tfds.features.FeaturesDict({ + 'image': tfds.features.Image(shape=(None, None, 3)), + 'image_id': tfds.features.Scalar(dtype=np.int64), + 'answer_type': tfds.features.Text(), + 'answers': tfds.features.Sequence(tfds.features.Text()), + 'answers_confidence': tfds.features.Tensor(shape=[10], dtype=np.bool_), + 'answers_raw': tfds.features.Sequence(tfds.features.Text()), + 'question_id': tfds.features.Scalar(dtype=np.int64), + 'question_type': tfds.features.Text(), + 'question_type_readable': tfds.features.Text(), + 'question': tfds.features.Text(), + }) + + return tfds.core.DatasetInfo( + builder=self, + features=features, + description=_DESCRIPTION, + supervised_keys=None, + homepage='https://okvqa.allenai.org/', + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager) -> ...: + """Call the function which defines the splits.""" + # data_dir = dl_manager.manual_dir + data_dir = _OKVQA_PATH + return { + 'train': self._generate_examples(data_dir, 'train'), + 'val': self._generate_examples(data_dir, 'val'), + } + + def _generate_examples(self, data_dir: str, split: str) -> ...: + annotations = get_okvqa_annotations(data_dir, split) + + for question_id, annotation in annotations.items(): + image_id = annotation['image_id'] + + # Sanity check. + if len(annotation['answers']) != 10: + num_answers = len(annotation['answers']) + raise ValueError( + f'The number of answers for {image_id} is not 10 but {num_answers}') + + feature_dict = { + 'image': self.get_image_path(data_dir, split, image_id), + 'image_id': image_id, + 'answer_type': annotation['answer_type'], + 'answers': [a['answer'] for a in annotation['answers']], + 'answers_confidence': _get_answer_confidence(annotation['answers']), + 'answers_raw': [a['raw_answer'] for a in annotation['answers']], + 'question_id': annotation['question_id'], + 'question_type': annotation['question_type'], + 'question_type_readable': QUESTION_TYPES[annotation['question_type']], + 'question': annotation['question'], + } + yield f'{question_id}', feature_dict + + def get_image_path(self, data_dir: str, split: str, image_id: int) -> str: + subdir = {'train': 'train2014', 'val': 'val2014'}[split] + return f'{data_dir}/{subdir}/COCO_{subdir}_{image_id:012d}.jpg' + + +def _get_answer_confidence(answers: list[dict[str, str]]) -> np.ndarray: + """Get OKVQA answer confidences as bool.""" + confidences = [] + for a in answers: + confidence = a['answer_confidence'] + if confidence == 'yes': + confidences.append(True) + elif confidence == 'no': + confidences.append(False) + else: + raise ValueError(f'Unknown confidence: {confidence}') + return np.array(confidences, dtype=bool) + + +def _read_json( + data_dir: str, file: str, key: str +) -> dict[int, dict[str, Any]]: + with open(os.path.join(data_dir, file)) as f: + data = json.load(f) + questions = {d['question_id']: d for d in data[key]} + return questions + + +def get_okvqa_annotations( + data_dir: str, split: str +) -> dict[int, dict[str, Any]]: + """Return okvqa annotations (quesions and answers) as dictionary.""" + questions = _read_json(data_dir, QUESTIONS_FILE[split], 'questions') + annotations = _read_json(data_dir, ANNOTATION_FILE[split], 'annotations') + + assert len(annotations) == len(questions) + for question_id, question in questions.items(): + assert question['image_id'] == annotations[question_id]['image_id'] + assert question['question_id'] == annotations[question_id]['question_id'] + annotations[question_id]['question'] = question['question'] + + return annotations diff --git a/Tipsomaly/model/big_vision/datasets/pope/pope.py b/Tipsomaly/model/big_vision/datasets/pope/pope.py new file mode 100644 index 0000000000000000000000000000000000000000..abeb41234b9fe4a00654dfbb77d8cc3e2d57b471 --- /dev/null +++ b/Tipsomaly/model/big_vision/datasets/pope/pope.py @@ -0,0 +1,145 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +r"""Implements POPE test-set in TFDS structure. + +It's small data, so simple to run locally. First, copy the data to local disk: +First download json files from https://github.com/AoiDragon/POPE; then download +MSCOCO (val 2014) images from https://cocodataset.org/#download + + mkdir -p /tmp/data/pope/ + mkdir -p /tmp/data/pope/pope/ + mkdir -p /tmp/data/pope/images/ + git clone https://github.com/AoiDragon/POPE.git + cp POPE/output/coco/* /tmp/data/pope/pope/ + wget http://images.cocodataset.org/zips/val2014.zip + unzip val2014.zip + cp -r val2014/ /tmp/data/pope/images/ + +Then, run conversion locally (make sure to install tensorflow-datasets for the `tfds` util): + + cd big_vision/datasets + env TFDS_DATA_DIR=/tmp/tfds tfds build --datasets=pope + +Example to load: + + import tensorflow_datasets as tfds + dataset_random = tfds.load('pope/pope_random', split='test', data_dir='/tmp/tfds') + dataset_popular = tfds.load('pope/pope_popular', split='test', data_dir='/tmp/tfds') + dataset_adversarial = tfds.load('pope/pope_adversarial', split='test', data_dir='/tmp/tfds') + +""" +import json +import os + +import numpy as np +import tensorflow_datasets as tfds + + +_DESCRIPTION = """POPE dataset.""" + +# pylint: disable=line-too-long +_CITATION = """ +@inproceedings{li-etal-2023-evaluating, + title = "Evaluating Object Hallucination in Large Vision-Language Models", + author = "Li, Yifan and + Du, Yifan and + Zhou, Kun and + Wang, Jinpeng and + Zhao, Xin and + Wen, Ji-Rong", + editor = "Bouamor, Houda and + Pino, Juan and + Bali, Kalika", + booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing", + month = dec, + year = "2023", + address = "Singapore", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.emnlp-main.20", + doi = "10.18653/v1/2023.emnlp-main.20", + pages = "292--305", + abstract = "Inspired by the superior language abilities of large language models (LLM), large vision-language models (LVLM) have been recently proposed by integrating powerful LLMs for improving the performance on complex multimodal tasks. Despite the promising progress on LVLMs, we find that they suffer from object hallucinations, i.e., they tend to generate objects inconsistent with the target images in the descriptions. To investigate it, this work presents the first systematic study on object hallucination of LVLMs. We conduct the evaluation experiments on several representative LVLMs, and show that they mostly suffer from severe object hallucination issues. We further discuss that the visual instructions may influence the hallucination, and find that: objects that frequently appear in the visual instructions or co-occur with the image objects are obviously prone to be hallucinated by LVLMs. Besides, we further design a polling-based query method called POPE for better evaluation of object hallucination. Experiment results show that our POPE can evaluate object hallucination in a more stable and flexible way.", +} +""" +# pylint: enable=line-too-long + +# When running locally (recommended), copy files as above and use these: +_POPE_PATH = '/tmp/data/pope/' + + +class POPEConfig(tfds.core.BuilderConfig): + """Configuration to build the dataset.""" + + pass + + +class POPE(tfds.core.GeneratorBasedBuilder): + """DatasetBuilder for POPE dataset.""" + + VERSION = tfds.core.Version('1.0.0') + RELEASE_NOTES = {'1.0.0': 'First release.'} + BUILDER_CONFIGS = [ + POPEConfig(name='pope_random', description='Random set'), + POPEConfig(name='pope_popular', description='Popular set'), + POPEConfig(name='pope_adversarial', description='Adversarial set'), + ] + + def _info(self): + """Returns the metadata.""" + return tfds.core.DatasetInfo( + builder=self, + description=_DESCRIPTION, + features=tfds.features.FeaturesDict({ + 'question_id': tfds.features.Scalar(np.int32), + 'image/filename': tfds.features.Text(), + 'image': tfds.features.Image(encoding_format='png'), + 'question': tfds.features.Text(), + 'answer': tfds.features.Text(), + 'thing': tfds.features.Text(), + }), + supervised_keys=None, + homepage='https://github.com/AoiDragon/POPE', + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + """Returns SplitGenerators.""" + return {'test': self._generate_examples('test', self.builder_config.name)} + + def _generate_examples(self, split: str, source: str): + """Yields (key, example) tuples from test set.""" + annot_fname = os.path.join( + _POPE_PATH, f'pope/coco_{source}.json' + ) + + with open(annot_fname, 'r') as f: + data = [json.loads(line) for line in f] + + for idx, v in enumerate(data): + question = v['text'] + thing = ( + question.replace('Is there an ', '') + .replace('Is there a ', '') + .replace(' in the image?', '') + ) + yield idx, { + 'question_id': idx, + 'image/filename': v['image'], + 'image': os.path.join(_POPE_PATH, 'images/val2014/', v['image']), + 'question': question, + 'answer': v['label'], + 'thing': thing, + } diff --git a/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/coco_caption.py b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/coco_caption.py new file mode 100644 index 0000000000000000000000000000000000000000..db53d9831c3ea8e877c0463cf1eca07290e2ddea --- /dev/null +++ b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/coco_caption.py @@ -0,0 +1,145 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluator for caption generation metrics used for the MS COCO dataset.""" +import collections +import functools +import os +import tempfile + +import big_vision.evaluators.common as c +import big_vision.input_pipeline +import big_vision.pp.builder +import big_vision.pp.tokenizer +import big_vision.utils as u + +from pycocoevalcap.bleu import bleu +from pycocoevalcap.cider import cider +from pycocoevalcap.meteor import meteor +from pycocoevalcap.rouge import rouge +from pycocoevalcap.spice import spice +from pycocoevalcap.tokenizer import ptbtokenizer + +import jax + +from tensorflow.io import gfile + +# Temporary global flag to facilitate backwards compatability. Will be removed +# by the end of year 2023. +API = "jit" + + +class Evaluator: + """Evaluator for caption generation metrics used for the MS COCO dataset. + + See https://arxiv.org/pdf/1504.00325.pdf or the repository implementing it + https://github.com/tylin/coco-caption for details on the metrics. This code + uses the python3 pip package from: https://github.com/salaniz/pycocoevalcap + + Note that both the model caption and the ground truth reference captions are + further processed with the PTBTokenizer before computing scores. + + `predict_fn` accepts arbitrary dictionaries of parameters and data, where + the data dictionary is produced by the `pp_fn` op. It is expected to output a + dict containing tokenized captions. + + `pp_fn` must have fields: "image/id" and "captions". + """ + + def __init__( + self, predict_fn, tokenizer=None, + metrics=("cider",), # Default to only cider. We often just look at that. + preds_outfile="{workdir}/{name}_{split}_preds.json", + annot_outfile="{workdir}/{name}_{split}_annotations.json", + *, data, devices, **kw + ): + self.get_data_iter, self.steps = c.eval_input_pipeline( + keep_on_cpu={"image/id", "captions"}, data=data, devices=devices, **kw) + + self.preds_outfile = c.resolve_outfile( + preds_outfile, name=data.get("name"), split=data.get("split")) + self.annot_outfile = c.resolve_outfile( + annot_outfile, name=data.get("name"), split=data.get("split")) + + self.metrics = metrics + self.tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer) + self.decode = functools.partial( + predict_fn, devices=devices, eos_token=self.tok.eos_token) + + def run(self, train_state): + """Run eval.""" + gts = [] + res = [] + + for _, batch in zip(range(self.steps), self.get_data_iter()): + # (batch, seqlen) array of decoded generated tokens. + tokens = self.decode(train_state, batch) + + # (local_batch,) + tokens = u.get_local_slice_from_fsarray(tokens) + ex_masks = u.get_local_slice_from_fsarray(batch["_mask"]) + + image_ids = batch["image/id"][ex_masks] + pred_captions = self.tok.to_str(tokens[ex_masks]) + + for image_id, caption in zip(image_ids, pred_captions): + res.append({"image_id": image_id.item(), "caption": caption}) + + for image_id, captions in zip(image_ids, batch["captions"]): + for caption in captions: + gts.append({"image_id": image_id.item(), "caption": caption.item()}) + + # Write model outputs following: https://cocodataset.org/#format-results + # Use same format for gt although that is not the usual format for them. + res = c.multiprocess_write_json(self.preds_outfile, res) + gts = c.multiprocess_write_json(self.annot_outfile, gts) + + if jax.process_index(): # Host0 gets all preds and does eval. + return + + outs = self.evaluate(gts, res) + for key, score in outs.items(): + yield key, score + + def evaluate(self, gt_annotations, res_annotations): + """Creates scorers and run evaluation.""" + scorers = { + "rouge": rouge.Rouge, + "cider": cider.Cider, + "bleu-4": bleu.Bleu, + "spice": spice.Spice, + "meteor": meteor.Meteor, + } + + # Reformat gts and res from [{"image_id": int|str, "caption": str}] to + # {int_image_id: [{"caption": str}]} as expected by tokenizer and scorers. + # Note there are multiple reference captions for the ground truth but only + # one for the model predictions. + iid_map = collections.defaultdict(lambda: len(iid_map)) + res = {iid_map[x["image_id"]]: [x] for x in res_annotations} + gts = collections.defaultdict(list) + for x in gt_annotations: + gts[iid_map[x["image_id"]]].append(x) + assert sorted(gts.keys()) == sorted(res.keys()) + + # Tokenize captions and predictions using coco tokenizer. + coco_tokenizer = ptbtokenizer.PTBTokenizer() + gts = coco_tokenizer.tokenize(gts) + res = coco_tokenizer.tokenize(res) + + scores = {} + for metric in self.metrics: + scorer = scorers[metric]() + scores[metric], _ = scorer.compute_score(gts, res) + return scores diff --git a/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/rsvqa.py b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/rsvqa.py new file mode 100644 index 0000000000000000000000000000000000000000..18e7899f694698d0e7f877d3003d0dbecf041587 --- /dev/null +++ b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/rsvqa.py @@ -0,0 +1,173 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluator for simple VQA variants with per answer-type metrics. + +According to the (A-)OKVAQ papers, the eval for these datasets should follow +VQAv2. But here we don't track different answer-types, and don't do any +leave-one-out averaging, as this isn't done in the official implementation at +https://github.com/allenai/aokvqa/blob/main/evaluation/eval_predictions.py +either. +""" + +import functools + +import big_vision.evaluators.common as c +import big_vision.pp.tokenizer +import big_vision.utils as u +import editdistance + + +# Temporary global flag to facilitate backwards compatability. Will be removed +# by the end of year 2023. +API = "jit" + +QUESTION_TYPES = ("comp", "count", "presence", "rural_urban", "area") + +ACC_SUBSETS = ( + ("nonum", ("comp", "presence", "rural_urban")), # rsvqa_lr + ("nonum", ("comp", "presence")), # rsvqa_hr +) + + +class Evaluator: + """Evaluator for simple VQA tasks.""" + + def __init__( + self, predict_fn, tokenizer, to_lower=False, + outfile="{workdir}/{split}.json", + *, data, devices, **kw): + self.get_data_iter, self.steps = c.eval_input_pipeline( + keep_on_cpu={"answers", "answer", "question_id", "question_type"}, + data=data, devices=devices, **kw) + + self.outfile = c.resolve_outfile(outfile, split=data.get("split")) + + # We'll need the tokenizer to detokenize the model outputs later. + self.tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer) + self.postproc = (lambda s: s.lower()) if to_lower else lambda s: s + self.decode = functools.partial( + predict_fn, devices=devices, eos_token=self.tok.eos_token) + + def run(self, train_state): + """Does one evaluation run, yields metrics.""" + + accuracies = [] + accuracies_any = [] + counts_per_type = {t: 0 for t in QUESTION_TYPES} + accuracies_per_type = {t: [] for t in QUESTION_TYPES} + anls_values = [] + json_out = [] + for _, batch in zip(range(self.steps), self.get_data_iter()): + # (batch, seqlen) array of decoded generated tokens. + tokens = self.decode(train_state, batch) # (B,L,E) + + # (local_batch,) that indicates padding examples (0) vs real examples (1). + tokens = u.get_local_slice_from_fsarray(tokens) + ex_masks = u.get_local_slice_from_fsarray(batch["_mask"]) + + # Turn predictions into texts and then scores, one by one. + for i in range(len(tokens)): + if ex_masks[i] == 0: # Skip last-batch padding examples + continue + + answer = self.postproc(self.tok.to_str(tokens[i], stop_at_eos=True)) + + # Now we have two commonly used VQA evaluation modes: + if "answer" in batch: + # single GT (eg ocrvqa): just compare to that answer, done. + gt = self.postproc(batch["answer"][i]) + gts = [gt] + accuracies.append(float(answer == gt)) + accuracies_any.append(float(answer == gt)) + anls_values.append(anls_metric(gt, answer)) + elif "answers" in batch and (gt_answers := batch["answers"][i]).size: + # multiple GTs (eg okvqa): introduced by VQA, compare to each of them + # with a threshold, see also: https://visualqa.org/evaluation.html + gts = [self.postproc(a) for a in gt_answers] + num_match = sum([answer == gt for gt in gts]) + accuracies.append(min(1.0, num_match / 3.0)) + accuracies_any.append(min(1.0, float(num_match))) + anls_values.append(max(anls_metric(gt, answer) for gt in gts)) + accuracies_per_type[batch["question_type"][i]].append( + accuracies_any[-1] + ) + counts_per_type[batch["question_type"][i]] += 1 + else: + gts = [] + + json_out.append({ + "question_id": batch["question_id"][i].item(), + "answer": answer} | ({"gts": gts} if gts else {})) + + # At this point `accuracies` is a list of per-example scores. However, + # remember that each host holds a different subset of the examples! So if + # we were to just return the mean accuracy here, we would effectively only + # have evaluated on the main host's (who writes metrics) subset! + # So now, we need to compute global means. + # There is one more caveat: `process_sum` needs the summands on each host + # to have the same size. So we either need to include dummy values for + # the padding examples (last batch, annoying), or we only sum scalars as in + # sufficient statistics, which we do here. + sum_accs, sum_accs_any, sum_anls, num_accs, num = c.process_sum( + [sum(accuracies), sum(accuracies_any), sum(anls_values), + len(accuracies), len(json_out)]) + + sum_accs_per_type, sum_cnts_per_type = c.process_sum( + [{k: sum(v) for k, v in accuracies_per_type.items()}, counts_per_type] + ) + + # Yielding metric_name, value means logging the metric. + if num_accs: + yield "acc", sum_accs / num_accs + yield "acc_any", sum_accs_any / num_accs # Overall Accuracy (OA). + yield "anls", sum_anls / num_accs + acc_types = {} + for k, v in sum_accs_per_type.items(): + if sum_cnts_per_type[k]: + acc_types[k] = v / sum_cnts_per_type[k] + yield f"acc_{k}", acc_types[k] + yield "acc_avg", sum(acc_types.values()) / len(acc_types) # Avg acc (AA). + for postfix, types in ACC_SUBSETS: + if all(t in acc_types for t in types): + yield f"acc_avg_{postfix}", sum( + [v for k, v in acc_types.items() if k in types] + ) / len(types) # Average accuracy per question types subset. + yield "num", num # Just for sanity checks. + c.multiprocess_write_json(self.outfile, json_out) + + +def anls_metric(target: str, prediction: str, theta: float = 0.5): + """Calculates ANLS for DocVQA. + + There does not seem to be an official evaluation script. + Public implementation on which this implementation is based: + https://github.com/herobd/layoutlmv2/blob/main/eval_docvqa.py#L92 + + Original paper (see Eq 1): https://arxiv.org/pdf/1907.00490.pdf + + Args: + target: Target string. + prediction: Predicted string. + theta: Filter threshold set to 0.5 for DocVQA. + + Returns: + ANLS score. + """ + if target: + edit_distance = editdistance.eval(target, prediction) + normalized_ld = edit_distance / max(len(target), len(prediction)) + return 1 - normalized_ld if normalized_ld < theta else 0 + else: + return float(prediction == "") diff --git a/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/science_qa.py b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/science_qa.py new file mode 100644 index 0000000000000000000000000000000000000000..c9f024cb305d377a2c8b7d9f3279d193af26d655 --- /dev/null +++ b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/science_qa.py @@ -0,0 +1,122 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluator for ScienceQA. + +based on the official implementation at +https://github.com/lupantech/ScienceQA/blob/main/models/run_gpt3.py +""" + +import functools +import re + +import big_vision.evaluators.common as c +import big_vision.pp.tokenizer +import big_vision.utils as u + + +# Temporary global flag to facilitate backwards compatability. Will be removed +# by the end of year 2023. +API = "jit" +FAILURE = "failed" + + +class Evaluator: + """Evaluator for simple VQA tasks.""" + + def __init__( + self, predict_fn, tokenizer, + outfile="{workdir}/{split}.json", + out_question_key="question_id", + *, data, devices, **kw): + self.get_data_iter, self.steps = c.eval_input_pipeline( + keep_on_cpu={"answer", "question_id"}, data=data, devices=devices, **kw) + + self.outfile = c.resolve_outfile(outfile, split=data.get("split")) + self.out_question_key = out_question_key + + # We'll need the tokenizer to detokenize the model outputs later. + self.tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer) + self.decode = functools.partial( + predict_fn, devices=devices, eos_token=self.tok.eos_token + ) + + def postproc(self, raw_answer): + """Post-processes the raw answer. extract a, b, c from the string.""" + match = re.match( + pattern=r"the answer is ([a-z])\.", string=raw_answer.lower() + ) + if match: + return match.groups()[0] # 'a', 'b', ... + else: + return FAILURE + + def run(self, train_state): + """Does one evaluation run, yields metrics.""" + + accuracies = [] + fail_parse = [] + json_out = [] + for _, batch in zip(range(self.steps), self.get_data_iter()): + # (batch, seqlen) array of decoded generated tokens. + tokens = self.decode(train_state, batch) + + # (local_batch,) that indicates padding examples (0) vs real examples (1). + tokens = u.get_local_slice_from_fsarray(tokens) + ex_masks = u.get_local_slice_from_fsarray(batch["_mask"]) + + # Turn predictions into texts and then scores, one by one. + for i in range(len(tokens)): + if ex_masks[i] == 0: # Skip last-batch padding examples + continue + + raw_answer = self.tok.to_str(tokens[i], stop_at_eos=True) + answer = self.postproc(raw_answer) + if "answer" in batch: + gt = self.postproc(batch["answer"][i]) + gts = [gt] + accuracies.append(float(answer == gt)) + fail_parse.append(float(answer == FAILURE)) + else: + gts = [] + + json_out.append( + { + self.out_question_key: batch["question_id"][i].item(), + "raw_answer": raw_answer, + "answer": answer, + } + | ({"gts": gts} if gts else {}) + ) + + # At this point `accuracies` is a list of per-example scores. However, + # remember that each host holds a different subset of the examples! So if + # we were to just return the mean accuracy here, we would effectively only + # have evaluated on the main host's (who writes metrics) subset! + # So now, we need to compute global means. + # There is one more caveat: `process_sum` needs the summands on each host + # to have the same size. So we either need to include dummy values for + # the padding examples (last batch, annoying), or we only sum scalars as in + # sufficient statistics, which we do here. + sum_accs, num_parsefail, num_accs, num = c.process_sum( + [sum(accuracies), sum(fail_parse), len(accuracies), len(json_out)] + ) + + # Yielding metric_name, value means logging the metric. + if num_accs > 0: + yield "acc", sum_accs / num_accs + yield "parsefail", num_parsefail / num_accs + + yield "num", num # Just for sanity checks. + c.multiprocess_write_json(self.outfile, json_out) diff --git a/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/segmentation.py b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..94ef8e7136893da3fef66f7bcef6ae63449fefab --- /dev/null +++ b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/segmentation.py @@ -0,0 +1,270 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluator for segmentation.""" + +import functools + +import big_vision.evaluators.common as c +import big_vision.pp.tokenizer +import big_vision.utils as u +import flax.linen as nn +import jax +import jax.numpy as jnp +import numpy as np +import PIL.Image + +from tensorflow.io import gfile + + +# Temporary global flag to facilitate backwards compatability. Will be removed +# by the end of year 2023. +API = 'jit' + + +def _inrange(a, min_value, max_value): + return (np.clip(a, min_value, max_value) == a).all() + + +def _area(y1, x1, y2, x2): + return max(x2 - x1, 0.0) * max(y2 - y1, 0.0) + + +class Evaluator: + """Evaluator for instance segmentation.""" + + def __init__(self, predict_fn, tokenizer, + model='oi', det_ious=(0.5, 0.75), + *, devices, **kw): + self.get_data_iter, self.steps = c.eval_input_pipeline( + keep_on_cpu={'prefix', 'suffix', 'objects/mask', 'objects/bbox'}, + devices=devices, **kw) + + self.tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer) + self.decode = functools.partial( + predict_fn, devices=devices, eos_token=self.tok.eos_token) + tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer) + self.loc0 = np.array(tok.to_int('')) + self.seg0 = np.array(tok.to_int('')) + # Verify tokenizer has `tokensets=("loc", "seg")` + assert self.loc0.shape == (1,), self.loc0 + assert self.seg0.shape == (1,), self.seg0 + self.reconstruct_masks = get_reconstruct_masks(model) + self.det_ious = det_ious + + def run(self, train_state): + """Does one evaluation run, yields metrics.""" + ious = [] # NOTE: no point to split in s/m/l: all objects are L (>96px²) + det_by_iou = {iou: [] for iou in self.det_ious} + invalid = total = 0 + for _, batch in zip(range(self.steps), self.get_data_iter()): + + decoded = self.decode(train_state, batch) + + not_padding = u.get_local_slice_from_fsarray(batch['_mask']) + decoded = u.get_local_slice_from_fsarray(decoded)[not_padding] + + # Note, gt masks are in full original image resolution. + gt_masks = [gt[:, :, 0] > 0 for gt in batch['objects/mask'][not_padding]] + gt_bbs = [gt for gt in batch['objects/bbox'][not_padding]] + + valid = [] + tokens = np.zeros([decoded.shape[0], 4 + 16], np.int32) + for i, dec in enumerate(decoded): + # TODO: b/andstein - do we need to optimize this loop? + t = np.r_[dec[:4] - self.loc0, dec[4:4 + 16] - self.seg0] # Ignore rest + if ( + len(t) == 4 + 16 # Full prediction + and _inrange(t[:4], 0, 1023) # Valid box tokens + and _inrange(t[4:], 0, 127) # Valid seg tokens + and t[2] > t[0] and t[3] > t[1] # Valid box + ): + valid.append(True) + tokens[i] = t + else: + valid.append(False) + + tocpu = lambda x: jax.device_put(x, jax.local_devices(backend='cpu')[0]) + seg_indices = np.array(tokens[:, 4:]) + mask64 = jax.device_get(self.reconstruct_masks(tocpu(seg_indices))) + mask64 = mask64[..., 0] + bbox = tokens[:, :4] / 1023 # Back to [0.0 ... 1.0] + + for v, m64, gtm, bb, gtbb in zip(valid, mask64, gt_masks, bbox, gt_bbs): + # TODO: b/andstein - do we need to optimize this loop? + total += 1 + h, w = gtm.shape # gt is full/original image resolution mask. + + # First, compute detection iou, in [0.0 ... 1.0] coordinate space. + y1, x1, y2, x2 = bb + gty1, gtx1, gty2, gtx2 = gtbb + ibb = max(y1, gty1), max(x1, gtx1), min(y2, gty2), min(x2, gtx2) + box_iou = _area(*ibb) / (_area(*bb) + _area(*gtbb) - _area(*ibb)) + for iou_thresh in det_by_iou: + det_by_iou[iou_thresh].append(iou_thresh <= box_iou) + + # Next, we convert to pixel coordinates and compute mask iou. + gt_area = gtm.sum() + y1, x1, y2, x2 = map(int, (y1 * h, x1 * w, y2 * h, x2 * w)) + + # Avoid compute-intensive mask stuff for invalid preds: + if not v or x2 <= x1 or y2 <= y1: # Can still happen after int(). + iou = 0.0 + invalid += 1 + else: + mi = np.asarray( + PIL.Image.fromarray(m64).resize( # pytype: disable=wrong-arg-types # pillow-102-upgrade + [x2 - x1, y2 - y1], resample=PIL.Image.BILINEAR # pytype: disable=module-attr + ) + ) # Predicted mask in box-sized image. + mi = mi > 0.0 # Mask decoder output in [-1.0 ... 1.0] + iarea = (gtm[y1:y2, x1:x2] & mi).sum() # Intersection pixels. + iou = iarea / (gt_area + mi.sum() - iarea) + ious.append(iou) + + # Done going over all batches, now collect results from all processes. + sum_ious, num_ious, sum_dets, num_dets, num_invalid, num = c.process_sum([ + sum(ious), len(ious), + {k: sum(v) for k, v in det_by_iou.items()}, + {k: len(v) for k, v in det_by_iou.items()}, + invalid, total + ]) + + yield 'miou', sum_ious / num_ious + for k in sum_dets: + yield f'boxacc/{k}', sum_dets[k] / num_dets[k] + yield 'invalid', num_invalid + yield 'total', num + + +_KNOWN_MODELS = { + # Trained on open images. + 'oi': 'gs://big_vision/paligemma/vae-oid.npz', +} + + +def _get_params(checkpoint): + """Converts PyTorch checkpoint to Flax params.""" + + def transp(kernel): + return np.transpose(kernel, (2, 3, 1, 0)) + + def conv(name): + return { + 'bias': checkpoint[name + '.bias'], + 'kernel': transp(checkpoint[name + '.weight']), + } + + def resblock(name): + return { + 'Conv_0': conv(name + '.0'), + 'Conv_1': conv(name + '.2'), + 'Conv_2': conv(name + '.4'), + } + + return { + '_embeddings': checkpoint['_vq_vae._embedding'], + 'Conv_0': conv('decoder.0'), + 'ResBlock_0': resblock('decoder.2.net'), + 'ResBlock_1': resblock('decoder.3.net'), + 'ConvTranspose_0': conv('decoder.4'), + 'ConvTranspose_1': conv('decoder.6'), + 'ConvTranspose_2': conv('decoder.8'), + 'ConvTranspose_3': conv('decoder.10'), + 'Conv_1': conv('decoder.12'), + } + + +def _quantized_values_from_codebook_indices(codebook_indices, embeddings): + batch_size, num_tokens = codebook_indices.shape + assert num_tokens == 16, codebook_indices.shape + unused_num_embeddings, embedding_dim = embeddings.shape + + encodings = jnp.take(embeddings, codebook_indices.reshape((-1)), axis=0) + encodings = encodings.reshape((batch_size, 4, 4, embedding_dim)) + return encodings + + +class ResBlock(nn.Module): + features: int + + @nn.compact + def __call__(self, x): + original_x = x + x = nn.Conv(features=self.features, kernel_size=(3, 3), padding=1)(x) + x = nn.relu(x) + x = nn.Conv(features=self.features, kernel_size=(3, 3), padding=1)(x) + x = nn.relu(x) + x = nn.Conv(features=self.features, kernel_size=(1, 1), padding=0)(x) + return x + original_x + + +class Decoder(nn.Module): + """Upscales quantized vectors to mask.""" + + @nn.compact + def __call__(self, x): + num_res_blocks = 2 + dim = 128 + num_upsample_layers = 4 + + x = nn.Conv(features=dim, kernel_size=(1, 1), padding=0)(x) + x = nn.relu(x) + + for _ in range(num_res_blocks): + x = ResBlock(features=dim)(x) + + for _ in range(num_upsample_layers): + x = nn.ConvTranspose( + features=dim, + kernel_size=(4, 4), + strides=(2, 2), + padding=2, + transpose_kernel=True, + )(x) + x = nn.relu(x) + dim //= 2 + + x = nn.Conv(features=1, kernel_size=(1, 1), padding=0)(x) + + return x + + +@functools.cache +def get_reconstruct_masks(model): + """Reconstructs masks from codebook indices. + + Based on code from https://arxiv.org/abs/2301.02229 + + Verified in + https://colab.research.google.com/drive/1AOr0cokOpM6-N9Z5HmxoeGxGj6jS37Vl + + Args: + model: Model to use for conversion. + + Returns: + A function that expects indices shaped `[B, 16]` of dtype int32, each + ranging from 0 to 127 (inclusive), and that returns a decoded masks sized + `[B, 64, 64, 1]`, of dtype float32, in range [-1, 1]. + """ + def reconstruct_masks(codebook_indices): + quantized = _quantized_values_from_codebook_indices( + codebook_indices, params['_embeddings'] + ) + return Decoder().apply({'params': params}, quantized) + + with gfile.GFile(_KNOWN_MODELS.get(model, model), 'rb') as f: + params = _get_params(dict(np.load(f))) + + return jax.jit(reconstruct_masks, backend='cpu') diff --git a/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/storepreds.py b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/storepreds.py new file mode 100644 index 0000000000000000000000000000000000000000..05230a54ae8b0c6bac2420c49d3bc444a6a548a4 --- /dev/null +++ b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/storepreds.py @@ -0,0 +1,77 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluator to run inference and store results.""" +import functools + +import big_vision.evaluators.common as c +import big_vision.input_pipeline +import big_vision.pp.builder +import big_vision.pp.tokenizer +import big_vision.utils as u + +import jax + +# Temporary global flag to facilitate backwards compatability. Will be removed +# by the end of year 2023. +API = "jit" + + +class Evaluator: + """Evaluator to run inference and store results.""" + + def __init__( + self, predict_fn, tokenizer=None, + preds_outfile="{workdir}/{name}_{split}_preds.json", + annot_outfile="{workdir}/{name}_{split}_annotations.json", + id_key="id", + *, data, devices, **kw + ): + self.id_key = id_key + self.get_data_iter, self.steps = c.eval_input_pipeline( + keep_on_cpu={id_key}, data=data, devices=devices, **kw) + + self.preds_outfile = c.resolve_outfile( + preds_outfile, name=data.get("name"), split=data.get("split", "")) + self.annot_outfile = c.resolve_outfile( + annot_outfile, name=data.get("name"), split=data.get("split", "")) + + self.tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer) + self.decode = functools.partial( + predict_fn, devices=devices, eos_token=self.tok.eos_token) + + def run(self, train_state): + """Run eval.""" + res = [] + + for _, batch in zip(range(self.steps), self.get_data_iter()): + # (batch, seqlen) array of decoded generated tokens. + tokens = self.decode(train_state, batch) + + # (local_batch,) + tokens = u.get_local_slice_from_fsarray(tokens) + ex_masks = u.get_local_slice_from_fsarray(batch["_mask"]) + + image_ids = batch[self.id_key][ex_masks] + pred_captions = self.tok.to_str(tokens[ex_masks]) + + for image_id, caption in zip(image_ids, pred_captions): + res.append({self.id_key: str(image_id), "caption": caption}) + + res = c.multiprocess_write_json(self.preds_outfile, res) + + if jax.process_index(): # Host0 gets all preds and does eval. + return + + yield "num_examples", len(res) diff --git a/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/tallyqa.py b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/tallyqa.py new file mode 100644 index 0000000000000000000000000000000000000000..9d82757f2047fee43c1d02d7112383addb07c82a --- /dev/null +++ b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/tallyqa.py @@ -0,0 +1,144 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluator for TallyQA dataset.""" + +import functools + +import big_vision.evaluators.common as c +import big_vision.pp.tokenizer +import big_vision.utils as u + + +# Temporary global flag to facilitate backwards compatability. Will be removed +# by the end of year 2023. +API = "jit" + + +# Largest count we want to track. +_LARGEST_COUNT = 15 + + +class Evaluator: + """TallyQA evaluator.""" + + def __init__(self, predict_fn, tokenizer, *, devices, **kw): + self.get_data_iter, self.steps = c.eval_input_pipeline( + keep_on_cpu={"answer", "issimple"}, devices=devices, **kw) + + # We'll need the tokenizer to detokenize the model outputs later. + self.tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer) + self.decode = functools.partial( + predict_fn, devices=devices, eos_token=self.tok.eos_token + ) + + def run(self, train_state): + """Does one evaluation run, yields metrics.""" + + accuracies_by_type = {"all": [], "simple": [], "complex": []} + # Add per-count entries. Cannot use a `defaultdict` as we need to `tree_map` + # over keys later in `c.process_sum`. + accuracies_by_type.update( + {f"count_{i}": [] for i in range(_LARGEST_COUNT + 1)} + ) + + for _, batch in zip(range(self.steps), self.get_data_iter()): + # (batch, seqlen) array of decoded (generated) token sequences suffixes. + tokens = self.decode(train_state, batch) + + # (local_batch,) that indicates padding examples (0) vs real examples (1). + tokens = u.get_local_slice_from_fsarray(tokens) + ex_masks = u.get_local_slice_from_fsarray(batch["_mask"]) + + # Turn predictions into texts and then scores, one by one. + # We always compare the gt (string digit, e.g. "1") to the answer by the + # model (e.g. "1"). + for i in range(len(tokens)): + if ex_masks[i] == 0: # Skip last-batch padding examples + continue + + # Extract the suffix/answer from the generated string, skip bos. + answer = self.tok.to_str(tokens[i], stop_at_eos=True) + # Standardize the reponse, i.e., convert number words ("one") to + # numerals ("1"). + answer = _number_word_to_numeral(answer) + + # Always need to do light space-processing: + gt = _number_word_to_numeral(batch["answer"][i]) + accuracies_by_type["all"].append(float(answer == gt)) + + if "issimple" in batch: + # Simple/complex split. + if batch["issimple"][i] == 1: + accuracies_by_type["simple"].append(float(answer == gt)) + elif batch["issimple"][i] == 0: + accuracies_by_type["complex"].append(float(answer == gt)) + else: + # Train set is not annotated with simple/complex (but has dummy + # value of `-1` in this field). + pass + + # Store accuracies per count. + accuracies_by_type[f"count_{gt}"].append(float(answer == gt)) + + # At this point `accuracies` is a list of per-example scores. However, + # remember that each host holds a different subset of the examples! So if + # we were to just return the mean accuracy here, we would effectively only + # have evaluated on the main host's (who writes metrics) subset! + # So now, we need to compute global means. + # There is one more caveat: `process_sum` needs the summands on each host + # to have the same size. So we either need to include dummy values for + # the padding examples (last batch, annoying), or we only sum scalars as in + # sufficient statistics, which we do here. + sum_accs = c.process_sum({k: sum(v) for k, v in accuracies_by_type.items()}) + num_accs = c.process_sum({k: len(v) for k, v in accuracies_by_type.items()}) + + if n := num_accs["all"]: + yield "acc", sum_accs["all"] / n + yield "num", n # Just for sanity checks. + for key in sum_accs.keys(): + if (key != "all") and (num_accs[key]): + yield f"acc/{key}", sum_accs[key] / num_accs[key] + yield f"num/{key}", num_accs[key] # Just for sanity checks. + + +def _number_word_to_numeral(s: str) -> str: + """Returns numeral for a given number word, e.g., "one" -> "1" (up to 20).""" + return REPLACEMENTS.get(s.lower(), s) + + +REPLACEMENTS = { + "none": "0", + "zero": "0", + "one": "1", + "two": "2", + "three": "3", + "four": "4", + "five": "5", + "six": "6", + "seven": "7", + "eight": "8", + "nine": "9", + "ten": "10", + "eleven": "11", + "twelve": "12", + "thirteen": "13", + "fourteen": "14", + "fifteen": "15", + "sixteen": "16", + "seventeen": "17", + "eighteen": "18", + "nineteen": "19", + "twenty": "20", +} diff --git a/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/vqa.py b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/vqa.py new file mode 100644 index 0000000000000000000000000000000000000000..bf837b4cdff4c7eaaa5ae1da734371e9118a2264 --- /dev/null +++ b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/vqa.py @@ -0,0 +1,163 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluator for simple VQA variants (OCR-VQA, OKVQA, A-OKVQA). + +According to the (A-)OKVAQ papers, the eval for these datasets should follow +VQAv2. But here we don't track different answer-types, and don't do any +leave-one-out averaging, as this isn't done in the official implementation at +https://github.com/allenai/aokvqa/blob/main/evaluation/eval_predictions.py +either. + +Please read the description of how evaluators work at (internal link). +This evaluator follows the pattern of also parallelizing the CPU computations +(ie postprocessing, score computation) across hosts for more scalability. + +For now, simple decoding is implemented as part of the evaluator. We'll soon +unify and move to a library of decoding functions, including fancier and more +efficient ones. +""" +import functools + +import big_vision.evaluators.common as c +import big_vision.pp.tokenizer +import big_vision.utils as u +import editdistance + + +# Temporary global flag to facilitate backwards compatability. Will be removed +# by the end of year 2023. +API = "jit" + + +class Evaluator: + """Evaluator for simple VQA tasks. + + This evaluator expects the batch to contain a field `question_id` and a field + `answer` for single ground truth or `answers` for multiple ground truths. + + The field names used when writting the json result can be controlled with + `out_question_key` and `out_answer_key`. + """ + + def __init__( + self, predict_fn, tokenizer, to_lower=False, + outfile="{workdir}/{split}.json", + out_question_key="question_id", out_answer_key="answer", + *, data, devices, **kw): + self.get_data_iter, self.steps = c.eval_input_pipeline( + keep_on_cpu={"answers", "answer", "question_id"}, + data=data, devices=devices, **kw) + + self.outfile = c.resolve_outfile(outfile, split=data.get("split")) + self.out_question_key = out_question_key + self.out_answer_key = out_answer_key + + # We'll need the tokenizer to detokenize the model outputs later. + self.tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer) + self.postproc = (lambda s: s.lower()) if to_lower else lambda s: s + self.decode = functools.partial( + predict_fn, devices=devices, eos_token=self.tok.eos_token) + + def run(self, train_state): + """Does one evaluation run, yields metrics.""" + + accuracies = [] + accuracies_any = [] + anls_values = [] + json_out = [] + for _, batch in zip(range(self.steps), self.get_data_iter()): + # (batch, seqlen) array of decoded generated tokens. + tokens = self.decode(train_state, batch) + + # (local_batch,) that indicates padding examples (0) vs real examples (1). + tokens = u.get_local_slice_from_fsarray(tokens) + ex_masks = u.get_local_slice_from_fsarray(batch["_mask"]) + + # Turn predictions into texts and then scores, one by one. + for i in range(len(tokens)): + if ex_masks[i] == 0: # Skip last-batch padding examples + continue + + answer = self.postproc(self.tok.to_str(tokens[i], stop_at_eos=True)) + + # Now we have two commonly used VQA evaluation modes: + if "answer" in batch: + # single GT (eg ocrvqa): just compare to that answer, done. + gt = self.postproc(batch["answer"][i]) + gts = [gt] + accuracies.append(float(answer == gt)) + accuracies_any.append(float(answer == gt)) + anls_values.append(anls_metric(gt, answer)) + elif "answers" in batch and (gt_answers := batch["answers"][i]).size: + # multiple GTs (eg okvqa): introduced by VQA, compare to each of them + # with a threshold, see also: https://visualqa.org/evaluation.html + gts = [self.postproc(a) for a in gt_answers] + num_match = sum([answer == gt for gt in gts]) + accuracies.append(min(1.0, num_match / 3.0)) + accuracies_any.append(min(1.0, float(num_match))) + anls_values.append(max(anls_metric(gt, answer) for gt in gts)) + else: + gts = [] + + json_out.append({ + self.out_question_key: batch["question_id"][i].item(), + self.out_answer_key: answer} | ({"gts": gts} if gts else {})) + + # At this point `accuracies` is a list of per-example scores. However, + # remember that each host holds a different subset of the examples! So if + # we were to just return the mean accuracy here, we would effectively only + # have evaluated on the main host's (who writes metrics) subset! + # So now, we need to compute global means. + # There is one more caveat: `process_sum` needs the summands on each host + # to have the same size. So we either need to include dummy values for + # the padding examples (last batch, annoying), or we only sum scalars as in + # sufficient statistics, which we do here. + sum_accs, sum_accs_any, sum_anls, num_accs, num = c.process_sum( + [sum(accuracies), sum(accuracies_any), sum(anls_values), + len(accuracies), len(json_out)]) + + # Yielding metric_name, value means logging the metric. + if num_accs: + yield "acc", sum_accs / num_accs + yield "acc_any", sum_accs_any / num_accs + yield "anls", sum_anls / num_accs + + yield "num", num # Just for sanity checks. + c.multiprocess_write_json(self.outfile, json_out) + + +def anls_metric(target: str, prediction: str, theta: float = 0.5): + """Calculates ANLS for DocVQA. + + There does not seem to be an official evaluation script. + Public implementation on which this implementation is based: + https://github.com/herobd/layoutlmv2/blob/main/eval_docvqa.py#L92 + + Original paper (see Eq 1): https://arxiv.org/pdf/1907.00490.pdf + + Args: + target: Target string. + prediction: Predicted string. + theta: Filter threshold set to 0.5 for DocVQA. + + Returns: + ANLS score. + """ + if target: + edit_distance = editdistance.eval(target, prediction) + normalized_ld = edit_distance / max(len(target), len(prediction)) + return 1 - normalized_ld if normalized_ld < theta else 0 + else: + return float(prediction == "") diff --git a/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/vqav2.py b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/vqav2.py new file mode 100644 index 0000000000000000000000000000000000000000..5de1d650d81c1a4cf44889d22b2edd270f8c05c1 --- /dev/null +++ b/Tipsomaly/model/big_vision/evaluators/proj/paligemma/transfers/vqav2.py @@ -0,0 +1,197 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluator for VQAV2 dataset. +""" +import functools +import re + +import big_vision.evaluators.common as c +import big_vision.pp.tokenizer +import big_vision.utils as u +import numpy as np + + +# Temporary global flag to facilitate backwards compatability. Will be removed +# by the end of year 2023. +API = "jit" + + +class Evaluator: + """VQAv2 evaluator.""" + + def __init__( + self, predict_fn, tokenizer, outfile="{workdir}/{split}.json", + *, data, devices, **kw): + self.get_data_iter, self.steps = c.eval_input_pipeline( + keep_on_cpu={"answers", "answer_type", "question_type", "question_id"}, + data=data, devices=devices, **kw) + + self.outfile = c.resolve_outfile(outfile, split=data.get("split")) + + # We'll need the tokenizer to detokenize the model outputs later. + self.tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer) + self.decode = functools.partial( + predict_fn, devices=devices, eos_token=self.tok.eos_token) + + def run(self, train_state): + """Does one evaluation run, yields metrics.""" + accuracies_by_type = {"yes/no": [], "number": [], "other": []} + json_out = [] + + for _, batch in zip(range(self.steps), self.get_data_iter()): + # (batch, seqlen) array of decoded (generated) token sequences suffixes. + tokens = self.decode(train_state, batch) + + # (local_batch,) that indicates padding examples (0) vs real examples (1). + tokens = u.get_local_slice_from_fsarray(tokens) + ex_masks = u.get_local_slice_from_fsarray(batch["_mask"]) + + # Turn predictions into texts and then scores, one by one. + for i in range(len(tokens)): + if ex_masks[i] == 0: # Skip last-batch padding examples + continue + + # Extract the suffix/answer from the generated string, skip bos. + answer = self.tok.to_str(tokens[i], stop_at_eos=True) + json = {"question_id": batch["question_id"][i].item(), "answer": answer} + + # The rest is computation of VQA-score which compares to multiple GTs. + # This is described better here: https://visualqa.org/evaluation.html + if (gt_answers := batch["answers"][i]).size: + # Always need to do light space-processing: + gt_answers = [stripspace_vqav2(a) for a in gt_answers] + answer = stripspace_vqav2(answer) + + # Only post-process if not all agree. Supposedly avoids postproc OCR: + # https://github.com/GT-Vision-Lab/VQA/issues/14#issuecomment-1334695361 + if len(set(gt_answers)) > 1: + answer = postprocess_vqav2_text(answer) + gt_answers = [postprocess_vqav2_text(a) for a in gt_answers] + + # Accuracy is avg over all ten leave-one-out GT's. + # https://github.com/GT-Vision-Lab/VQA/issues/1#issuecomment-199921352 + # An answer is counted 100% correct as soon as 3 GT's agree with it. + matches = answer == np.array(gt_answers) + acc = np.mean([ + np.clip(np.sum(np.delete(matches, i_leave_out)) / 3, 0, 1) + for i_leave_out in range(10) + ]) + + accuracies_by_type[batch["answer_type"][i]].append(acc) + + # Update json with fully post-processed answer and gt: + json["answer_raw"] = json["answer"] + json["answer"] = answer + json["gts"] = gt_answers + + json_out.append(json) + + # At this point `accuracies` is a list of per-example scores. However, + # remember that each host holds a different subset of the examples! So if + # we were to just return the mean accuracy here, we would effectively only + # have evaluated on the main host's (who writes metrics) subset! + # So now, we need to compute global means. + # There is one more caveat: `process_sum` needs the summands on each host + # to have the same size. So we either need to include dummy values for + # the padding examples (last batch, annoying), or we only sum scalars as in + # sufficient statistics, which we do here. + sum_accs = c.process_sum({k: sum(v) for k, v in accuracies_by_type.items()}) + num_accs = c.process_sum({k: len(v) for k, v in accuracies_by_type.items()}) + num = c.process_sum(len(json_out)) + + # Yielding metric_name, value means logging the metric. + if n := sum(num_accs.values()): + yield "acc", sum(sum_accs.values()) / n + if n := num_accs["yes/no"]: + yield "acc/yesno", sum_accs["yes/no"] / n + yield "num/yesno", n + if n := num_accs["number"]: + yield "acc/number", sum_accs["number"] / n + yield "num/number", n + if n := num_accs["other"]: + yield "acc/other", sum_accs["other"] / n + yield "num/other", n + + yield "num", num # Just for sanity checks. + c.multiprocess_write_json(self.outfile, json_out) + + +# Post-processing required is described at https://visualqa.org/evaluation.html + + +def stripspace_vqav2(txt): + return txt.replace("\n", " ").replace("\t", " ").strip() + + +def postprocess_vqav2_text(txt): + """Cleanup string according to VQA.""" + has_digit_comma = re.search(r"(\d)(\,)(\d)", txt) is not None + + out = txt + for p in PUNCT: + # NOTE: digit_comma here looks like a bug in official code, so we follow it. + if has_digit_comma or f"{p} " in txt or f" {p}" in txt: + out = out.replace(p, "") + else: + out = out.replace(p, " ") + + # Remove full-stops that aren't part of a number. + out = re.sub(r"(?!<=\d)(\.)(?!\d)", "", out, flags=re.UNICODE) + + words = [] + for word in out.lower().split(): + if word not in ARTICLES: + words.append(REPLACEMENTS.get(word, word)) + return " ".join(words) + + +# pylint: disable=line-too-long +REPLACEMENTS = { + # CONTRACTIONS + "aint": "ain't", "arent": "aren't", "cant": "can't", "couldve": "could've", "couldnt": "couldn't", + "couldn'tve": "couldn't've", "couldnt've": "couldn't've", "didnt": "didn't", "doesnt": "doesn't", "dont": "don't", "hadnt": "hadn't", + "hadnt've": "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent": "haven't", "hed": "he'd", "hed've": "he'd've", + "he'dve": "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll", "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", + "Im": "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've": "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's", + "maam": "ma'am", "mightnt": "mightn't", "mightnt've": "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've", + "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't", "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't", + "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat": "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve": "she'd've", + "she's": "she's", "shouldve": "should've", "shouldnt": "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve": "shouldn't've", + "somebody'd": "somebodyd", "somebodyd've": "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll": "somebody'll", + "somebodys": "somebody's", "someoned": "someone'd", "someoned've": "someone'd've", "someone'dve": "someone'd've", + "someonell": "someone'll", "someones": "someone's", "somethingd": "something'd", "somethingd've": "something'd've", + "something'dve": "something'd've", "somethingll": "something'll", "thats": "that's", "thered": "there'd", "thered've": "there'd've", + "there'dve": "there'd've", "therere": "there're", "theres": "there's", "theyd": "they'd", "theyd've": "they'd've", + "they'dve": "they'd've", "theyll": "they'll", "theyre": "they're", "theyve": "they've", "twas": "'twas", "wasnt": "wasn't", + "wed've": "we'd've", "we'dve": "we'd've", "weve": "we've", "werent": "weren't", "whatll": "what'll", "whatre": "what're", + "whats": "what's", "whatve": "what've", "whens": "when's", "whered": "where'd", "wheres": "where's", "whereve": "where've", + "whod": "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl": "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll", + "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve": "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've", + "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll": "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've", + "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd": "you'd", "youd've": "you'd've", "you'dve": "you'd've", + "youll": "you'll", "youre": "you're", "youve": "you've", + # NUMBERS + "none": "0", "zero": "0", "one": "1", "two": "2", + "three": "3", "four": "4", "five": "5", "six": "6", + "seven": "7", "eight": "8", "nine": "9", "ten": "10", +} +# pylint: enable=line-too-long + +PUNCT = [ + ";", "/", "[", "]", "\"", "{", "}", + "(", ")", "=", "+", "\\", "_", "-", + ">", "<", "@", "`", ",", "?", "!" +] +ARTICLES = {"a", "an", "the"} diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/__init__.cpython-311.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53c3c10e163b684cc99b2d7e000730385d8585e6 Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/__init__.cpython-311.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/__init__.cpython-312.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..445c9ddab0f16de93ba63504659bcd93129cb54f Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/__init__.cpython-312.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/__init__.cpython-39.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef041fc83c1cd243fae5b2444715f0a9f753ab37 Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/__init__.cpython-39.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/builder.cpython-311.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/builder.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..56dbc7c8002c1a88a91737e3ee14d267610d93ce Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/builder.cpython-311.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/builder.cpython-312.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/builder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..885648c36f83682a0b83fc65139f615a37b791aa Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/builder.cpython-312.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/builder.cpython-39.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/builder.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a68a6ff7f6869c7ae6172128de3c8b62a63e651b Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/builder.cpython-39.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/ops_general.cpython-311.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/ops_general.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39ae48ce5651d77927ef7ea0f06eed6c49fff9a1 Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/ops_general.cpython-311.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/ops_general.cpython-312.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/ops_general.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d75bd7b3f069bd165796ec0bc2cc2048c258f2df Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/ops_general.cpython-312.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/ops_general.cpython-39.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/ops_general.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88f846d488d7d9805e8cafda03798b682361faeb Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/ops_general.cpython-39.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/ops_image.cpython-311.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/ops_image.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b007f92d93ffab44a770e025e656e2e3257d8771 Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/ops_image.cpython-311.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/ops_image.cpython-312.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/ops_image.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a7ede87239c536049906a1466ab0a58f28f289e Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/ops_image.cpython-312.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/ops_image.cpython-39.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/ops_image.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..135cbf967c0c32d0752aee0de8776afe99dc8177 Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/ops_image.cpython-39.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/ops_text.cpython-311.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/ops_text.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6cfa0eae14c24a6c45a500cedcc92f2cf54e1fe Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/ops_text.cpython-311.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/ops_text.cpython-312.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/ops_text.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d7255b635fb6427bb21cb864e2aa4f8e76878d1a Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/ops_text.cpython-312.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/ops_text.cpython-39.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/ops_text.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..31039a2efd6ea176cdfd9988b60bf82954031a0d Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/ops_text.cpython-39.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/registry.cpython-311.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/registry.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee00f881258690adc65dbdbd2edb9af0141dedb9 Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/registry.cpython-311.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/registry.cpython-312.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/registry.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d240b68d08cba6c4faa07b43308425417aa61c1 Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/registry.cpython-312.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/registry.cpython-39.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/registry.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b86060cb10f3c9f5b396fce146a33237c64325a Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/registry.cpython-39.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/tokenizer.cpython-311.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/tokenizer.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f6872db91e8e2d0d469ae894418bb2037a367e7 Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/tokenizer.cpython-311.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/tokenizer.cpython-312.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/tokenizer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7003e80d68c03fa9243dc3f7dbf79c1a6a39029f Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/tokenizer.cpython-312.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/tokenizer.cpython-39.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/tokenizer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d7d876b2ac4825ea8de5fbea50b5e11fe330bea Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/tokenizer.cpython-39.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/utils.cpython-311.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/utils.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ce3fac011650eb1f90dd68a81dd91003708aa79 Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/utils.cpython-311.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/utils.cpython-312.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9369f176a3bd43391fc391a0d77ae9d4c4d886e2 Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/utils.cpython-312.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/__pycache__/utils.cpython-39.pyc b/Tipsomaly/model/big_vision/pp/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..613d129afa4ef4af59089877c7b4c7607dd4064c Binary files /dev/null and b/Tipsomaly/model/big_vision/pp/__pycache__/utils.cpython-39.pyc differ diff --git a/Tipsomaly/model/big_vision/pp/archive/__init__.py b/Tipsomaly/model/big_vision/pp/archive/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Tipsomaly/model/big_vision/pp/archive/autoaugment.py b/Tipsomaly/model/big_vision/pp/archive/autoaugment.py new file mode 100644 index 0000000000000000000000000000000000000000..1e90e199b4c2385e452a7d1d26b475fcb2c998da --- /dev/null +++ b/Tipsomaly/model/big_vision/pp/archive/autoaugment.py @@ -0,0 +1,700 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoAugment and RandAugment policies for enhanced image preprocessing. + +AutoAugment Reference: https://arxiv.org/abs/1805.09501 +RandAugment Reference: https://arxiv.org/abs/1909.13719 + +This code is forked from +https://github.com/tensorflow/tpu/blob/11d0db15cf1c3667f6e36fecffa111399e008acd/models/official/efficientnet/autoaugment.py +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import dataclasses +import inspect +import math +import tensorflow.compat.v1 as tf +from tensorflow_addons import image as contrib_image + +# This signifies the max integer that the controller RNN could predict for the +# augmentation scheme. +_MAX_LEVEL = 10. + + +@dataclasses.dataclass +class HParams: + """Parameters for AutoAugment and RandAugment.""" + cutout_const: int + translate_const: int + + +def policy_v0(): + """Autoaugment policy that was used in AutoAugment Paper.""" + # Each tuple is an augmentation operation of the form + # (operation, probability, magnitude). Each element in policy is a + # sub-policy that will be applied sequentially on the image. + policy = [ + [('Equalize', 0.8, 1), ('ShearY', 0.8, 4)], + [('Color', 0.4, 9), ('Equalize', 0.6, 3)], + [('Color', 0.4, 1), ('Rotate', 0.6, 8)], + [('Solarize', 0.8, 3), ('Equalize', 0.4, 7)], + [('Solarize', 0.4, 2), ('Solarize', 0.6, 2)], + [('Color', 0.2, 0), ('Equalize', 0.8, 8)], + [('Equalize', 0.4, 8), ('SolarizeAdd', 0.8, 3)], + [('ShearX', 0.2, 9), ('Rotate', 0.6, 8)], + [('Color', 0.6, 1), ('Equalize', 1.0, 2)], + [('Invert', 0.4, 9), ('Rotate', 0.6, 0)], + [('Equalize', 1.0, 9), ('ShearY', 0.6, 3)], + [('Color', 0.4, 7), ('Equalize', 0.6, 0)], + [('Posterize', 0.4, 6), ('AutoContrast', 0.4, 7)], + [('Solarize', 0.6, 8), ('Color', 0.6, 9)], + [('Solarize', 0.2, 4), ('Rotate', 0.8, 9)], + [('Rotate', 1.0, 7), ('TranslateY', 0.8, 9)], + [('ShearX', 0.0, 0), ('Solarize', 0.8, 4)], + [('ShearY', 0.8, 0), ('Color', 0.6, 4)], + [('Color', 1.0, 0), ('Rotate', 0.6, 2)], + [('Equalize', 0.8, 4), ('Equalize', 0.0, 8)], + [('Equalize', 1.0, 4), ('AutoContrast', 0.6, 2)], + [('ShearY', 0.4, 7), ('SolarizeAdd', 0.6, 7)], + [('Posterize', 0.8, 2), ('Solarize', 0.6, 10)], + [('Solarize', 0.6, 8), ('Equalize', 0.6, 1)], + [('Color', 0.8, 6), ('Rotate', 0.4, 5)], + ] + return policy + + +def policy_vtest(): + """Autoaugment test policy for debugging.""" + # Each tuple is an augmentation operation of the form + # (operation, probability, magnitude). Each element in policy is a + # sub-policy that will be applied sequentially on the image. + policy = [ + [('TranslateX', 1.0, 4), ('Equalize', 1.0, 10)], + ] + return policy + + +def blend(image1, image2, factor): + """Blend image1 and image2 using 'factor'. + Factor can be above 0.0. A value of 0.0 means only image1 is used. + A value of 1.0 means only image2 is used. A value between 0.0 and + 1.0 means we linearly interpolate the pixel values between the two + images. A value greater than 1.0 "extrapolates" the difference + between the two pixel values, and we clip the results to values + between 0 and 255. + Args: + image1: An image Tensor of type uint8. + image2: An image Tensor of type uint8. + factor: A floating point value above 0.0. + Returns: + A blended image Tensor of type uint8. + """ + if factor == 0.0: + return tf.convert_to_tensor(image1) + if factor == 1.0: + return tf.convert_to_tensor(image2) + + image1 = tf.to_float(image1) + image2 = tf.to_float(image2) + + difference = image2 - image1 + scaled = factor * difference + + # Do addition in float. + temp = tf.to_float(image1) + scaled + + # Interpolate + if factor > 0.0 and factor < 1.0: + # Interpolation means we always stay within 0 and 255. + return tf.cast(temp, tf.uint8) + + # Extrapolate: + # + # We need to clip and then cast. + return tf.cast(tf.clip_by_value(temp, 0.0, 255.0), tf.uint8) + + +def cutout(image, pad_size, replace=0): + """Apply cutout (https://arxiv.org/abs/1708.04552) to image. + This operation applies a (2*pad_size x 2*pad_size) mask of zeros to + a random location within `img`. The pixel values filled in will be of the + value `replace`. The located where the mask will be applied is randomly + chosen uniformly over the whole image. + Args: + image: An image Tensor of type uint8. + pad_size: Specifies how big the zero mask that will be generated is that + is applied to the image. The mask will be of size + (2*pad_size x 2*pad_size). + replace: What pixel value to fill in the image in the area that has + the cutout mask applied to it. + Returns: + An image Tensor that is of type uint8. + """ + image_height = tf.shape(image)[0] + image_width = tf.shape(image)[1] + + # Sample the center location in the image where the zero mask will be applied. + cutout_center_height = tf.random_uniform( + shape=[], minval=0, maxval=image_height, + dtype=tf.int32) + + cutout_center_width = tf.random_uniform( + shape=[], minval=0, maxval=image_width, + dtype=tf.int32) + + lower_pad = tf.maximum(0, cutout_center_height - pad_size) + upper_pad = tf.maximum(0, image_height - cutout_center_height - pad_size) + left_pad = tf.maximum(0, cutout_center_width - pad_size) + right_pad = tf.maximum(0, image_width - cutout_center_width - pad_size) + + cutout_shape = [image_height - (lower_pad + upper_pad), + image_width - (left_pad + right_pad)] + padding_dims = [[lower_pad, upper_pad], [left_pad, right_pad]] + mask = tf.pad( + tf.zeros(cutout_shape, dtype=image.dtype), + padding_dims, constant_values=1) + mask = tf.expand_dims(mask, -1) + mask = tf.tile(mask, [1, 1, 3]) + image = tf.where( + tf.equal(mask, 0), + tf.ones_like(image, dtype=image.dtype) * replace, + image) + return image + + +def solarize(image, threshold=128): + # For each pixel in the image, select the pixel + # if the value is less than the threshold. + # Otherwise, subtract 255 from the pixel. + return tf.where(image < threshold, image, 255 - image) + + +def solarize_add(image, addition=0, threshold=128): + # For each pixel in the image less than threshold + # we add 'addition' amount to it and then clip the + # pixel value to be between 0 and 255. The value + # of 'addition' is between -128 and 128. + added_image = tf.cast(image, tf.int64) + addition + added_image = tf.cast(tf.clip_by_value(added_image, 0, 255), tf.uint8) + return tf.where(image < threshold, added_image, image) + + +def color(image, factor): + """Equivalent of PIL Color.""" + degenerate = tf.image.grayscale_to_rgb(tf.image.rgb_to_grayscale(image)) + return blend(degenerate, image, factor) + + +def contrast(image, factor): + """Equivalent of PIL Contrast.""" + degenerate = tf.image.rgb_to_grayscale(image) + # Cast before calling tf.histogram. + degenerate = tf.cast(degenerate, tf.int32) + + # Compute the grayscale histogram, then compute the mean pixel value, + # and create a constant image size of that value. Use that as the + # blending degenerate target of the original image. + hist = tf.histogram_fixed_width(degenerate, [0, 255], nbins=256) + mean = tf.reduce_sum(tf.cast(hist, tf.float32)) / 256.0 + degenerate = tf.ones_like(degenerate, dtype=tf.float32) * mean + degenerate = tf.clip_by_value(degenerate, 0.0, 255.0) + degenerate = tf.image.grayscale_to_rgb(tf.cast(degenerate, tf.uint8)) + return blend(degenerate, image, factor) + + +def brightness(image, factor): + """Equivalent of PIL Brightness.""" + degenerate = tf.zeros_like(image) + return blend(degenerate, image, factor) + + +def posterize(image, bits): + """Equivalent of PIL Posterize.""" + shift = 8 - bits + return tf.bitwise.left_shift(tf.bitwise.right_shift(image, shift), shift) + + +def rotate(image, degrees, replace): + """Rotates the image by degrees either clockwise or counterclockwise. + Args: + image: An image Tensor of type uint8. + degrees: Float, a scalar angle in degrees to rotate all images by. If + degrees is positive the image will be rotated clockwise otherwise it will + be rotated counterclockwise. + replace: A one or three value 1D tensor to fill empty pixels caused by + the rotate operation. + Returns: + The rotated version of image. + """ + # Convert from degrees to radians. + degrees_to_radians = math.pi / 180.0 + radians = degrees * degrees_to_radians + + # In practice, we should randomize the rotation degrees by flipping + # it negatively half the time, but that's done on 'degrees' outside + # of the function. + image = contrib_image.rotate(wrap(image), radians) + return unwrap(image, replace) + + +def translate_x(image, pixels, replace): + """Equivalent of PIL Translate in X dimension.""" + image = contrib_image.translate(wrap(image), [-pixels, 0]) + return unwrap(image, replace) + + +def translate_y(image, pixels, replace): + """Equivalent of PIL Translate in Y dimension.""" + image = contrib_image.translate(wrap(image), [0, -pixels]) + return unwrap(image, replace) + + +def shear_x(image, level, replace): + """Equivalent of PIL Shearing in X dimension.""" + # Shear parallel to x axis is a projective transform + # with a matrix form of: + # [1 level + # 0 1]. + image = contrib_image.transform( + wrap(image), [1., level, 0., 0., 1., 0., 0., 0.]) + return unwrap(image, replace) + + +def shear_y(image, level, replace): + """Equivalent of PIL Shearing in Y dimension.""" + # Shear parallel to y axis is a projective transform + # with a matrix form of: + # [1 0 + # level 1]. + image = contrib_image.transform( + wrap(image), [1., 0., 0., level, 1., 0., 0., 0.]) + return unwrap(image, replace) + + +def autocontrast(image): + """Implements Autocontrast function from PIL using TF ops. + Args: + image: A 3D uint8 tensor. + Returns: + The image after it has had autocontrast applied to it and will be of type + uint8. + """ + + def scale_channel(image): + """Scale the 2D image using the autocontrast rule.""" + # A possibly cheaper version can be done using cumsum/unique_with_counts + # over the histogram values, rather than iterating over the entire image. + # to compute mins and maxes. + lo = tf.to_float(tf.reduce_min(image)) + hi = tf.to_float(tf.reduce_max(image)) + + # Scale the image, making the lowest value 0 and the highest value 255. + def scale_values(im): + scale = 255.0 / (hi - lo) + offset = -lo * scale + im = tf.to_float(im) * scale + offset + im = tf.clip_by_value(im, 0.0, 255.0) + return tf.cast(im, tf.uint8) + + result = tf.cond(hi > lo, lambda: scale_values(image), lambda: image) + return result + + # Assumes RGB for now. Scales each channel independently + # and then stacks the result. + s1 = scale_channel(image[:, :, 0]) + s2 = scale_channel(image[:, :, 1]) + s3 = scale_channel(image[:, :, 2]) + image = tf.stack([s1, s2, s3], 2) + return image + + +def sharpness(image, factor): + """Implements Sharpness function from PIL using TF ops.""" + orig_image = image + image = tf.cast(image, tf.float32) + # Make image 4D for conv operation. + image = tf.expand_dims(image, 0) + # SMOOTH PIL Kernel. + kernel = tf.constant( + [[1, 1, 1], [1, 5, 1], [1, 1, 1]], dtype=tf.float32, + shape=[3, 3, 1, 1]) / 13. + # Tile across channel dimension. + kernel = tf.tile(kernel, [1, 1, 3, 1]) + strides = [1, 1, 1, 1] + with tf.device('/cpu:0'): + # Some augmentation that uses depth-wise conv will cause crashing when + # training on GPU. See ((internal link)) for details. + degenerate = tf.nn.depthwise_conv2d( + image, kernel, strides, padding='VALID', rate=[1, 1]) + degenerate = tf.clip_by_value(degenerate, 0.0, 255.0) + degenerate = tf.squeeze(tf.cast(degenerate, tf.uint8), [0]) + + # For the borders of the resulting image, fill in the values of the + # original image. + mask = tf.ones_like(degenerate) + padded_mask = tf.pad(mask, [[1, 1], [1, 1], [0, 0]]) + padded_degenerate = tf.pad(degenerate, [[1, 1], [1, 1], [0, 0]]) + result = tf.where(tf.equal(padded_mask, 1), padded_degenerate, orig_image) + + # Blend the final result. + return blend(result, orig_image, factor) + + +def equalize(image): + """Implements Equalize function from PIL using TF ops.""" + def scale_channel(im, c): + """Scale the data in the channel to implement equalize.""" + im = tf.cast(im[:, :, c], tf.int32) + # Compute the histogram of the image channel. + histo = tf.histogram_fixed_width(im, [0, 255], nbins=256) + + # For the purposes of computing the step, filter out the nonzeros. + nonzero = tf.where(tf.not_equal(histo, 0)) + nonzero_histo = tf.reshape(tf.gather(histo, nonzero), [-1]) + step = (tf.reduce_sum(nonzero_histo) - nonzero_histo[-1]) // 255 + + def build_lut(histo, step): + # Compute the cumulative sum, shifting by step // 2 + # and then normalization by step. + lut = (tf.cumsum(histo) + (step // 2)) // step + # Shift lut, prepending with 0. + lut = tf.concat([[0], lut[:-1]], 0) + # Clip the counts to be in range. This is done + # in the C code for image.point. + return tf.clip_by_value(lut, 0, 255) + + # If step is zero, return the original image. Otherwise, build + # lut from the full histogram and step and then index from it. + result = tf.cond(tf.equal(step, 0), + lambda: im, + lambda: tf.gather(build_lut(histo, step), im)) + + return tf.cast(result, tf.uint8) + + # Assumes RGB for now. Scales each channel independently + # and then stacks the result. + s1 = scale_channel(image, 0) + s2 = scale_channel(image, 1) + s3 = scale_channel(image, 2) + image = tf.stack([s1, s2, s3], 2) + return image + + +def invert(image): + """Inverts the image pixels.""" + image = tf.convert_to_tensor(image) + return 255 - image + + +def wrap(image): + """Returns 'image' with an extra channel set to all 1s.""" + shape = tf.shape(image) + extended_channel = tf.ones([shape[0], shape[1], 1], image.dtype) + extended = tf.concat([image, extended_channel], 2) + return extended + + +def unwrap(image, replace): + """Unwraps an image produced by wrap. + Where there is a 0 in the last channel for every spatial position, + the rest of the three channels in that spatial dimension are grayed + (set to 128). Operations like translate and shear on a wrapped + Tensor will leave 0s in empty locations. Some transformations look + at the intensity of values to do preprocessing, and we want these + empty pixels to assume the 'average' value, rather than pure black. + Args: + image: A 3D Image Tensor with 4 channels. + replace: A one or three value 1D tensor to fill empty pixels. + Returns: + image: A 3D image Tensor with 3 channels. + """ + image_shape = tf.shape(image) + # Flatten the spatial dimensions. + flattened_image = tf.reshape(image, [-1, image_shape[2]]) + + # Find all pixels where the last channel is zero. + alpha_channel = flattened_image[:, 3] + + replace = tf.concat([replace, tf.ones([1], image.dtype)], 0) + + # Where they are zero, fill them in with 'replace'. + flattened_image = tf.where( + tf.equal(alpha_channel, 0), + tf.ones_like(flattened_image, dtype=image.dtype) * replace, + flattened_image) + + image = tf.reshape(flattened_image, image_shape) + image = tf.slice(image, [0, 0, 0], [image_shape[0], image_shape[1], 3]) + return image + + +NAME_TO_FUNC = { + 'AutoContrast': autocontrast, + 'Equalize': equalize, + 'Invert': invert, + 'Rotate': rotate, + 'Posterize': posterize, + 'Solarize': solarize, + 'SolarizeAdd': solarize_add, + 'Color': color, + 'Contrast': contrast, + 'Brightness': brightness, + 'Sharpness': sharpness, + 'ShearX': shear_x, + 'ShearY': shear_y, + 'TranslateX': translate_x, + 'TranslateY': translate_y, + 'Cutout': cutout, +} + + +def _randomly_negate_tensor(tensor): + """With 50% prob turn the tensor negative.""" + should_flip = tf.cast(tf.floor(tf.random_uniform([]) + 0.5), tf.bool) + final_tensor = tf.cond(should_flip, lambda: tensor, lambda: -tensor) + return final_tensor + + +def _rotate_level_to_arg(level): + level = (level/_MAX_LEVEL) * 30. + level = _randomly_negate_tensor(level) + return (level,) + + +def _shrink_level_to_arg(level): + """Converts level to ratio by which we shrink the image content.""" + if level == 0: + return (1.0,) # if level is zero, do not shrink the image + # Maximum shrinking ratio is 2.9. + level = 2. / (_MAX_LEVEL / level) + 0.9 + return (level,) + + +def _enhance_level_to_arg(level): + return ((level/_MAX_LEVEL) * 1.8 + 0.1,) + + +def _shear_level_to_arg(level): + level = (level/_MAX_LEVEL) * 0.3 + # Flip level to negative with 50% chance. + level = _randomly_negate_tensor(level) + return (level,) + + +def _translate_level_to_arg(level, translate_const): + level = (level/_MAX_LEVEL) * float(translate_const) + # Flip level to negative with 50% chance. + level = _randomly_negate_tensor(level) + return (level,) + + +def level_to_arg(hparams): + return { + 'AutoContrast': lambda level: (), + 'Equalize': lambda level: (), + 'Invert': lambda level: (), + 'Rotate': _rotate_level_to_arg, + 'Posterize': lambda level: (int((level/_MAX_LEVEL) * 4),), + 'Solarize': lambda level: (int((level/_MAX_LEVEL) * 256),), + 'SolarizeAdd': lambda level: (int((level/_MAX_LEVEL) * 110),), + 'Color': _enhance_level_to_arg, + 'Contrast': _enhance_level_to_arg, + 'Brightness': _enhance_level_to_arg, + 'Sharpness': _enhance_level_to_arg, + 'ShearX': _shear_level_to_arg, + 'ShearY': _shear_level_to_arg, + 'Cutout': lambda level: (int((level/_MAX_LEVEL) * hparams.cutout_const),), + 'TranslateX': lambda level: _translate_level_to_arg( + level, hparams.translate_const), + 'TranslateY': lambda level: _translate_level_to_arg( + level, hparams.translate_const), + # pylint:enable=g-long-lambda + } + + +def _parse_policy_info(name, prob, level, replace_value, augmentation_hparams): + """Return the function that corresponds to `name` and update `level` param.""" + func = NAME_TO_FUNC[name] + args = level_to_arg(augmentation_hparams)[name](level) + + # Check to see if prob is passed into function. This is used for operations + # where we alter bboxes independently. + # pytype:disable=wrong-arg-types + if 'prob' in inspect.getfullargspec(func).args: + args = tuple([prob] + list(args)) + # pytype:enable=wrong-arg-types + + # Add in replace arg if it is required for the function that is being called. + # pytype:disable=wrong-arg-types + if 'replace' in inspect.getfullargspec(func).args: + # Make sure replace is the final argument + assert 'replace' == inspect.getfullargspec(func).args[-1] + args = tuple(list(args) + [replace_value]) + # pytype:enable=wrong-arg-types + + return (func, prob, args) + + +def _apply_func_with_prob(func, image, args, prob): + """Apply `func` to image w/ `args` as input with probability `prob`.""" + assert isinstance(args, tuple) + + # If prob is a function argument, then this randomness is being handled + # inside the function, so make sure it is always called. + # pytype:disable=wrong-arg-types + if 'prob' in inspect.getfullargspec(func).args: + prob = 1.0 + # pytype:enable=wrong-arg-types + + # Apply the function with probability `prob`. + should_apply_op = tf.cast( + tf.floor(tf.random_uniform([], dtype=tf.float32) + prob), tf.bool) + augmented_image = tf.cond( + should_apply_op, + lambda: func(image, *args), + lambda: image) + return augmented_image + + +def select_and_apply_random_policy(policies, image): + """Select a random policy from `policies` and apply it to `image`.""" + policy_to_select = tf.random_uniform([], maxval=len(policies), dtype=tf.int32) + # Note that using tf.case instead of tf.conds would result in significantly + # larger graphs and would even break export for some larger policies. + for (i, policy) in enumerate(policies): + image = tf.cond( + tf.equal(i, policy_to_select), + lambda selected_policy=policy: selected_policy(image), + lambda: image) + return image + + +def build_and_apply_nas_policy(policies, image, + augmentation_hparams): + """Build a policy from the given policies passed in and apply to image. + Args: + policies: list of lists of tuples in the form `(func, prob, level)`, `func` + is a string name of the augmentation function, `prob` is the probability + of applying the `func` operation, `level` is the input argument for + `func`. + image: tf.Tensor that the resulting policy will be applied to. + augmentation_hparams: Hparams associated with the NAS learned policy. + Returns: + A version of image that now has data augmentation applied to it based on + the `policies` pass into the function. + """ + replace_value = [128, 128, 128] + + # func is the string name of the augmentation function, prob is the + # probability of applying the operation and level is the parameter associated + # with the tf op. + + # tf_policies are functions that take in an image and return an augmented + # image. + tf_policies = [] + for policy in policies: + tf_policy = [] + # Link string name to the correct python function and make sure the correct + # argument is passed into that function. + for policy_info in policy: + policy_info = list(policy_info) + [replace_value, augmentation_hparams] + + tf_policy.append(_parse_policy_info(*policy_info)) + # Now build the tf policy that will apply the augmentation procedue + # on image. + def make_final_policy(tf_policy_): + def final_policy(image_): + for func, prob, args in tf_policy_: + image_ = _apply_func_with_prob( + func, image_, args, prob) + return image_ + return final_policy + tf_policies.append(make_final_policy(tf_policy)) + + augmented_image = select_and_apply_random_policy( + tf_policies, image) + return augmented_image + + +def distort_image_with_autoaugment(image, augmentation_name): + """Applies the AutoAugment policy to `image`. + AutoAugment is from the paper: https://arxiv.org/abs/1805.09501. + Args: + image: `Tensor` of shape [height, width, 3] representing an image. + augmentation_name: The name of the AutoAugment policy to use. The available + options are `v0` and `test`. `v0` is the policy used for + all of the results in the paper and was found to achieve the best results + on the COCO dataset. `v1`, `v2` and `v3` are additional good policies + found on the COCO dataset that have slight variation in what operations + were used during the search procedure along with how many operations are + applied in parallel to a single image (2 vs 3). + Returns: + A tuple containing the augmented versions of `image`. + """ + available_policies = {'v0': policy_v0, + 'test': policy_vtest} + if augmentation_name not in available_policies: + raise ValueError('Invalid augmentation_name: {}'.format(augmentation_name)) + + policy = available_policies[augmentation_name]() + # Hparams that will be used for AutoAugment. + augmentation_hparams = HParams( + cutout_const=100, translate_const=250) + + return build_and_apply_nas_policy(policy, image, augmentation_hparams) + + +def distort_image_with_randaugment(image, num_layers, magnitude): + """Applies the RandAugment policy to `image`. + RandAugment is from the paper https://arxiv.org/abs/1909.13719, + Args: + image: `Tensor` of shape [height, width, 3] representing an image. + num_layers: Integer, the number of augmentation transformations to apply + sequentially to an image. Represented as (N) in the paper. Usually best + values will be in the range [1, 3]. + magnitude: Integer, shared magnitude across all augmentation operations. + Represented as (M) in the paper. Usually best values are in the range + [5, 30]. + Returns: + The augmented version of `image`. + """ + replace_value = [128] * 3 + tf.logging.info('Using RandAug.') + augmentation_hparams = HParams( + cutout_const=40, translate_const=100) + available_ops = [ + 'AutoContrast', 'Equalize', 'Invert', 'Rotate', 'Posterize', + 'Solarize', 'Color', 'Contrast', 'Brightness', 'Sharpness', + 'ShearX', 'ShearY', 'TranslateX', 'TranslateY', 'Cutout', 'SolarizeAdd'] + + for layer_num in range(num_layers): + op_to_select = tf.random_uniform( + [], maxval=len(available_ops), dtype=tf.int32) + random_magnitude = float(magnitude) + with tf.name_scope('randaug_layer_{}'.format(layer_num)): + for (i, op_name) in enumerate(available_ops): + prob = tf.random_uniform([], minval=0.2, maxval=0.8, dtype=tf.float32) + func, _, args = _parse_policy_info(op_name, prob, random_magnitude, + replace_value, augmentation_hparams) + image = tf.cond( + tf.equal(i, op_to_select), + lambda selected_func=func, selected_args=args: selected_func( + image, *selected_args), + # pylint:enable=g-long-lambda + lambda: image) + return image diff --git a/Tipsomaly/model/big_vision/pp/archive/randaug.py b/Tipsomaly/model/big_vision/pp/archive/randaug.py new file mode 100644 index 0000000000000000000000000000000000000000..39508a801ef03dcae694ba8e4678733f12562cf1 --- /dev/null +++ b/Tipsomaly/model/big_vision/pp/archive/randaug.py @@ -0,0 +1,46 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""RandAug depends on deprecated tfa.image package, now defunct.""" + +from big_vision.pp import registry +from big_vision.pp import utils +from big_vision.pp.archive import autoaugment + + +@registry.Registry.register("preprocess_ops.randaug") +@utils.InKeyOutKey() +def get_randaug(num_layers: int = 2, magnitude: int = 10): + """Creates a function that applies RandAugment. + + RandAugment is from the paper https://arxiv.org/abs/1909.13719, + + Args: + num_layers: Integer, the number of augmentation transformations to apply + sequentially to an image. Represented as (N) in the paper. Usually best + values will be in the range [1, 3]. + magnitude: Integer, shared magnitude across all augmentation operations. + Represented as (M) in the paper. Usually best values are in the range [5, + 30]. + + Returns: + a function that applies RandAugment. + """ + + def _randaug(image): + return autoaugment.distort_image_with_randaugment( + image, num_layers, magnitude + ) + + return _randaug diff --git a/Tipsomaly/model/big_vision/pp/proj/clippo/download_unifont.sh b/Tipsomaly/model/big_vision/pp/proj/clippo/download_unifont.sh new file mode 100644 index 0000000000000000000000000000000000000000..cbb0364316e0f0d47dded16cee36efc7b969b777 --- /dev/null +++ b/Tipsomaly/model/big_vision/pp/proj/clippo/download_unifont.sh @@ -0,0 +1,21 @@ +# Copyright 2022 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +# This is intended to be run from the big_vision repository root: +# +# bash big_vision/pp/proj/clippo/download_unifont.sh +wget https://unifoundry.com/pub/unifont/unifont-9.0.06/font-builds/unifont-9.0.06.hex.gz https://unifoundry.com/pub/unifont/unifont-9.0.06/font-builds/unifont_upper-9.0.06.hex.gz +gunzip unifont-9.0.06.hex.gz unifont_upper-9.0.06.hex.gz +mv unifont-9.0.06.hex unifont_upper-9.0.06.hex big_vision/pp/proj/clippo/ \ No newline at end of file diff --git a/Tipsomaly/model/big_vision/pp/proj/givt/pp_ops.py b/Tipsomaly/model/big_vision/pp/proj/givt/pp_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..5c2013cb99b29fd836517cafa5d3854d81de4e01 --- /dev/null +++ b/Tipsomaly/model/big_vision/pp/proj/givt/pp_ops.py @@ -0,0 +1,36 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GIVT-specific preprocessing ops.""" + +from big_vision.pp import registry +from big_vision.pp import utils +import tensorflow as tf + + +@registry.Registry.register("preprocess_ops.bin_nyu_depth") +@utils.InKeyOutKey(indefault="labels", outdefault="labels") +def get_bin_nyu_depth(min_depth=0.001, max_depth=10.0, num_bins=256): + """Binning of NYU depth for UViM in preprocessing rather than model.""" + + def _bin_depth(labels): # pylint: disable=missing-docstring + labels = (labels - min_depth) / (max_depth - min_depth) + labels *= num_bins + labels = tf.cast(tf.floor(labels), tf.int32) + labels = tf.minimum(labels, num_bins - 1) + labels = tf.maximum(labels, 0) + return labels + + return _bin_depth + diff --git a/Tipsomaly/model/big_vision/pp/proj/image_text/ops_naflex.py b/Tipsomaly/model/big_vision/pp/proj/image_text/ops_naflex.py new file mode 100644 index 0000000000000000000000000000000000000000..48d3959e38b7c74705c999c8ac76bd649381a96b --- /dev/null +++ b/Tipsomaly/model/big_vision/pp/proj/image_text/ops_naflex.py @@ -0,0 +1,202 @@ +# Copyright 2024 Big Vision Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""NaFlex (NaViT + FlexiViT) preprocessing ops.""" + +from big_vision.pp import utils +from big_vision.pp.registry import Registry +import big_vision.utils as u +import tensorflow as tf + + +def _get_image_size_for_seq( + image_hw, + patch_size: int, + max_sequence_len: int, + divisible_by_patch: bool = True, + eps: float = 1e-5): + """Determine scaling ratio and image size for `get_resize_to_sequence`. + + Did not prove monotonicity necessary for binary search correctness, but this + works well in practice. + + Args: + image_hw: Image height and width. + patch_size: Patchification patch size. + max_sequence_len: Maximum allowed sequence length for the resulting image. + divisible_by_patch: If True, the resulting image height and width must be + divisible by patch size. + eps: Small number used for binary search convergence. + + Returns: + ratio: Scaling ratio to applied to image. + target_hw: Target image height and width taking into account the scaling + ratio and the `divisible_by_patch` constraint. + """ + def search_not_done(lb, rb): + return (rb - lb) >= eps + + def prepare_target_hw(ratio): + target_hw = tf.cast(image_hw, tf.float32) * ratio + if divisible_by_patch: + # Round to multiple of patch size as we want to avoid dropping patches. + target_hw = patch_size * tf.math.ceil(target_hw / patch_size) + # Ensure that the image is at least 1 patch in height / width. + target_hw = tf.maximum(target_hw, patch_size) + target_hw = tf.cast(target_hw, tf.int32) + return target_hw + + def is_feasible(ratio): + target_hw = prepare_target_hw(ratio) + num_patches = target_hw / patch_size + sequence_len = tf.math.reduce_prod(num_patches) + return sequence_len <= max_sequence_len + + def _search_fn(lb, rb): + mid = (lb + rb) / 2 + return tf.cond(is_feasible(mid), lambda: (mid, rb), lambda: (lb, mid)) + + # Left and right boundaries for the binary search. + state = (tf.constant(eps / 10.), tf.constant(100.0)) + ratio, _ = tf.while_loop( + search_not_done, _search_fn, state, parallel_iterations=1) + tf.assert_greater( + ratio, eps, message="Binary search failed - image too large?") + tf.assert_less( + ratio, 100.0, message="Binary search failed - image too small?") + + return ratio, prepare_target_hw(ratio) + + +@Registry.register("preprocess_ops.resize_to_sequence") +@utils.InKeyOutKey(indefault="image", outdefault="image") +def get_resize_to_sequence( + patch_size: int, + max_sequence_len: int, + divisible_by_patch: bool = True, + eps: float = 1e-5): + """Resizes image if it violates restrictions on sequence/side length. + + This op attempts to resize the image in an AR-preserving manner such that: + - The sequence length of the resulting image (after patchification) is + maximized, but <= `max_sequence_len`. + + This op *violates* the AR-preserving property if: + - Image size resulting from the above procedure is not a multiple of patch + size. In this case AR is distorted to ensure this condition is satisfied. + + Args: + patch_size: Patchification patch size. + max_sequence_len: Maximum allowed sequence length for the resulting image. + divisible_by_patch: If True, the resulting image height and width must be + divisible by patch size. + eps: Small number used for binary search convergence. + + Returns: + Pre-processing op. + """ + def _resize_fn(image): + """Performs binary search to find a feasible image size.""" + image_hw = tf.shape(image)[:2] + _, target_hw = _get_image_size_for_seq( + image_hw, + patch_size, + max_sequence_len, + divisible_by_patch=divisible_by_patch, + eps=eps) + + # Actually resize image. + image = tf.image.resize( + image, + target_hw, + preserve_aspect_ratio=False, + antialias=True) + return tf.ensure_shape(image, [None, None, 3]) + return _resize_fn + + +@Registry.register("preprocess_ops.central_crop_to_sequence") +@utils.InKeyOutKey(indefault="image", outdefault="image") +def get_central_crop_to_sequence( + patch_size: int, + max_sequence_len: int, + divisible_by_patch: bool = True, + eps: float = 1e-5): + """Central crops image such that patch sequence length satisfies constraints. + + Constraints used are the as in `resize_to_sequence`. + + Args: + patch_size: Patchification patch size. + max_sequence_len: Maximum allowed sequence length for the resulting image. + divisible_by_patch: If True, the resulting image height and width must be + divisible by patch size. + eps: Small number used for binary search convergence. + + Returns: + Pre-processing op. + """ + def _central_crop_fn(image): + image_hw = tf.shape(image)[:2] + _, target_hw = _get_image_size_for_seq( + image_hw, + patch_size, + max_sequence_len, + divisible_by_patch=divisible_by_patch, + eps=eps) + + tf.assert_greater( + image_hw + 1, target_hw, + "For central crop the image must be larger than target HW.") + offset_hw = (image_hw - target_hw) // 2 + image = image[ + offset_hw[0]:offset_hw[0] + target_hw[0], + offset_hw[1]:offset_hw[1] + target_hw[1], + :] + return tf.ensure_shape(image, [None, None, 3]) + return _central_crop_fn + + +@Registry.register("preprocess_ops.patchify") +@utils.InKeyOutKey(indefault="image", outdefault="image") +def get_patchify(patch_size): + """Reshapes image into patches and provides patch coordinates.""" + ph, pw = utils.maybe_repeat(patch_size, 2) + + def _patchify(img): + patches = tf.image.extract_patches( + img[None, ...], sizes=[1, ph, pw, 1], strides=[1, ph, pw, 1], + rates=[1, 1, 1, 1], padding="VALID")[0] + # Patches is now (nh, nw, ph*pw*3), i.e. contains flattened patches. + nh, nw, d = tf.shape(patches)[0], tf.shape(patches)[1], tf.shape(patches)[2] + + # Get two (nh, nw) tensors of y/x indices of the patches. + gy, gx = tf.meshgrid(tf.range(nh), tf.range(nw), indexing="ij") + + return { + "patches": tf.reshape(patches, (nh * nw, d)), + "yidx": tf.reshape(gy, [nh * nw]), + "xidx": tf.reshape(gx, [nh * nw]), + "type": tf.fill([nh * nw], 1), + } + return _patchify + + +@Registry.register("preprocess_ops.tuplify") +def get_tuplify(inkeys: list[str], outkey: str): + """Create a tuple of multiple inputs.""" + def tuplify(data): + data[outkey] = tuple(u.tree_get(data, k) for k in inkeys) + return data + return tuplify diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/README.md b/Tipsomaly/model/big_vision/tools/lit_demo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..029f7bbea31d0ffaf2dd378cf3de3e6b35ed8828 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/README.md @@ -0,0 +1,26 @@ +# LiT-Demo + +See https://blog.tensorflow.org/2022/08/jax-on-web-with-tensorflowjs.html + +Demo originally appeared on Twitter +https://twitter.com/AndreasPSteiner/status/1514722383818543106 + +App published at +https://google-research.github.io/vision_transformer/lit + +## Build + +Install packages (tested with node v16.17.0 and yarn 1.22.19) + +```bash +yarn +``` + + +## Run + +The web app will appear on http://localhost:8000 + +``` +node build.js +``` diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/build.js b/Tipsomaly/model/big_vision/tools/lit_demo/build.js new file mode 100644 index 0000000000000000000000000000000000000000..3441c8b01c32f6fbcb19ff2c64aeed180e1200a2 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/build.js @@ -0,0 +1,39 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const sassPlugin = require('esbuild-sass-plugin').sassPlugin; + +require('esbuild').serve({ + servedir: 'src', + port: 8000, +}, { + entryPoints: ['src/app.ts'], + bundle: true, + outfile: 'src/index.js', + plugins: [ + sassPlugin({ + filter: /style.scss$/, + type: 'style' + }), + sassPlugin({ + type: 'lit-css', + }), + ], + sourcemap: true, +}).then(() => { + console.log('Serving on port 8000'); +}).catch(() => process.exit(1)); diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/package.json b/Tipsomaly/model/big_vision/tools/lit_demo/package.json new file mode 100644 index 0000000000000000000000000000000000000000..7db7c516a532bbe90e1ef95b30e6c16f56462171 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/package.json @@ -0,0 +1,54 @@ +{ + "name": "lit-demo", + "version": "0.0.2", + "description": "", + "main": "src/app.ts", + "license": "Apache-2.0", + "private": true, + "engines": { + "node": ">=8.9.0" + }, + "scripts": { + "serve": "node build.js", + "test": "ts-node --skip-ignore --project tsconfig.test.json run_tests.ts" + }, + "devDependencies": { + "@babel/core": "^7.7.5", + "@babel/plugin-transform-runtime": "^7.7.6", + "@babel/polyfill": "^7.10.4", + "@babel/preset-env": "^7.7.6", + "@tensorflow/tfjs-backend-cpu": "^3.15.0", + "@tensorflow/tfjs-backend-webgl": "^3.15.0", + "@tensorflow/tfjs-converter": "3.20.0", + "@tensorflow/tfjs-core": "3.20.0", + "babel-preset-env": "^1.7.0", + "esbuild": "^0.15.5", + "esbuild-sass-plugin": "^2.3.2", + "jasmine": "^3.3.1", + "lit": "^2.3.1", + "naughty-words": "^1.2.0", + "sass": "^1.50.0", + "ts-node": "~5.0.0", + "typescript": "4.1.3" + }, + "resolutions": { + "is-svg": "4.3.1" + }, + "eslintConfig": { + "extends": "google", + "rules": { + "require-jsdoc": 0, + "valid-jsdoc": 0 + }, + "env": { + "es6": true + }, + "parserOptions": { + "ecmaVersion": 8, + "sourceType": "module" + } + }, + "eslintIgnore": [ + "dist/" + ] +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/app.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/app.ts new file mode 100644 index 0000000000000000000000000000000000000000..e63ee4a8f31e920caf88c1ea22672e6bbd9b5d84 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/app.ts @@ -0,0 +1,22 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {LitDemoApp} from './components/lit-demo-app'; +import './style.scss'; + +// tslint:disable-next-line:no-any +(window as any).LitDemoApp = LitDemoApp; diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-carousel.scss b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-carousel.scss new file mode 100644 index 0000000000000000000000000000000000000000..962f59a884a2b625a39d2a0f3805953b51092ade --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-carousel.scss @@ -0,0 +1,32 @@ +@import '../style/mixins'; + +.selector { + overflow: scroll; + padding-bottom: 10px; // OS X scroll bar + + .inner { + white-space: nowrap; + + .thumb { + display: inline-block; + + img { + cursor: pointer; + + width: 20vmin; + height: 20vmin; + max-width: 200px; + max-height: 200px; + + @include phone-portrait { + width: 33vmin; + height: 33vmin; + } + + margin: 10px; + + box-shadow: 0 0 10px #888; + } + } + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-carousel.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-carousel.ts new file mode 100644 index 0000000000000000000000000000000000000000..5cc411d48150d873674499505e39ced15caf1830 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-carousel.ts @@ -0,0 +1,70 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Carousel of images. + */ + +import {html, LitElement} from 'lit'; + +import {app} from '../lit_demo/app'; +import {getImageUrl} from '../lit_demo/constants'; +import {ImageRow} from '../lit_demo/data'; + +import {customElement} from 'lit/decorators.js'; +import styles from './image-carousel.scss'; + +/** + * Shows multiple images in a horizontal carousel. + * + * Dispatches `'image-select'` event when an image is clicked/tapped. + */ +@customElement('image-carousel') +export class ImageCarousel extends LitElement { + static override styles = [styles]; + + onClick(id: string) { + const event = + new CustomEvent('image-select', {composed: true, detail: {id}}); + this.dispatchEvent(event); + } + + override render() { + const images = app.imageData.rows.map( + (row: ImageRow) => html` +
+ { + this.onClick(row.id); + }} data-id=${row.id} src="${getImageUrl(row.id)}"> +
+ `); + return html` +
+
+ ${images} +
+
+

Select an image 👆 to get started.

+ `; + } +} + +declare global { + interface HTMLElementTagNameMap { + 'image-carousel': ImageCarousel; + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-prompts.scss b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-prompts.scss new file mode 100644 index 0000000000000000000000000000000000000000..8e5d66d6f211d83be87482d8952bac5bd8718b38 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-prompts.scss @@ -0,0 +1,124 @@ +@import '../style/mixins'; + +.image-prompt { + display: flex; + gap: 1.5em; + align-items: flex-start; + margin-top: 2rem; + + @include phone-portrait { + align-items: center; + flex-direction: column; + gap: 0; + margin-bottom: 5rem; + } + + .left { + display: flex; + flex-direction: column; + + .wrapper { + position: relative; + + .src { + position: absolute; + right: 2rem; + bottom: 2rem; + color: white; + font-size: 1.5rem; + text-shadow: 2px 2px black; + text-decoration: none; + } + } + + .animation { + position: relative; + width: 224px; + height: 15px; + opacity: 0; + + .computing { + text-align: center; + } + } + } + + .right { + display: flex; + flex-grow: 1; + flex-direction: column; + gap: 0.5em; + + .top { + text-align: right; + height: 30px; + } + + .buttons { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 1em; + align-items: center; + } + + .item { + position: relative; + display: flex; + + .pct { + display: inline-block; + margin-right: 1em; + width: 3.5em; + text-align: right; + opacity: 0; + transition: opacity 0.5s; + } + + input { + flex-grow: 1; + max-width: 70vw; + border-radius: 0; + background: transparent; + border: 0; + border-bottom: 1px solid var(--text-fg); + color: var(--text-fg); + outline: none; + + &.toolong { + border-bottom: 1px solid var(--text-red); + color: var(--text-red); + } + } + + .bar { + position: absolute; + display: inline-block; + top: 5%; + left: 0; + z-index: -1; + background: var(--bar-col); + height: 90%; + width: 0; + transition: width 0.5s; + } + } + + .bottom { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 1em; + align-items: center; + opacity: 0; + + .tweet { + background: rgb(18, 150, 223); + color: white; + text-decoration: none; + padding: 0px 15px; + border-radius: 16px; + } + } + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-prompts.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-prompts.ts new file mode 100644 index 0000000000000000000000000000000000000000..d6eac95cf5b47701f49b98cca82ccfb3f02a397b --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/image-prompts.ts @@ -0,0 +1,250 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Image and text prompts. + */ + +import {html, LitElement} from 'lit'; +import * as naughtyWords from 'naughty-words'; + +import {app} from '../lit_demo/app'; +import {getBackend} from '../lit_demo/compute'; +import {getImageUrl} from '../lit_demo/constants'; +import {getUrl} from '../lit_demo/url_utils'; + +import {MessageList} from './message-list'; + +import {customElement, query} from 'lit/decorators.js'; +import styles from './image-prompts.scss'; + +const setHref = (anchorEl: HTMLAnchorElement, href:string) => { + anchorEl. href = href; +}; + +const HTML_TEMPLATE = ` + We cannot include the word "{word}" as it is found on the list + naughty-words/{lang}. + We understand blocklists are an imperfect solution but we believe it's + important to ensure these models are not misused, and hope that in this + instance it does not serve to marginalise anybody. If you don't agree, + please reach out via + form link. +`; + + +/** + * Shows image and text prompts, and computes similarities. + * + * Also dispatches some events like `'duplicate'` to the parent. + */ +@customElement('image-prompts') +export class ImagePrompts extends LitElement { + + static override styles = [styles]; + + @query('message-list') + messageList!: MessageList; + @query('.animation') + animation!: HTMLElement; + @query('.bottom') + bottom!: HTMLElement; + + lastPrompts?: string[]; + + constructor(private readonly imageId: string) { + super(); + } + + override firstUpdated() { + if (getBackend() !== 'webgl') { + this.messageList.warning( + 'Please activate WebGL. Running ML demos on ' + + 'CPU will drain your battery in no time...'); + } + } + + onDuplicate() { + this.dispatchEvent(new Event('duplicate')); + } + + onRemove() { + this.remove(); + } + + onClear() { + this.shadowRoot!.querySelectorAll('.prompt').forEach((input: Element) => { + (input as HTMLInputElement).value = ''; + }); + (this.shadowRoot!.querySelector('.prompt') as HTMLInputElement).focus(); + } + + onKeyup(event: KeyboardEvent) { + if (event.key === 'Enter') { + this.onCompute(); + } + } + + async setPrompts(prompts: string[]) { + await this.updateComplete; + this.shadowRoot!.querySelectorAll('.prompt').forEach((input: Element, idx: number) => { + (input as HTMLInputElement).value = prompts[idx] || ''; + }); + } + + getPrompts(): string[] { + return [...this.shadowRoot!.querySelectorAll('.prompt')].map((input: Element) => + (input as HTMLInputElement).value + ); + } + + override render() { + const row = app.imageData.get(this.imageId); + const inputs = row.prompts.split(',').map((prompt: string, idx: number) => { + return html` +
+
+ +
+
+ `; + }); + return html` +
+
+
+ + source +
+
+
✨✨Computing✨✨
+
+
+
+ +
+ + + + +
+ ${inputs} +
+ Model: ? + tweet +
+
+
+ `; + } + + onCompute() { + if (!app.models.ready) { + this.messageList.warning('Model not ready yet.'); + return; + } + + const model = app.models.model!; + const zimgIdx = model.zimgIds!.indexOf(this.imageId); + if (zimgIdx === -1) { + this.messageList.warning('Model is missing this image embedding'); + return; + } + + const texts = this.getPrompts(); + for (const text of texts) { + for (const word of text.toLocaleLowerCase().split(/\s+/g)) { + // tslint:disable-next-line:ban-module-namespace-object-escape + for (const lang of Object.keys(naughtyWords)) { + if (lang === 'default') { + continue; + } + // tslint:disable-next-line:ban-module-namespace-object-escape + const words = (naughtyWords as {[key: string]: string[]})[lang]; + if (words.indexOf(word) !== -1) { + const msg = HTML_TEMPLATE.replace(/\{word\}/g, word).replace(/\{lang\}/g, lang); + this.messageList.warning(msg, {rawHtml: true}); + return; + } + } + } + } + + const compute = () => { + let probs: number[]|undefined; + try { + // ??? how to move into webworker (to avoid freezing UI) ? + // https://github.com/tensorflow/tfjs/issues/102 + probs = model.computeProbabilities(texts, zimgIdx); + } catch (error) { + if ((error as Error).message.toLocaleLowerCase().match(/greater than .* maximum/)) { + this.messageList.warning('Model too large for Browser!'); + return; + } + throw error; + } + this.setProbabilities(probs); + this.lastPrompts = this.getPrompts(); + this.animation.style.opacity = '0'; + }; + + this.animation.style.opacity = '1'; + this.messageList.clear(); + setTimeout(compute, 10); // Give UI some time to update. + } + + setProbabilities(probs: number[]) { + const pcts = [...this.shadowRoot!.querySelectorAll('.pct')] as HTMLElement[]; + const bars = [...this.shadowRoot!.querySelectorAll('.bar')] as HTMLElement[]; + this.hideBottom(); + for(let i = 0; i < Math.max(probs.length, pcts.length, bars.length); i++) { + const prob = probs[i] || 0; + const pct = `${Math.round(prob * 1e3) / 1e1}%`; + bars[i].style.width = pct; + if (prob) { + pcts[i].innerText = pct; + pcts[i].style.opacity = '1'; + } else { + pcts[i].style.opacity = '0'; + } + } + this.updateBottom(); + } + + updateBottom() { + const tweet = this.shadowRoot!.querySelector('.tweet') as HTMLAnchorElement; + const url = getUrl(app.models.model!.name, this.imageId, this.getPrompts()); + const description = app.imageData.get(this.imageId).description; + const text = `LiT matching prompts to an image of "${description}"\n\n#lit_demo\n`; + setHref(tweet, 'https://twitter.com/intent/tweet' + + '?url=' + encodeURIComponent(url) + + '&text=' + encodeURIComponent(text)); + this.bottom.style.opacity = '1'; + const model = this.shadowRoot!.querySelector('.model') as HTMLAnchorElement; + model.innerText = app.models.model!.name; + } + + hideBottom() { + this.bottom.style.opacity = '0'; + } +} + +declare global { + interface HTMLElementTagNameMap { + 'image-prompts': ImagePrompts; + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/lit-demo-app.scss b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/lit-demo-app.scss new file mode 100644 index 0000000000000000000000000000000000000000..5553e48184466a434e0cb876bfca5e77d6129681 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/lit-demo-app.scss @@ -0,0 +1,3 @@ +.loading-container { + text-align: center; +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/lit-demo-app.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/lit-demo-app.ts new file mode 100644 index 0000000000000000000000000000000000000000..5c9df378f65ca0caff33f23f0670c2e22271d4ce --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/lit-demo-app.ts @@ -0,0 +1,127 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Main application. + */ + +import {html, LitElement} from 'lit'; + +import {app} from '../lit_demo/app'; +import {parseUrl, State} from '../lit_demo/url_utils'; + +import './image-carousel'; +import {ImagePrompts} from './image-prompts'; +import './loading-animation'; +import {MessageList} from './message-list'; +import {ModelControls} from './model-controls'; + +import {customElement, property, query} from 'lit/decorators.js'; +import styles from './lit-demo-app.scss'; + +/** + * Main application container. + */ +@customElement('lit-demo-app') +export class LitDemoApp extends LitElement { + + static override styles = [styles]; + + @property({type: Boolean}) + loading: boolean = true; + + @query('message-list') + messageList!: MessageList; + @query('model-controls') + modelControls!: ModelControls; + @query('#examples') + examples!: HTMLElement; + + state?: State; + lingeringWarning?: string; + + constructor() { + super(); + window.onerror = this.onglobalerror.bind(this); + this.load(); + } + + onglobalerror(message: string|Event, source: string|undefined, lineno: number|undefined) { + source = source || ''; + source = source.substring(source.lastIndexOf('/') + 1); + this.messageList.error( + `Javascript error at ${source}:${lineno}
` + + `${message}`, + {rawHtml: true}); + } + + async load() { + await app.load(); + this.loading = false; + try { + this.state = parseUrl(); + } catch (error) { + this.messageList.warning(`Could not parse URL: ${error}`); + } + } + + override updated() { + if (this.state && this.examples) { + this.modelControls.setModel(this.state.modelName); + this.addFromState(this.state); + this.state = undefined; + } + } + + override render() { + return html` + ${this.loading ? html`` : html` + + + `} + + ${this.loading ? html` +
+ +
+ ` : html` +
+
+ `} + `; + } + + onImageSelect(event: CustomEvent) { + this.addImagePrompts(event.detail.id); + } + + addFromState(state: State) { + const imagePrompts = new ImagePrompts(state.imageId); + imagePrompts.setPrompts(state.prompts); + this.examples.insertBefore(imagePrompts, this.examples.childNodes[0]); + } + + addImagePrompts(id: string): ImagePrompts { + const imagePrompts = new ImagePrompts(id); + imagePrompts.addEventListener('duplicate', (event: Event) => { + const duplicated = this.addImagePrompts(id); + duplicated.setPrompts(imagePrompts.getPrompts()); + }); + this.examples.insertBefore(imagePrompts, this.examples.childNodes[0]); + return imagePrompts; + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/loading-animation.scss b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/loading-animation.scss new file mode 100644 index 0000000000000000000000000000000000000000..30da6314d6d1a85888ad4c5ba89455272a13ae11 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/loading-animation.scss @@ -0,0 +1,65 @@ +// CC0 from https://loading.io/css/ + +@import '../style/colors'; + +.lds-ellipsis { + display: inline-block; + position: relative; + width: 80px; + height: 80px; + + div { + position: absolute; + top: 33px; + width: 13px; + height: 13px; + border-radius: 50%; + background: var(--text-fg); + animation-timing-function: cubic-bezier(0, 1, 1, 0); + } + + div:nth-child(1) { + left: 8px; + animation: lds-ellipsis1 0.6s infinite; + } + + div:nth-child(2) { + left: 8px; + animation: lds-ellipsis2 0.6s infinite; + } + + div:nth-child(3) { + left: 32px; + animation: lds-ellipsis2 0.6s infinite; + } + + div:nth-child(4) { + left: 56px; + animation: lds-ellipsis3 0.6s infinite; + } +} + +@keyframes lds-ellipsis1 { + 0% { + transform: scale(0); + } + 100% { + transform: scale(1); + } +} +@keyframes lds-ellipsis3 { + 0% { + transform: scale(1); + } + 100% { + transform: scale(0); + } +} +@keyframes lds-ellipsis2 { + 0% { + transform: translate(0, 0); + } + 100% { + transform: translate(24px, 0); + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/loading-animation.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/loading-animation.ts new file mode 100644 index 0000000000000000000000000000000000000000..9f605fc09e381b2e0f8f637b664beb72b01b2de2 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/loading-animation.ts @@ -0,0 +1,51 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Carousel of images. + */ + +import {html, LitElement} from 'lit'; + +import {customElement} from 'lit/decorators.js'; +import styles from './loading-animation.scss'; + +/** + * Shows an animated loading animation. + */ +@customElement('loading-animation') +export class LoadingAnimation extends LitElement { + + static override styles = [styles]; + + override render() { + return html` +
+
+
+
+
+
+ `; + } +} + +declare global { + interface HTMLElementTagNameMap { + 'loading-animation': LoadingAnimation; + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/message-list.scss b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/message-list.scss new file mode 100644 index 0000000000000000000000000000000000000000..bb686520409ac2d515a93d32cfe70498cb61f344 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/message-list.scss @@ -0,0 +1,26 @@ +@import '../style/colors'; + +.message { + padding: 0.1rem 0.5rem; + margin-bottom: 1rem; +} + +.warning { + background: var(--warn-bg); + color: var(--warn-fg); +} + +.error { + background: var(--error-bg); + color: var(--error-fg); +} + +.info { + background: var(--note-bg); + color: var(--note-fg); +} + +.close { + float: right; + cursor: pointer; +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/message-list.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/message-list.ts new file mode 100644 index 0000000000000000000000000000000000000000..9b4ff2dae692e14f415334b9418bed29b3d9506b --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/message-list.ts @@ -0,0 +1,97 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview A list of dismissable info/warning/error messages. + */ + +import {html, LitElement} from 'lit'; + +import {unsafeHTML} from 'lit/directives/unsafe-html.js'; + +import {customElement} from 'lit/decorators.js'; +import styles from './message-list.scss'; + +enum MessageType { + INFO = 'info', + WARNING = 'warning', + ERROR = 'error', +} + +interface Message { + message: string; + type: MessageType; + rawHtml: boolean; +} + + +/** + * Shows info/warning/error messages that remain until closed by user. + */ +@customElement('message-list') +export class MessageList extends LitElement { + static override styles = [styles]; + + messages: Message[] = []; + + addMessage(message: Message) { + this.messages.push(message); + this.requestUpdate(); + } + + info(message: string, {rawHtml = false}: {rawHtml?: boolean} = {}) { + this.addMessage({message, type: MessageType.INFO, rawHtml}); + } + + warning(message: string, {rawHtml = false}: {rawHtml?: boolean} = {}) { + this.addMessage({message, type: MessageType.WARNING, rawHtml}); + } + + error(message: string, {rawHtml = false}: {rawHtml?: boolean} = {}) { + this.addMessage({message, type: MessageType.ERROR, rawHtml}); + } + + removeMessage(event: Event, idx: number) { + this.messages.splice(idx, 1); + (event.target! as HTMLElement).closest('.message')!.remove(); + } + + clear() { + this.messages = []; + while (this.firstChild) this.firstChild.remove(); + } + + override render() { + return this.messages.map( + (message: Message, idx: number) => html` +
+ ${ + message.rawHtml ? unsafeHTML(message.message) : + message.message} + { + this.removeMessage(e, idx); + }} class="close">✖ +
+ `); + } +} + +declare global { + interface HTMLElementTagNameMap { + 'message-list': MessageList; + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/model-controls.scss b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/model-controls.scss new file mode 100644 index 0000000000000000000000000000000000000000..14e760000926b675ac8cab7f434c9525dd4c533e --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/model-controls.scss @@ -0,0 +1,12 @@ +.controls { + margin: 1em 0; + display: flex; + + select { + margin-left: 0.5em; + } + + progress { + margin: 0 1em; + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/components/model-controls.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/model-controls.ts new file mode 100644 index 0000000000000000000000000000000000000000..21c98fd33cb804a0ac6af5e10543e48ae35d719f --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/components/model-controls.ts @@ -0,0 +1,93 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Controls to choose model. + */ + +import {html, LitElement} from 'lit'; + +import {getModels} from '../lit_demo/constants'; +import {app} from '../lit_demo/app'; + +import {customElement, property} from 'lit/decorators.js'; +import styles from './model-controls.scss'; + +/** + * Shows controls for model selection, progress bar, and status text. + */ +@customElement('model-controls') +export class ModelControls extends LitElement { + + static override styles = [styles]; + + @property({attribute: false}) + progress: number = 0; + + @property({attribute: false}) + status: string = 'Initializing...'; + + constructor() { + super(); + app.models.addListener(this.onModelUpdate.bind(this)); + app.models.load(getModels()[0]); + } + + onModelUpdate(progress: number, message?: string) { + this.progress = progress; + if (message) this.status = message; + } + + onModelChange(event: Event) { + const target = event.target as HTMLSelectElement; + const name = target.value; + app.models.load(name).catch((error) => { + this.status = `ERROR loading model "${name}": ${error}`; + }); + } + + async setModel(model: string) { + if (getModels().indexOf(model) === -1) { + throw new Error(`Model "${model}" not found!`); + } + await this.updateComplete; + const dropdown = this.shadowRoot!.querySelector('#model_dropdown') as HTMLSelectElement; + dropdown.value = model; + dropdown.dispatchEvent(new Event('change')); + } + + override render() { + const options = getModels().map((model: string) => + html``); + return html` +
+ + + +
${this.status}
+
+ `; + } +} + +declare global { + interface HTMLElementTagNameMap { + 'model-controls': ModelControls; + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/exports.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/exports.ts new file mode 100644 index 0000000000000000000000000000000000000000..5512a56337844e9cc8e20883f1286592c7527578 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/exports.ts @@ -0,0 +1,38 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview some useful exports to play around with the models & + * tokenizers. + * + * Simple usage (see ./playground.html for more complete usage example): + * + * model = lit.Model('tiny'); + * model.load(progress => console.log('loading...', progress)); + * console.log(model.computeProbabilities(['a dog', 'a cat'], '0')); + */ + +import {Model} from './lit_demo/compute'; +import {getImageUrl, setBaseUrl} from './lit_demo/constants'; +import {ImageData} from './lit_demo/data'; +import * as tf from '@tensorflow/tfjs-core'; + +// tslint:disable-next-line:no-any Export symbols into global namespace. +(window as any).lit = { Model, getImageUrl, ImageData, setBaseUrl }; +// tslint:disable-next-line:no-any Export symbols into global namespace. +// tslint:disable-next-line:ban-module-namespace-object-escape Export all of TF. +(window as any).tf = tf; diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/index.html b/Tipsomaly/model/big_vision/tools/lit_demo/src/index.html new file mode 100644 index 0000000000000000000000000000000000000000..db5d628d66fbb33f9519056727148ca1e6f33d38 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/index.html @@ -0,0 +1,80 @@ + + + + + + + Lit Demo App + + + + + + + +

LiT: Zero-Shot Transfer with Locked-image Tuning

+ +

+ This page is an interactive demo of the Google AI blog post + LiT: adding language understanding to image models + – please refer to that page for a detailed explanation of how a LiT model works. + If you're interested in how this demo makes a JAX model run on device in your + browser, check out our other blog post + JAX on the Web with TensorFlow.js. +

+ +

+ Below you can choose an image from a selection and then write free-form + text prompts that are matched to the image. Once you hit return on your + keyboard or press the "compute" button, a text encoder implemented in + TensorFlow.js + will compute embeddings for the provided text on your local device, and the + similarity of these text embeddings to the image embedding will be displayed. +

+ +

+ The prompts can be used to classify an image into multiple categories, listing + each category individually with a prompt "an image of a X". But you can also + probe the model interactively with more detailed prompts, comparing the + different results when small details change in the text. +

+ +

+ Please use this demo responsibly. The models will always compare the image to + the prompts you provide, and it is therefore trivial to construct situations + where the model picks from a bunch of bad options. +

+ +

+ Note: + The models available in this interactive demo are not those from the + paper. + We had to train much smaller text towers and tokenizers to avoid + overloading your browser. Please see + our GitHub repository + for the models from the paper pre-trained on public datasets. + Multilingual models coming soon. +

+ + + + \ No newline at end of file diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/app.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/app.ts new file mode 100644 index 0000000000000000000000000000000000000000..51c15986eb43baf64e4a574e256a02133ea666a0 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/app.ts @@ -0,0 +1,47 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Global app state. + */ + +import {ImageData} from './data'; +import {Models} from './compute'; + +/** + * Container class holding image data and models. + * + * The main application component would typically call `load()` and then show + * the components depending on this class asynchronously. + */ +export class App { + + imageData = new ImageData(); + models = new Models(); + + ready: boolean = false; + + async load() { + await this.imageData.load(); + this.ready = true; + } +} + +/** + * Global app state. + */ +export const app = new App(); diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/compute.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/compute.ts new file mode 100644 index 0000000000000000000000000000000000000000..85b0edf1d17a89c578319355bfab013793585ca2 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/compute.ts @@ -0,0 +1,293 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Model code. + */ + +import '@tensorflow/tfjs-backend-webgl'; + +import * as tfconv from '@tensorflow/tfjs-converter'; +import * as tf from '@tensorflow/tfjs-core'; +import {MathBackendWebGL} from '@tensorflow/tfjs-backend-webgl'; + +import {getTokenizer, Tokenizer} from '../tokenizers/index'; + +import {getModelFileUrl} from './constants'; + +/** + * Callback to be updated with model load status. + * + * @param progress: the callback function is repeatedly called with values from + * 0 to 1 (both endpoints are guaranteed to be included) + * @param message: optional message to be displayed to user + */ +export type StatusCallback = (progress: number, message?: string) => void; + +const canonicalize = (s: string) => { + s = s.toLocaleLowerCase(); + s = s.replace(/[^\w ]/g, ''); + s = s.replace(/\s+/g, ' '); + return s.trim(); +}; + +/** + * The model definition is read from a JSON and specifies model details. + */ +// tslint:disable:enforce-name-casing +export interface ModelDefinition { + /** Human-readable description of the model. */ + description: string; + /** Tokenizer type. See ./tokenizers/index */ + tokenizer_type: string; + /** Temperature for computing softmax. */ + temperature: number; + /** Token used for padding. */ + pad_value: number; + /** Maximum token length. */ + max_len: number; + /** Dimensionality of image/text embeddings. */ + embedding_size: number; +} +// tslint:enable:enforce-name-casing + +/** + * TFJS model to compute text embeddings and similarities. + */ +export class Model { + def?: ModelDefinition; + tokenizer?: Tokenizer; + model?: tfconv.GraphModel; + /** Pre-computed image embeddings. */ + zimgs?: tf.Tensor; + /** Pre-computed text embeddings. */ + ztxts?: tf.Tensor; + /** IDs for pre-computed image embeddings. */ + zimgIds?: string[]; + /** Prompts for pre-computed text embeddings. */ + ztxtPrompts?: string[]; + /** Will be set to `true` when `load()` has completed successfully. */ + ready: boolean = false; + + /** + * @param name: Name of the model to be loaded. Will be used to construct the + * model URL. Note that the model must be loaded via calling `load()` before + * it can be used. + */ + constructor(public name: string) { + } + + /** + * Loads model, tokenizer, and pre-computed embeddings. + */ + async load(callback?: StatusCallback) { + this.def = + await fetch(getModelFileUrl(this.name, 'def.json')).then(resp => { + if (resp.ok) return resp.json(); + throw new Error(`Could not load model def: ${resp.status}`); + }); + console.log('def', this.def); + + const tokenizer = fetch(getModelFileUrl(this.name, 'vocabulary.json')) + .then(resp => resp.json()) + .then( + vocabulary => getTokenizer( + this.def!.tokenizer_type, vocabulary)); + + const model = + tfconv.loadGraphModel(getModelFileUrl(this.name, 'tfjs/model.json'), { + onProgress: (progress: number) => { + callback && callback(progress); + } + }); + + const fetchBin = async (name: string) => { + const response = await fetch(getModelFileUrl(this.name, `${name}.bin`)); + const blob = await response.blob(); + const data = await new Promise(resolve => { + const reader = new FileReader(); + reader.addEventListener('loadend', () => { + resolve(reader.result); + }); + reader.readAsArrayBuffer(blob); + }); + const arr = new Float32Array(data as Iterable); + const n = arr.length / this.def!.embedding_size; + return tf.tensor(arr, [n, this.def!.embedding_size]); + }; + const fetchTxt = (name: string) => + fetch(getModelFileUrl(this.name, `${name}.txt`)) + .then(response => response.text()) + .then(text => text.split(/\n/g)); + + [this.tokenizer, + this.model, + this.zimgs, + this.ztxts, + this.zimgIds, + this.ztxtPrompts, + ] = + [ + await tokenizer, + await model, + await fetchBin('zimgs'), + await fetchBin('ztxts'), + await fetchTxt('zimgs'), + await fetchTxt('ztxts'), + ]; + this.ready = true; + await this.warmup(); + if (callback) callback(1, 'Done.'); + } + + private async warmup() { + if (getBackend() !== 'webgl') return; + + const webGLBackend = tf.backend() as MathBackendWebGL; + tf.env().set('ENGINE_COMPILE_ONLY', true); + const tokens = tf.zeros([5, this.def!.max_len], 'int32'); + const preCompileResults = + this.model!.predict({inputs: tokens}) as tf.Tensor; + webGLBackend.checkCompileCompletion(); + webGLBackend.getUniformLocations(); + + tf.env().set('ENGINE_COMPILE_ONLY', false); + const warmUpResults = this.model!.predict({inputs: tokens}) as tf.Tensor; + await warmUpResults.data(); + + preCompileResults.dispose(); + warmUpResults.dispose(); + } + + /** + * Tokenizes strings with the model's tokenizer. + */ + tokenize(texts: string[]): tf.Tensor { + if (!this.ready) throw new Error('Cannot tokenize: not ready'); + const tokenize = (text: string) => { + const maxLen = this.def!.max_len || 16; + const tokens = this.tokenizer!.encode(text).slice(0, maxLen); + // eos="sticky" + const tokenEos = tf.tensor( + [ + ...tokens, + ...new Array(16 - tokens.length).fill(this.def!.pad_value), + ], + undefined, 'int32'); + return tokenEos; + }; + return tf.stack(texts.map(tokenize)); + } + + /** + * Computes embeddings for text tokenized via `tokenize()`. + */ + embed(tokens: tf.Tensor): tf.Tensor { + if (!this.ready) throw new Error('Cannot embed: not ready'); + return this.model!.execute({inputs: tokens}) as tf.Tensor; + } + + /** + * Computes similarities between specified prompts and images. Images are + * referenced by their ID. + */ + computeSimilarities(texts: string[], imgidxs: number[]) { + if (!this.ready) throw new Error('Cannot compute similarities: not ready'); + texts = texts.map(canonicalize); + const precomputed = + texts + .map(text => { + const idx = this.ztxtPrompts!.indexOf(text); + return idx === -1 ? null : tf.slice(this.ztxts!, idx, 1); + }) + .filter((x: tf.Tensor|null) => !!x) as tf.Tensor[]; + console.log(texts.length, 'texts, ', precomputed.length, 'precomputed'); + const textEmbeddings = texts.length === precomputed.length ? + tf.concat(precomputed) : + this.embed(this.tokenize(texts)); + const imageEmbeddingsTransposed = tf.transpose( + tf.concat(imgidxs.map(idx => tf.slice(this.zimgs!, idx, 1)))); + const sims = tf.matMul(textEmbeddings, imageEmbeddingsTransposed); + sims.print(); + return sims; + } + + /** + * Computes probabilities between a set of prompts and a single image + * (identified by its ID). + */ + computeProbabilities(texts: string[], imgidx: number): number[] { + const sims = this.computeSimilarities(texts, [imgidx]); + const row = tf.squeeze(tf.slice(tf.transpose(sims), 0, 1)); + return [...tf.softmax(tf.mul(this.def!.temperature, row)).dataSync()]; + } +} + +/** + * Container that holds a set of models. + */ +export class Models { + private readonly map = new Map(); + private readonly listeners = new Set(); + model?: Model; + + /** + * Adds a listener to be updated about individual models' loading progress. + */ + addListener(callback: StatusCallback) { + this.listeners.add(callback); + } + + /** + * Updates all listeners wth `progress` and `message`. + */ + onUpdate(progress: number, message?: string) { + if (progress === 1) { + message = `Loaded model "${this.model?.name}".`; + } + for (const callback of this.listeners) { + callback(progress, message); + } + } + + /** + * Loads model and sets `model` attribute when ready. + */ + async load(name: string) { + if (this.map.has(name)) { + this.model = this.map.get(name); + this.onUpdate(1, `Loaded "${name}".`); + return; + } + this.onUpdate(0, 'Loading...'); + this.model = new Model(name); + await this.model.load(this.onUpdate.bind(this)); + this.map.set(name, this.model); + } + + /** + * Whether model referenced by `model` attribute is ready. + */ + get ready(): boolean { + return !!this.model?.ready; + } +} + +/** Returns backend, such as "cpu" or "webgl". */ +export function getBackend(): string { + return tf.getBackend(); +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/constants.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/constants.ts new file mode 100644 index 0000000000000000000000000000000000000000..fa0ac59161a8a67e5e7a528911d8bbbd2f0820c4 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/constants.ts @@ -0,0 +1,50 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Project-wide constants. + */ + +// Can be overwritten with setBaseUrl() below. +// let baseUrl = 'https://google-research.github.io/vision_transformer/lit'; +let baseUrl = 'https://figur.li/jax2tfjs'; +// Can be overwritten with setModels() below. +let models = ['tiny', 'small']; + +/** Allows to set abnew base URL. ase URL on which all other. */ +export const setBaseUrl = (newBaseUrl: string) => { + baseUrl = newBaseUrl; +}; + +/** Retrieves URL for a model-specific file (vocabulary, embeddings, ...). */ +export const getModelFileUrl = (name: string, relativePath: string) => ( + `${baseUrl}/data/models/${name}/${relativePath}` +); + +/** Retrieves the URL for images information JSON file. */ +export const getImagesInfoUrl = () => `${baseUrl}/data/images/info.json`; + +/** Retrieves the URL for an image. */ +export const getImageUrl = (id: string) => `${baseUrl}/data/images/${id}.jpg`; + +/** Returns names of available models. */ +export const getModels = () => models; + +/** Sets names of available models. */ +export const setModels = (newModels: string[]) => { + models = newModels; +}; diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/data.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..3f041499504c7bf55ba0900c45045df1fdb72643 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/data.ts @@ -0,0 +1,76 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Accessing additional data. + */ + +import {getImagesInfoUrl} from './constants'; + +/** + * Information about a single image. + */ +export interface ImageRow { + /** Stable ID of the image. */ + id: string; + /** Set of example prompts for this image. */ + prompts: string; + /** License of the image. */ + license: string; + /** Where the image was originally downloaded from. */ + source: string; + /** Short description of image. */ + description: string; +} +/** + * Contains information about all images. + */ +export class ImageData { + + rows: ImageRow[] = []; + /** Will be set to `true` when `load()` finishes. */ + ready = false; + + /** + * Gets an image by ID. Throws an error if image is not found, data is not + * loaded, or ID is not unique. + */ + get(id: string): ImageRow { + if (!this.ready) { + throw new Error('ImageData not loaded!'); + } + const matching = this.rows.filter(row => row.id === id); + if (matching.length !== 1) { + throw new Error(`Got unexpected ${matching.length} matches for id="${id}"`); + } + return matching[0]; + } + + /** + * Loads image data asynchronously. + */ + async load() { + this.rows = ( + await fetch(getImagesInfoUrl()) + .then(response => { + console.log('response', response); + return response.json(); + }) + ); + this.ready = true; + } +} diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/url_utils.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/url_utils.ts new file mode 100644 index 0000000000000000000000000000000000000000..f914dbf5227d114c0b2efcdaaf606c675a20f8d8 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/lit_demo/url_utils.ts @@ -0,0 +1,92 @@ +/** + * @license + * Copyright Big Vision Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview (De)serialize state from/to URL. + */ + +// Should be updated whenever URLs are not compatible anymore +// (e.g. adding new images) +export const VERSION = 'v2'; +// version history: +// v1 used row number instead of image id + +const V1_IMAGE_IDS = [ + '1', '48', '43', '22', '2', '3', '4', '5', '6', '7', '8', '9', + '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', + '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', '34', + '35', '36', '37', '38', '39', '40', '41', '42', '44', '45', '46', '47', + '49', '50', '51', '52', '53', '54', '55', '56', '57', '58', '59', '60' +]; + +/** + * State that can be stored in the URL. + */ +export interface State { + /** Name of the model. */ + modelName: string; + /** ID Of the image. */ + imageId: string; + /** List of text prompts. */ + prompts: string[]; +} + +/** + * Returns a URL for provided model/image/prompts. + */ +export const getUrl = + (modelName: string, imageId: string, prompts: string[]): string => { + let href = window.location.href; + if (href.indexOf('#') !== -1) { + href = href.substring(0, href.indexOf('#')); + } + const parts = [ + VERSION, + modelName, + imageId, + ...prompts, + ]; + return href + '#' + parts.map(encodeURIComponent).join('|'); + }; + +/** + * Parses an URL and returns a `State`, or undefined if no state is spefified. + * + * Raises an exception if there was a problem with the parsing of the URL. + */ +export const parseUrl = (): State|undefined => { + const hash = window.location.hash.substring(1); + if (!hash) return; + const parts = hash.split(/\|/g); + if (parts.length < 4) { + throw new Error(`Invalid URL: "${hash}"`); + } + let [version, modelName, imageId, ...texts] = parts; + if (version === VERSION) { + } else if (version === 'v1') { + const idx = Number(imageId); + if (isNaN(idx)) throw new Error(`Expected idx="${idx}" to be numerical!`); + imageId = V1_IMAGE_IDS[idx]; + } else { + throw new Error(`Incompatible version: ${version} (supported: ${VERSION})`); + } + return { + modelName, + imageId, + prompts: texts.map(decodeURIComponent), + }; +}; diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/playground.html b/Tipsomaly/model/big_vision/tools/lit_demo/src/playground.html new file mode 100644 index 0000000000000000000000000000000000000000..347d62d28e742e904d52679c48770fa43b006623 --- /dev/null +++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/playground.html @@ -0,0 +1,92 @@ + + + + + + +

+ A simple demonstration how to use LiT models in a JS application using global exports. + See source code of this file for API usage. +

+ +

+    
+
+
+

+
+
+
+
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/style.scss b/Tipsomaly/model/big_vision/tools/lit_demo/src/style.scss
new file mode 100644
index 0000000000000000000000000000000000000000..17de8432776354e76fa7588633ed048f903a9055
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/style.scss
@@ -0,0 +1,80 @@
+// General styles for the page.
+
+@import './style/colors';
+@import './style/mixins';
+
+html {
+  font-size: 14px;
+  line-height: 1.6em;
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen,
+    Ubuntu, Cantarell, 'Fira Sans', 'Droid Sans', 'Helvetica Neue', Arial,
+    sans-serif;
+  text-size-adjust: 100%;
+  -ms-text-size-adjust: 100%;
+  -webkit-text-size-adjust: 100%;
+
+  @media (min-width: 1200px) {
+    width: 1024px;
+    margin: 0 auto;
+  }
+  @media (min-width: 768px) {
+    font-size: 16px;
+  }
+
+  color: var(--text-fg);
+  background: var(--text-bg);
+
+  body {
+    margin: 0;
+    padding: 0rem 1rem 10rem;
+  }
+}
+
+a,
+a:visited {
+  color: var(--link-col);
+}
+
+h1 {
+  font-weight: 700;
+  font-size: 2rem;
+  line-height: 1.3em;
+}
+
+p {
+  font-size: 1.06rem;
+  line-height: 1.3em;
+}
+
+input {
+  font-size: 1rem;
+
+  &::placeholder {
+    color: var(--placeholder-col);
+  }
+}
+
+.note {
+  font-style: normal;
+  border: none;
+  border-radius: 2px;
+  margin-left: auto;
+  margin-right: auto;
+
+  padding: 0.5rem 0.5rem 0.5rem 2rem;
+  width: 90%;
+
+  @include phone-portrait {
+    width: 100%;
+    padding: 0.5rem;
+    box-sizing: border-box;
+  }
+
+  background-color: var(--note-bg);
+  color: var(--note-fg);
+
+  &.warning {
+    background-color: var(--warn-bg);
+    color: var(--warn-fg);
+  }
+}
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/style/colors.scss b/Tipsomaly/model/big_vision/tools/lit_demo/src/style/colors.scss
new file mode 100644
index 0000000000000000000000000000000000000000..107023ceac594458b3e4be9562dded4325ce8abb
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/style/colors.scss
@@ -0,0 +1,35 @@
+// Dark and light mode colors.
+
+:root {
+  --text-bg: hsl(0, 0%, 97%);
+  --gray-border: hsla(0, 0%, 0%, 0.1);
+  --gray: rgba(0, 0, 0, 0.6);
+  --border-radius: 5px;
+  --orange: hsl(24, 100%, 50%);
+  --distill-blue: hsl(200, 50%, 25%);
+  --blue: #337699;
+  --green: #3db867;
+  --text-fg: rgb(15, 15, 15);
+  --text-red: rgb(220, 0, 0);
+  --bar-col: rgb(171, 199, 227);
+  --link-col: rgb(0, 0, 238);
+  --placeholder-col: rgb(166, 166, 166);
+  --note-bg: #e1f5fe;
+  --note-fg: #1a6ebb;
+  --warn-bg: #ffe1aa;
+  --warn-fg: #a16800;
+  --error-bg: #850000;
+  --error-fg: white;
+
+  @media (prefers-color-scheme: dark) {
+    --text-bg: rgb(56, 56, 56);
+    --text-fg: rgb(213, 213, 213);
+    --bar-col: rgb(20, 109, 163);
+    --link-col: rgb(66, 165, 245);
+
+    --note-fg: rgb(121 157 190);
+    --note-bg: rgb(2 59 85);
+    --warn-bg: #784e00;
+    --warn-fg: #edbe68;
+  }
+}
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/style/mixins.scss b/Tipsomaly/model/big_vision/tools/lit_demo/src/style/mixins.scss
new file mode 100644
index 0000000000000000000000000000000000000000..83886328def7e15ff3de56455810a5915738b087
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/style/mixins.scss
@@ -0,0 +1,8 @@
+// Useful mixins.
+
+// To wrap styles that should only trigger for phones in portrait mode.
+@mixin phone-portrait {
+  @media only screen and (max-device-width: 800px) and (orientation: portrait) {
+    @content;
+  }
+}
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/common.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/common.ts
new file mode 100644
index 0000000000000000000000000000000000000000..b3988465d86282ba10b3adb476854fe8009fd945
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/common.ts
@@ -0,0 +1,58 @@
+/**
+ * @license
+ * Copyright Big Vision Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @fileoverview Utility code shared between tokenizers.
+ */
+
+/**
+ * A vocabulary consists of a list of tokens, and optional numerical value.
+ * The numerical value is used by the unigram algorithnm to find the best
+ * tokenizaion, and is ignored by the BPE algorithm.
+ */
+export type Vocabulary = Array<[string, number]>;
+
+/**
+ * Converts a string to a sequence of tokens.
+ */
+export interface Tokenizer {
+  encode(input: string): number[];
+}
+
+/**
+ * Factory for new `Tokenizer`.
+ */
+export interface TokenizerConstructor {
+  new (vocabulary: Vocabulary): Tokenizer;
+}
+
+/**
+ * Unicode-aware character iteration of strings.
+ */
+export const stringToChars = (input: string): string[] => {
+  const symbols = [];
+  for (const symbol of input) {
+    symbols.push(symbol);
+  }
+  return symbols;
+};
+
+/**
+ * Special separator character used to delimit sub-word tokens.
+ */
+export const TOKEN_SEPARATOR =
+  '\u2581';  // This is the unicode character 'lower one eighth block'.
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/index.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/index.ts
new file mode 100644
index 0000000000000000000000000000000000000000..1263e678a4ad5672e7e7d2cfd629162f25fceab7
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/index.ts
@@ -0,0 +1,40 @@
+/**
+ * @license
+ * Copyright Big Vision Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @fileoverview Tokenizers and tokenizer mappings.
+ */
+
+import {Tokenizer, TokenizerConstructor, Vocabulary} from './common';
+import * as sentencepieceBpe from './sentencepiece_bpe';
+import * as sentencepieceUnigram from './sentencepiece_unigram';
+
+export {Tokenizer, Vocabulary} from './common';
+
+const TOKENIZERS = new Map([
+  ['BPE', sentencepieceBpe.Tokenizer],
+  ['UNIGRAM', sentencepieceUnigram.Tokenizer],
+]);
+
+/**
+ * Returns a tokenizer of type `name` using `vocabulary`.
+ */
+export const getTokenizer = (name: string, vocabulary: Vocabulary): Tokenizer => {
+  const ctor = TOKENIZERS.get(name);
+  if (!ctor) throw new Error(`Unknown tokenizer: ${name}`);
+  return new ctor(vocabulary);
+};
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_bpe.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_bpe.ts
new file mode 100644
index 0000000000000000000000000000000000000000..d3466f5f22194bd12210a085babbe8bd08ba8ac8
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_bpe.ts
@@ -0,0 +1,80 @@
+/**
+ * @license
+ * Copyright Big Vision Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {stringToChars, TOKEN_SEPARATOR, Vocabulary, Tokenizer as TokenizerInterface} from './common';
+
+interface Candidate {
+  piece: string;
+  pos: number;
+  score: number;
+}
+
+const scoreDesc = (a: Candidate, b: Candidate) => b.score - a.score;
+
+function processInput(str: string): string {
+  const normalized = str.normalize('NFKC');
+  return normalized.length > 0 ?
+    TOKEN_SEPARATOR + normalized.replace(/ /g, TOKEN_SEPARATOR) :
+    normalized;
+}
+
+/**
+ * Sentencepiece tokenizer implementing the BPE algorithm.
+ */
+export class Tokenizer implements TokenizerInterface {
+
+  // piece -> [score, index]
+  private readonly map: Map;
+
+  constructor(vocabulary: Vocabulary) {
+    this.map = new Map();
+    vocabulary.forEach(([piece, score], idx) => {
+      if (this.map.has(piece)) {
+        throw new Error(`Piece "${piece}" occurs multiple times in vocabulary`);
+      }
+      this.map.set(piece, [score, idx]);
+    });
+  }
+
+  encode(input: string): number[] {
+    const processed: string = processInput(input);
+    let pieces: string[] = stringToChars(processed);
+
+    while (true) {
+      const candidates: Candidate[] = [];
+      for (let i = 0; i < pieces.length - 1; i++) {
+        const fused = pieces[i] + pieces[i + 1];
+        const el = this.map.get(fused);
+        if (el) {
+          candidates.push({ piece: fused, pos: i, score: el[0] });
+        }
+      }
+      if (candidates.length === 0) {
+        break;
+      }
+      candidates.sort(scoreDesc);
+      const best = candidates[0];
+      pieces = [
+        ...pieces.slice(0, best.pos),
+        best.piece,
+        ...pieces.slice(best.pos + 2)
+      ];
+    }
+
+    return pieces.map(piece => this.map.get(piece)![1]);
+  }
+}
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_bpe_test.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_bpe_test.ts
new file mode 100644
index 0000000000000000000000000000000000000000..5009ff758c56d8f5ee79479e5d5385932ad070e3
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_bpe_test.ts
@@ -0,0 +1,48 @@
+/**
+ * @license
+ * Copyright Big Vision Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import 'jasmine';
+
+describe('sentencepiece bpe test', () => {
+  it('computes a thing when asked', () => {});
+});
+
+import * as bpe from './sentencepiece_bpe';
+import {TOKEN_SEPARATOR, Vocabulary} from './common';
+
+const vocab: Vocabulary = [
+  [TOKEN_SEPARATOR, 0],  // 0
+  ['a', 0],              // 1
+  ['e', 0],              // 2
+  ['s', 0],              // 3
+  ['t', 0],              // 4
+  ['te', -1],            // 5
+  ['st', -2],            // 6
+  ['test', -3],          // 7
+  ['tes', -4],           // 8
+];
+
+describe('BPE Tokenizer', () => {
+  let tokenizer: bpe.Tokenizer;
+  beforeAll(() => {
+    tokenizer = new bpe.Tokenizer(vocab);
+  });
+
+  it('should tokenize correctly', () => {
+    expect(tokenizer.encode('a test')).toEqual([0, 1, 0, 7]);
+  });
+});
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_unigram.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_unigram.ts
new file mode 100644
index 0000000000000000000000000000000000000000..1ec1315f458ed85d7789db8f6355fb94301aa4d5
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_unigram.ts
@@ -0,0 +1,134 @@
+/**
+ * @license
+ * Copyright Big Vision Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Copied & adapted from
+// https://github.com/tensorflow/tfjs-models/blob/master/universal-sentence-encoder/src/tokenizer/index.ts
+
+import {TOKEN_SEPARATOR, stringToChars, Tokenizer as TokenizerInterface, Vocabulary} from './common';
+import {Trie} from './trie';
+
+function processInput(str: string): string {
+  const normalized = str.normalize('NFKC');
+  return normalized.length > 0 ?
+      TOKEN_SEPARATOR + normalized.replace(/ /g, TOKEN_SEPARATOR) :
+      normalized;
+}
+
+// The first tokens are reserved for unk, control symbols, and user-defined
+// symbols.
+const RESERVED_SYMBOLS_COUNT = 6;
+
+interface Score {
+  key: string[];
+  score: number;
+  index: number;
+}
+
+/**
+ * Sentencepiece tokenizer implementing the UNIGRAM algorithm.
+ *
+ * `Tokenizer.encode()` is a port of `EncodeAsIds` from the SentencePiece
+ * library (https://github.com/google/sentencepiece). Encode uses the Viterbi
+ * algorithm to find the most likely sequence of tokens that comprise the input.
+ * For more details, refer to https://arxiv.org/pdf/1804.10959.pdf.
+ */
+export class Tokenizer implements TokenizerInterface {
+  trie: Trie;
+
+  constructor(
+      private readonly vocabulary: Vocabulary,
+      private readonly reservedSymbolsCount = RESERVED_SYMBOLS_COUNT) {
+    this.trie = new Trie();
+
+    for (let i = this.reservedSymbolsCount; i < this.vocabulary.length; i++) {
+      this.trie.insert(this.vocabulary[i][0], this.vocabulary[i][1], i);
+    }
+  }
+
+  encode(input: string): number[] {
+    const nodes: Array<{[index: number]: Score[]}> = [];
+    const words: number[] = [];
+    const best: number[] = [];
+
+    input = processInput(input);
+
+    const symbols = stringToChars(input);
+
+    for (let i = 0; i <= symbols.length; i++) {
+      nodes.push({});
+      words.push(0);
+      best.push(0);
+    }
+
+    // Construct the lattice.
+    for (let i = 0; i < symbols.length; i++) {
+      const matches = this.trie.commonPrefixSearch(symbols.slice(i));
+
+      for (let j = 0; j < matches.length; j++) {
+        const piece = matches[j];
+        const obj = {key: piece[0], score: piece[1], index: piece[2]};
+
+        const endPos = piece[0].length;
+        if (nodes[i + endPos][i] == null) {
+          nodes[i + endPos][i] = [];
+        }
+
+        nodes[i + endPos][i].push(obj);
+      }
+    }
+
+    for (let endPos = 0; endPos <= symbols.length; endPos++) {
+      for (const startPos in nodes[endPos]) {
+        if (!nodes[endPos].hasOwnProperty(startPos)) continue;
+        const arr = nodes[endPos][startPos];
+
+        for (let j = 0; j < arr.length; j++) {
+          const word = arr[j];
+          const score = word.score + best[endPos - word.key.length];
+
+          if (best[endPos] === 0 || score >= best[endPos]) {
+            best[endPos] = score;
+            words[endPos] = arr[j].index;
+          }
+        }
+      }
+    }
+
+    const results: number[] = [];
+
+    // Backward pass.
+    let iter = words.length - 1;
+    while (iter > 0) {
+      results.push(words[iter]);
+      iter -= this.vocabulary[words[iter]][0].length;
+    }
+
+    // Merge consecutive unks.
+    const merged = [];
+    let isPreviousUnk = false;
+    for (let i = 0; i < results.length; i++) {
+      const id = results[i];
+      if (!(isPreviousUnk && id === 0)) {
+        merged.push(id);
+      }
+
+      isPreviousUnk = id === 0;
+    }
+
+    return merged.reverse();
+  }
+}
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_unigram_test.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_unigram_test.ts
new file mode 100644
index 0000000000000000000000000000000000000000..66a29072e7080a3c83695371e69ac0947de87f6a
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/sentencepiece_unigram_test.ts
@@ -0,0 +1,71 @@
+/**
+ * @license
+ * Copyright Big Vision Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Tokenizer} from './sentencepiece_unigram';
+
+const stubbedTokenizerVocab = [
+  ['�', 0],
+  ['', 0],
+  ['', 0],
+  ['extra_token_id_1', 0],
+  ['extra_token_id_2', 0],
+  ['extra_token_id_3', 0],
+  ['▁', -2],
+  ['▁a', -1],
+  ['▁ç', -2],
+  ['a', -3],
+  ['.', -1],
+  ['▁I', -1],
+  ['▁like', -1],
+  ['▁it', -1],
+  ['I', -2],
+  ['like', -2],
+  ['it', -2],
+  ['l', -3],
+  ['i', -3],
+  ['k', -3],
+  ['e', -3],
+  ['i', -3],
+  ['t', -3]
+];
+
+describe('Universal Sentence Encoder tokenizer', () => {
+  let tokenizer: Tokenizer;
+  beforeAll(() => {
+    tokenizer = new Tokenizer(stubbedTokenizerVocab as Array<[string, number]>);
+  });
+
+  it('basic usage', () => {
+    expect(tokenizer.encode('Ilikeit.')).toEqual([11, 15, 16, 10]);
+  });
+
+  it('handles whitespace', () => {
+    expect(tokenizer.encode('I like it.')).toEqual([11, 12, 13, 10]);
+  });
+
+  it('should normalize inputs', () => {
+    expect(tokenizer.encode('ça')).toEqual(tokenizer.encode('c\u0327a'));
+  });
+
+  it('should handle unknown inputs', () => {
+    expect(() => tokenizer.encode('😹')).not.toThrow();
+  });
+
+  it('should treat consecutive unknown inputs as a single word', () => {
+    expect(tokenizer.encode('a😹😹')).toEqual([7, 0]);
+  });
+});
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/trie.ts b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/trie.ts
new file mode 100644
index 0000000000000000000000000000000000000000..4f7ca0709ead8aad4dd6d05efbda108fa587cd5c
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/tokenizers/trie.ts
@@ -0,0 +1,96 @@
+/**
+ * @license
+ * Copyright Big Vision Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Copied from
+// https://github.com/tensorflow/tfjs-models/blob/master/universal-sentence-encoder/src/tokenizer/trie.ts
+
+import {stringToChars} from './common';
+
+// [token, score, index]
+type OutputNode = [string[], number, number];
+
+class TrieNode {
+  parent: TrieNode|null;
+  end: boolean;
+  children: {[firstSymbol: string]: TrieNode};
+  word: OutputNode;
+
+  constructor() {
+    this.parent = null;
+    this.children = {};
+    this.end = false;
+    this.word = [[], 0, 0];
+  }
+}
+
+/**
+ * Simple Trie datastructure.
+ */
+export class Trie {
+  root: TrieNode;
+
+  constructor() {
+    this.root = new TrieNode();
+  }
+
+  /**
+   * Inserts a token into the trie.
+   */
+  insert(word: string, score: number, index: number) {
+    let node = this.root;
+
+    const symbols = stringToChars(word);
+
+    for (let i = 0; i < symbols.length; i++) {
+      if (!node.children[symbols[i]]) {
+        node.children[symbols[i]] = new TrieNode();
+        node.children[symbols[i]].parent = node;
+        node.children[symbols[i]].word[0] = node.word[0].concat(symbols[i]);
+      }
+
+      node = node.children[symbols[i]];
+      if (i === symbols.length - 1) {
+        node.end = true;
+        node.word[1] = score;
+        node.word[2] = index;
+      }
+    }
+  }
+
+  /**
+   * Returns an array of all tokens starting with ss.
+   *
+   * @param ss The prefix to match on.
+   */
+  commonPrefixSearch(ss: string[]): OutputNode[] {
+    const output: OutputNode[] = [];
+    let node = this.root.children[ss[0]];
+
+    for (let i = 0; i < ss.length && node; i++) {
+      if (node.end) {
+        output.push(node.word);
+      }
+      node = node.children[ss[i + 1]];
+    }
+
+    if (!output.length) {
+      output.push([[ss[0]], 0, 0]);
+    }
+
+    return output;
+  }
+}
diff --git a/Tipsomaly/model/big_vision/tools/lit_demo/src/tsconfig.json b/Tipsomaly/model/big_vision/tools/lit_demo/src/tsconfig.json
new file mode 100644
index 0000000000000000000000000000000000000000..03c56e010a861561584208b99821017e2f338944
--- /dev/null
+++ b/Tipsomaly/model/big_vision/tools/lit_demo/src/tsconfig.json
@@ -0,0 +1,41 @@
+{
+  "compilerOptions": {
+    "outDir": "dist",
+    "target": "es6",
+    "module": "commonjs",
+    "lib": ["dom", "DOM.Iterable", "es2019", "es2020.string"],
+    "types": ["node", "jasmine", "resize-observer-browser"],
+    "moduleResolution": "node",
+    "allowJs": false,
+    "pretty": true,
+    "resolveJsonModule": true,
+    "sourceMap": false,
+    "skipLibCheck": true,
+    "removeComments": true,
+    "esModuleInterop": true,
+    "importsNotUsedAsValues": "preserve",
+    "downlevelIteration": true,
+    "skipDefaultLibCheck": true,
+    "preserveConstEnums": false,
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
+    "noErrorTruncation": false,
+    "noEmitOnError": false,
+    "declaration": false,
+    "stripInternal": true,
+    "inlineSourceMap": true,
+    "inlineSources": true,
+    "importHelpers": true,
+    "allowUnreachableCode": false,
+    "noFallthroughCasesInSwitch": true,
+    "noImplicitAny": true,
+    "noImplicitReturns": false,
+    "noImplicitThis": true,
+    "strictBindCallApply": true,
+    "strictFunctionTypes": true,
+    "strictNullChecks": false,
+    "strictPropertyInitialization": false
+  },
+  "include": ["./client", "./examples"],
+  "compileOnSave": false
+}
diff --git a/Tipsomaly/model/big_vision/trainers/proj/gsam/gsam.py b/Tipsomaly/model/big_vision/trainers/proj/gsam/gsam.py
new file mode 100644
index 0000000000000000000000000000000000000000..00eb68aa52b29f24d057ef79c520c9b5e719baf7
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/gsam/gsam.py
@@ -0,0 +1,122 @@
+# Copyright 2022 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''This file provides jax implementation of GSAM.'''
+
+import jax
+import jax.numpy as jnp
+
+def dual_vector(y):
+  """Returns the solution of max_x y^T x s.t. ||x||_2 <= 1.
+  Args:
+    y: A pytree of numpy ndarray, vector y in the equation above.
+  """
+  gradient_norm = jnp.sqrt(sum(
+        jnp.sum(jnp.square(e)) for e in jax.tree_util.tree_leaves(y)))
+  normalized_gradient = jax.tree_map(lambda x: x / gradient_norm, y)
+  return normalized_gradient, gradient_norm
+
+def gsam_gradient(loss_fn, params, inputs, targets,
+                  rho_max, rho_min, alpha, lr, lr_max, lr_min, eps=1e-12,
+                  adaptive_perturbation=False, minimize_fp=True):
+  """
+  Get the GSAM gradient (https://openreview.net/pdf?id=edONMAnhLu-).
+  Args:
+    loss_fn: the loss function.
+    params: the model weights.
+    inputs: the inputs to the loss function.
+    targets: the targets to the loss function.
+    rho_max: the maximum rho value for perturbation of weights.
+    rho_min: the minimum rho value for perturbation of weights.
+    alpha: the alpha value for the rho schedule, see Algorithm 1 in the paper.
+    lr: current learning rate.
+    lr_max: the maximum learning rate.
+    lr_min: the minimum learning rate.
+    eps: the epsilon value for numerical stability.
+    adaptive_perturbation: if False, same perturbation as SAM,
+        treat all parameters as a single vector,
+        perturbation norm is calculated as the norm of the whole vector;
+        If True, perturbation norm is proportional to parameter norm,
+        this stabilizes training when different layers have weights
+        of different scales.
+        Emprically, setting it to True can handle 10x larger rho than
+        setting it to False.
+    minimize_fp: if True, min(f_p, h), original GSAM;
+        if False, min(f, h), where f is the clean loss.
+        f_p is the perturbed loss, h is the surrogate gap.
+        If True, training dynamics is closer to SAM than conventional training,
+        you might observe several loss spikes during training.
+        If False, the training dynamics is closer to conventional training,
+        and is often more stable (fewer loss spikes) during training.
+  Returns:
+    l_clean: the loss function value.
+    g_gsam: the GSAM gradient. g_gsam is not averaged across workers,
+        need to call "jax.lax.pmean" to average.
+
+  Note:
+    Setting `rho_max=rho_min` and `alpha=0` reduces GSAM to SAM.
+  """
+  l_clean, g_clean = jax.value_and_grad(loss_fn)(params, inputs, targets)
+  g_clean_normalized, g_clean_length = dual_vector(g_clean)
+
+  if lr_max == lr_min:
+    sam_rho = rho_max
+  else:
+    sam_rho = rho_min + (rho_max - rho_min) * (lr - lr_min) / (lr_max - lr_min)
+
+  # Per-worker perturbation.
+  if adaptive_perturbation:
+    param_sam = jax.tree_map(lambda a, b: a + \
+        jnp.abs(a) * sam_rho * b / (g_clean_length + eps), params, g_clean)
+  else:
+    param_sam = jax.tree_map(lambda a, b: a + \
+        sam_rho * b / (g_clean_length + eps), params, g_clean)
+
+  # Get gradients at perturbed weights.
+  _, g_robust = jax.value_and_grad(loss_fn)(param_sam, inputs, targets)
+
+  # Decompose gradients.
+  g_clean_flatten, _ = jax.tree_util.tree_flatten(g_clean)
+  g_robust_flatten, _ = jax.tree_util.tree_flatten(g_robust)
+
+  if minimize_fp:
+    # Decompose g_clean onto parallel and vertical to g_robust.
+    g_robust_normalized, _ = dual_vector(g_robust)
+    g_robust_normalized_flatten, _ = jax.tree_util.tree_flatten(
+        g_robust_normalized)
+
+    g_clean_projection_norm = sum(jnp.vdot(p, q) for (p,q) in
+        zip(g_robust_normalized_flatten, g_clean_flatten))
+    g_clean_residual = jax.tree_map(lambda a, b:
+        a - g_clean_projection_norm * b, g_clean, g_robust_normalized)
+
+    # Get GSAM gradient.
+    g_gsam = jax.tree_map(lambda a, b: a - b * alpha,
+        g_robust, g_clean_residual)
+  else:
+    # Decompose g_robust onto parallel and vertical to g_clean.
+    g_clean_normalized, g_clean_length = dual_vector(g_clean)
+    g_clean_normalized_flatten, _ = jax.tree_util.tree_flatten(
+        g_clean_normalized)
+
+    g_robust_projection_norm = sum(jnp.vdot(p, q) for (p,q) in
+        zip(g_clean_normalized_flatten, g_robust_flatten))
+    g_robust_residual = jax.tree_map(lambda a, b:
+        a - g_robust_projection_norm * b, g_robust, g_clean_normalized)
+
+    # Get GSAM gradient.
+    g_gsam = jax.tree_map(lambda a, b: a + b * alpha,
+        g_clean, g_robust_residual)
+
+  # Always return the clean loss (rather than the perturbed loss).
+  return l_clean, g_gsam
diff --git a/Tipsomaly/model/big_vision/trainers/proj/gsam/train.py b/Tipsomaly/model/big_vision/trainers/proj/gsam/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..66099ea114ba95c5c545ac42d58c8b1c4870f695
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/gsam/train.py
@@ -0,0 +1,370 @@
+# Copyright 2022 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Training loop example.
+Trainer that implements SAM/GSAM optimizers.
+"""
+# pylint: disable=consider-using-from-import
+from functools import partial
+import importlib
+import multiprocessing.pool
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+import big_vision.evaluators.common as eval_common
+import big_vision.input_pipeline as input_pipeline
+import big_vision.optax as bv_optax
+import big_vision.pp.builder as pp_builder
+from big_vision.trainers.proj.gsam.gsam import gsam_gradient
+import big_vision.utils as u
+from clu import parameter_overview
+import flax
+import jax
+import jax.numpy as jnp
+from ml_collections import config_flags
+import numpy as np
+import optax
+import tensorflow as tf
+import tensorflow.io.gfile as gfile
+
+# pylint: disable=logging-fstring-interpolation
+
+
+config_flags.DEFINE_config_file(
+    "config", None, "Training configuration.", lock_config=True)
+
+flags.DEFINE_string("workdir", default=None, help="Work unit directory.")
+flags.DEFINE_boolean("cleanup", default=False,
+                     help="Delete workdir (only) after successful completion.")
+
+# Adds jax flags to the program.
+jax.config.parse_flags_with_absl()
+
+
+def main(argv):
+  del argv
+  tf.config.experimental.set_visible_devices([], "GPU")
+
+  config = flags.FLAGS.config
+  workdir = flags.FLAGS.workdir
+  logging.info(
+      f"\u001b[33mHello from process {jax.process_index()} holding "
+      f"{jax.local_device_count()}/{jax.device_count()} devices and "
+      f"writing to workdir {workdir}.\u001b[0m")
+
+  assert not config.get("grad_accum_steps"), "Grad-acc not supported anymore."
+
+  save_checkpoint_path = None
+  if workdir and config.get("checkpoint_steps"):
+    gfile.makedirs(workdir)
+    save_checkpoint_path = os.path.join(workdir, "checkpoint.npz")
+
+  # The pool is used to perform misc operations such as logging in async way.
+  pool = multiprocessing.pool.ThreadPool()
+
+  # Here we register preprocessing ops from modules listed on `pp_modules`.
+  for m in config.get("pp_modules", ["ops_general", "ops_image"]):
+    importlib.import_module(f"big_vision.pp.{m}")
+
+  # This seed makes the Jax part of things (like model init) deterministic.
+  # However, full training still won't be deterministic, for example due to the
+  # tf.data pipeline not being deterministic even if we would set TF seed.
+  # See (internal link) for a fun read on what it takes.
+  rng = jax.random.PRNGKey(config.get("seed", 0))
+
+  # These functions do more stuff internally, for OSS release we mock them by
+  # trivial alternatives in order to minize disruptions in the code.
+  xid, wid = -1, -1
+  fillin = lambda s: s
+  def info(s, *a):
+    logging.info("\u001b[33mNOTE\u001b[0m: " + s, *a)
+  def write_note(note):
+    if jax.process_index() == 0:
+      info("%s", note)
+
+  # Verify settings to make sure no checkpoints are accidentally missed.
+  if config.get("keep_checkpoint_steps"):
+    assert config.get("checkpoint_steps"), "Specify `checkpoint_steps`."
+    assert config.keep_checkpoint_steps % config.checkpoint_steps == 0, (
+        f"`keep_checkpoint_steps` ({config.checkpoint_steps}) should be"
+        f"divisible by `checkpoint_steps ({config.checkpoint_steps}).`")
+
+  batch_size = config.batch_size
+  if batch_size % jax.device_count() != 0:
+    raise ValueError(f"Batch size ({batch_size}) must "
+                     f"be divisible by device number ({jax.device_count()})")
+  info("Global batch size %d on %d hosts results in %d local batch size. With "
+       "%d dev per host (%d dev total), that's a %d per-device batch size.",
+       batch_size, jax.process_count(), batch_size // jax.process_count(),
+       jax.local_device_count(), jax.device_count(),
+       batch_size // jax.device_count())
+
+  # First thing after above sanity checks, so we can log "start" ticks.
+  mw = u.BigVisionMetricWriter(xid, wid, workdir)
+  chrono = u.Chrono()
+
+  write_note("Initializing train dataset...")
+  train_ds = input_pipeline.make_for_train(
+      dataset=config.dataset,
+      split=config.train_split,
+      batch_size=config.batch_size,
+      preprocess_fn=pp_builder.get_preprocess_fn(config.pp_train),
+      shuffle_buffer_size=config.get("shuffle_buffer_size"),
+      cache_raw=config.get("cache_raw", False),
+      data_dir=fillin(config.get("dataset_dir")))
+
+  n_prefetch = config.get("prefetch_to_device", 1)
+  train_iter = input_pipeline.start_input_pipeline(train_ds, n_prefetch)
+
+  ntrain_img = input_pipeline.get_num_examples(
+      config.dataset, config.train_split,
+      data_dir=fillin(config.get("dataset_dir")))
+  steps_per_epoch = ntrain_img / batch_size
+
+  if config.get("num_epochs"):
+    total_steps = int(config.num_epochs * steps_per_epoch)
+    assert not config.get("total_steps"), "Set either num_epochs or total_steps"
+  else:
+    total_steps = config.total_steps
+
+  info("Running for %d steps, that means %f epochs and %f steps per epoch",
+       total_steps, total_steps * batch_size / ntrain_img, steps_per_epoch)
+
+  write_note(f"Initializing {config.model_name} model...")
+  model_mod = importlib.import_module(f"big_vision.models.{config.model_name}")
+  model = model_mod.Model(
+      num_classes=config.num_classes, **config.get("model", {}))
+
+  # We want all parameters to be created in host RAM, not on any device, they'll
+  # be sent there later as needed, otherwise we already encountered two
+  # situations where we allocate them twice.
+  @partial(jax.jit, backend="cpu")
+  def init(rng):
+    shape = tuple(train_ds.element_spec["image"].shape[1:])
+    bs = config.batch_size // jax.device_count()
+    dummy_input = jnp.zeros((bs,) + shape, jnp.float32)
+    params = flax.core.unfreeze(model.init(rng, dummy_input))["params"]
+
+    # Set bias in the head to a low value, such that loss is small initially.
+    if "init_head_bias" in config:
+      params["head"]["bias"] = jnp.full_like(params["head"]["bias"],
+                                             config["init_head_bias"])
+
+    return params
+
+  rng, rng_init = jax.random.split(rng)
+  params_cpu = init(rng_init)
+
+  if jax.process_index() == 0:
+    num_params = sum(p.size for p in jax.tree_util.tree_leaves(params_cpu))
+    parameter_overview.log_parameter_overview(params_cpu, msg="init params")
+    mw.measure("num_params", num_params)
+
+  write_note(f"Initializing {config.optax_name} optimizer...")
+  tx, sched_fns = bv_optax.make(config, params_cpu, sched_kw=dict(
+      global_batch_size=batch_size,
+      total_steps=total_steps,
+      steps_per_epoch=steps_per_epoch))
+
+  assert len(sched_fns) == 1, "Current GSAM supports one global learning-rate."
+
+  # We jit this, such that the arrays are created on the CPU, not device[0].
+  opt_cpu = jax.jit(tx.init, backend="cpu")(params_cpu)
+  sched_fns_cpu = [jax.jit(sched_fn, backend="cpu") for sched_fn in sched_fns]
+
+  @partial(jax.pmap, axis_name="batch", donate_argnums=(0, 1))
+  def update_fn(params, opt, rng, images, labels, step):
+    """Update step."""
+
+    measurements = {}
+
+    if config.get("mixup") and config.mixup.p:
+      rng, (images, labels), _ = u.mixup(rng, images, labels, **config.mixup)
+
+    # Get device-specific loss rng.
+    rng, rng_model = jax.random.split(rng, 2)
+    rng_model_local = jax.random.fold_in(rng_model, jax.lax.axis_index("batch"))
+
+    def loss_fn(params, images, labels):
+      logits, _ = model.apply(
+          {"params": flax.core.freeze(params)}, images,
+          train=True, rngs={"dropout": rng_model_local})
+      return getattr(u, config.get("loss", "sigmoid_xent"))(
+          logits=logits, labels=labels)
+
+    learning_rate = sched_fns[0](step) * config.lr
+    l, grads = gsam_gradient(loss_fn=loss_fn, params=params, inputs=images,
+        targets=labels, lr=learning_rate, **config.gsam)
+    l, grads = jax.lax.pmean((l, grads), axis_name="batch")
+    updates, opt = tx.update(grads, opt, params)
+    params = optax.apply_updates(params, updates)
+
+    gs = jax.tree_leaves(bv_optax.replace_frozen(config.schedule, grads, 0.))
+    measurements["l2_grads"] = jnp.sqrt(sum(jnp.vdot(g, g) for g in gs))
+    ps = jax.tree_util.tree_leaves(params)
+    measurements["l2_params"] = jnp.sqrt(sum(jnp.vdot(p, p) for p in ps))
+    us = jax.tree_util.tree_leaves(updates)
+    measurements["l2_updates"] = jnp.sqrt(sum(jnp.vdot(u, u) for u in us))
+
+    return params, opt, rng, l, measurements
+
+  # We do not jit/pmap this function, because it is passed to evaluator that
+  # does it later. We output as many intermediate tensors as possible for
+  # maximal flexibility. Later `jit` will prune out things that are not needed.
+  def predict_fn(params, image):
+    logits, out = model.apply({"params": params}, image)
+    return logits, out
+
+  # Decide how to initialize training. The order is important.
+  # 1. Always resumes from the existing checkpoint, e.g. resumes a finetune job.
+  # 2. Resume from a previous checkpoint, e.g. start a cooldown training job.
+  # 3. Initialize model from something, e,g, start a fine-tuning job.
+  # 4. Train from scratch.
+  resume_checkpoint_path = None
+  if save_checkpoint_path and gfile.exists(save_checkpoint_path):
+    resume_checkpoint_path = save_checkpoint_path
+  elif config.get("resume"):
+    resume_checkpoint_path = fillin(config.resume)
+  if resume_checkpoint_path:
+    write_note("Resume training from checkpoint...")
+    checkpoint = {
+        "params": params_cpu,
+        "opt": opt_cpu,
+        "chrono": chrono.save(),
+    }
+    checkpoint_tree = jax.tree_structure(checkpoint)
+    loaded = u.load_checkpoint(checkpoint_tree, resume_checkpoint_path)
+    # bfloat16 type gets lost when data is saved to disk, so we recover it.
+    checkpoint = jax.tree_map(u.recover_dtype, loaded)
+    params_cpu, opt_cpu = checkpoint["params"], checkpoint["opt"]
+    chrono.load(checkpoint["chrono"])
+  elif config.get("model_init"):
+    write_note(f"Initialize model from {config.model_init}...")
+    params_cpu = model_mod.load(
+        params_cpu, config.model_init, config.get("model"),
+        **config.get("model_load", {}))
+    if jax.process_index() == 0:
+      parameter_overview.log_parameter_overview(
+          params_cpu, msg="restored params")
+
+  write_note("Kicking off misc stuff...")
+  first_step = bv_optax.get_count(opt_cpu)
+  chrono.inform(first_step, total_steps, batch_size, steps_per_epoch)
+  prof = None  # Keeps track of start/stop of profiler state.
+
+  write_note(f"Replicating...\n{chrono.note}")
+  params_repl = flax.jax_utils.replicate(params_cpu)
+  opt_repl = flax.jax_utils.replicate(opt_cpu)
+
+  evaluators = eval_common.from_config(
+      config, {"predict": predict_fn},
+      lambda s: write_note(f"Initializing evaluator: {s}...\n{chrono.note}"))
+
+  rng, rng_loop = jax.random.split(rng, 2)
+  rngs_loop = flax.jax_utils.replicate(rng_loop)
+  checkpoint_writer = None
+
+  write_note(f"First step compilations...\n{chrono.note}")
+  error = None  # For exiting with an error after cleanup. Avoids indentation.
+  # Using a python integer for step here, because opt.state.step is allocated
+  # on TPU during replication.
+  for step, train_batch in zip(
+      range(first_step + 1, total_steps + 1), train_iter):
+    mw.step_start(step)
+
+    with jax.profiler.StepTraceAnnotation("train_step", step_num=step):
+      params_repl, opt_repl, rngs_loop, loss_value, measurements = update_fn(
+          params_repl, opt_repl, rngs_loop,
+          train_batch["image"],
+          train_batch["labels"],
+          flax.jax_utils.replicate(step))
+
+    # On the first host, let's always profile a handful of early steps.
+    if jax.process_index() == 0:
+      prof = u.startstop_prof(prof, step, first_step, config.log_training_steps)
+
+    # Report training progress
+    if (u.itstime(step, config.log_training_steps, total_steps, host=0)
+        or chrono.warmup and jax.process_index() == 0):
+      for i, sched_fn_cpu in enumerate(sched_fns_cpu):
+        mw.measure(f"global_schedule{i if i else ''}", sched_fn_cpu(step - 1))
+      l = mw.measure("training_loss", loss_value[0])
+      for name, value in measurements.items():
+        mw.measure(name, value[0])
+      chrono.tick(step, mw.measure, write_note)
+      if not np.isfinite(l):
+        error = (f"The loss became nan or inf somewhere within steps "
+                 f"[{step - config.log_training_steps}, {step}]")
+        break
+
+    # Checkpoint saving
+    if (save_checkpoint_path and
+        u.itstime(step, config.get("checkpoint_steps"), total_steps, host=0)):
+      chrono.pause(wait_for=(params_repl, opt_repl))
+      u.checkpointing_timeout(checkpoint_writer,
+                              config.get("checkpoint_timeout", 1))
+      # We need to transfer the weights over now or else we risk keeping them
+      # alive while they'll be updated in a future step, creating hard to debug
+      # memory errors (see (internal link)). Also, takes device 0's params only.
+      params_cpu = jax.tree_map(lambda x: np.array(x[0]), params_repl)
+      opt_cpu = jax.tree_map(lambda x: np.array(x[0]), opt_repl)
+
+      # Check whether we want to keep a copy of the current checkpoint.
+      copy_step = None
+      if u.itstime(step, config.get("keep_checkpoint_steps"), total_steps):
+        copy_step = step
+
+      ckpt = {"params": params_cpu, "opt": opt_cpu, "chrono": chrono.save()}
+      checkpoint_writer = pool.apply_async(
+          u.save_checkpoint, (ckpt, save_checkpoint_path, copy_step))
+      chrono.resume()
+
+    for (name, evaluator, log_steps, prefix) in evaluators:
+      if u.itstime(step, log_steps, total_steps):
+        chrono.pause(wait_for=params_repl)
+        write_note(f"{name} evaluation...\n{chrono.note}")
+        for key, value in evaluator.run(params_repl):
+          mw.measure(f"{prefix}{key}", value)
+        chrono.resume()
+    mw.step_end()
+
+  # Always give a chance to stop the profiler, no matter how things ended.
+  # TODO: can we also do this when dying of an exception like OOM?
+  if jax.process_index() == 0 and prof is not None:
+    u.startstop_prof(prof)
+
+  # Last note needs to happen before the pool's closed =)
+  if not error:
+    write_note(f"Done!\n{chrono.note}")
+  else:
+    write_note(f"Failed!\n{error}\n{chrono.note}")
+
+  pool.close()
+  pool.join()
+  mw.close()
+
+  # Make sure all hosts stay up until the end of main.
+  u.sync_all_hosts()
+
+  # Before cleanup, as cleanup should only run for successful jobs.
+  if error is not None:
+    raise RuntimeError(error)
+
+  u.maybe_cleanup_workdir(workdir, flags.FLAGS.cleanup, info)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/Tipsomaly/model/big_vision/trainers/proj/image_text/_deprecated_contrastive.py b/Tipsomaly/model/big_vision/trainers/proj/image_text/_deprecated_contrastive.py
new file mode 100644
index 0000000000000000000000000000000000000000..50d4570c9b7defd0f9d5d102c1e1c97bf1916968
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/image_text/_deprecated_contrastive.py
@@ -0,0 +1,514 @@
+# Copyright 2023 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contrastive training loop.
+
+For models Like
+- LiT (https://arxiv.org/abs/2111.07991)
+- CLIP (https://arxiv.org/abs/2103.00020)
+- SigLIP (https://arxiv.org/abs/2303.15343)
+"""
+# pylint: disable=consider-using-from-import
+import functools
+import importlib
+import multiprocessing.pool
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+import big_vision.evaluators.common as eval_common
+import big_vision.input_pipeline as input_pipeline
+import big_vision.optax as bv_optax
+import big_vision.utils as u
+from clu import parameter_overview
+import flax
+import jax
+import jax.numpy as jnp
+from ml_collections import config_flags
+import numpy as np
+import optax
+import tensorflow as tf
+
+from tensorflow.io import gfile
+
+# pylint: disable=logging-fstring-interpolation
+
+
+config_flags.DEFINE_config_file(
+    "config", None, "Training configuration.", lock_config=True)
+
+flags.DEFINE_string("workdir", default=None, help="Work unit directory.")
+flags.DEFINE_boolean("cleanup", default=False,
+                     help="Delete workdir (only) after successful completion.")
+
+# Adds jax flags to the program.
+jax.config.parse_flags_with_absl()
+
+
+def clip(x, *, a_max=None, a_min=None):
+  """Like jnp.clip, but allows all-None to mean don't clip."""
+  if a_max is None and a_min is None:
+    return x
+  return jnp.clip(x, a_max=a_max, a_min=a_min)
+
+
+def all_gather(z, roll=False, only_others=False):
+  """All gather and flatten first two dims."""
+  def gather_flat(x):
+    x = jax.lax.all_gather(x, "batch")
+    if roll or only_others:
+      # Each device moves "its" chunk to the beginning. Simplies loss/acc calcs.
+      x = jnp.roll(x, -jax.lax.axis_index("batch"), axis=0)
+      if only_others:
+        x = x[1:]
+    return jnp.concatenate(x, 0)  # Fold in "device" and "batch" dims.
+  return jax.tree_map(gather_flat, z)
+
+
+def softmax_loss(zimg, ztxt, temperature):
+  """Softmax loss following the CLIP paper. Factorized to reduce memory cost."""
+
+  def unidirectional_loss(z1, z2, t):
+    z2 = all_gather(z2, roll=True)
+    logits = jnp.dot(z1, z2.T) * t
+    # This a softmax across the larger gathered axis, taking advantage of the
+    # fact that positives are known to be on the diagonal.
+    loss = -(jnp.diag(logits) - jax.scipy.special.logsumexp(logits, axis=-1))
+    acc = jnp.argmax(logits, axis=1) == jnp.arange(z1.shape[0])
+    return loss.mean(), acc.mean()
+
+  extras = {}
+  loss = 0
+  for name, row, col in [("i2t", zimg, ztxt), ("t2i", ztxt, zimg)]:
+    loss_dir, acc_dir = unidirectional_loss(row, col, temperature)
+    loss += 0.5 * loss_dir
+    extras[f"{name}_acc"] = acc_dir
+    extras[f"{name}_loss"] = loss_dir
+
+  loss = jax.lax.pmean(loss, "batch")
+  return loss, extras
+
+
+def _avg_pos_logit(x_me):
+  return jnp.mean(jnp.diag(x_me))
+
+
+def _avg_neg_logit(x_me, x_ot=None):
+  nom = jnp.sum(x_me) - jnp.sum(jnp.diag(x_me))
+  den = x_me.size - len(x_me)
+  if x_ot is not None and x_ot.size:
+    nom += jnp.sum(x_ot)
+    den += x_ot.size
+  return nom / den
+
+
+def sigmoid_loss(zimg, ztxt, temperature, bias=0.0):
+  """Sigmoid loss from SigLIP: https://arxiv.org/abs/2303.15343."""
+  # Sigmoid loss. Since it's unidirectional, image embeddings stick to
+  # "me", i.e. the device they are computed on, and text embeddings travel.
+  ztxt_me = ztxt  # Text embeddings on my devices: (n, D)
+  ztxt_ot = all_gather(ztxt, only_others=True)  # Text emb from others: (N, D)
+
+  logits_me = jnp.dot(zimg, ztxt_me.T)  # (n, D) . (D, n) -> (n, n)
+  logits_ot = jnp.dot(zimg, ztxt_ot.T)  # (n, D) . (D, N) -> (n, N)
+  logits_me = logits_me * temperature + bias
+  logits_ot = logits_ot * temperature + bias
+
+  eye = jnp.eye(zimg.shape[0])
+  # Standard sigmoid computes everything twice, once assuming positive
+  # labels and once assuming negative ones. But here we know exactly where
+  # to find positives (on "me" diagonal) and negatives (everywhere else),
+  # so compute each one's loss only once:
+  m1_diag1 = -jnp.ones_like(logits_me) + 2 * eye
+  loglik_me = jax.nn.log_sigmoid(m1_diag1 * logits_me)
+  loglik_ot = jax.nn.log_sigmoid(-logits_ot)
+
+  # Normalize by npos per column, but that's one, so just sum.
+  nll_me = -loglik_me.sum(axis=-1)
+  nll_ot = -loglik_ot.sum(axis=-1)
+  l = nll_me.mean() + nll_ot.mean()  # == concat'ing me/ot along axis -1 above.
+
+  return l, {
+      # Only local device metrics for now, as last time I tried, there was
+      # some funny unimplemented business with jax.lax.pmin/pmax!
+      # So what's reported here is average of per-device min/max/avg.
+      "pos_min_logit": jnp.min(jnp.diag(logits_me)),
+      "pos_max_logit": jnp.max(jnp.diag(logits_me)),
+      "pos_avg_logit": _avg_pos_logit(logits_me),
+      "local_neg_min_logit": jnp.min(logits_me + 1e9 * eye),
+      "local_neg_max_logit": jnp.max(logits_me - 1e9 * eye),
+      "local_neg_avg_logit": _avg_neg_logit(logits_me),
+      "neg_min_logit": jnp.minimum(
+          jnp.min(logits_me + 1e9 * eye),
+          jnp.min(logits_ot) if logits_ot.size else jnp.inf),
+      "neg_max_logit": jnp.maximum(
+          jnp.max(logits_me - 1e9 * eye),
+          jnp.max(logits_ot) if logits_ot.size else -jnp.inf),
+      "neg_avg_logit": _avg_neg_logit(logits_me, logits_ot),
+  }
+
+
+def _gather_from_device(x, device_id, axis_name="batch"):
+  return jax.lax.psum((jax.lax.axis_index(axis_name) == device_id) * x,
+                      axis_name)
+
+
+def chunked_sigmoid_loss(zimg, ztxt, temperature, bias=0.0):
+  """Loss computation from section 3.1 of arxiv.org/abs/2303.15343."""
+
+  # Calculate loss for representations on this device, which includes positives.
+  logits_me = jnp.dot(zimg, ztxt.T)  # (n, D) . (D, n) -> (n, n)
+  logits_me = logits_me * temperature + bias
+  m1_diag1 = -jnp.ones_like(logits_me) + 2 * jnp.eye(zimg.shape[0])
+  loglik_me = jax.nn.log_sigmoid(m1_diag1 * logits_me)
+  nll_me = -loglik_me.sum(axis=-1).mean()
+
+  def negative_loss(ztxt_other_device):
+    logits_ot = jnp.dot(zimg, ztxt_other_device.T)  # (n, D) . (D, n) -> (n, n)
+    logits_ot = logits_ot * temperature + bias
+    loglik_ot = jax.nn.log_sigmoid(-logits_ot)
+    return -jnp.sum(loglik_ot, axis=-1).mean()
+
+  me = jax.lax.axis_index("batch")
+  # All other devices are negatives. Hot-potato swap ztxt across devices.
+  # Interestingly, ppermute based implementation was memory intensive, so using
+  # all-reduce to gather representations.
+  nll_others = 0
+  for device_id in range(jax.device_count()):
+    skip = jnp.not_equal(device_id, me)
+    nll_others += skip * negative_loss(_gather_from_device(ztxt, device_id))
+
+  eye = jnp.eye(zimg.shape[0])
+  return nll_me + nll_others, {
+      "pos_min_logit": jnp.min(jnp.diag(logits_me)),
+      "pos_max_logit": jnp.max(jnp.diag(logits_me)),
+      "pos_avg_logit": _avg_pos_logit(logits_me),
+      "local_neg_min_logit": jnp.min(logits_me + 1e9 * eye),
+      "local_neg_max_logit": jnp.max(logits_me - 1e9 * eye),
+      "local_neg_avg_logit": _avg_neg_logit(logits_me),}
+
+
+def main(argv):
+  del argv
+  tf.config.experimental.set_visible_devices([], "GPU")
+
+  config = flags.FLAGS.config
+  workdir = flags.FLAGS.workdir
+  logging.info(  # pylint: disable=logging-fstring-interpolation
+      f"\u001b[33mHello from process {jax.process_index()} holding "
+      f"{jax.local_device_count()}/{jax.device_count()} devices and "
+      f"writing to workdir {workdir}.\u001b[0m")
+
+  save_ckpt_path = None
+  if workdir:  # Always create if requested, even if we may not write into it.
+    gfile.makedirs(workdir)
+    save_ckpt_path = os.path.join(workdir, "checkpoint.npz")
+
+  # The pool is used to perform misc operations such as logging in async way.
+  pool = multiprocessing.pool.ThreadPool()
+
+  # Here we register preprocessing ops from modules listed on `pp_modules`.
+  for m in config.get("pp_modules", ["ops_general", "ops_image", "ops_text"]):
+    importlib.import_module(f"big_vision.pp.{m}")
+
+  # This seed makes the Jax part of things (like model init) deterministic.
+  # However, full training still won't be deterministic, for example due to the
+  # tf.data pipeline not being deterministic even if we would set TF seed.
+  # See (internal link) for a fun read on what it takes.
+  rng = jax.random.PRNGKey(config.get("seed", 0))
+
+  # These functions do more stuff internally, for OSS release we mock them by
+  # trivial alternatives in order to minize disruptions in the code.
+  xid, wid = -1, -1
+  def info(s, *a):
+    logging.info("\u001b[33mNOTE\u001b[0m: " + s, *a)
+  def write_note(note):
+    if jax.process_index() == 0:
+      info("%s", note)
+
+  write_note("Initializing...")
+
+  batch_size = config.input.batch_size
+  if batch_size % jax.device_count() != 0:
+    raise ValueError(f"Batch size ({batch_size}) must "
+                     f"be divisible by device number ({jax.device_count()})")
+  info("Global batch size %d on %d hosts results in %d local batch size. With "
+       "%d dev per host (%d dev total), that's a %d per-device batch size.",
+       batch_size, jax.process_count(), batch_size // jax.process_count(),
+       jax.local_device_count(), jax.device_count(),
+       batch_size // jax.device_count())
+
+  # First thing after above sanity checks, so we can log "start" ticks.
+  mw = u.BigVisionMetricWriter(xid, wid, workdir, config)
+
+  write_note("Initializing train dataset...")
+  train_ds, ntrain_img = input_pipeline.training(config.input)
+
+  # Start prefetching already.
+  n_prefetch = config.get("prefetch_to_device", 1)
+  train_iter = input_pipeline.start_input_pipeline(train_ds, n_prefetch)
+
+  total_steps = u.steps("total", config, ntrain_img, batch_size)
+  def get_steps(name, default=ValueError, cfg=config):
+    return u.steps(name, cfg, ntrain_img, batch_size, total_steps, default)
+
+  u.chrono.inform(total_steps=total_steps, global_bs=batch_size,
+                  steps_per_epoch=ntrain_img / batch_size,
+                  measure=mw.measure, write_note=write_note)
+
+  info("Running for %d steps, that means %f epochs",
+       total_steps, total_steps * batch_size / ntrain_img)
+
+  write_note(f"Initializing {config.model_name} model...")
+  model_mod = importlib.import_module(f"big_vision.models.{config.model_name}")
+  model = model_mod.Model(**config.get("model", {}))
+
+  # We want all parameters to be created in host RAM, not on any device, they'll
+  # be sent there later as needed, otherwise we already encountered two
+  # situations where we allocate them twice.
+  @functools.partial(jax.jit, backend="cpu")
+  def init(rng):
+    bs = batch_size // jax.device_count()
+    image_size = tuple(train_ds.element_spec["image"].shape[1:])
+    no_image = jnp.zeros((bs,) + image_size, jnp.float32)
+    text_size = tuple(train_ds.element_spec["labels"].shape[1:])
+    no_text = jnp.zeros((bs,) + text_size, jnp.int32)
+    params = flax.core.unfreeze(model.init(rng, no_image, no_text))["params"]
+    return params
+
+  rng, rng_init = jax.random.split(rng)
+  with u.chrono.log_timing("z/secs/init"):
+    params_cpu = init(rng_init)
+
+  if jax.process_index() == 0:
+    num_params = sum(p.size for p in jax.tree_leaves(params_cpu))
+    parameter_overview.log_parameter_overview(params_cpu, msg="init params")
+    mw.measure("num_params", num_params)
+
+  write_note(f"Initializing {config.optax_name} optimizer...")
+  tx, sched_fns = bv_optax.make(config, params_cpu, sched_kw=dict(
+      total_steps=total_steps, batch_size=batch_size, data_size=ntrain_img))
+
+  # We jit this, such that the arrays are created on the CPU, not device[0].
+  opt_cpu = jax.jit(tx.init, backend="cpu")(params_cpu)
+  sched_fns_cpu = [jax.jit(sched_fn, backend="cpu") for sched_fn in sched_fns]
+
+  @functools.partial(jax.pmap, axis_name="batch", donate_argnums=(0, 1))
+  def update_fn(params, opt, rng, batch):
+    """Update step."""
+    assert "mixup" not in config, "We still have to figure out mixup."
+
+    # Get device-specific loss rng.
+    rng, rng_model = jax.random.split(rng, 2)
+    rng_model_local = jax.random.fold_in(rng_model, jax.lax.axis_index("batch"))
+
+    def loss_fn(params, images, labels):
+      zimg, ztxt, extras = model.apply(
+          {"params": params}, images, labels,
+          train=True, rngs={"dropout": rng_model_local})
+
+      match config.get("loss_fn", "softmax"):
+        case "softmax":
+          l, l_extras = softmax_loss(zimg, ztxt, extras["t"])
+        case "sigmoid":
+          l, l_extras = sigmoid_loss(zimg, ztxt, extras["t"], bias=extras["b"])
+        case "chunked_sigmoid":
+          l, l_extras = chunked_sigmoid_loss(zimg, ztxt, extras["t"],
+                                             bias=extras["b"])
+        case _:
+          raise NotImplementedError(f"Unrecognized loss {config.loss_fn=}")
+
+      return l, {
+          "t": extras["t"],
+          "t/parameter": extras["t/parameter"],
+          "train/nimg": jnp.mean(extras["img/norm"]),
+          "train/ntxt": jnp.mean(extras["txt/norm"]),
+          **{f"train/{k}": v for k, v in l_extras.items()},
+      }
+
+    (l, measurements), grads = jax.value_and_grad(
+        loss_fn, has_aux=True)(params, batch["image"], batch["labels"])
+    l, measurements, grads = jax.lax.pmean((l, measurements, grads),
+                                           axis_name="batch")
+    updates, opt = tx.update(grads, opt, params)
+    params = optax.apply_updates(params, updates)
+
+    gs = jax.tree_leaves(bv_optax.replace_frozen(config.schedule, grads, 0.))
+    measurements["l2_grads"] = jnp.sqrt(sum([jnp.vdot(g, g) for g in gs]))
+    ps = jax.tree_leaves(params)
+    measurements["l2_params"] = jnp.sqrt(sum([jnp.vdot(p, p) for p in ps]))
+    us = jax.tree_leaves(updates)
+    measurements["l2_updates"] = jnp.sqrt(sum([jnp.vdot(u, u) for u in us]))
+
+    return params, opt, rng, l, measurements
+
+  # We require hashable function reference for evaluator.
+  # We do not jit/pmap this function, because it is passed to evaluator that
+  # does it later. We output as many intermediate tensors as possible for
+  # maximal flexibility. Later `jit` will prune out things that are not needed.
+  def predict_fn(params, image=None, text=None, **unused_kwargs):
+    del unused_kwargs  # `unused_kwargs` is to be compatible with few-shot
+    zimg, ztxt, out = model.apply({"params": params}, image, text)
+    return zimg, ztxt, out
+
+  # Only initialize evaluators when they are first needed.
+  @functools.lru_cache(maxsize=None)
+  def evaluators():
+    return eval_common.from_config(
+        config, {"predict": predict_fn},
+        lambda s: write_note(f"Init evaluator: {s}…\n{u.chrono.note}"),
+        lambda key, cfg: get_steps(key, default=None, cfg=cfg),
+    )
+
+  # Decide how to initialize training. The order is important.
+  # 1. Always resumes from the existing checkpoint, e.g. resumes a finetune job.
+  # 2. Resume from a previous checkpoint, e.g. start a cooldown training job.
+  # 3. Initialize model from something, e,g, start a fine-tuning job.
+  # 4. Train from scratch.
+  resume_ckpt_path = None
+  if save_ckpt_path and gfile.exists(save_ckpt_path):
+    resume_ckpt_path = save_ckpt_path
+  elif config.get("resume"):
+    resume_ckpt_path = config.resume.format(wid=xm_wu.id)
+  if resume_ckpt_path:
+    write_note("Resume training from checkpoint...")
+    checkpoint = {
+        "params": params_cpu,
+        "opt": opt_cpu,
+        "chrono": u.chrono.save(),
+    }
+    checkpoint_tree = jax.tree_structure(checkpoint)
+    loaded = u.load_checkpoint_np(resume_ckpt_path, checkpoint_tree)
+    # bfloat16 type gets lost when data is saved to disk, so we recover it.
+    checkpoint = jax.tree_map(u.recover_dtype, loaded)
+    params_cpu, opt_cpu = checkpoint["params"], checkpoint["opt"]
+    u.chrono.load(checkpoint["chrono"])
+  elif config.get("model_init"):
+    write_note(f"Initialize model from {config.model_init}...")
+    params_cpu = model_mod.load(
+        params_cpu, config.model_init, config.get("model"),
+        **config.get("model_load", {}))
+    if jax.process_index() == 0:
+      parameter_overview.log_parameter_overview(
+          params_cpu, msg="restored params")
+
+  write_note("Kicking off misc stuff...")
+  first_step = bv_optax.get_count(opt_cpu)
+  u.chrono.inform(first_step=first_step)
+  prof = None  # Keeps track of start/stop of profiler state.
+
+  write_note(f"Replicating...\n{u.chrono.note}")
+  params_repl = flax.jax_utils.replicate(params_cpu)
+  opt_repl = flax.jax_utils.replicate(opt_cpu)
+
+  rng, rng_loop = jax.random.split(rng, 2)
+  rngs_loop = flax.jax_utils.replicate(rng_loop)
+  ckpt_writer = None
+
+  write_note(f"First step compilations...\n{u.chrono.note}")
+
+  # Note that training can be pre-empted during the final evaluation (i.e.
+  # just after the final checkpoint has been written to disc), in which case we
+  # want to run the evals.
+  if first_step in (total_steps, 0):
+    mw.step_start(first_step)
+    for (name, evaluator, _, prefix) in evaluators():
+      if config.evals[name].get("skip_first") and first_step != total_steps:
+        continue
+      write_note(f"{name} evaluation...\n{u.chrono.note}")
+      with u.chrono.log_timing(f"z/secs/eval/{name}"):
+        for key, value in evaluator.run(params_repl):
+          mw.measure(f"{prefix}{key}", value)
+
+  # Using a python integer for step here, because opt.state.step is allocated
+  # on TPU during replication.
+  for step, batch in zip(range(first_step + 1, total_steps + 1), train_iter):
+    mw.step_start(step)
+
+    with jax.profiler.StepTraceAnnotation("train_step", step_num=step):
+      with u.chrono.log_timing("z/secs/update0", noop=step > first_step + 1):
+        params_repl, opt_repl, rngs_loop, loss_value, measurements = update_fn(
+            params_repl, opt_repl, rngs_loop, batch)
+
+    # On the first host, let's always profile a handful of early steps.
+    if jax.process_index() == 0:
+      prof = u.startstop_prof(prof, step, first_step, get_steps("log_training"))
+
+    # Report training progress
+    if (u.itstime(step, get_steps("log_training"), total_steps, host=0)
+        or u.chrono.warmup and jax.process_index() == 0):
+      for i, sched_fn_cpu in enumerate(sched_fns_cpu):
+        mw.measure(f"global_schedule{i if i else ''}", sched_fn_cpu(step - 1))
+      l = mw.measure("training_loss", loss_value[0])
+      for name, value in measurements.items():
+        mw.measure(name, value[0])
+      u.chrono.tick(step)
+      if not np.isfinite(l):
+        raise RuntimeError(f"The loss became nan or inf somewhere within steps "
+                           f"[{step - get_steps('log_training')}, {step}]")
+
+    # Checkpoint saving
+    if (save_ckpt_path and
+        (u.itstime(step, get_steps("ckpt", None), total_steps, host=0) or
+         u.itstime(step, get_steps("keep_ckpt", None), total_steps, host=0))):
+      u.chrono.pause(wait_for=(params_repl, opt_repl))
+      u.checkpointing_timeout(ckpt_writer, config.get("ckpt_timeout", 1))
+      # We need to transfer the weights over now or else we risk keeping them
+      # alive while they'll be updated in a future step, creating hard to debug
+      # memory errors (see (internal link)). Also, takes device 0's params only.
+      params_cpu = jax.tree_map(lambda x: np.array(x[0]), params_repl)
+      opt_cpu = jax.tree_map(lambda x: np.array(x[0]), opt_repl)
+
+      # Check whether we want to keep a copy of the current checkpoint.
+      copy_step = None
+      if u.itstime(step, get_steps("keep_ckpt", None), total_steps):
+        copy_step = step
+
+      ckpt = {"params": params_cpu, "opt": opt_cpu, "chrono": u.chrono.save()}
+      ckpt_writer = pool.apply_async(
+          u.save_checkpoint, (ckpt, save_ckpt_path, copy_step))
+      u.chrono.resume()
+
+    for (name, evaluator, log_steps, prefix) in evaluators():
+      if u.itstime(step, log_steps, total_steps, first=False, last=True):
+        u.chrono.pause(wait_for=params_repl)
+        u.chrono.tick(step)  # Record things like epoch number, core hours etc.
+        write_note(f"{name} evaluation...\n{u.chrono.note}")
+        with u.chrono.log_timing(f"z/secs/eval/{name}"):
+          for key, value in evaluator.run(params_repl):
+            mw.measure(f"{prefix}{key}", value)
+        u.chrono.resume()
+    mw.step_end()
+
+  # Always give a chance to stop the profiler, no matter how things ended.
+  # TODO: can we also do this when dying of an exception like OOM?
+  if jax.process_index() == 0 and prof is not None:
+    u.startstop_prof(prof)
+
+  # Last note needs to happen before the pool's closed =)
+  write_note(f"Done!\n{u.chrono.note}")
+
+  pool.close()
+  pool.join()
+  mw.close()
+
+  # Make sure all hosts stay up until the end of main.
+  u.sync()
+
+  u.maybe_cleanup_workdir(workdir, flags.FLAGS.cleanup, info)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/Tipsomaly/model/big_vision/trainers/proj/image_text/siglip.py b/Tipsomaly/model/big_vision/trainers/proj/image_text/siglip.py
new file mode 100644
index 0000000000000000000000000000000000000000..f632c7a0ce6960ae4bfbe59c6d46066fbf7f573a
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/image_text/siglip.py
@@ -0,0 +1,527 @@
+# Copyright 2024 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Trainer for "Sigmoid Loss for Language Image Pre-Training".
+
+SigLIP (https://arxiv.org/abs/2303.15343)
+
+TODO: implement chunked version with shard_map.
+"""
+# pylint: disable=consider-using-from-import
+# pylint: disable=logging-fstring-interpolation
+
+import functools
+import importlib
+import multiprocessing.pool
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+import big_vision.evaluators.common as eval_common
+import big_vision.input_pipeline as input_pipeline
+import big_vision.optax as bv_optax
+import big_vision.sharding as bv_sharding
+import big_vision.utils as u
+from clu import parameter_overview
+import flax.linen as nn
+import jax
+from jax.experimental import mesh_utils
+from jax.experimental import multihost_utils
+from jax.experimental.array_serialization import serialization as array_serial
+import jax.numpy as jnp
+from ml_collections import config_flags
+import numpy as np
+import optax
+import tensorflow as tf
+
+from tensorflow.io import gfile
+
+
+config_flags.DEFINE_config_file(
+    "config", None, "Training configuration.", lock_config=True)
+
+flags.DEFINE_string("workdir", default=None, help="Work unit directory.")
+flags.DEFINE_boolean("cleanup", default=False,
+                     help="Delete workdir (only) after successful completion.")
+
+# Adds jax flags to the program.
+jax.config.parse_flags_with_absl()
+# Transfer guard will fail the program whenever that data between a host and
+# a device is transferred implicitly. This often catches subtle bugs that
+# cause slowdowns and memory fragmentation. Explicit transfers are done
+# with jax.device_put and jax.device_get.
+jax.config.update("jax_transfer_guard", "disallow")
+# Fixes design flaw in jax.random that may cause unnecessary d2d comms.
+jax.config.update("jax_threefry_partitionable", True)
+
+
+NamedSharding = jax.sharding.NamedSharding
+P = jax.sharding.PartitionSpec
+
+
+def main(argv):
+  del argv
+
+  jax.distributed.initialize()
+
+  # Make sure TF does not touch GPUs.
+  tf.config.set_visible_devices([], "GPU")
+
+  config = flags.FLAGS.config
+
+################################################################################
+#                                                                              #
+#                                Set up logging                                #
+#                                                                              #
+################################################################################
+
+  # Set up work directory and print welcome message.
+  workdir = flags.FLAGS.workdir
+  logging.info(
+      f"\u001b[33mHello from process {jax.process_index()} holding "
+      f"{jax.local_device_count()}/{jax.device_count()} devices and "
+      f"writing to workdir {workdir}.\u001b[0m")
+
+  save_ckpt_path = None
+  if workdir:  # Always create if requested, even if we may not write into it.
+    gfile.makedirs(workdir)
+    save_ckpt_path = os.path.join(workdir, "checkpoint.bv")
+
+  # The pool is used to perform misc operations such as logging in async way.
+  pool = multiprocessing.pool.ThreadPool()
+
+  # Here we register preprocessing ops from modules listed on `pp_modules`.
+  for m in config.get("pp_modules", ["ops_general", "ops_image", "ops_text"]):
+    importlib.import_module(f"big_vision.pp.{m}")
+
+  # Setup up logging and experiment manager.
+  xid, wid = -1, -1
+  fillin = lambda s: s
+  def info(s, *a):
+    logging.info("\u001b[33mNOTE\u001b[0m: " + s, *a)
+  def write_note(note):
+    if jax.process_index() == 0:
+      info("%s", note)
+
+  mw = u.BigVisionMetricWriter(xid, wid, workdir, config)
+
+  # Allow for things like timings as early as possible!
+  u.chrono.inform(measure=mw.measure, write_note=write_note)
+
+################################################################################
+#                                                                              #
+#                                Set up Mesh                                   #
+#                                                                              #
+################################################################################
+
+  # We rely on jax mesh_utils to organize devices, such that communication
+  # speed is the fastest for the last dimension, second fastest for the
+  # penultimate dimension, etc.
+  config_mesh = config.get("mesh", [("data", jax.device_count())])
+
+  # Sharding rules with default
+  sharding_rules = config.get("sharding_rules", [("act_batch", "data")])
+
+  mesh_axes, mesh_size = tuple(zip(*config_mesh))
+
+  # Because jax.utils do not support `-1` shape size.
+  mesh_size = np.array(jax.devices()).reshape(mesh_size).shape
+
+  device_mesh = mesh_utils.create_device_mesh(mesh_size)
+
+  # Consistent device order is important to ensure correctness of various train
+  # loop components, such as input pipeline, update step, evaluators. The
+  # order presribed by the `devices_flat` variable should be used throughout
+  # the program.
+  devices_flat = device_mesh.flatten()
+
+################################################################################
+#                                                                              #
+#                                Input Pipeline                                #
+#                                                                              #
+################################################################################
+
+  write_note("Initializing train dataset...")
+  batch_size = config.input.batch_size
+  if batch_size % jax.device_count() != 0:
+    raise ValueError(f"Batch size ({batch_size}) must "
+                     f"be divisible by device number ({jax.device_count()})")
+  info("Global batch size %d on %d hosts results in %d local batch size. With "
+       "%d dev per host (%d dev total), that's a %d per-device batch size.",
+       batch_size, jax.process_count(), batch_size // jax.process_count(),
+       jax.local_device_count(), jax.device_count(),
+       batch_size // jax.device_count())
+
+  train_ds, ntrain_img = input_pipeline.training(config.input)
+
+  total_steps = u.steps("total", config, ntrain_img, batch_size)
+  def get_steps(name, default=ValueError, cfg=config):
+    return u.steps(name, cfg, ntrain_img, batch_size, total_steps, default)
+
+  u.chrono.inform(total_steps=total_steps, global_bs=batch_size,
+                  steps_per_epoch=ntrain_img / batch_size)
+
+  info("Running for %d steps, that means %f epochs",
+       total_steps, total_steps * batch_size / ntrain_img)
+
+  # Start input pipeline as early as possible.
+  n_prefetch = config.get("prefetch_to_device", 1)
+  train_iter = input_pipeline.start_global(train_ds, devices_flat, n_prefetch)
+
+################################################################################
+#                                                                              #
+#                           Create Model & Optimizer                           #
+#                                                                              #
+################################################################################
+
+  write_note("Creating model...")
+  model_mod = importlib.import_module(f"big_vision.models.{config.model_name}")
+  model = model_mod.Model(**config.get("model", {}))
+
+  def init(rng):
+    batch = jax.tree_map(lambda x: jnp.zeros(x.shape, x.dtype.as_numpy_dtype),
+                         train_ds.element_spec)
+    params = model.init(rng, batch["image"], batch["labels"])["params"]
+
+    # Set bias in the head to a low value, such that loss is small initially.
+    if "init_head_bias" in config:
+      params["head"]["bias"] = jnp.full_like(params["head"]["bias"],
+                                             config["init_head_bias"])
+
+    return params
+
+  # This seed makes the Jax part of things (like model init) deterministic.
+  # However, full training still won't be deterministic, for example due to the
+  # tf.data pipeline not being deterministic even if we would set TF seed.
+  # See (internal link) for a fun read on what it takes.
+  rng = jax.random.PRNGKey(u.put_cpu(config.get("seed", 0)))
+
+  write_note("Inferring parameter shapes...")
+  rng, rng_init = jax.random.split(rng)
+  params_shape = jax.eval_shape(init, rng_init)
+
+  write_note("Inferring optimizer state shapes...")
+  tx, sched_fns = bv_optax.make(config, params_shape, sched_kw=dict(
+      total_steps=total_steps, batch_size=batch_size, data_size=ntrain_img))
+  opt_shape = jax.eval_shape(tx.init, params_shape)
+  # We jit this, such that the arrays are created on the CPU, not device[0].
+  sched_fns_cpu = [u.jit_cpu()(sched_fn) for sched_fn in sched_fns]
+
+  if jax.process_index() == 0:
+    num_params = sum(np.prod(p.shape) for p in jax.tree_leaves(params_shape))
+    mw.measure("num_params", num_params)
+
+################################################################################
+#                                                                              #
+#                               Shard & Transfer                               #
+#                                                                              #
+################################################################################
+
+  write_note("Creating device mesh...")
+  mesh = jax.sharding.Mesh(device_mesh, mesh_axes)
+  repl_sharding = jax.sharding.NamedSharding(mesh, P())
+
+  write_note("Inferring shardings...")
+  train_state_shape = {"params": params_shape, "opt": opt_shape}
+
+  strategy = config.get("sharding_strategy", [(".*", "replicate")])
+  train_state_sharding = bv_sharding.infer_sharding(
+      train_state_shape, strategy=strategy, mesh=mesh)
+
+  write_note("Transferring train_state to devices...")
+  # RNG is always replicated
+  rng_init = u.reshard(rng_init, repl_sharding)
+
+  # Parameters and the optimizer are now global (distributed) jax arrays.
+  params = jax.jit(init, out_shardings=train_state_sharding["params"])(rng_init)
+  opt = jax.jit(tx.init, out_shardings=train_state_sharding["opt"])(params)
+
+  rng, rng_loop = jax.random.split(rng, 2)
+  rng_loop = u.reshard(rng_loop, repl_sharding)
+  del rng  # not used anymore, so delete it.
+
+  # At this point we have everything we need to form a train state. It contains
+  # all the parameters that are passed and updated by the main training step.
+  train_state = {"params": params, "opt": opt}
+  del params, opt  # Delete to avoid memory leak or accidental reuse.
+
+  write_note("Logging parameter overview...")
+  parameter_overview.log_parameter_overview(
+      train_state["params"], msg="Init params",
+      include_stats="global", jax_logging_process=0)
+
+################################################################################
+#                                                                              #
+#                                 Update Step                                  #
+#                                                                              #
+################################################################################
+
+  @functools.partial(
+      jax.jit,
+      donate_argnums=(0,),
+      out_shardings=(train_state_sharding, repl_sharding))
+  def update_fn(train_state, rng, batch):
+    """Update step."""
+
+    images, labels = batch["image"], batch["labels"]
+
+    step_count = bv_optax.get_count(train_state["opt"], jittable=True)
+    rng = jax.random.fold_in(rng, step_count)
+    assert "mixup" not in config, "Mixup is not supported for SigLIP."
+
+    # Get device-specific loss rng.
+    rng, rng_model = jax.random.split(rng, 2)
+
+    def loss_fn(params):
+      zimg, ztxt, extras = model.apply(
+          {"params": params}, images, labels,
+          train=True, rngs={"dropout": rng_model})
+      logits = jnp.dot(zimg, ztxt.T)
+      logits = logits * extras["t"] + extras["b"]
+      eye = jnp.eye(zimg.shape[0])
+
+      # Standard sigmoid computes everything twice, once assuming positive
+      # labels and once assuming negative ones. But here we know exactly where
+      # to find positives (on "me" diagonal) and negatives (everywhere else),
+      # so compute each one's loss only once:
+      m1_diag1 = -jnp.ones_like(logits) + 2 * eye
+      loglik = jax.nn.log_sigmoid(m1_diag1 * logits)
+
+      # Normalize by npos per column, but that's one, so just sum.
+      nll = -jnp.sum(loglik, axis=-1)
+
+      # NOTE: same as concat'ing me/ot along axis -1 above.
+      l = jnp.mean(nll)
+
+      return l
+
+    params, opt = train_state["params"], train_state["opt"]
+    loss, grads = jax.value_and_grad(loss_fn)(params)
+    updates, opt = tx.update(grads, opt, params)
+    params = optax.apply_updates(params, updates)
+
+    measurements = {"training_loss": loss}
+    gs = jax.tree_leaves(bv_optax.replace_frozen(config.schedule, grads, 0.))
+    measurements["l2_grads"] = jnp.sqrt(sum([jnp.sum(g * g) for g in gs]))
+    ps = jax.tree_leaves(params)
+    measurements["l2_params"] = jnp.sqrt(sum([jnp.sum(p * p) for p in ps]))
+    us = jax.tree_leaves(updates)
+    measurements["l2_updates"] = jnp.sqrt(sum([jnp.sum(u * u) for u in us]))
+
+    return {"params": params, "opt": opt}, measurements
+
+################################################################################
+#                                                                              #
+#                               Load Checkpoint                                #
+#                                                                              #
+################################################################################
+
+  # Decide how to initialize training. The order is important.
+  # 1. Always resumes from the existing checkpoint, e.g. resumes a finetune job.
+  # 2. Resume from a previous checkpoint, e.g. start a cooldown training job.
+  # 3. Initialize model from something, e,g, start a fine-tuning job.
+  # 4. Train from scratch.
+  resume_ckpt_path = None
+  if save_ckpt_path and gfile.exists(f"{save_ckpt_path}-LAST"):
+    resume_ckpt_path = save_ckpt_path
+  elif config.get("resume"):
+    resume_ckpt_path = fillin(config.resume)
+
+  ckpt_mngr = None
+  if save_ckpt_path or resume_ckpt_path:
+    ckpt_mngr = array_serial.GlobalAsyncCheckpointManager()
+
+  if resume_ckpt_path:
+    write_note(f"Resuming training from checkpoint {resume_ckpt_path}...")
+    jax.tree_map(lambda x: x.delete(), train_state)
+    del train_state
+    shardings = {
+        **train_state_sharding,
+        "chrono": jax.tree_map(lambda _: repl_sharding,
+                               u.chrono.save()),
+    }
+    loaded = u.load_checkpoint_ts(
+        resume_ckpt_path, tree=shardings, shardings=shardings)
+    train_state = {key: loaded[key] for key in train_state_sharding.keys()}
+
+    u.chrono.load(jax.device_get(loaded["chrono"]))
+    del loaded
+  elif config.get("model_init"):
+    write_note(f"Initialize model from {config.model_init}...")
+    # TODO: when updating the `load` API soon, do pass and request the
+    # full `train_state` from it. Examples where useful: VQVAE, BN.
+    train_state["params"] = model_mod.load(
+        train_state["params"], config.model_init, config.get("model"),
+        **config.get("model_load", {}))
+
+    # load has the freedom to return params not correctly sharded. Think of for
+    # example ViT resampling position embedings on CPU as numpy arrays.
+    train_state["params"] = u.reshard(
+        train_state["params"], train_state_sharding["params"])
+
+    parameter_overview.log_parameter_overview(
+        train_state["params"], msg="restored params",
+        include_stats="global", jax_logging_process=0)
+
+
+################################################################################
+#                                                                              #
+#                                 Setup Evals                                  #
+#                                                                              #
+################################################################################
+
+  # We do not jit/pmap this function, because it is passed to evaluator that
+  # does it later. We output as many intermediate tensors as possible for
+  # maximal flexibility. Later `jit` will prune out things that are not needed.
+  def eval_logits_fn(train_state, batch):
+    zimg, ztxt, out = model.apply(
+        {"params": train_state["params"]},
+        batch.get("image", None), batch.get("labels", None))
+    return zimg, ztxt, out
+
+  def eval_loss_fn(train_state, batch):
+    logits, _ = model.apply({"params": train_state["params"]}, batch["image"])
+    loss_fn = getattr(u, config.get("loss", "sigmoid_xent"))
+    return {
+        "loss": loss_fn(logits=logits, labels=batch["labels"], reduction=False)
+    }
+
+  eval_fns = {
+      "predict": eval_logits_fn,
+      "loss": eval_loss_fn,
+  }
+
+  # Only initialize evaluators when they are first needed.
+  @functools.lru_cache(maxsize=None)
+  def evaluators():
+    return eval_common.from_config(
+        config, eval_fns,
+        lambda s: write_note(f"Init evaluator: {s}…\n{u.chrono.note}"),
+        lambda key, cfg: get_steps(key, default=None, cfg=cfg),
+        devices_flat,
+    )
+
+  # At this point we need to know the current step to see whether to run evals.
+  write_note("Inferring the first step number...")
+  first_step_device = bv_optax.get_count(train_state["opt"], jittable=True)
+  first_step = int(jax.device_get(first_step_device))
+  u.chrono.inform(first_step=first_step)
+
+  # Note that training can be pre-empted during the final evaluation (i.e.
+  # just after the final checkpoint has been written to disc), in which case we
+  # want to run the evals.
+  if first_step in (total_steps, 0):
+    write_note("Running initial or final evals...")
+    mw.step_start(first_step)
+    for (name, evaluator, _, prefix) in evaluators():
+      if config.evals[name].get("skip_first") and first_step != total_steps:
+        continue
+      write_note(f"{name} evaluation...\n{u.chrono.note}")
+      with u.chrono.log_timing(f"z/secs/eval/{name}"):
+        with mesh, nn.logical_axis_rules(sharding_rules):
+          for key, value in evaluator.run(train_state):
+            mw.measure(f"{prefix}{key}", value)
+
+################################################################################
+#                                                                              #
+#                                  Train Loop                                  #
+#                                                                              #
+################################################################################
+
+  prof = None  # Keeps track of start/stop of profiler state.
+
+  write_note("Starting training loop, compiling the first step...")
+  for step, batch in zip(range(first_step + 1, total_steps + 1), train_iter):
+    mw.step_start(step)
+
+    with jax.profiler.StepTraceAnnotation("train_step", step_num=step):
+      with u.chrono.log_timing("z/secs/update0", noop=step > first_step + 1):
+        with mesh, nn.logical_axis_rules(sharding_rules):
+          train_state, measurements = update_fn(train_state, rng_loop, batch)
+
+    # On the first host, let's always profile a handful of early steps.
+    if jax.process_index() == 0:
+      prof = u.startstop_prof(prof, step, first_step, get_steps("log_training"))
+
+    # Report training progress
+    if (u.itstime(step, get_steps("log_training"), total_steps, host=0)
+        or u.chrono.warmup and jax.process_index() == 0):
+      for i, sched_fn_cpu in enumerate(sched_fns_cpu):
+        mw.measure(f"global_schedule{i if i else ''}",
+                   sched_fn_cpu(u.put_cpu(step - 1)))
+      measurements = jax.device_get(measurements)
+      for name, value in measurements.items():
+        mw.measure(name, value)
+      u.chrono.tick(step)
+      if not np.isfinite(measurements["training_loss"]):
+        raise RuntimeError(f"The loss became nan or inf somewhere within steps "
+                           f"[{step - get_steps('log_training')}, {step}]")
+
+    # Checkpoint saving
+    keep_ckpt_steps = get_steps("keep_ckpt", None) or total_steps
+    if save_ckpt_path and (
+        (keep := u.itstime(step, keep_ckpt_steps, total_steps, first=False))
+        or u.itstime(step, get_steps("ckpt", None), total_steps, first=True)
+    ):
+      u.chrono.pause(wait_for=train_state)
+
+      # Copy because we add extra stuff to the checkpoint.
+      ckpt = {**train_state}
+
+      # To save chrono state correctly and safely in a multihost setup, we
+      # broadcast the state to all hosts and convert it to a global array.
+      with jax.transfer_guard("allow"):
+        chrono_ckpt = multihost_utils.broadcast_one_to_all(u.chrono.save())
+      chrono_shardings = jax.tree_map(lambda _: repl_sharding, chrono_ckpt)
+      ckpt = ckpt | {"chrono": u.reshard(chrono_ckpt, chrono_shardings)}
+
+      u.save_checkpoint_ts(ckpt_mngr, ckpt, save_ckpt_path, step, keep)
+      u.chrono.resume()
+
+    for (name, evaluator, log_steps, prefix) in evaluators():
+      if u.itstime(step, log_steps, total_steps, first=False, last=True):
+        u.chrono.pause(wait_for=train_state)
+        u.chrono.tick(step)  # Record things like epoch number, core hours etc.
+        write_note(f"{name} evaluation...\n{u.chrono.note}")
+        with u.chrono.log_timing(f"z/secs/eval/{name}"):
+          with mesh, nn.logical_axis_rules(sharding_rules):
+            for key, value in evaluator.run(train_state):
+              mw.measure(f"{prefix}{key}", jax.device_get(value))
+        u.chrono.resume()
+    mw.step_end()
+
+  # Always give a chance to stop the profiler, no matter how things ended.
+  # TODO: can we also do this when dying of an exception like OOM?
+  if jax.process_index() == 0 and prof is not None:
+    u.startstop_prof(prof)
+
+  # Last note needs to happen before the pool's closed =)
+  write_note(f"Done!\n{u.chrono.note}")
+
+  pool.close()
+  pool.join()
+  mw.close()
+
+  if ckpt_mngr:
+    ckpt_mngr.wait_until_finished()
+
+  # Make sure all hosts stay up until the end of main.
+  u.sync()
+
+  u.maybe_cleanup_workdir(workdir, flags.FLAGS.cleanup, info)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/Tipsomaly/model/big_vision/trainers/proj/jet/train.py b/Tipsomaly/model/big_vision/trainers/proj/jet/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..46c82542409e0b15a9c25f20b9ffac7e1be72cf7
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/jet/train.py
@@ -0,0 +1,535 @@
+# Copyright 2024 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Training loop for Jet."""
+# pylint: disable=consider-using-from-import
+# pylint: disable=logging-fstring-interpolation
+
+import functools
+import importlib
+import multiprocessing.pool
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+import big_vision.evaluators.common as eval_common
+import big_vision.input_pipeline as input_pipeline
+import big_vision.optax as bv_optax
+import big_vision.sharding as bv_sharding
+import big_vision.utils as u
+from clu import parameter_overview
+import distrax
+import flax.linen as nn
+import jax
+from jax.experimental import mesh_utils
+from jax.experimental import multihost_utils
+from jax.experimental.array_serialization import serialization as array_serial
+import jax.numpy as jnp
+from ml_collections import config_flags
+import numpy as np
+import optax
+import tensorflow as tf
+
+
+from tensorflow.io import gfile
+
+
+config_flags.DEFINE_config_file(
+    "config", None, "Training configuration.", lock_config=True)
+
+flags.DEFINE_string("workdir", default=None, help="Work unit directory.")
+flags.DEFINE_boolean("cleanup", default=False,
+                     help="Delete workdir (only) after successful completion.")
+
+# Adds jax flags to the program.
+jax.config.parse_flags_with_absl()
+# Transfer guard will fail the program whenever that data between a host and
+# a device is transferred implicitly. This often catches subtle bugs that
+# cause slowdowns and memory fragmentation. Explicit transfers are done
+# with jax.device_put and jax.device_get.
+jax.config.update("jax_transfer_guard", "disallow")
+# Fixes design flaw in jax.random that may cause unnecessary d2d comms.
+jax.config.update("jax_threefry_partitionable", True)
+
+
+NamedSharding = jax.sharding.NamedSharding
+P = jax.sharding.PartitionSpec
+
+
+def main(argv):
+  del argv
+
+  # This is needed on multihost systems, but crashes on non-TPU single-host.
+  if os.environ.get("BV_JAX_INIT"):
+    jax.distributed.initialize()
+
+  # Make sure TF does not touch GPUs.
+  tf.config.set_visible_devices([], "GPU")
+
+  config = flags.FLAGS.config
+
+################################################################################
+#                                                                              #
+#                                Set up logging                                #
+#                                                                              #
+################################################################################
+
+  # Set up work directory and print welcome message.
+  workdir = flags.FLAGS.workdir
+  logging.info(
+      f"\u001b[33mHello from process {jax.process_index()} holding "
+      f"{jax.local_device_count()}/{jax.device_count()} devices and "
+      f"writing to workdir {workdir}.\u001b[0m")
+
+  save_ckpt_path = None
+  if workdir:  # Always create if requested, even if we may not write into it.
+    gfile.makedirs(workdir)
+    save_ckpt_path = os.path.join(workdir, "checkpoint.bv")
+
+  # The pool is used to perform misc operations such as logging in async way.
+  pool = multiprocessing.pool.ThreadPool()
+
+  # Here we register preprocessing ops from modules listed on `pp_modules`.
+  for m in config.get("pp_modules", ["ops_general", "ops_image", "ops_text"]):
+    importlib.import_module(f"big_vision.pp.{m}")
+
+  # Setup up logging and experiment manager.
+  xid, wid = -1, -1
+  fillin = lambda s: s
+  def info(s, *a):
+    logging.info("\u001b[33mNOTE\u001b[0m: " + s, *a)
+  def write_note(note):
+    if jax.process_index() == 0:
+      info("%s", note)
+
+  mw = u.BigVisionMetricWriter(xid, wid, workdir, config)
+
+  # Allow for things like timings as early as possible!
+  u.chrono.inform(measure=mw.measure, write_note=write_note)
+
+################################################################################
+#                                                                              #
+#                                Set up Mesh                                   #
+#                                                                              #
+################################################################################
+
+  # We rely on jax mesh_utils to organize devices, such that communication
+  # speed is the fastest for the last dimension, second fastest for the
+  # penultimate dimension, etc.
+  config_mesh = config.get("mesh", [("data", jax.device_count())])
+
+  # Sharding rules with default
+  sharding_rules = config.get("sharding_rules", [("act_batch", "data")])
+
+  mesh_axes, mesh_size = tuple(zip(*config_mesh))
+
+  # Because jax.utils do not support `-1` shape size.
+  mesh_size = np.array(jax.devices()).reshape(mesh_size).shape
+
+  device_mesh = mesh_utils.create_device_mesh(
+      mesh_size,
+      allow_split_physical_axes=config.get("mesh_allow_split_physical_axes",
+                                           False))
+
+  # Consistent device order is important to ensure correctness of various train
+  # loop components, such as input pipeline, update step, evaluators. The
+  # order presribed by the `devices_flat` variable should be used throughout
+  # the program.
+  devices_flat = device_mesh.flatten()
+
+################################################################################
+#                                                                              #
+#                                Input Pipeline                                #
+#                                                                              #
+################################################################################
+
+  write_note("Initializing train dataset...")
+  batch_size = config.input.batch_size
+  if batch_size % jax.device_count() != 0:
+    raise ValueError(f"Batch size ({batch_size}) must "
+                     f"be divisible by device number ({jax.device_count()})")
+  info("Global batch size %d on %d hosts results in %d local batch size. With "
+       "%d dev per host (%d dev total), that's a %d per-device batch size.",
+       batch_size, jax.process_count(), batch_size // jax.process_count(),
+       jax.local_device_count(), jax.device_count(),
+       batch_size // jax.device_count())
+
+  train_ds, ntrain_img = input_pipeline.training(config.input)
+
+  total_steps = u.steps("total", config, ntrain_img, batch_size)
+  def get_steps(name, default=ValueError, cfg=config):
+    return u.steps(name, cfg, ntrain_img, batch_size, total_steps, default)
+
+  u.chrono.inform(total_steps=total_steps, global_bs=batch_size,
+                  steps_per_epoch=ntrain_img / batch_size)
+
+  info("Running for %d steps, that means %f epochs",
+       total_steps, total_steps * batch_size / ntrain_img)
+
+  # Start input pipeline as early as possible.
+  n_prefetch = config.get("prefetch_to_device", 1)
+  train_iter = input_pipeline.start_global(train_ds, devices_flat, n_prefetch)
+
+################################################################################
+#                                                                              #
+#                           Create Model & Optimizer                           #
+#                                                                              #
+################################################################################
+
+  write_note("Creating model...")
+  model_mod = importlib.import_module(f"big_vision.models.{config.model_name}")
+  model = model_mod.Model(**config.get("model", {}))
+
+  first_batch = next(train_iter)
+  def init(rng, batch):
+    context = batch["label"][:, None] if "label" in batch else None
+    return model.init(rng,
+                      batch["image"],
+                      context=context,
+                      method=model.forward)["params"]
+
+  # This seed makes the Jax part of things (like model init) deterministic.
+  # However, full training still won't be deterministic, for example due to the
+  # tf.data pipeline not being deterministic even if we would set TF seed.
+  # See (internal link) for a fun read on what it takes.
+  rng = jax.random.PRNGKey(u.put_cpu(config.get("seed", 0)))
+
+  write_note("Inferring parameter shapes...")
+  rng, rng_init = jax.random.split(rng)
+  params_shape = jax.eval_shape(init, rng_init, first_batch)
+
+  write_note("Inferring optimizer state shapes...")
+  tx, sched_fns = bv_optax.make(config, nn.unbox(params_shape), sched_kw=dict(
+      total_steps=total_steps, batch_size=batch_size, data_size=ntrain_img))
+  opt_shape = jax.eval_shape(tx.init, params_shape)
+  # We jit this, such that the arrays are created on the CPU, not device[0].
+  sched_fns_cpu = [u.jit_cpu()(sched_fn) for sched_fn in sched_fns]
+
+  if jax.process_index() == 0:
+    num_params = sum(np.prod(p.shape) for p in jax.tree.leaves(params_shape))
+    mw.measure("num_params", num_params)
+
+################################################################################
+#                                                                              #
+#                               Shard & Transfer                               #
+#                                                                              #
+################################################################################
+
+  write_note("Creating device mesh...")
+  mesh = jax.sharding.Mesh(device_mesh, mesh_axes)
+  repl_sharding = jax.sharding.NamedSharding(mesh, P())
+
+  write_note("Inferring shardings...")
+  train_state_shape = {"params": params_shape, "opt": opt_shape}
+
+  strategy = config.get("sharding_strategy", [(".*", "replicate")])
+  with nn.logical_axis_rules(sharding_rules):
+    train_state_sharding = bv_sharding.infer_sharding(
+        train_state_shape, strategy=strategy, mesh=mesh)
+
+  write_note("Transferring train_state to devices...")
+  # RNG is always replicated
+  rng_init = u.reshard(rng_init, repl_sharding)
+
+  # Parameters and the optimizer are now global (distributed) jax arrays.
+  params = jax.jit(init, out_shardings=train_state_sharding["params"])(
+      rng_init, first_batch)
+  opt = jax.jit(tx.init, out_shardings=train_state_sharding["opt"])(params)
+
+  rng, rng_loop = jax.random.split(rng, 2)
+  rng_loop = u.reshard(rng_loop, repl_sharding)
+  del rng  # not used anymore, so delete it.
+
+  # At this point we have everything we need to form a train state. It contains
+  # all the parameters that are passed and updated by the main training step.
+  # From here on, we have no need for Flax AxisMetadata (such as partitioning).
+  train_state = nn.unbox({"params": params, "opt": opt})
+  del params, opt  # Delete to avoid memory leak or accidental reuse.
+
+  write_note("Logging parameter overview...")
+  parameter_overview.log_parameter_overview(
+      train_state["params"], msg="Init params",
+      include_stats="global", jax_logging_process=0)
+
+################################################################################
+#                                                                              #
+#                                 Update Step                                  #
+#                                                                              #
+################################################################################
+
+  def _bits_per_dim(logits, logdet, dim_count, reduce=True):
+    normal = distrax.Normal(0.0, 1.0)
+    nll = -normal.log_prob(logits)
+    nll = jnp.sum(nll + np.log(127.5), axis=range(1, nll.ndim))
+
+    bits = nll - logdet
+
+    reduce_fn = jnp.mean if reduce else lambda x: x
+    normalizer = np.log(2) * dim_count
+
+    logging.info("nll: %s", nll.shape)
+    return (reduce_fn(bits) / normalizer,
+            reduce_fn(nll) / normalizer,
+            reduce_fn(logdet) / normalizer)
+
+  @functools.partial(
+      jax.jit,
+      donate_argnums=(0,),
+      out_shardings=(train_state_sharding, repl_sharding))
+  def update_fn(train_state, rng, batch):
+    """Update step."""
+
+    step_count = bv_optax.get_count(train_state["opt"], jittable=True)
+    rng = jax.random.fold_in(rng, step_count)
+
+    rng_input, rng_model, rng_cond_drop = jax.random.split(rng, 3)
+    images = (batch["image"] +
+              jax.random.uniform(
+                  rng_input,
+                  shape=batch["image"].shape,
+                  minval=0.0,
+                  maxval=1.0 / 127.5))
+
+    def loss_fn(params):
+      context = None
+      if "label" in batch:
+        drop = (config.get("condition_drop_prob", 0.1) >
+                jax.random.uniform(rng_cond_drop, (), jnp.float32))
+        context = batch["label"][:, None] * (~drop)
+      logits, logdet = model.apply(
+          {"params": params}, images,
+          rngs={"dropout": rng_model},
+          context=context,
+          method=model.forward)
+      bits, nll, logdet = _bits_per_dim(logits, logdet,
+                                        np.prod(images.shape[1:]))
+      return bits, {"bits": bits, "nll": nll, "logdet": logdet}
+
+    params, opt = train_state["params"], train_state["opt"]
+    (loss, extra), grads = jax.value_and_grad(loss_fn, has_aux=True)(params)
+    updates, opt = tx.update(grads, opt, params)
+    params = optax.apply_updates(params, updates)
+
+    measurements = {"training_loss": loss, **extra}
+    gs = jax.tree.leaves(bv_optax.replace_frozen(config.schedule, grads, 0.))
+    measurements["l2_grads"] = jnp.sqrt(sum([jnp.sum(g * g) for g in gs]))
+    ps = jax.tree.leaves(params)
+    measurements["l2_params"] = jnp.sqrt(sum([jnp.sum(p * p) for p in ps]))
+    us = jax.tree.leaves(updates)
+    measurements["l2_updates"] = jnp.sqrt(sum([jnp.sum(u * u) for u in us]))
+
+    return {"params": params, "opt": opt}, measurements
+
+################################################################################
+#                                                                              #
+#                               Load Checkpoint                                #
+#                                                                              #
+################################################################################
+
+  # Decide how to initialize training. The order is important.
+  # 1. Always resumes from the existing checkpoint, e.g. resumes a finetune job.
+  # 2. Resume from a previous checkpoint, e.g. start a cooldown training job.
+  # 3. Initialize model from something, e,g, start a fine-tuning job.
+  # 4. Train from scratch.
+  resume_ckpt_path = None
+  if save_ckpt_path and gfile.exists(f"{save_ckpt_path}-LAST"):
+    resume_ckpt_path = save_ckpt_path
+  elif config.get("resume"):
+    resume_ckpt_path = fillin(config.resume)
+
+  ckpt_mngr = None
+  if save_ckpt_path or resume_ckpt_path:
+    ckpt_mngr = array_serial.GlobalAsyncCheckpointManager()
+
+  if resume_ckpt_path:
+    write_note(f"Resuming training from checkpoint {resume_ckpt_path}...")
+    jax.tree.map(lambda x: x.delete(), train_state)
+    del train_state
+    shardings = {
+        **train_state_sharding,
+        "chrono": jax.tree.map(lambda _: repl_sharding,
+                               u.chrono.save()),
+    }
+    loaded = u.load_checkpoint_ts(
+        resume_ckpt_path, tree=shardings, shardings=shardings)
+    train_state = {key: loaded[key] for key in train_state_sharding.keys()}
+
+    u.chrono.load(jax.device_get(loaded["chrono"]))
+    del loaded
+  elif config.get("model_init"):
+    write_note(f"Initialize model from {config.model_init}...")
+    # TODO: when updating the `load` API soon, do pass and request the
+    # full `train_state` from it. Examples where useful: VQVAE, BN.
+    train_state["params"] = model_mod.load(
+        train_state["params"], config.model_init, config.get("model"),
+        **config.get("model_load", {}))
+
+    # load has the freedom to return params not correctly sharded. Think of for
+    # example ViT resampling position embedings on CPU as numpy arrays.
+    train_state["params"] = u.reshard(
+        train_state["params"], train_state_sharding["params"])
+
+    parameter_overview.log_parameter_overview(
+        train_state["params"], msg="restored params",
+        include_stats="global", jax_logging_process=0)
+
+
+################################################################################
+#                                                                              #
+#                                 Setup Evals                                  #
+#                                                                              #
+################################################################################
+
+  # We do not jit/pmap this function, because it is passed to evaluator that
+  # does it later. We output as many intermediate tensors as possible for
+  # maximal flexibility. Later `jit` will prune out things that are not needed.
+  def eval_loss_fn(train_state, batch):
+    noise = jax.lax.rng_uniform(0.0, 1.0, batch["image"].shape) / 127.5
+    logits, logdet = model.apply(
+        {"params": train_state["params"]},
+        batch["image"] + noise,
+        context=batch["label"][:, None] if "label" in batch else None,
+        method=model.forward)
+    bits, nll, logdet = _bits_per_dim(
+        logits, logdet,
+        np.prod(batch["image"].shape[1:]),
+        reduce=False)
+    return {"bits": bits, "nll": nll, "logdet": logdet}
+
+  eval_fns = {
+      "loss": eval_loss_fn,
+  }
+
+  # Only initialize evaluators when they are first needed.
+  @functools.lru_cache(maxsize=None)
+  def evaluators():
+    return eval_common.from_config(
+        config, eval_fns,
+        lambda s: write_note(f"Init evaluator: {s}…\n{u.chrono.note}"),
+        lambda key, cfg: get_steps(key, default=None, cfg=cfg),
+        devices_flat,
+    )
+
+  # At this point we need to know the current step to see whether to run evals.
+  write_note("Inferring the first step number...")
+  first_step_device = bv_optax.get_count(train_state["opt"], jittable=True)
+  first_step = int(jax.device_get(first_step_device))
+  u.chrono.inform(first_step=first_step)
+
+  # Note that training can be pre-empted during the final evaluation (i.e.
+  # just after the final checkpoint has been written to disc), in which case we
+  # want to run the evals.
+  if first_step in (total_steps, 0):
+    write_note("Running initial or final evals...")
+    mw.step_start(first_step)
+    for (name, evaluator, _, prefix) in evaluators():
+      if config.evals[name].get("skip_first") and first_step != total_steps:
+        continue
+      write_note(f"{name} evaluation...\n{u.chrono.note}")
+      with u.chrono.log_timing(f"z/secs/eval/{name}"):
+        with mesh, nn.logical_axis_rules(sharding_rules):
+          for key, value in evaluator.run(train_state):
+            mw.measure(f"{prefix}{key}", value)
+
+################################################################################
+#                                                                              #
+#                                  Train Loop                                  #
+#                                                                              #
+################################################################################
+
+  prof = None  # Keeps track of start/stop of profiler state.
+
+  write_note("Starting training loop, compiling the first step...")
+  for step, batch in zip(range(first_step + 1, total_steps + 1), train_iter):
+    mw.step_start(step)
+
+    with jax.profiler.StepTraceAnnotation("train_step", step_num=step):
+      with u.chrono.log_timing("z/secs/update0", noop=step > first_step + 1):
+        with mesh, nn.logical_axis_rules(sharding_rules):
+          train_state, measurements = update_fn(train_state, rng_loop, batch)
+
+    # On the first host, let's always profile a handful of early steps.
+    if jax.process_index() == 0:
+      prof = u.startstop_prof(prof, step, first_step, get_steps("log_training"))
+
+    # Report training progress
+    if (u.itstime(step, get_steps("log_training"), total_steps, host=0)
+        or u.chrono.warmup and jax.process_index() == 0):
+      for i, sched_fn_cpu in enumerate(sched_fns_cpu):
+        mw.measure(f"global_schedule{i if i else ''}",
+                   sched_fn_cpu(u.put_cpu(step - 1)))
+      measurements = jax.device_get(measurements)
+      for name, value in measurements.items():
+        mw.measure(name, value)
+      u.chrono.tick(step)
+      if not np.isfinite(measurements["training_loss"]):
+        raise RuntimeError(f"The loss became nan or inf somewhere within steps "
+                           f"[{step - get_steps('log_training')}, {step}]")
+
+    # Checkpoint saving
+    keep_ckpt_steps = get_steps("keep_ckpt", None) or total_steps
+    if save_ckpt_path and (
+        (keep := u.itstime(step, keep_ckpt_steps, total_steps, first=False))
+        or u.itstime(step, get_steps("ckpt", None), total_steps, first=True)
+    ):
+      u.chrono.pause(wait_for=train_state)
+
+      # Copy because we add extra stuff to the checkpoint.
+      ckpt = {**train_state}
+
+      # To save chrono state correctly and safely in a multihost setup, we
+      # broadcast the state to all hosts and convert it to a global array.
+      with jax.transfer_guard("allow"):
+        chrono_ckpt = multihost_utils.broadcast_one_to_all(u.chrono.save())
+      chrono_shardings = jax.tree.map(lambda _: repl_sharding, chrono_ckpt)
+      ckpt = ckpt | {"chrono": u.reshard(chrono_ckpt, chrono_shardings)}
+
+      u.save_checkpoint_ts(ckpt_mngr, ckpt, save_ckpt_path, step, keep)
+      u.chrono.resume()
+
+    for (name, evaluator, log_steps, prefix) in evaluators():
+      if u.itstime(step, log_steps, total_steps, first=False, last=True):
+        u.chrono.pause(wait_for=train_state)
+        u.chrono.tick(step)  # Record things like epoch number, core hours etc.
+        write_note(f"{name} evaluation...\n{u.chrono.note}")
+        with u.chrono.log_timing(f"z/secs/eval/{name}"):
+          with mesh, nn.logical_axis_rules(sharding_rules):
+            for key, value in evaluator.run(train_state):
+              mw.measure(f"{prefix}{key}", jax.device_get(value))
+        u.chrono.resume()
+    mw.step_end()
+
+  # Always give a chance to stop the profiler, no matter how things ended.
+  # TODO: can we also do this when dying of an exception like OOM?
+  if jax.process_index() == 0 and prof is not None:
+    u.startstop_prof(prof)
+
+  # Last note needs to happen before the pool's closed =)
+  write_note(f"Done!\n{u.chrono.note}")
+
+  pool.close()
+  pool.join()
+  mw.close()
+  if ckpt_mngr:
+    ckpt_mngr.wait_until_finished()
+
+  # Make sure all hosts stay up until the end of main.
+  u.sync()
+
+  u.maybe_cleanup_workdir(workdir, flags.FLAGS.cleanup, info)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/Tipsomaly/model/big_vision/trainers/proj/jetformer/predict_fns.py b/Tipsomaly/model/big_vision/trainers/proj/jetformer/predict_fns.py
new file mode 100644
index 0000000000000000000000000000000000000000..dc311551ec2c2853b57573663d59e62f2dfb1415
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/jetformer/predict_fns.py
@@ -0,0 +1,297 @@
+# Copyright 2024 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Prediction functions for JetFormer."""
+# pylint: disable=consider-using-from-import
+from absl import logging
+import big_vision.models.proj.givt.parallel_decode as parallel_decode
+import einops
+import flax.linen as nn
+import jax
+import jax.numpy as jnp
+import numpy as np
+
+
+# Utils to encode and decode images as latents.
+def encode_images(  # pylint: disable=missing-function-docstring
+    params, images, *, adaptor, patch_pca, rngs, reparametrize: bool):
+  # Apply patch_pca module.
+  x, logvar = patch_pca.apply({}, images, method=patch_pca.encode, rngs=rngs)
+  if reparametrize:
+    x = patch_pca.apply(
+        {}, x, logvar, method=patch_pca.reparametrize, rngs=rngs)
+
+  # Apply invertible network.
+  if adaptor is not None:
+    x = unflatten_latents(x)
+    x, _ = adaptor.apply({"params": params}, x, method=adaptor.forward)
+    x = flatten_latents(x)
+  return x
+
+
+def decode_images(params, x, *, adaptor, patch_pca):
+  # Apply invertible network backwards.
+  if adaptor is not None:
+    x = unflatten_latents(x)
+    x, _ = adaptor.apply({"params": params}, x, method=adaptor.inverse)
+    x = flatten_latents(x)
+
+  # Apply patch_pca module backwards.
+  images = patch_pca.apply({"params": {}}, x, method=patch_pca.decode)
+  return images
+
+
+def unflatten_latents(x):
+  hw = int(x.shape[1] ** 0.5)
+  return einops.rearrange(x, "b (h w) c -> b h w c", h=hw, w=hw)
+
+
+def flatten_latents(x):
+  return einops.rearrange(x, "b h w c -> b (h w) c")
+
+
+# Utils to sample the decoder.
+def sample_image_latents(
+    params, batch, *, model,
+    decode_len=256, temperature=1.0, temperature_probs=1.0,
+    cfg_weight=None, rng=None):
+  """Sample image latents conditioned on text prompt."""
+  rng = rng if rng is not None else jax.random.PRNGKey(
+      jax.lax.rng_uniform(0, np.iinfo(np.int32).max, tuple()))
+  # The following makes sure to skip CFG if cfg_weight is a Python float or int
+  # and is 0, or is None, but performs CFG if cfg_weight is a traced array.
+  if isinstance(cfg_weight, (int, float)):
+    do_cfg = (cfg_weight != 0)
+  else:
+    do_cfg = cfg_weight is not None
+  logging.info("Sampling with cfg_weight=%r", cfg_weight)
+
+  def _sample_prelogits(model, pre_logits):
+    rng = model.make_rng("sample")
+    logits = model.img_logits(pre_logits)
+    if do_cfg:
+      # get_pdf is not jax-friendly as it returns an opaque object so we have
+      # to split logits before calling it.
+      logits_cond, logits_uncond = einops.rearrange(
+          logits, "(b s) ... -> s b ...", s=2)
+      pdf_cond = model.get_pdf(logits_cond,
+                               temperature_scales=temperature,
+                               temperature_probs=temperature_probs)
+      pdf_uncond = model.get_pdf(logits_uncond,
+                                 temperature_scales=temperature,
+                                 temperature_probs=temperature_probs)
+      pdf = parallel_decode.CFGDensity(
+          pdf_cond, pdf_uncond,
+          w=cfg_weight, rng=rng)
+      samples = pdf.sample(seed=rng)
+      logprobs = pdf.log_prob(samples)
+      logprobs = jnp.sum(logprobs, axis=2)  # [B, N, C] -> [B, N]
+      return jnp.repeat(samples, 2, axis=0), jnp.repeat(logprobs, 2, axis=0)
+    else:
+      pdf = model.get_pdf(logits,
+                          temperature_scales=temperature,
+                          temperature_probs=temperature_probs)
+      samples = pdf.sample(seed=rng)
+      logprobs = pdf.log_prob(samples)
+      return samples, logprobs
+
+  # Main sample logic where "model" has been bound.
+  def _sample(model):
+    text = batch["text"]
+    text_mask = batch.get("text_mask", jnp.full(text.shape, True))
+
+    # Add unconditional sequences if needed [x, x_uncond, y, y_uncond, ...]
+    if do_cfg:
+      drop_prefix = jnp.tile(jnp.array([False, True]), text.shape[0])
+      text = jnp.repeat(text, 2, axis=0)
+      # Overridden to full mask when dropping the label in embed_image_and_text.
+      text_mask = jnp.repeat(text_mask, 2, axis=0)
+    else:
+      drop_prefix = None
+
+    # Prepare inputs to prefill the decoder. Pass images of seq_len=0 so it
+    # prefills up to the BOI of images.
+    batch_size, _ = text.shape
+    images = jnp.zeros((batch_size, 0, model.out_dim))  # zero-len images.
+    text_first_mask = jnp.full((batch_size,), True)
+    x, attn_mask, input_mask = model.embed_image_and_text(
+        text, images,
+        text_first_mask=text_first_mask,
+        text_input_mask=text_mask,
+        drop_prefix=drop_prefix, shift=False)
+
+    # Prefill the decoder cache with 'x' and sample the first output.
+    cache_size = x.shape[1] + decode_len - 1
+    last_prelogits = model.prefill_cache(x, attn_mask, input_mask,
+                                         cache_size=cache_size)[:, -1:]
+    tokens, logp = _sample_prelogits(model, last_prelogits)
+
+    # Init loop state with the token decoded during prefill.
+    batch_size, _, prelogits_dim = last_prelogits.shape
+    out_prelogits = jnp.zeros((batch_size, decode_len, prelogits_dim))
+    out_logp = jnp.zeros((batch_size, decode_len))
+    out_tokens = jnp.zeros((batch_size, decode_len, tokens.shape[2]))
+
+    out_prelogits = out_prelogits.at[:, 0:1].set(last_prelogits)
+    out_tokens = out_tokens.at[:, 0:1].set(tokens)
+    out_logp = out_logp.at[:, 0:1].set(logp)
+
+    # Most callers will only need the out_tokens (i.e. the sample latents).
+    # This code pattern allows one to easily add other outputs if needed.
+    # last_tokens is a carry variable to avoid a dynamic lookup in the loop.
+    state = {
+        "out_prelogits": out_prelogits,  # [B, decode_len, D]
+        "out_tokens": out_tokens,        # [B, decode_len, H]
+        "out_logp": out_logp,            # [B, decode_len]
+        "last_tokens": tokens,           # [B, 1, H]
+    }
+
+    # Loop to decode remaining tokens. This function will be called by nn.scan
+    # with xs = 1, 2, ..., decode_len and update the state accordingly.
+    def loop_step(model, state, xs):
+      x = model.img_emb(state["last_tokens"])
+      prelogits = model.extend_cache(x)
+      tokens, logp = _sample_prelogits(model, prelogits)
+      # Update state with the new tokens.
+      state["out_prelogits"] = jax.lax.dynamic_update_slice(
+          state["out_prelogits"], prelogits, (0, xs, 0))
+      state["out_tokens"] = jax.lax.dynamic_update_slice(
+          state["out_tokens"], tokens, (0, xs, 0))
+      state["out_logp"] = jax.lax.dynamic_update_slice(
+          state["out_logp"], logp, (0, xs))
+      state["last_tokens"] = tokens
+      return state, None
+    # Note that besides "state", the "cache" variables are also carried and
+    # that the sample rng state is splitted so each call sees a different rng.
+    xs = jnp.arange(1, decode_len)
+    state, _ = nn.scan(
+        loop_step, variable_broadcast="params", variable_carry="cache",
+        split_rngs={"sample": True})(model, state, xs)
+    del state["last_tokens"]  # Not needed anymore.
+
+    # Remove unconditional sequences if needed.
+    if do_cfg:
+      state = jax.tree.map(lambda x: x[::2], state)
+
+    return state
+
+  out, _ = nn.apply(_sample, model, mutable=["cache"])(
+      {"params": params}, rngs={"sample": rng})
+
+  return out
+
+
+def sample_text(
+    params, batch, *, model,
+    decode_len=64, temperature=1.0, rng=None):
+  """Sample text continuation conditioned on image."""
+  rng = rng if rng is not None else jax.random.PRNGKey(
+      jax.lax.rng_uniform(0, np.iinfo(np.int32).max, tuple()))
+
+  def _sample_prelogits(model, pre_logits):
+    rng = model.make_rng("sample")
+    logits = model.text_logits(pre_logits)
+    # Sample using temperature.
+    # TODO: Add top-k/top-p/etc...
+    modified_pmf = model.get_pmf(logits / temperature)
+    samples = modified_pmf.sample(seed=rng)
+    # Return logp according to the original unmodified distribution.
+    pmf = model.get_pmf(logits)
+    logprobs = pmf.log_prob(samples)
+    return samples, logprobs
+
+  # Main sample logic where "model" has been bound.
+  def _sample(model):
+    images = batch["image_latents"]
+
+    # TODO: Add support for CFG.
+    drop_prefix = None
+
+    # Prepare inputs to prefill the decoder with [image, Optional[text]].
+    batch_size, _, _ = images.shape
+    text_first_mask = jnp.full((batch_size,), False)
+
+    if batch["text"] is None:
+      # Zero-len text, so it prefills up to the BOS of text.
+      text = jnp.full((batch_size, 0), 0)
+      text_input_mask = jnp.full((batch_size, 0), True)
+    else:
+      # If text is present, it will prefill the BOS of text and the tokens
+      # which are true in the text_mask (i.e. each example can have a variable
+      # number of text tokens prefilled).
+      text = batch["text"]
+      text_input_mask = batch["text_mask"]
+
+    x, attn_mask, input_mask = model.embed_image_and_text(
+        text, images,
+        text_first_mask=text_first_mask,
+        text_input_mask=text_input_mask,
+        drop_prefix=drop_prefix, shift=False)
+
+    # Prefill the decoder cache with 'x' and sample the first output.
+    cache_size = x.shape[1] + decode_len - 1
+    last_prelogits = model.prefill_cache(
+        x, attn_mask, input_mask, cache_size=cache_size)[:, -1:]
+    tokens, logp = _sample_prelogits(model, last_prelogits)
+
+    # Init loop state with the token decoded during prefill.
+    batch_size, _, prelogits_dim = last_prelogits.shape
+    out_prelogits = jnp.zeros((batch_size, decode_len, prelogits_dim),
+                              dtype=last_prelogits.dtype)
+    out_logp = jnp.zeros((batch_size, decode_len), dtype=logp.dtype)
+    out_tokens = jnp.zeros((batch_size, decode_len), dtype=tokens.dtype)
+
+    out_prelogits = out_prelogits.at[:, 0:1].set(last_prelogits)
+    out_tokens = out_tokens.at[:, 0:1].set(tokens)
+    out_logp = out_logp.at[:, 0:1].set(logp)
+
+    # Most callers will only need the out_tokens (i.e. the sample latents).
+    # This code pattern allows one to easily add other outputs if needed.
+    # last_tokens is a carry variable to avoid a dynamic lookup in the loop.
+    state = {
+        "out_prelogits": out_prelogits,  # [B, decode_len, D]
+        "out_tokens": out_tokens,        # [B, decode_len]
+        "out_logp": out_logp,            # [B, decode_len]
+        "last_tokens": tokens,           # [B, 1]
+    }
+
+    # Loop to decode remaining tokens. This function will be called by nn.scan
+    # with xs = 1, 2, ..., decode_len and update the state accordingly.
+    def loop_step(model, state, xs):
+      x = model.text_emb(state["last_tokens"])
+      prelogits = model.extend_cache(x)
+      tokens, logp = _sample_prelogits(model, prelogits)
+      # Update state with the new tokens.
+      state["out_prelogits"] = jax.lax.dynamic_update_slice(
+          state["out_prelogits"], prelogits, (0, xs, 0))
+      state["out_tokens"] = jax.lax.dynamic_update_slice(
+          state["out_tokens"], tokens, (0, xs))
+      state["out_logp"] = jax.lax.dynamic_update_slice(
+          state["out_logp"], logp, (0, xs))
+      state["last_tokens"] = tokens
+      return state, None
+    # Note that besides "state", the "cache" variables are also carried and
+    # that the sample rng state is splitted so each call sees a different rng.
+    xs = jnp.arange(1, decode_len)
+    state, _ = nn.scan(
+        loop_step, variable_broadcast="params", variable_carry="cache",
+        split_rngs={"sample": True})(model, state, xs)
+    del state["last_tokens"]  # Not needed anymore.
+
+    return state
+
+  out, _ = nn.apply(_sample, model, mutable=["cache"])(
+      {"params": params}, rngs={"sample": rng})
+
+  return out
diff --git a/Tipsomaly/model/big_vision/trainers/proj/jetformer/train.py b/Tipsomaly/model/big_vision/trainers/proj/jetformer/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..83b94d161bc278153a8838f13ecef10e633358a8
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/jetformer/train.py
@@ -0,0 +1,887 @@
+# Copyright 2024 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Training loop for JetFormer."""
+# pylint: disable=consider-using-from-import
+# pylint: disable=logging-fstring-interpolation
+
+import functools
+import importlib
+import multiprocessing.pool
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+import big_vision.evaluators.common as eval_common
+import big_vision.input_pipeline as input_pipeline
+import big_vision.optax as bv_optax
+import big_vision.sharding as bv_sharding
+import big_vision.trainers.proj.jetformer.predict_fns as predict_fns
+import big_vision.utils as u
+from clu import parameter_overview
+import distrax
+import flax
+import flax.linen as nn
+import jax
+from jax.experimental import multihost_utils
+from jax.experimental.array_serialization import serialization as array_serial
+import jax.numpy as jnp
+from ml_collections import config_flags
+import numpy as np
+import optax
+import tensorflow as tf
+
+from tensorflow.io import gfile
+
+
+config_flags.DEFINE_config_file(
+    "config", None, "Training configuration.", lock_config=True)
+
+flags.DEFINE_string("workdir", default=None, help="Work unit directory.")
+flags.DEFINE_boolean("cleanup", default=False,
+                     help="Delete workdir (only) after successful completion.")
+
+# Adds jax flags to the program.
+jax.config.parse_flags_with_absl()
+# Transfer guard will fail the program whenever that data between a host and
+# a device is transferred implicitly. This often catches subtle bugs that
+# cause slowdowns and memory fragmentation. Explicit transfers are done
+# with jax.device_put and jax.device_get.
+jax.config.update("jax_transfer_guard", "disallow")
+# Fixes design flaw in jax.random that may cause unnecessary d2d comms.
+jax.config.update("jax_threefry_partitionable", True)
+
+
+NamedSharding = jax.sharding.NamedSharding
+P = jax.sharding.PartitionSpec
+
+
+def main(argv):
+  del argv
+
+  # This is needed on multihost systems, but crashes on non-TPU single-host.
+  if os.environ.get("BV_JAX_INIT"):
+    jax.distributed.initialize()
+
+  # Make sure TF does not touch GPUs.
+  tf.config.set_visible_devices([], "GPU")
+
+  config = flags.FLAGS.config
+
+################################################################################
+#                                                                              #
+#                                Set up logging                                #
+#                                                                              #
+################################################################################
+
+  # Set up work directory and print welcome message.
+  workdir = flags.FLAGS.workdir
+  logging.info(
+      f"\u001b[33mHello from process {jax.process_index()} holding "
+      f"{jax.local_device_count()}/{jax.device_count()} devices and "
+      f"writing to workdir {workdir}.\u001b[0m")
+
+  save_ckpt_path = None
+  if workdir:  # Always create if requested, even if we may not write into it.
+    gfile.makedirs(workdir)
+    save_ckpt_path = os.path.join(workdir, "checkpoint.bv")
+
+  # The pool is used to perform misc operations such as logging in async way.
+  pool = multiprocessing.pool.ThreadPool()
+
+  # Here we register preprocessing ops from modules listed on `pp_modules`.
+  for m in config.get("pp_modules", ["ops_general", "ops_image", "ops_text"]):
+    importlib.import_module(f"big_vision.pp.{m}")
+
+  # Setup up logging and experiment manager.
+  xid, wid = -1, -1
+  fillin = lambda s: s
+  def info(s, *a):
+    logging.info("\u001b[33mNOTE\u001b[0m: " + s, *a)
+  def write_note(note):
+    if jax.process_index() == 0:
+      info("%s", note)
+
+  mw = u.BigVisionMetricWriter(xid, wid, workdir, config)
+
+  # Allow for things like timings as early as possible!
+  u.chrono.inform(measure=mw.measure, write_note=write_note)
+
+################################################################################
+#                                                                              #
+#                                Set up Mesh                                   #
+#                                                                              #
+################################################################################
+
+  # We rely on jax mesh_utils to organize devices, such that communication
+  # speed is the fastest for the last dimension, second fastest for the
+  # penultimate dimension, etc.
+  config_mesh = config.get("mesh", [("data", jax.device_count())])
+
+  # Sharding rules with default
+  sharding_rules = config.get("sharding_rules", [("act_batch", "data")])
+
+  write_note("Creating device mesh...")
+  mesh = u.create_device_mesh(
+      config_mesh,
+      allow_split_physical_axes=config.get("mesh_allow_split_physical_axes",
+                                           False))
+  repl_sharding = jax.sharding.NamedSharding(mesh, P())
+
+  # Consistent device order is important to ensure correctness of various train
+  # loop components, such as input pipeline, update step, evaluators. The
+  # order presribed by the `devices_flat` variable should be used throughout
+  # the program.
+  devices_flat = mesh.devices.flatten()
+
+################################################################################
+#                                                                              #
+#                                Input Pipeline                                #
+#                                                                              #
+################################################################################
+
+  write_note("Initializing train dataset...")
+  batch_size = config.input.batch_size
+  if batch_size % jax.device_count() != 0:
+    raise ValueError(f"Batch size ({batch_size}) must "
+                     f"be divisible by device number ({jax.device_count()})")
+  info("Global batch size %d on %d hosts results in %d local batch size. With "
+       "%d dev per host (%d dev total), that's a %d per-device batch size.",
+       batch_size, jax.process_count(), batch_size // jax.process_count(),
+       jax.local_device_count(), jax.device_count(),
+       batch_size // jax.device_count())
+
+  train_ds, ntrain_img = input_pipeline.training(config.input)
+
+  total_steps = u.steps("total", config, ntrain_img, batch_size)
+  def get_steps(name, default=ValueError, cfg=config):
+    return u.steps(name, cfg, ntrain_img, batch_size, total_steps, default)
+
+  u.chrono.inform(total_steps=total_steps, global_bs=batch_size,
+                  steps_per_epoch=ntrain_img / batch_size)
+
+  info("Running for %d steps, that means %f epochs",
+       total_steps, total_steps * batch_size / ntrain_img)
+
+  # Start input pipeline as early as possible.
+  n_prefetch = config.get("prefetch_to_device", 1)
+  train_iter = input_pipeline.start_global(train_ds, devices_flat, n_prefetch)
+
+################################################################################
+#                                                                              #
+#                           Create Model & Optimizer                           #
+#                                                                              #
+################################################################################
+
+  assert config.patch_pca.model_name == "proj.jetformer.patch_pca", (
+      "This trainer only supports proj.jetformer.patch_pca as an embedder."
+  )
+  write_note(f"Creating {config.patch_pca.model_name} model...")
+  pca_mod = importlib.import_module(
+      f"big_vision.models.{config.patch_pca.model_name}")
+  patch_pca = pca_mod.Model(**config.patch_pca.get("model", {}))
+
+  # Apply PCA - does not require any learnable parameters.
+  def patch_pca_encode(images, rng=None, reparametrize=True):
+    mu, logvar = patch_pca.apply(
+        {"params": {}}, images, method=patch_pca.encode, rngs=rng)
+    if reparametrize:
+      assert rng is not None and "dropout" in rng
+      return patch_pca.apply({"params": {}}, mu, logvar,
+                             method=patch_pca.reparametrize, rngs=rng)
+    return mu
+
+  write_note(f"Creating {config.model_name} model...")
+  model_mod = importlib.import_module(f"big_vision.models.{config.model_name}")
+  model_config = config.get("model", {})
+  model = model_mod.Model(**model_config)
+
+  if config.get("adaptor_name"):
+    write_note(f"Creating {config.adaptor_name} model...")
+    adaptor_mod = importlib.import_module(
+        f"big_vision.models.{config.adaptor_name}")
+    adaptor = adaptor_mod.Model(**config.adaptor.model)
+  else:
+    adaptor = None
+
+  def adaptor_apply(params, sequence, inverse=False):
+    # Apply NVP and ensure compatible input/output format.
+    sequence = predict_fns.unflatten_latents(sequence)
+    assert hasattr(adaptor, "forward") and hasattr(adaptor, "inverse")
+    sequence, sum_log_det = adaptor.apply(
+        {"params": params}, sequence,
+        method=adaptor.inverse if inverse else adaptor.forward)
+    sequence = predict_fns.flatten_latents(sequence)
+    return sequence, sum_log_det
+
+  def _maybe_remove_latent_noise_dims(image_tokens):
+    if (noise_dim := config.get("latent_noise_dim", 0)) > 0:
+      image_tokens = image_tokens[..., :-noise_dim]
+      assert image_tokens.shape[-1] == model.out_dim
+    return image_tokens
+
+  def init(rng, batch=None):
+    # TODO: Update init function with new arguments
+    def _get_dummy_input(input_name, dtype=jnp.int64):
+      if batch is not None:
+        return batch.get(input_name)
+      elif input_name in train_ds.element_spec:
+        return jnp.zeros(train_ds.element_spec[input_name].shape, dtype=dtype)
+      return None
+
+    images = _get_dummy_input("image", dtype=jnp.float32)
+    text = _get_dummy_input("text")
+    assert images is not None and text is not None
+
+    image_tokens = patch_pca_encode(images, rng={"dropout": rng})
+
+    if adaptor is not None:
+      rng, rng_adaptor = jax.random.split(rng)
+      image_tokens = predict_fns.unflatten_latents(image_tokens)
+      (image_tokens, _), adaptor_variables = adaptor.init_with_output(
+          rng_adaptor, image_tokens, method=adaptor.forward)
+      params_adaptor = flax.core.unfreeze(adaptor_variables["params"])
+      image_tokens = predict_fns.flatten_latents(image_tokens)
+    else:
+      params_adaptor = {}
+
+    image_tokens = _maybe_remove_latent_noise_dims(image_tokens)
+
+    text_first = jnp.full(images.shape[0], 0)
+    params = model.init(rng, text, image_tokens,
+                        text_input_mask=_get_dummy_input("text_mask"),
+                        text_first_mask=text_first)["params"]
+    params["params_adaptor"] = params_adaptor
+    return params
+
+  # This seed makes the Jax part of things (like model init) deterministic.
+  # However, full training still won't be deterministic, for example due to the
+  # tf.data pipeline not being deterministic even if we would set TF seed.
+  # See (internal link) for a fun read on what it takes.
+  rng = jax.random.PRNGKey(u.put_cpu(config.get("seed", 0)))
+
+  write_note("Inferring parameter shapes...")
+  rng, rng_init = jax.random.split(rng)
+  params_shape = jax.eval_shape(init, rng_init)
+
+  write_note("Inferring optimizer state shapes...")
+  tx, sched_fns = bv_optax.make(config, nn.unbox(params_shape), sched_kw=dict(
+      total_steps=total_steps, batch_size=batch_size, data_size=ntrain_img))
+  opt_shape = jax.eval_shape(tx.init, params_shape)
+  # We jit this, such that the arrays are created on the CPU, not device[0].
+  sched_fns_cpu = [u.jit_cpu()(sched_fn) for sched_fn in sched_fns]
+
+  if jax.process_index() == 0:
+    num_params = sum(np.prod(p.shape) for p in jax.tree.leaves(params_shape))
+    mw.measure("num_params", num_params)
+
+################################################################################
+#                                                                              #
+#                               Shard & Transfer                               #
+#                                                                              #
+################################################################################
+
+  write_note("Inferring shardings...")
+  train_state_shape = {"params": params_shape, "opt": opt_shape}
+
+  if config.get("ema_decay", 0.0) > 0.0:
+    write_note(f"Tracking parameter EMA with decay {config.ema_decay}.")
+    train_state_shape["params_ema"] = params_shape
+
+  strategy = config.get("sharding_strategy", [(".*", "replicate")])
+  with nn.logical_axis_rules(sharding_rules):
+    train_state_sharding = bv_sharding.infer_sharding(
+        train_state_shape, strategy=strategy, mesh=mesh)
+
+  write_note("Transferring train_state to devices...")
+  # RNG is always replicated
+  rng_init = u.reshard(rng_init, repl_sharding)
+
+  # Parameters and the optimizer are now global (distributed) jax arrays.
+  first_batch = next(train_iter)
+  params = jax.jit(init, out_shardings=train_state_sharding["params"])(
+      rng_init, first_batch)
+  opt = jax.jit(tx.init, out_shardings=train_state_sharding["opt"])(params)
+
+  rng, rng_loop = jax.random.split(rng, 2)
+  rng_loop = u.reshard(rng_loop, repl_sharding)
+  del rng  # not used anymore, so delete it.
+
+  train_state = {"params": params, "opt": opt}
+  if config.get("ema_decay", 0.0) > 0.0:
+    # Copy model parameters for EMA
+    train_state["params_ema"] = jax.tree.map(jnp.array, train_state["params"])
+
+  # At this point we have everything we need to form a train state. It contains
+  # all the parameters that are passed and updated by the main training step.
+  # From here on, we have no need for Flax AxisMetadata (such as partitioning).
+  train_state = nn.unbox(train_state)
+  del params, opt  # Delete to avoid memory leak or accidental reuse.
+
+  write_note("Logging parameter overview...")
+  parameter_overview.log_parameter_overview(
+      train_state["params"], msg="Init params",
+      include_stats="global", jax_logging_process=0)
+
+################################################################################
+#                                                                              #
+#                                 Update Step                                  #
+#                                                                              #
+################################################################################
+
+  # Define the loss function
+  def loss_fn(params, batch, rng, noise_scale=None, train=True):
+    text, images = batch["text"], batch["image"]
+    text_mask, text_loss = batch["text_mask"], batch["text_loss"]
+
+    rng, rng_dropout, rng_order, rng_droplabels, rng_noise = (
+        jax.random.split(rng, 5))
+
+    rng_dropout = {"dropout": rng_dropout}
+
+    batch_size = images.shape[0]
+    # 0 -> image first, 1 -> text first
+    text_first_mask = jax.random.bernoulli(
+        rng_order, config.get("text_prefix_prob", 0.5), (batch_size,))
+
+    if noise_scale is not None:
+      # Maybe skip noise on image prefix.
+      if not config.get("rgb_noise_on_image_prefix", True):
+        noise_scale = jnp.where(text_first_mask, noise_scale, 0.0)
+        noise_scale = noise_scale[:, None, None, None]  # [bs, h, w, 3]
+      # Convert images to [0, 255] add noise with std scale round to int and
+      # convert back to [-1, 1]. This way it is as if the input was added
+      # to the uint8 images in the preprocessing before the value_range(-1, 1).
+      images = jnp.round((images+1)*127.5)
+      images = images + noise_scale * jax.random.normal(rng_noise, images.shape)
+      images = jnp.round(images)
+      images = images/127.5 - 1
+
+    image_tokens = patch_pca_encode(images, rng_dropout)
+    if adaptor is not None:
+      # Use the (invertible) adaptor to map to a new latent sequence
+      image_tokens, sum_log_det = adaptor_apply(
+          params["params_adaptor"], image_tokens)
+    else:
+      sum_log_det = jnp.zeros((images.shape[0],),)
+
+    if (noise_dim := config.get("latent_noise_dim", 0)) > 0:
+      # Mapping the last noise_dim dimensions to a standard normal prior.
+      assert model.out_dim + noise_dim == image_tokens.shape[-1]
+      image_tokens, noise = jnp.split(image_tokens, [model.out_dim], axis=-1)
+      noise_pdf = distrax.Normal(0.0, 1.0)
+      noise_nll = -noise_pdf.log_prob(noise).sum(axis=(1, 2))
+    else:
+      noise_nll = 0.0
+
+    if train and (input_noise_std := config.get("input_noise_std", 0.0)) > 0.0:
+      # Add noise on the input during teacher forcing to make autoregressive
+      # sampling more robust: Sample a noise std uniformly at random per example
+      # and add Gaussian noise with that std to the input.
+      _, rng_std, rng_input_noise = jax.random.split(rng, 3)
+      sampled_input_noise_std = jax.random.uniform(
+          rng_std, (batch_size, 1, 1), minval=0.0, maxval=input_noise_std)
+      # Only apply noise for image generation (i.e. when text is first).
+      sampled_input_noise_std = jnp.where(
+          text_first_mask[:, None, None], sampled_input_noise_std, 0.0)
+      image_tokens = image_tokens + (
+          sampled_input_noise_std
+          * jax.random.normal(rng_input_noise, image_tokens.shape))
+
+    # TODO: Do cfg for text and don't apply prefix loss when dropped.
+    # For now only drop when text is first.
+    if train:
+      drop_prefix = model.get_drop_labels(rng_droplabels, batch_size=batch_size)
+    else:
+      drop_prefix = None
+    if drop_prefix is None:
+      drop_prefix = jnp.full((batch_size,), False)
+    drop_prefix = drop_prefix & text_first_mask
+
+    # Stop gradients to NVP when it is used as an encoder to get an image prefix
+    if config.get("stop_grad_nvp_prefix", False):
+      image_tokens = jnp.where(
+          text_first_mask[:, None, None],
+          image_tokens,
+          jax.lax.stop_gradient(image_tokens)
+      )
+
+    *_, pmf, pdf, _ = model.apply(
+        {"params": params},
+        text,
+        image_tokens,
+        train=train,
+        text_first_mask=text_first_mask,
+        text_input_mask=text_mask,
+        drop_prefix=drop_prefix,
+        rngs=rng_dropout)
+
+    def _log_prob(value):
+      # Re-implementing distrax.Categorical.log_prob() without logic to ignore
+      # NaNs, which is not required and seems to produce a compilation error
+      # with recent jax versions.
+      value_one_hot = jax.nn.one_hot(
+          value, pmf.num_categories, dtype=pmf.logits.dtype)
+      mask_outside_domain = jnp.logical_or(
+          value < 0, value > pmf.num_categories - 1)
+      return jnp.where(
+          mask_outside_domain, -jnp.inf,
+          jnp.sum(pmf.logits * value_one_hot, axis=-1))
+    nll_txt = -_log_prob(text)  # [BS, TXT_LEN]
+    nll_txt = jnp.mean(nll_txt, axis=1, where=text_loss)
+
+    # Report image related loss in log2/subpixels (bits per subpixels).
+    # When using PCA this value is off, as it does not accounts for logdet
+    # of PCA and ignores the perplexity of dropped PCA components.
+    num_subpixels = np.prod(images.shape[1:])  # H*W*C
+    nll_image_tokens = -pdf.log_prob(image_tokens)  # [BS, IMG_LEN]
+    nll_image_tokens = (
+        jnp.sum(nll_image_tokens, axis=1) + noise_nll) / num_subpixels
+    nll_image_tokens /= jnp.log(2)
+    # Convert logdet sum to be per subpixel and account for conversion
+    # [0, 255]->[-1, 1] (i.e. by divide by 127.5).
+    logdet = sum_log_det / num_subpixels - jnp.log(127.5)
+    logdet /= jnp.log(2)
+    nll_image = nll_image_tokens - logdet
+
+    def mean(x, where=None):
+      if valid_example_mask := batch.get("_mask", None) is not None:
+        if where is not None:
+          where = where & valid_example_mask
+        else:
+          where = valid_example_mask
+      return jnp.mean(x, where=where)
+
+    metrics = {
+        "nll_text_prefix": mean(
+            nll_txt, where=text_first_mask & ~drop_prefix),
+        "nll_text_suffix": mean(nll_txt, where=~text_first_mask),
+        # Currently, we never drop the image prefix, but we already consider
+        # this case here.
+        "nll_image_prefix": mean(
+            nll_image, where=~text_first_mask & ~drop_prefix),
+        "nll_image_suffix": mean(nll_image, where=text_first_mask),
+    }
+
+    text_w = config.get("text_loss_weight", 1.0)
+    if config.get("loss_on_prefix", True):
+      valid_txt_nll = (text_first_mask & ~drop_prefix) | ~text_first_mask
+      valid_img_nll = (~text_first_mask & ~drop_prefix) | text_first_mask
+      metrics.update({
+          "nll_text": mean(nll_txt, where=valid_txt_nll),
+          "nll_image": mean(nll_image, where=valid_img_nll),
+          "logdet": mean(logdet),
+      })
+      loss = (mean(nll_txt, where=valid_txt_nll) * text_w
+              + mean(nll_image, where=valid_img_nll))
+    else:
+      text_suffix = ~text_first_mask
+      image_suffix = text_first_mask
+      metrics.update({
+          "nll_text": mean(nll_txt, where=text_suffix),
+          "nll_image": mean(nll_image, where=image_suffix),
+          "nll_image_tokens": mean(nll_image_tokens, where=image_suffix),
+          "logdet": mean(logdet, where=image_suffix),
+      })
+      example_loss = jnp.where(text_suffix, nll_txt*text_w, nll_image)
+      loss = mean(example_loss)
+
+    metrics["loss"] = loss
+    return loss, metrics
+
+  @functools.partial(
+      jax.jit,
+      donate_argnums=(0,),
+      out_shardings=(train_state_sharding, repl_sharding))
+  def update_fn(train_state, rng, batch):
+    """Update step."""
+    step_count = bv_optax.get_count(train_state["opt"], jittable=True)
+    rng = jax.random.fold_in(rng, step_count)
+
+    measurements = {}
+    progress = step_count / total_steps
+
+    if config.get("noise_scale", 0.0) > 0.0:
+      noise_min = config.get("noise_min", 0.0)
+      noise_scale = ((config.noise_scale - noise_min)
+                     * (1+jnp.cos(jnp.pi*progress)) * 0.5) + noise_min
+      measurements["noise_scale"] = noise_scale
+    else:
+      noise_scale = None
+
+    # Get device-specific loss rng.
+    _, rng_model = jax.random.split(rng, 2)
+    params, opt = train_state["params"], train_state["opt"]
+
+    (loss, metrics), grads = jax.value_and_grad(loss_fn, has_aux=True)(
+        params, batch, rng_model, noise_scale=noise_scale)
+    updates, opt = tx.update(grads, opt, params)
+    params = optax.apply_updates(params, updates)
+    new_train_state = {"params": params, "opt": opt}
+    # Update EMA parameters.
+    if (ema_decay := config.get("ema_decay", 0.0)) > 0.0:
+      new_params_ema = jax.tree.map(
+          lambda pe, p: ema_decay * pe + (1 - ema_decay) * p,
+          train_state["params_ema"], params)
+      new_train_state["params_ema"] = new_params_ema
+
+    measurements["training_loss"] = loss
+    gs = jax.tree.leaves(bv_optax.replace_frozen(config.schedule, grads, 0.))
+    measurements["l2_grads"] = jnp.sqrt(sum([jnp.vdot(g, g) for g in gs]))
+    ps = jax.tree.leaves(params)
+    measurements["l2_params"] = jnp.sqrt(sum([jnp.vdot(p, p) for p in ps]))
+    us = jax.tree.leaves(updates)
+    measurements["l2_updates"] = jnp.sqrt(sum([jnp.vdot(u, u) for u in us]))
+
+    if adaptor is not None:
+      ps_a = jax.tree.leaves(params["params_adaptor"])
+      measurements["l2_params_adaptor"] = jnp.sqrt(sum([jnp.vdot(p, p)
+                                                        for p in ps_a]))
+
+    measurements.update({f"train/{k}": v.mean() for k, v in metrics.items()})
+
+    return new_train_state, measurements
+
+################################################################################
+#                                                                              #
+#                               Load Checkpoint                                #
+#                                                                              #
+################################################################################
+
+  # Decide how to initialize training. The order is important.
+  # 1. Always resumes from the existing checkpoint, e.g. resumes a finetune job.
+  # 2. Resume from a previous checkpoint, e.g. start a cooldown training job.
+  # 3. Initialize model from something, e,g, start a fine-tuning job.
+  # 4. Train from scratch.
+  resume_ckpt_path = None
+  if save_ckpt_path and gfile.exists(f"{save_ckpt_path}-LAST"):
+    resume_ckpt_path = save_ckpt_path
+  elif config.get("resume"):
+    resume_ckpt_path = fillin(config.resume)
+
+  ckpt_mngr = None
+  if save_ckpt_path or resume_ckpt_path:
+    ckpt_mngr = array_serial.GlobalAsyncCheckpointManager()
+
+  if resume_ckpt_path:
+    write_note(f"Resuming training from checkpoint {resume_ckpt_path}...")
+    jax.tree.map(lambda x: x.delete(), train_state)
+    del train_state
+    shardings = {
+        **train_state_sharding,
+        "chrono": jax.tree.map(lambda _: repl_sharding,
+                               u.chrono.save()),
+    }
+    loaded = u.load_checkpoint_ts(
+        resume_ckpt_path, tree=shardings, shardings=shardings)
+    train_state = {key: loaded[key] for key in train_state_sharding.keys()}
+
+    u.chrono.load(jax.device_get(loaded["chrono"]))
+    del loaded
+  elif config.get("model_init"):
+    write_note(f"Initialize model from {config.model_init}...")
+    train_state["params"] = model_mod.load(
+        train_state["params"], config.model_init, config.get("model"),
+        **config.get("model_load", {}))
+
+    # load has the freedom to return params not correctly sharded
+    train_state["params"] = u.reshard(
+        train_state["params"], train_state_sharding["params"])
+
+    parameter_overview.log_parameter_overview(
+        train_state["params"], msg="restored params",
+        include_stats="global", jax_logging_process=0)
+
+
+################################################################################
+#                                                                              #
+#                                 Setup Evals                                  #
+#                                                                              #
+################################################################################
+
+  def validation_fn(train_state, batch, *, use_ema=False):
+    params = train_state["params_ema"] if use_ema else train_state["params"]
+    rng = jax.random.PRNGKey(
+        jax.lax.rng_uniform(0, np.iinfo(np.int32).max, tuple()))
+
+    _, aux = loss_fn(params, batch, rng, train=False)
+    # The metrics produced by loss_fn may already be averaged over the batch,
+    # since some of them only apply for certain batches. Here we broadcast the
+    # metrics across the batch dimension, which might introduce some errors when
+    # the batch size is not divisible by the number of devices.
+    aux = jax.tree.map(
+        lambda x: jnp.broadcast_to(x, batch["text"].shape[:1]), aux)
+    return aux
+
+  def sample_images_fn(train_state, batch, *, decode_len=256, use_ema=False):
+    params = train_state["params_ema"] if use_ema else train_state["params"]
+    cfg_weight = config.sample_images.get("cfg_inference_weight", 0.0)
+    temperature = config.sample_images.get("temperature", 1.0)
+    temperature_probs = config.sample_images.get("temperature_probs", 1.0)
+
+    if batch["text"].ndim < 2:
+      batch["text"] = batch["text"][:, None]
+
+    out = predict_fns.sample_image_latents(
+        params, batch, model=model, decode_len=decode_len,
+        cfg_weight=cfg_weight, temperature=temperature,
+        temperature_probs=temperature_probs)
+
+    image_tokens = out["out_tokens"]
+    if (noise_dim := config.get("latent_noise_dim", 0)) > 0:
+      rng = jax.random.PRNGKey(
+          jax.lax.rng_uniform(0, np.iinfo(np.int32).max, tuple()))
+      noise = jax.random.normal(rng, image_tokens.shape[:-1] + (noise_dim,))
+      image_tokens = jnp.concatenate([image_tokens, noise], axis=-1)
+
+    images = predict_fns.decode_images(
+        params["params_adaptor"], image_tokens,
+        adaptor=adaptor, patch_pca=patch_pca)
+    out["logits"] = images
+    return out
+
+  def sample_text_fn(params, batch, *, temperature, decode_len):
+    """Jittable sampling of text."""
+    image_latents = predict_fns.encode_images(
+        params["params_adaptor"], batch["image"],
+        adaptor=adaptor, patch_pca=patch_pca,
+        rngs={"dropout": jax.random.key(0)},
+        reparametrize=False)
+
+    image_latents = _maybe_remove_latent_noise_dims(image_latents)
+
+    out = predict_fns.sample_text(
+        params, {"image_latents": image_latents, **batch}, model=model,
+        temperature=temperature, decode_len=decode_len)
+    return out["out_tokens"]
+
+  def sample_text(train_state, batch, *,
+                  use_ema=False, decode_len, temperature=1e-5, devices,
+                  eos_token=None):
+    """Predict fn that does the jitting of sample_text_fn for evaluators."""
+    del eos_token  # Unused. We always sample decode_len tokens.
+    params = train_state["params_ema"] if use_ema else train_state["params"]
+    mesh = jax.sharding.Mesh(devices, ("devices",))
+    data_sharding = jax.sharding.NamedSharding(mesh, P("devices"))
+    new_batch = {
+        "image": batch["image"],
+        "text": batch.get("text", None),
+        "text_mask": batch.get("text_mask", None),
+    }
+    tokens = jax.jit(sample_text_fn, out_shardings=data_sharding,
+                     static_argnames=("decode_len", "temperature"))(
+                         params, new_batch,
+                         decode_len=decode_len, temperature=temperature)
+    return tokens
+
+  def score_captions_fn(train_state, batch, *, use_ema=False):
+    # TODO: Enable caching of the prefix to speed up evaluation.
+    params = train_state["params_ema"] if use_ema else train_state["params"]
+    images = batch["image"]
+    all_labels = batch["_label_tokens"]
+    all_labels_mask = batch["_label_masks"]
+    all_loss_masks = batch["_loss_masks"]
+    batch_size = images.shape[0]
+
+    rng = jax.random.PRNGKey(
+        jax.lax.rng_uniform(0, np.iinfo(np.int32).max, tuple()))
+    rng_dropout = {"dropout": rng}
+    image_tokens = patch_pca_encode(images, rng_dropout)
+
+    if adaptor is not None:
+      image_tokens, _ = adaptor_apply(params["params_adaptor"], image_tokens)
+
+    image_tokens = _maybe_remove_latent_noise_dims(image_tokens)
+
+    def _score_label(label_and_mask):
+      label, mask, loss_mask = label_and_mask
+      label_rep = jnp.tile(label, (batch_size, 1))
+      masks_rep = jnp.tile(mask, (batch_size, 1))
+      loss_masks_rep = jnp.tile(loss_mask, (batch_size, 1))
+      _, _, pmf, *_ = model.apply(
+          {"params": params},
+          text_tokens=label_rep,
+          image_tokens=image_tokens,
+          text_first_mask=jnp.full((batch_size,), False),  # images always first
+          text_input_mask=masks_rep,
+      )
+      return jnp.sum(pmf.log_prob(label_rep), axis=-1, where=loss_masks_rep)
+
+    ll_labels = jax.lax.map(
+        _score_label, (all_labels, all_labels_mask, all_loss_masks))
+    return ll_labels.T
+
+  def image_rep_fn(train_state, batch, *, use_ema=False):
+    params = train_state["params_ema"] if use_ema else train_state["params"]
+    images = batch["image"]
+
+    rng = jax.random.PRNGKey(
+        jax.lax.rng_uniform(0, np.iinfo(np.int32).max, tuple()))
+    rng_dropout = {"dropout": rng}
+    image_tokens = patch_pca_encode(images, rng_dropout)
+
+    out = {"patch_emb": image_tokens}
+    if adaptor is not None:
+      image_tokens, _ = adaptor_apply(params["params_adaptor"], image_tokens)
+      out["nvp"] = image_tokens
+
+    image_tokens = _maybe_remove_latent_noise_dims(image_tokens)
+
+    # TODO: Allow the code to be called without text tokens and think
+    # better what representations to use here... For now its the representation
+    # obtained by feeding the 257 tokens [BOI, image_tokens] to the model. The
+    # text tokens are not used. It is only [BOS] but since its the last token in
+    # the input it is not feed to the model (dropped by shift).
+    *_, decoder_out = model.apply(
+        {"params": params},
+        text_tokens=jnp.full((batch_size, 0,), 0),  # [BS, 0]: empty text.
+        image_tokens=image_tokens,
+        text_first_mask=jnp.full((batch_size,), False),  # images always first
+    )
+    out.update(decoder_out)
+
+    # Average pool intermediate representations.
+    out = jax.tree.map(lambda x: x.mean(axis=-2), out)
+
+    return out["pre_logits"], out
+
+  # Only initialize evaluators when they are first needed.
+  @functools.lru_cache(maxsize=None)
+  def evaluators():
+    return eval_common.from_config(
+        config,
+        {
+            "validation": validation_fn,
+            "sample_images": sample_images_fn,
+            "sample_text": sample_text,
+            "score_captions": score_captions_fn,
+            "image_representation": image_rep_fn,
+        },
+        lambda s: write_note(f"Init evaluator: {s}…\n{u.chrono.note}"),
+        lambda key, cfg: get_steps(key, default=None, cfg=cfg),
+        devices_flat,
+    )
+
+  # At this point we need to know the current step to see whether to run evals.
+  write_note("Inferring the first step number...")
+  first_step_device = bv_optax.get_count(train_state["opt"], jittable=True)
+  first_step = int(jax.device_get(first_step_device))
+  u.chrono.inform(first_step=first_step)
+
+  # Note that training can be pre-empted during the final evaluation (i.e.
+  # just after the final checkpoint has been written to disc), in which case we
+  # want to run the evals.
+  if first_step in (total_steps, 0):
+    write_note("Running initial or final evals...")
+    mw.step_start(first_step)
+    for (name, evaluator, _, prefix) in evaluators():
+      if config.evals[name].get("skip_first") and first_step != total_steps:
+        continue
+      write_note(f"{name} evaluation...\n{u.chrono.note}")
+      with u.chrono.log_timing(f"z/secs/eval/{name}"):
+        with mesh, nn.logical_axis_rules(sharding_rules):
+          for key, value in evaluator.run(train_state):
+            mw.measure(f"{prefix}{key}", value)
+
+################################################################################
+#                                                                              #
+#                                  Train Loop                                  #
+#                                                                              #
+################################################################################
+
+  prof = None  # Keeps track of start/stop of profiler state.
+
+  write_note("Starting training loop, compiling the first step...")
+  for step, batch in zip(range(first_step + 1, total_steps + 1), train_iter):
+    mw.step_start(step)
+
+    with jax.profiler.StepTraceAnnotation("train_step", step_num=step):
+      with u.chrono.log_timing("z/secs/update0", noop=step > first_step + 1):
+        with mesh, nn.logical_axis_rules(sharding_rules):
+          train_state, measurements = update_fn(train_state, rng_loop, batch)
+
+    # On the first host, let's always profile a handful of early steps.
+    if jax.process_index() == 0:
+      prof = u.startstop_prof(prof, step, first_step, get_steps("log_training"))
+
+    # Report training progress
+    if (u.itstime(step, get_steps("log_training"), total_steps, host=0)
+        or u.chrono.warmup and jax.process_index() == 0):
+      for i, sched_fn_cpu in enumerate(sched_fns_cpu):
+        mw.measure(f"global_schedule{i if i else ''}",
+                   sched_fn_cpu(u.put_cpu(step - 1)))
+      measurements = jax.device_get(measurements)
+      for name, value in measurements.items():
+        mw.measure(name, value)
+      u.chrono.tick(step)
+      if not np.isfinite(measurements["training_loss"]):
+        raise RuntimeError(f"The loss became nan or inf somewhere within steps "
+                           f"[{step - get_steps('log_training')}, {step}]")
+
+    # Checkpoint saving
+    keep_ckpt_steps = get_steps("keep_ckpt", None) or total_steps
+    if save_ckpt_path and (
+        (keep := u.itstime(step, keep_ckpt_steps, total_steps, first=False))
+        or u.itstime(step, get_steps("ckpt", None), total_steps, first=True)
+    ):
+      u.chrono.pause(wait_for=train_state)
+
+      # Copy because we add extra stuff to the checkpoint.
+      ckpt = {**train_state}
+
+      # To save chrono state correctly and safely in a multihost setup, we
+      # broadcast the state to all hosts and convert it to a global array.
+      with jax.transfer_guard("allow"):
+        chrono_ckpt = multihost_utils.broadcast_one_to_all(u.chrono.save())
+      chrono_shardings = jax.tree.map(lambda _: repl_sharding, chrono_ckpt)
+      ckpt = ckpt | {"chrono": u.reshard(chrono_ckpt, chrono_shardings)}
+
+      u.save_checkpoint_ts(ckpt_mngr, ckpt, save_ckpt_path, step, keep)
+      u.chrono.resume()
+
+    for (name, evaluator, log_steps, prefix) in evaluators():
+      if u.itstime(step, log_steps, total_steps, first=False, last=True):
+        u.chrono.pause(wait_for=train_state)
+        u.chrono.tick(step)  # Record things like epoch number, core hours etc.
+        write_note(f"{name} evaluation...\n{u.chrono.note}")
+        with u.chrono.log_timing(f"z/secs/eval/{name}"):
+          with mesh, nn.logical_axis_rules(sharding_rules):
+            for key, value in evaluator.run(train_state):
+              mw.measure(f"{prefix}{key}", jax.device_get(value))
+        u.chrono.resume()
+    mw.step_end()
+
+  # Always give a chance to stop the profiler, no matter how things ended.
+  # TODO: can we also do this when dying of an exception like OOM?
+  if jax.process_index() == 0 and prof is not None:
+    u.startstop_prof(prof)
+
+  # Last note needs to happen before the pool's closed =)
+  write_note(f"Done!\n{u.chrono.note}")
+
+  pool.close()
+  pool.join()
+  mw.close()
+  if ckpt_mngr:
+    ckpt_mngr.wait_until_finished()
+
+  # Make sure all hosts stay up until the end of main.
+  u.sync()
+
+  u.maybe_cleanup_workdir(workdir, flags.FLAGS.cleanup, info)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/Tipsomaly/model/big_vision/trainers/proj/paligemma/predict_fns.py b/Tipsomaly/model/big_vision/trainers/proj/paligemma/predict_fns.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a27b3eb00a3739965bdaafa02f1b9be01eefb49
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/paligemma/predict_fns.py
@@ -0,0 +1,486 @@
+# Copyright 2024 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Prediction functions for PaliGemma."""
+
+import collections
+import functools
+
+from big_vision.pp import registry
+import big_vision.utils as u
+import einops
+import jax
+import jax.numpy as jnp
+import numpy as np
+
+
+P = jax.sharding.PartitionSpec
+
+# pylint: disable=missing-function-docstring
+
+
+def get_all(model):
+  """Returns `predict_fns` for evaluators."""
+  fns = {
+      "logits": _logits,
+      "image_avg_repr": _image_avg_repr,
+      "decode": _decode,
+      "decode_with_logp": _decode_with_logp,
+      "beam_decode": _beam_decode,
+  }
+  return {name: functools.partial(fn, model=model) for name, fn in fns.items()}
+
+
+def _logits(train_state, batch, *, model):
+  images, text, mask = batch["image"], batch["text"], batch["mask_ar"]
+  text_logits, out = model.apply(
+      {"params": train_state["params"]},
+      images, text[:, :-1], mask[:, :-1],
+  )
+  return text_logits, out
+
+
+def _image_avg_repr(train_state, batch, *, model, key="img/pre_logits"):
+  zimg, out = model.apply(
+      {"params": train_state["params"]},
+      image=batch["image"],
+      method=model.embed_image,
+  )
+  if key:
+    zimg = u.tree_get(out, key)
+  # At this point, zimg is a (batch of) sequence of image tokens, because we
+  # assume the model is a vit with "none" head. This predict-fn is for fewshot
+  # evaluator, so we need to turn it into reasonably-sized vector -> avg.
+  zimg = jnp.mean(zimg, axis=range(1, zimg.ndim - 1))
+  return zimg, out
+
+
+def _decode_with_logp(
+    train_state, batch, *, model, devices, max_decode_len, eos_token,
+    best_of_n=1, sampler="greedy", eos_look_behind=0):
+  """Sample token continuations to the input sequences."""
+  mesh = jax.sharding.Mesh(devices, ("devices",))
+  replicate_sharding = jax.sharding.NamedSharding(mesh, P())
+  bs_shardable = len(batch["image"]) % jax.device_count() == 0
+  out_sharding = jax.sharding.NamedSharding(
+      mesh, P("devices") if bs_shardable else P()
+  )
+
+  # Prefill the model cache and generate logits for first token.
+  logits, cache = jax.jit(
+      _prefill_cache,
+      out_shardings=(None, out_sharding),
+      static_argnames=("model", "max_decode_len"),
+  )(
+      train_state["params"],
+      {
+          "image": batch["image"],
+          "text": batch["text"],
+          "mask_input": batch["mask_input"],
+          "mask_ar": batch["mask_ar"],
+      },
+      model=model,
+      max_decode_len=max_decode_len,
+  )
+
+  # Mask indicating real examples. False if example is used to pad the batch.
+  mask = batch["_mask"]
+
+  # Mask indicating tokens for which the logits will be set to -Inf. Can be a
+  # Boolean mask or indices.
+  tok_mask = batch.get("mask_logits", None)
+
+  # Repeat example in case we are picking the best of n.
+  logits, cache, mask = jax.jit(
+      _bon_repeat,
+      static_argnames=("n",)
+  )((logits, cache, mask), n=best_of_n)
+
+  decode_sample_output = jax.jit(
+      _decode_sample_output,
+      static_argnames=("max_decode_len", "sampler"),
+  )
+  decode_early_stop = jax.jit(
+      _decode_early_stop,
+      out_shardings=replicate_sharding,
+      static_argnames=("eos_token",),
+  )
+  extend_cache = jax.jit(
+      _extend_cache,
+      donate_argnums=1,
+      out_shardings=(None, out_sharding),
+      static_argnames=("model",),
+  )
+
+  # Keep sampling tokens from last logits until EOS or max_decode_len.
+  state = None
+  # Setting `eos_look_behind>0` removes blocking transfer with small batches.
+  stops = collections.deque(maxlen=1 + eos_look_behind)
+  for idx in range(max_decode_len):
+    tokens, state = decode_sample_output(
+        state, logits, tok_mask, max_decode_len=max_decode_len, sampler=sampler
+    )
+
+    if idx + 1 >= max_decode_len:
+      break
+
+    stops.append(decode_early_stop(state, mask, eos_token=eos_token))
+    if len(stops) == stops.maxlen and jax.device_get(stops[0]):
+      break
+
+    # Compute logits for next token
+    logits, cache = extend_cache(
+        train_state["params"], cache, tokens, model=model
+    )
+
+  # Select the best of n sample for each example.
+  _, tokens, logp = jax.jit(
+      _bon_select,
+      out_shardings=out_sharding,
+      static_argnames=("n", "eos_token"),
+  )(state, n=best_of_n, eos_token=eos_token)
+
+  return tokens, logp
+
+
+def _decode(train_state, batch, **kwargs):
+  tokens, _ = _decode_with_logp(train_state, batch, **kwargs)
+  return tokens
+
+
+def _bon_repeat(tree, *, n):
+  return jax.tree.map(lambda x: jnp.repeat(x, n, axis=0), tree)
+
+
+def _compute_score(tokens, logp, eos_token):
+  """Compute log-probability of each sequence up to first eos (including it)."""
+  seqlen = jnp.sum(jnp.cumsum(tokens == eos_token, axis=-1) == 0, axis=-1) + 1
+  token_mask = jnp.arange(tokens.shape[-1]) < seqlen[..., None]
+  scores = jnp.sum(logp * token_mask, axis=-1)
+  return scores
+
+
+def _bon_select(state, *, n, eos_token):
+  """Pick the sampled sequence with the highest likelihood for each example."""
+  (_, tokens, logp) = state
+
+  # Filter state to only keep the best of each example.
+  scores = _compute_score(tokens, logp, eos_token)
+  scores = einops.rearrange(scores, "(b n) -> b n", n=n)
+  state = jax.tree.map(
+      lambda x: einops.rearrange(x, "(b n) l -> b n l", n=n), state)
+  best_indices = jnp.argmax(scores, -1)  # [b]
+  state = jax.tree.map(
+      lambda x: jnp.take_along_axis(x, best_indices[:, None, None], axis=1),
+      state)
+  state = jax.tree.map(lambda x: x[:, 0], state)
+
+  return state
+
+
+def _decode_sample_output(state, logits, tok_mask, *, max_decode_len, sampler):
+  if state is None:
+    # Decode state keeps track of sampled tokens and their logp.
+    bs = logits.shape[0]
+    seqlen = jnp.zeros((bs, 1), dtype=jnp.int32)
+    tokens = jnp.zeros((bs, max_decode_len), dtype=jnp.int32)
+    logp = jnp.zeros((bs, max_decode_len), dtype=logits.dtype)
+  else:
+    (seqlen, tokens, logp) = state
+
+  # Sample tokens.
+  sampled_tokens, sampled_logp = _sample_logits(logits, sampler=sampler,
+                                                tok_mask=tok_mask)
+
+  # Update state with sampled outputs.
+  new_len = seqlen + 1
+  new_tokens = _put_along_last_axis(tokens, seqlen, sampled_tokens)
+  new_logp = _put_along_last_axis(logp, seqlen, sampled_logp)
+  new_state = (new_len, new_tokens, new_logp)
+
+  return sampled_tokens, new_state
+
+
+def _decode_early_stop(state, mask, *, eos_token):
+  (seqlen, tokens, unused_logp) = state
+  token_mask = jnp.arange(tokens.shape[-1])[None, :] < seqlen
+  has_eos = jnp.any(jnp.logical_and(tokens == eos_token, token_mask), axis=-1)
+  done = jnp.logical_or(has_eos, jnp.logical_not(mask))
+  return jnp.all(done)
+
+
+def _put_along_last_axis(arr, indices, values):
+  """Like np.put_along_axis(..., axis=-1), since jax is missing it."""
+  assert arr.ndim == indices.ndim == values.ndim, (
+      arr.ndim, indices.ndim, values.ndim)
+  onehot = jax.nn.one_hot(indices, arr.shape[-1], dtype=values.dtype)
+  put_mask = jnp.einsum("...i,...in->...n",
+                        jnp.ones(values.shape, jnp.int32), onehot)
+  put_values = jnp.einsum("...i,...in->...n", values, onehot)
+  return jnp.where(put_mask, put_values, arr)
+
+
+def _prefill_cache(params, batch, *, model, max_decode_len):
+  """Initialize the model cache for decoding with the prompts."""
+  variables = {"params": params}
+  (x, input_mask, mask_ar), _ = model.apply(
+      variables, batch["image"], batch["text"],
+      input_mask=batch["mask_input"],
+      mask_ar=batch["mask_ar"],
+      method=model.embed_image_and_text)
+  last_logits, variables = model.apply(
+      variables, x, input_mask, mask_ar,
+      cache_size=x.shape[1] + max_decode_len,
+      method=model.prefill_cache,
+      mutable=("cache",))
+  return last_logits, variables["cache"]
+
+
+def _extend_cache(params, cache, tokens, *, model):
+  """Extend the model cache for decoding with one token per sequence."""
+  variables = {"params": params, "cache": cache}
+  x, _ = model.apply(variables, tokens, method=model.embed_text)
+  last_logits, variables = model.apply(
+      variables, x, method=model.extend_cache, mutable=("cache",))
+  return last_logits, variables["cache"]
+
+
+def _sample_logits(logits, sampler, tok_mask=None):
+  """Returns a sampled token and its logp from logits."""
+  # Note: Consider making it possible for evaluators to pass rng seed to
+  # decode functions. For now generate it from jax.lax and avoid evaluators
+  # having to deal with it.
+  rng = jax.random.PRNGKey(
+      jax.lax.rng_uniform(0, np.iinfo(np.int32).max, tuple()))
+
+  masked_logits = logits
+  if tok_mask is not None:
+    masked_logits = masked_logits.at[..., tok_mask].set(-jnp.inf)
+
+  # Use Registry to support specifying things like:
+  #  "greedy", "nucleus(0.2)", "temperature(t=1.0)"
+  sampled_tokens = registry.Registry.lookup("paligemma_sampler." + sampler)(
+      logits=masked_logits, rng=rng)
+
+  # Find the log probability (normalized logits) of selected tokens.
+  # NOTE: If you use tok_mask this returns the probability of the tokens while
+  # ignoring the masking. This is useful for pix2seq-style which has tokens like
+  # "noise" which it does not want to sample but it wants to use it to affect
+  # the score/logp of classes being sampled and it wants to intrepet as a
+  # confidence.
+  sampled_logp = jnp.take_along_axis(
+      jax.nn.log_softmax(logits, axis=-1),
+      sampled_tokens[..., None], -1)[..., 0]
+
+  return sampled_tokens, sampled_logp
+
+
+@registry.Registry.register("paligemma_sampler.greedy")
+def _greedy_sampling(*, logits, rng):
+  del rng
+  return jnp.argmax(logits, axis=-1)
+
+
+@registry.Registry.register("paligemma_sampler.temperature")
+def _temperature_sampling(t, *, logits, rng):
+  return jax.random.categorical(rng, logits / t)
+
+
+@registry.Registry.register("paligemma_sampler.nucleus")
+def _nucleus_sampling(p: float, t: float = 1.0, *, logits, rng):
+  logits = logits / t
+  neg_inf = np.array(-1.0e7)  # Effective negative infinity.
+  logits_sorted = jnp.sort(logits, axis=-1, descending=True)
+  sorted_cum_probs = jnp.cumsum(
+      jax.nn.softmax(logits_sorted, axis=-1), axis=-1)
+  cutoff_index = jnp.sum(sorted_cum_probs < p, axis=-1, keepdims=True)
+  cutoff_logit = jnp.take_along_axis(logits_sorted, cutoff_index, axis=-1)
+  logits = jnp.where(logits < cutoff_logit,
+                     jnp.full_like(logits, neg_inf), logits)
+  return jax.random.categorical(rng, logits)
+
+
+def _beam_decode(train_state, batch, *,
+                 model, devices, max_decode_len,
+                 eos_token, beam_size):
+  """Beam search (greedy/top-k exploration)."""
+  mesh = jax.sharding.Mesh(devices, ("devices",))
+  replicate_sharding = jax.sharding.NamedSharding(mesh, P())
+  bs_shardable = len(batch["image"]) % jax.device_count() == 0
+  out_sharding = jax.sharding.NamedSharding(
+      mesh, P("devices") if bs_shardable else P()
+  )
+
+  # Prefill the model cache and generate logits for first token.
+  logits, cache = jax.jit(
+      _prefill_cache,
+      out_shardings=(None, out_sharding),
+      static_argnames=("model", "max_decode_len"),
+  )(
+      train_state["params"],
+      {
+          "image": batch["image"],
+          "text": batch["text"],
+          "mask_input": batch["mask_input"],
+          "mask_ar": batch["mask_ar"],
+      },
+      model=model,
+      max_decode_len=max_decode_len,
+  )
+
+  # Mask indicating real examples. False if example is used to pad the batch.
+  mask = batch["_mask"]
+
+  beam_sample_output = jax.jit(
+      _beam_sample_output,
+      donate_argnums=2,
+      out_shardings=(None, None, out_sharding),
+      static_argnames=("max_decode_len", "beam_size", "eos_token"),
+  )
+  beam_early_stop = jax.jit(
+      _beam_early_stop,
+      out_shardings=replicate_sharding,
+      static_argnames=("eos_token",),
+  )
+  extend_cache = jax.jit(
+      _extend_cache,
+      donate_argnums=1,
+      out_shardings=(None, out_sharding),
+      static_argnames=("model",),
+  )
+
+  # Keep sampling tokens from last logits until EOS or max_decode_len.
+  state = None
+  for idx in range(max_decode_len):
+    tokens, state, cache = beam_sample_output(
+        state, logits, cache,
+        max_decode_len=max_decode_len, beam_size=beam_size, eos_token=eos_token)
+
+    early_stop = beam_early_stop(state, mask, eos_token=eos_token)
+    if jax.device_get(early_stop) or (idx + 1 >= max_decode_len):
+      break
+
+    # Compute logits for next token
+    logits, cache = extend_cache(
+        train_state["params"], cache, tokens, model=model)
+
+  return jax.jit(_beam_make_output, out_shardings=out_sharding)(state)
+
+
+def _beam_early_stop(state, mask, eos_token):
+  (best_tokens, best_logp, seqlen, unused_tokens, logp) = state
+
+  # Scores of finalized sequences.
+  best_scores = _compute_score(best_tokens, best_logp, eos_token)
+
+  # Scores of live sequences.
+  live_mask = jnp.arange(logp.shape[-1])[None, None] < seqlen
+  live_scores = jnp.sum(logp * live_mask, axis=-1)
+  live_scores = jnp.max(live_scores, axis=1)
+
+  done = live_scores < best_scores
+  return jnp.all(jnp.logical_or(done, jnp.logical_not(mask)))
+
+
+def _beam_make_output(state):
+  (best_tokens, *_) = state
+  return best_tokens[:, 0, ...]
+
+
+def _beam_sample_output(state, logits, cache, *,
+                        beam_size, max_decode_len, eos_token):
+  assert logits.shape[1] == 1
+  logits = jax.nn.log_softmax(logits[:, 0, :])  # Normalize logits
+
+  if state is None:
+    bs = logits.shape[0]
+    # Beam decode state keeps track of:
+    # A) Best sampled output for each example. At initialization these have
+    # shape[1]=0, but end up with shape[1]=1 after first call.
+    best_tokens = jnp.zeros((bs, 0, max_decode_len), dtype=jnp.int32)
+    best_logp = jnp.zeros((bs, 0, max_decode_len), dtype=logits.dtype)
+    # B) N candidate sequences for each example. At initialization these have
+    # beam_size=1, but end up with correct beam_size when expanded.
+    seqlen = jnp.zeros((bs, 1, 1), dtype=jnp.int32)
+    tokens = jnp.zeros((bs, 1, max_decode_len), dtype=jnp.int32)
+    logp = jnp.zeros((bs, 1, max_decode_len), dtype=logits.dtype)
+  else:
+    (best_tokens, best_logp, seqlen, tokens, logp) = state
+    bs = logits.shape[0] // beam_size
+    assert best_tokens.shape[0] == bs
+
+  # Reshape cache to [example, candidate, ...].
+  # Note: on first call the number of candidates is 1. Later it is beam_size.
+  cache, logits = jax.tree.map(
+      lambda x: einops.rearrange(x, "(b n) ... -> b n ...", b=bs),
+      (cache, logits))
+
+  # Consider a live sequence could end now and update the best finished
+  # sequences so far for each example. This strategy is found in some beam
+  # implementations such as in praxis.
+  # The code below also adjusts the best shape[1]=0 -> 1 during first call.
+  eos_tokens = jnp.array(eos_token)[None, None, None]
+  new_tokens = _put_along_last_axis(tokens, seqlen, eos_tokens)
+  new_logp = _put_along_last_axis(logp, seqlen, logits[:, :, eos_token, None])
+
+  best_tokens = jnp.concatenate([best_tokens, new_tokens], axis=1)
+  best_logp = jnp.concatenate([best_logp, new_logp], axis=1)
+  best_scores = _compute_score(best_tokens, best_logp, eos_token=eos_token)
+  _, top_indices = jax.lax.top_k(best_scores, k=1)
+
+  best_tokens = jnp.take_along_axis(best_tokens, top_indices[..., None], axis=1)
+  best_logp = jnp.take_along_axis(best_logp, top_indices[..., None], axis=1)
+
+  # To find the next best N live candidates we expand each candidate and keep
+  # the best N (ignoring EOS tokens). In this case we expand into (N+1)
+  # candidates and set their likelihood to "-inf" (if EOS) after the fact.
+  live_mask = jnp.arange(logp.shape[-1])[None, None] < seqlen
+  live_scores = jnp.sum(logp * live_mask, axis=-1)
+  topk_logits, topk_tokens = jax.lax.top_k(logits, beam_size+1)
+  scores = live_scores[..., None] + topk_logits
+  scores = jnp.where(
+      topk_tokens != eos_token, scores, jnp.finfo(scores.dtype).min)
+
+  # From the N*(N+1) candidates find the top N for each example.
+  topk_logits, topk_tokens, scores = jax.tree.map(
+      lambda x: einops.rearrange(x, "b n np1 -> b (n np1)"),
+      (topk_logits, topk_tokens, scores))
+  _, topk_indices = jax.lax.top_k(scores, k=beam_size)
+  sampled_indices = topk_indices // (beam_size+1)
+  sampled_tokens = jnp.take_along_axis(
+      topk_tokens, topk_indices, axis=-1)[..., None]
+  sampled_logits = jnp.take_along_axis(
+      topk_logits, topk_indices, axis=-1)[..., None]
+
+  # Adjust cache and state so it matches the selected top N input candidates.
+  # This also adjusts the beam_size=1->n during first call.
+  def take_candidates(x):
+    one_hot_matrix = jax.nn.one_hot(sampled_indices, x.shape[1], dtype=x.dtype)
+    return jnp.einsum("bi...,boi->bo...", x, one_hot_matrix)
+  cache, seqlen, tokens, logp = jax.tree.map(
+      take_candidates, (cache, seqlen, tokens, logp))
+
+  # Write the sampled tokens/logits on the reshuffled state.
+  tokens = _put_along_last_axis(tokens, seqlen, sampled_tokens)
+  logp = _put_along_last_axis(logp, seqlen, sampled_logits)
+  seqlen = seqlen + 1
+
+  state = (best_tokens, best_logp, seqlen, tokens, logp)
+
+  # Reshape to [(example, candidate), ...].
+  sampled_tokens, cache = jax.tree.map(
+      lambda x: einops.rearrange(x, "b n ... -> (b n) ..."),
+      (sampled_tokens, cache))
+
+  return sampled_tokens, state, cache
diff --git a/Tipsomaly/model/big_vision/trainers/proj/paligemma/run.py b/Tipsomaly/model/big_vision/trainers/proj/paligemma/run.py
new file mode 100644
index 0000000000000000000000000000000000000000..f128280c836aa206b96f890eee964f44f262c5be
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/paligemma/run.py
@@ -0,0 +1,153 @@
+# Copyright 2024 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Load and run the PaliGemma model."""
+import functools
+import sys
+
+from absl import app
+from absl import flags
+from absl import logging
+
+# pylint: disable=all
+import jax
+from jax.sharding import Mesh, NamedSharding, PartitionSpec
+import ml_collections
+import numpy as np
+
+import big_vision.models.proj.paligemma.gemma_bv
+import big_vision.models.proj.paligemma.paligemma as model_mod
+import big_vision.models.vit
+import big_vision.pp.builder
+import big_vision.pp.tokenizer
+import big_vision.pp.ops_image
+import big_vision.pp.ops_general
+import big_vision.pp.ops_text
+import big_vision.pp.proj.paligemma.ops
+import big_vision.sharding
+import big_vision.trainers.proj.paligemma.predict_fns
+import big_vision.utils as u
+# pylint: enable=all
+
+# We always want to be explicit about any host-device transfers.
+jax.config.update("jax_transfer_guard", "disallow")
+
+CKPT = flags.DEFINE_string(
+    "ckpt", default=None, help="Path to checkpoint.")
+IMAGE = flags.DEFINE_string(
+    "image", default=None, help="Path to input image.")
+
+SAMPLER = flags.DEFINE_string(
+    "sampler", default="greedy", help="Decoding strategy. Try `nucleus(0.1)`")
+RES = flags.DEFINE_integer(
+    "res", default=224, help="Image resolution (224, 448, 896).")
+MAX_DECODE_LEN = flags.DEFINE_integer(
+    "max_decode_len", default=128, help="Max total generation steps.")
+PREFILL_LEN = flags.DEFINE_integer(
+    "prefill_len", default=32, help="Size of prefill (prompt). "
+    "Shorter is faster, but too short will cut off your prompt.")
+CKPT_DTYPE = flags.DEFINE_string(
+    "ckpt_dtype", default=None,
+    help="Convert ckpt to dtype before using it (e.g. float16).")
+
+TOKENIZER = "gemma(tokensets=['loc', 'seg'])"
+
+
+def load_model(ckpt):
+  model_cfg = ml_collections.FrozenConfigDict(dict(
+      img=dict(variant="So400m/14", pool_type="none", scan=True),
+      llm=dict(vocab_size=256_000 + 1024 + 128),
+  ))
+  model = model_mod.Model(**model_cfg)
+  params = model_mod.load(None, ckpt, model_cfg)
+  return model, params
+
+
+def info(s, *a):
+  logging.info("\u001b[33mNOTE\u001b[0m: " + s, *a)
+  logging.flush()
+
+
+def cast_params(params):
+  return jax.tree.map(lambda x: x.astype(CKPT_DTYPE.value) if np.issubdtype(x.dtype, np.floating) else x, params)
+
+
+def main(argv):
+  info(f"{argv=}")
+  info("Loading model...")
+  model, params = load_model(CKPT.value)
+
+  predict_fns = big_vision.trainers.proj.paligemma.predict_fns.get_all(model)
+
+  info("Loading tokenizer...")
+  tokzr = big_vision.pp.tokenizer.get_tokenizer(TOKENIZER)
+
+  info("Creating mesh and sharding params...")
+  mesh = Mesh(jax.devices(), ("data"))
+  repl_sharding = NamedSharding(mesh, PartitionSpec())
+  data_sharding = NamedSharding(mesh, PartitionSpec("data"))
+  params_sharding = big_vision.sharding.infer_sharding(
+      params, strategy=[(".*", "fsdp(axis='data')")], mesh=mesh)
+
+  # Ship the params to device(s)
+  params = jax.tree.map(lambda x, sh: u.reshard(x, sh), params, params_sharding)
+  if CKPT_DTYPE.value:
+    # Note: if running out of HBM or memory consider convert the checkpoint
+    # ahead of time, or add feature to cast while loading.
+    params = jax.jit(cast_params, donate_argnums=(0,),
+                     out_shardings=params_sharding)(params)
+
+  # Mostly go through pp ops to build our batch:
+  pp_fn = big_vision.pp.builder.get_preprocess_fn("|".join([
+      f"decode|resize({RES.value})|value_range(-1, 1)",
+      f"tok(key='prefix', bos='yes', model={repr(TOKENIZER)})",
+      f"tok(key='septok', text='\\n', model={repr(TOKENIZER)})",
+      'masked_concat(["prefix", "septok"], mask_ar=[0, 0], mask_input=[1, 1])',
+      f'tolen({PREFILL_LEN.value}, pad_value=0, key="text")',
+      f'tolen({PREFILL_LEN.value}, pad_value=1, key="mask_ar")',
+      f'tolen({PREFILL_LEN.value}, pad_value=0, key="mask_input")',
+      'keep("image", "text", "mask_ar", "mask_input")',
+  ]), log_data=False)
+
+  decode = functools.partial(
+      predict_fns["decode"], devices=jax.devices(),
+      eos_token=tokzr.eos_token, max_decode_len=MAX_DECODE_LEN.value,
+      sampler=SAMPLER.value)
+
+  def make_batch(fname, prompt):
+    image = open(fname, "rb").read()
+
+    # Create an example
+    example = pp_fn({"image": image, "prefix": np.array(prompt)})
+    example["_mask"] = np.array(True)  # True means valid non-pad example
+
+    batch = jax.tree.map(lambda x: x[None], example)
+    return u.reshard(batch, repl_sharding)  # Move to device(s)
+
+  info("Precompiling inference function...")
+  decode({"params": params}, batch=make_batch(IMAGE.value, "caption en"))
+
+  info("Type a prompt and press enter, for example 'caption en': ")
+  for line in map(str.strip, sys.stdin):
+    tokens = decode({"params": params}, batch=make_batch(IMAGE.value, line))
+    tokens = jax.device_get(tokens)[0]  # First batch entry.
+
+    # TODO: b/lbeyer - flip around: output on stdout, logs on stderr.
+    print(tokzr.to_str(tokens), file=sys.stderr, flush=True)
+
+
+if __name__ == "__main__":
+  flags.mark_flag_as_required("ckpt")
+  flags.mark_flag_as_required("image")
+  app.run(main)
diff --git a/Tipsomaly/model/big_vision/trainers/proj/paligemma/train.py b/Tipsomaly/model/big_vision/trainers/proj/paligemma/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..44ec18ad4a81b975712e4320e3f8dc4b941c506a
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/paligemma/train.py
@@ -0,0 +1,521 @@
+# Copyright 2024 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Training loop for PaliGemma-style VLM."""
+# pylint: disable=consider-using-from-import
+# pylint: disable=logging-fstring-interpolation
+
+import functools
+import importlib
+import multiprocessing.pool
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+import big_vision.datasets.core as ds_core
+import big_vision.evaluators.common as eval_common
+import big_vision.input_pipeline as input_pipeline
+import big_vision.optax as bv_optax
+import big_vision.sharding as bv_sharding
+import big_vision.trainers.proj.paligemma.predict_fns as predict_fns
+import big_vision.utils as u
+from clu import parameter_overview
+import flax
+import flax.linen as nn
+import jax
+from jax.experimental import multihost_utils
+from jax.experimental.array_serialization import serialization as array_serial
+import jax.numpy as jnp
+import ml_collections as mlc
+from ml_collections import config_flags
+import numpy as np
+import optax
+import tensorflow as tf
+
+from tensorflow.io import gfile
+
+
+config_flags.DEFINE_config_file(
+    "config", None, "Training configuration.", lock_config=True)
+
+flags.DEFINE_string("workdir", default=None, help="Work unit directory.")
+flags.DEFINE_boolean("cleanup", default=False,
+                     help="Delete workdir (only) after successful completion.")
+
+# Adds jax flags to the program.
+jax.config.parse_flags_with_absl()
+# Transfer guard will fail the program whenever that data between a host and
+# a device is transferred implicitly. This often catches subtle bugs that
+# cause slowdowns and memory fragmentation. Explicit transfers are done
+# with jax.device_put and jax.device_get.
+jax.config.update("jax_transfer_guard", "disallow")
+# Fixes design flaw in jax.random that may cause unnecessary d2d comms.
+jax.config.update("jax_threefry_partitionable", True)
+
+
+NamedSharding = jax.sharding.NamedSharding
+P = jax.sharding.PartitionSpec
+
+
+def main(argv):
+  del argv
+
+  # This is needed on multihost systems, but crashes on non-TPU single-host.
+  if os.environ.get("BV_JAX_INIT"):
+    jax.distributed.initialize()
+
+  # Make sure TF does not touch GPUs.
+  tf.config.set_visible_devices([], "GPU")
+
+################################################################################
+#                                                                              #
+#                                Set up logging                                #
+#                                                                              #
+################################################################################
+
+  # Set up work directory and print welcome message.
+  config = flags.FLAGS.config
+  workdir = flags.FLAGS.workdir
+  logging.info(
+      f"\u001b[33mHello from process {jax.process_index()} holding "
+      f"{jax.local_device_count()}/{jax.device_count()} devices and "
+      f"writing to workdir {workdir}.\u001b[0m")
+  logging.info(f"The config:\n{config}")
+
+  save_ckpt_path = None
+  if workdir:  # Always create if requested, even if we may not write into it.
+    gfile.makedirs(workdir)
+    save_ckpt_path = os.path.join(workdir, "checkpoint.bv")
+
+  # The pool is used to perform misc operations such as logging in async way.
+  pool = multiprocessing.pool.ThreadPool(1)
+
+  # Here we register preprocessing ops from modules listed on `pp_modules`.
+  for m in config.get("pp_modules", ["ops_general", "ops_image", "ops_text"]):
+    importlib.import_module(f"big_vision.pp.{m}")
+
+  # Setup up logging and experiment manager.
+  xid, wid = -1, -1
+  fillin = lambda s: s
+  def info(s, *a):
+    logging.info("\u001b[33mNOTE\u001b[0m: " + s, *a)
+  def write_note(note):
+    if jax.process_index() == 0:
+      info("%s", note)
+
+  mw = u.BigVisionMetricWriter(xid, wid, workdir, config)
+
+  # Allow for things like timings as early as possible!
+  u.chrono.inform(measure=mw.measure, write_note=write_note)
+
+################################################################################
+#                                                                              #
+#                                Set up Mesh                                   #
+#                                                                              #
+################################################################################
+
+  # We rely on jax mesh_utils to organize devices, such that communication
+  # speed is the fastest for the last dimension, second fastest for the
+  # penultimate dimension, etc.
+  config_mesh = config.get("mesh", [("data", jax.device_count())])
+
+  # Sharding rules with the default of doing full data sharding.
+  sharding_rules = config.get("sharding_rules", [("act_batch", "data")])
+
+  write_note("Creating device mesh...")
+  mesh = u.create_device_mesh(
+      config_mesh,
+      allow_split_physical_axes=config.get("mesh_allow_split_physical_axes",
+                                           False))
+  repl_sharding = jax.sharding.NamedSharding(mesh, P())
+
+  # Consistent device order is important to ensure correctness of various train
+  # loop components, such as input pipeline, update step, evaluators. The
+  # order prescribed by the `devices_flat` variable should be used throughout
+  # the program.
+  devices_flat = mesh.devices.flatten()
+
+################################################################################
+#                                                                              #
+#                                Input Pipeline                                #
+#                                                                              #
+################################################################################
+
+  write_note("Initializing train dataset...")
+  batch_size = config.input.batch_size
+  if batch_size % jax.device_count() != 0:
+    raise ValueError(f"Batch size ({batch_size}) must "
+                     f"be divisible by device number ({jax.device_count()})")
+  info("Global batch size %d on %d hosts results in %d local batch size. With "
+       "%d dev per host (%d dev total), that's a %d per-device batch size.",
+       batch_size, jax.process_count(), batch_size // jax.process_count(),
+       jax.local_device_count(), jax.device_count(),
+       batch_size // jax.device_count())
+
+  train_ds, ntrain_img = input_pipeline.training(config.input)
+
+  total_steps = u.steps("total", config, ntrain_img, batch_size)
+  def get_steps(name, default=ValueError, cfg=config):
+    return u.steps(name, cfg, ntrain_img, batch_size, total_steps, default)
+
+  u.chrono.inform(total_steps=total_steps, global_bs=batch_size,
+                  steps_per_epoch=ntrain_img / batch_size)
+
+  info("Running for %d steps, that means %f epochs",
+       total_steps, total_steps * batch_size / ntrain_img)
+
+  # Start input pipeline as early as possible, this will kick-start filling
+  # shuffle buffers and get the first batch in a background thread.
+  n_prefetch = config.get("prefetch_to_device", 1)
+  train_iter = input_pipeline.start_global(
+      train_ds, devices_flat, n_prefetch, warmup=n_prefetch > 0)
+
+  # For mixed data, add per-dataset epoch and examples seen measurements.
+  if isinstance(config.input.data.get("name"), str):
+    measure_per_dataset_times = lambda step: None  # No-op
+  else:
+    nexamples = {
+        name: ds_core.get(**config.input[name].data).total_examples
+        for name in config.input.data
+    }
+    def measure_per_dataset_times(step):
+      total = sum(config.input.data.values())
+      for name, w in config.input.data.items():
+        w = w / total
+        mw.measure(f"examples_seen_{name}", u.chrono.accum_examples_seen * w)
+        mw.measure(f"epoch_{name}", step * batch_size * w / nexamples[name])
+
+################################################################################
+#                                                                              #
+#                           Create Model & Optimizer                           #
+#                                                                              #
+################################################################################
+
+  write_note(f"Initializing {config.model_name} model...")
+  model_mod = importlib.import_module(f"big_vision.models.{config.model_name}")
+  model = model_mod.Model(**mlc.FrozenConfigDict(config.get("model", {})))
+
+  def init(rng, partial_params=None):
+    batch = jax.tree.map(lambda x: jnp.zeros(x.shape, x.dtype.as_numpy_dtype),
+                         train_ds.element_spec)
+    _, variables = model.apply(  # flax init is just apply with mutable.
+        {"params": partial_params or {}},
+        batch["image"], batch["text"][:, :-1], batch["mask_ar"][:, :-1],
+        rngs={"params": rng, "dropout": rng},
+        mutable=["params"])
+    return flax.core.unfreeze(variables["params"])
+
+  # This seed makes the Jax part of things (like model init) deterministic.
+  # However, full training still won't be deterministic, for example due to the
+  # tf.data pipeline not being deterministic even if we would set TF seed.
+  # See (internal link) for a fun read on what it takes.
+  rng = jax.random.PRNGKey(u.put_cpu(config.get("seed", 0)))
+
+  write_note("Inferring parameter shapes...")
+  rng, rng_init = jax.random.split(rng)
+  params_shape = jax.eval_shape(init, rng_init)
+  params_shape = nn.unbox(params_shape)
+
+  write_note("Inferring optimizer state shapes...")
+  tx, sched_fns = bv_optax.make(config, params_shape, sched_kw=dict(
+      total_steps=total_steps, batch_size=batch_size, data_size=ntrain_img))
+  opt_shape = jax.eval_shape(tx.init, params_shape)
+  # We jit this, such that the arrays are created on the CPU, not device[0].
+  sched_fns_cpu = [u.jit_cpu()(sched_fn) for sched_fn in sched_fns]
+
+  if jax.process_index() == 0:
+    num_params = sum(np.prod(p.shape) for p in jax.tree.leaves(params_shape))
+    mw.measure("num_params", num_params)
+
+################################################################################
+#                                                                              #
+#                    Init and/or load model onto devices                       #
+#                                                                              #
+################################################################################
+
+  write_note("Inferring shardings...")
+  train_state_shape = {"params": params_shape, "opt": opt_shape}
+
+  strategy = config.get("sharding_strategy", [(".*", "replicate")])
+  train_state_sharding = bv_sharding.infer_sharding(
+      train_state_shape, strategy=strategy, mesh=mesh)
+
+  # Decide how to initialize training. The order is important.
+  # 1. Always resumes from the existing checkpoint, e.g. resumes a finetune job.
+  # 2. Resume from a previous checkpoint, e.g. start a cooldown training job.
+  # 3. Initialize model from scratch or from something, e.g. fine-tuning job.
+  resume_ckpt_path = None
+  if save_ckpt_path and gfile.exists(f"{save_ckpt_path}-LAST"):
+    resume_ckpt_path = save_ckpt_path
+  elif config.get("resume"):
+    resume_ckpt_path = fillin(config.resume)
+
+  if resume_ckpt_path:
+    write_note(f"Resuming training from checkpoint {resume_ckpt_path}...")
+    shardings = {
+        **train_state_sharding,
+        "chrono": jax.tree.map(lambda _: repl_sharding, u.chrono.save()),
+    }
+    loaded = u.load_checkpoint_ts(
+        resume_ckpt_path, tree=shardings, shardings=shardings)
+    train_state = {key: loaded[key] for key in train_state_sharding.keys()}
+    u.chrono.load(jax.device_get(loaded["chrono"]))
+    del loaded
+  else:
+    write_note(
+        f"Initialize model from {config.get('model_init') or 'scratch'}...")
+
+    # To avoid holding two copies of parameters we first call `model.load`
+    # and then initialize the missing variables.
+    if config.get("model_init"):
+      # We call `model.load` with params shape, so it can know all model params
+      # including their shapes and dtypes (also shardings once wired).
+      params = model_mod.load(
+          params_shape, config.model_init, config.get("model"),
+          **config.get("model_load", {}))
+
+      # Keep only params loaded by `model.load` and shard them into devices.
+      mask = jax.tree.map(
+          lambda x: not isinstance(x, jax.ShapeDtypeStruct), params)
+      params = u.reshard(u.tree_filter(params, mask),
+                         u.tree_filter(train_state_sharding["params"], mask))
+
+      parameter_overview.log_parameter_overview(
+          params, msg="Restored params",
+          include_stats="global", jax_logging_process=0)
+    else:
+      params = {}
+
+    # Init will initialize any missing params.
+    rng_init = u.reshard(rng_init, repl_sharding)
+    params = jax.jit(
+        init, donate_argnums=1, out_shardings=train_state_sharding["params"])(
+            rng_init, params)
+    params = nn.unbox(params)
+
+    # Initialize optimizer and construct train_state.
+    opt = jax.jit(tx.init, out_shardings=train_state_sharding["opt"])(params)
+    train_state = {"params": params, "opt": opt}
+    del params, opt  # Delete to avoid memory leak or accidental reuse.
+
+  parameter_overview.log_parameter_overview(
+      train_state["params"], msg="Parameter overview",
+      include_stats="global", jax_logging_process=0)
+
+  rng, rng_loop = jax.random.split(rng, 2)
+  rng_loop = u.reshard(rng_loop, repl_sharding)
+  del rng, rng_init  # not used anymore, so delete it.
+
+################################################################################
+#                                                                              #
+#                                 Update Step                                  #
+#                                                                              #
+################################################################################
+
+  @functools.partial(
+      jax.jit,
+      donate_argnums=(0,),
+      out_shardings=(train_state_sharding, repl_sharding))
+  def update_fn(train_state, rng, batch):
+    """Update step."""
+
+    step_count = bv_optax.get_count(train_state["opt"], jittable=True)
+    rng = jax.random.fold_in(rng, step_count)
+    assert "mixup" not in config, "Mixup is not supported for SigLIP."
+
+    # Get device-specific loss rng.
+    _, rng_model = jax.random.split(rng, 2)
+
+    imgs, txts, mask_ar = batch["image"], batch["text"], batch["mask_ar"]
+
+    def loss_fn(params):
+      text_logits, _ = model.apply(
+          {"params": params}, imgs, txts[:, :-1], mask_ar[:, :-1],
+          train=True, rngs={"dropout": rng_model})
+
+      logp = jax.nn.log_softmax(text_logits, axis=-1)
+      targets = jax.nn.one_hot(txts[:, 1:], text_logits.shape[-1])
+      off_value = config.get("label_smoothing", 0.0)
+      if off_value > 0:
+        denom = text_logits.shape[-1] - 1
+        targets = jnp.where(
+            targets == 1.0, 1.0 - off_value, off_value / denom)
+
+      # Sum across vocab.
+      token_pplx = jnp.sum(logp * targets, axis=-1)
+
+      # Shift by one since the loss is on the _next_ token.
+      mask_loss = batch["mask_loss"][:, 1:]
+      token_pplx = token_pplx * mask_loss
+      pplx = -jnp.sum(token_pplx, axis=-1)
+      pplx /= jnp.clip(jnp.sum(mask_loss, axis=-1), 1)
+
+      # In this dict the (outer) reduction is along batch.
+      measurements = dict(
+          training_loss=jnp.mean(pplx),
+          avg_sup_seqlen=jnp.mean(jnp.sum(mask_loss, axis=-1)),
+          max_sup_seqlen=jnp.max(jnp.sum(mask_loss, axis=-1)),
+      )
+
+      return measurements["training_loss"], measurements
+
+    params, opt = train_state["params"], train_state["opt"]
+    (_, measurements), grads = jax.value_and_grad(loss_fn, has_aux=True)(params)
+    updates, opt = tx.update(grads, opt, params)
+    params = optax.apply_updates(params, updates)
+
+    gs = jax.tree.leaves(bv_optax.replace_frozen(config.schedule, grads, 0.))
+    measurements["l2_grads"] = jnp.sqrt(sum([jnp.sum(g * g) for g in gs]))
+    ps = jax.tree.leaves(params)
+    measurements["l2_params"] = jnp.sqrt(sum([jnp.sum(p * p) for p in ps]))
+    us = jax.tree.leaves(updates)
+    measurements["l2_updates"] = jnp.sqrt(sum([jnp.sum(u * u) for u in us]))
+
+    return {"params": params, "opt": opt}, measurements
+
+################################################################################
+#                                                                              #
+#                                 Setup Evals                                  #
+#                                                                              #
+################################################################################
+
+  # Only initialize evaluators when they are first needed.
+  @functools.lru_cache(maxsize=None)
+  def evaluators():
+    return eval_common.from_config(
+        config,
+        predict_fns.get_all(model),
+        lambda s: write_note(f"Init evaluator: {s}…\n{u.chrono.note}"),
+        lambda key, cfg: get_steps(key, default=None, cfg=cfg),
+        devices_flat,
+    )
+
+  # At this point we need to know the current step to see whether to run evals.
+  write_note("Inferring the first step number...")
+  first_step_device = bv_optax.get_count(train_state["opt"], jittable=True)
+  first_step = int(jax.device_get(first_step_device))
+  u.chrono.inform(first_step=first_step)
+
+  # Note that training can be pre-empted during the final evaluation (i.e.
+  # just after the final checkpoint has been written to disc), in which case we
+  # want to run the evals.
+  if first_step in (total_steps, 0):
+    write_note("Running initial or final evals...")
+    mw.step_start(first_step)
+    for (name, evaluator, _, prefix) in evaluators():
+      if config.evals[name].get("skip_first") and first_step != total_steps:
+        continue
+      write_note(f"{name} evaluation...\n{u.chrono.note}")
+      with u.chrono.log_timing(f"z/secs/eval/{name}"):
+        with mesh, nn.logical_axis_rules(sharding_rules):
+          for key, value in evaluator.run(train_state):
+            mw.measure(f"{prefix}{key}", value)
+
+################################################################################
+#                                                                              #
+#                                  Train Loop                                  #
+#                                                                              #
+################################################################################
+
+  prof = None  # Keeps track of start/stop of profiler state.
+  ckpt_mngr = None
+
+  write_note("Starting training loop, compiling the first step...")
+  for step, batch in zip(range(first_step + 1, total_steps + 1), train_iter):
+    mw.step_start(step)
+
+    with jax.profiler.StepTraceAnnotation("train_step", step_num=step):
+      with u.chrono.log_timing("z/secs/update0", noop=step > first_step + 1):
+        with mesh, nn.logical_axis_rules(sharding_rules):
+          train_state, measurements = update_fn(train_state, rng_loop, batch)
+
+    # On the first host, let's always profile a handful of early steps.
+    if jax.process_index() == 0:
+      prof = u.startstop_prof(prof, step, first_step, get_steps("log_training"))
+
+    # Report training progress
+    if (u.itstime(step, get_steps("log_training"), total_steps, host=0)
+        or u.chrono.warmup and jax.process_index() == 0):
+      for i, sched_fn_cpu in enumerate(sched_fns_cpu):
+        mw.measure(f"global_schedule{i if i else ''}",
+                   sched_fn_cpu(u.put_cpu(step - 1)))
+      measurements = jax.device_get(measurements)
+      for name, value in measurements.items():
+        mw.measure(name, value)
+      u.chrono.tick(step)
+      measure_per_dataset_times(step)
+
+      for k in ("training_loss", "l2_grads", "l2_updates", "l2_params"):
+        if not np.isfinite(measurements.get(k, 0.0)):
+          raise RuntimeError(f"{k} became nan or inf somewhere within steps "
+                             f"[{step - get_steps('log_training')}, {step}]")
+
+    # Checkpoint saving
+    keep_last = total_steps if get_steps("ckpt", None) else None
+    keep_ckpt_steps = get_steps("keep_ckpt", None) or keep_last
+    if save_ckpt_path and (
+        (keep := u.itstime(step, keep_ckpt_steps, total_steps, first=False))
+        or u.itstime(step, get_steps("ckpt", None), total_steps, first=True)
+    ):
+      u.chrono.pause(wait_for=train_state)
+
+      # Copy because we add extra stuff to the checkpoint.
+      ckpt = {**train_state}
+
+      # To save chrono state correctly and safely in a multihost setup, we
+      # broadcast the state to all hosts and convert it to a global array.
+      with jax.transfer_guard("allow"):
+        chrono_ckpt = multihost_utils.broadcast_one_to_all(u.chrono.save())
+      chrono_shardings = jax.tree.map(lambda _: repl_sharding, chrono_ckpt)
+      ckpt = ckpt | {"chrono": u.reshard(chrono_ckpt, chrono_shardings)}
+
+      ckpt_mngr = ckpt_mngr or array_serial.GlobalAsyncCheckpointManager()
+      u.save_checkpoint_ts(ckpt_mngr, ckpt, save_ckpt_path, step, keep)
+      u.chrono.resume()
+
+    for (name, evaluator, log_steps, prefix) in evaluators():
+      if u.itstime(step, log_steps, total_steps, first=False, last=True):
+        u.chrono.pause(wait_for=train_state)
+        u.chrono.tick(step)  # Record things like epoch number, core hours etc.
+        write_note(f"{name} evaluation...\n{u.chrono.note}")
+        with u.chrono.log_timing(f"z/secs/eval/{name}"):
+          with mesh, nn.logical_axis_rules(sharding_rules):
+            for key, value in evaluator.run(train_state):
+              mw.measure(f"{prefix}{key}", jax.device_get(value))
+        u.chrono.resume()
+    mw.step_end()
+
+  # Always give a chance to stop the profiler, no matter how things ended.
+  # TODO: can we also do this when dying of an exception like OOM?
+  if jax.process_index() == 0 and prof is not None:
+    u.startstop_prof(prof)
+
+  # Last note needs to happen before the pool's closed =)
+  write_note(f"Done!\n{u.chrono.note}")
+
+  pool.close()
+  pool.join()
+  mw.close()
+  if ckpt_mngr:
+    ckpt_mngr.wait_until_finished()
+
+  # Make sure all hosts stay up until the end of main.
+  u.sync()
+
+  u.maybe_cleanup_workdir(workdir, flags.FLAGS.cleanup, info)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/Tipsomaly/model/big_vision/trainers/proj/uvim/coco_utils.py b/Tipsomaly/model/big_vision/trainers/proj/uvim/coco_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..701fbdebefbbc4b8b29b958c9ae0bb5c4e2760c2
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/uvim/coco_utils.py
@@ -0,0 +1,75 @@
+# Copyright 2022 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities to inspect coco data and predictions in notebooks."""
+# pylint: disable=consider-using-from-import
+import functools
+import json
+
+import numpy as np
+from panopticapi import utils as pycoco_utils
+from skimage import segmentation
+
+import tensorflow.io.gfile as gfile
+
+
+import os
+ROOT = os.environ.get('COCO_DATA_DIR', '.')
+
+
+PANOPTIC_COCO_CATS_FILE = f'{ROOT}/panoptic_coco_categories.json'
+
+
+@functools.lru_cache(maxsize=None)
+def _coco_panoptic_categories():
+  with gfile.GFile(PANOPTIC_COCO_CATS_FILE, 'r') as f:
+    categories_list = json.load(f)
+  return tuple(categories_list)
+
+
+def rgb_panoptic_from_twochannels(twochannels, boundaries: bool = False):
+  """Makes a RGB panoptic output and segments_info from a twochannels view."""
+  semantics = twochannels[..., 0]
+  instances = twochannels[..., 1]
+  max_instances = np.max(instances) + 1
+  merged = semantics * max_instances + instances
+  merged = np.where(semantics < 0, semantics, merged)
+
+  categories_list = _coco_panoptic_categories()
+  categories = {category['id']: category for category in categories_list}
+  id_generator = pycoco_utils.IdGenerator(categories)
+  segments_info = {}
+  rgb = np.zeros((*instances.shape[:2], 3), dtype=np.uint8)
+
+  for merged_id in np.unique(merged):
+    if merged_id // max_instances > 0:
+      category = categories_list[int(merged_id // max_instances) - 1]
+      segment_id, color = id_generator.get_id_and_color(category['id'])
+    else:
+      category = {'id': -1, 'name': 'void', 'isthing': False}
+      segment_id, color = -1, np.array([0, 0, 0])
+    segments_info[segment_id] = {
+        'id': segment_id,
+        'color': color,
+        'category_id': category['id'],
+        'name': category['name'],
+        'isthing': category['isthing'],
+    }
+    rgb[merged == merged_id] = color
+
+  if boundaries:
+    boundaries = segmentation.find_boundaries(
+        pycoco_utils.rgb2id(rgb), mode='thick')
+    rgb[boundaries] = 0
+  return rgb, segments_info
diff --git a/Tipsomaly/model/big_vision/trainers/proj/uvim/colorization_task.py b/Tipsomaly/model/big_vision/trainers/proj/uvim/colorization_task.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d6ead98c67113fe37ed08ea5b7262dcff4846b2
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/uvim/colorization_task.py
@@ -0,0 +1,62 @@
+# Copyright 2022 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Inputs, outputs and losses for colorization task."""
+import einops
+import jax.numpy as jnp
+import numpy as np
+
+ONE_HOT_AXIS = -2
+
+
+def input_pp(batch, config):
+  """Make inputs for colorization task."""
+  if "labels" not in batch:
+    # During predict of phase2 there is no 'labels' field.
+    x = None
+  else:
+    hp, wp = config.model.patch_size
+    x = {
+        "color": batch["labels"],
+    }
+    # Convert labels from (B, H, W) to (B, num_patches, C, patch_size)
+    x["color"] = einops.rearrange(
+        x["color"], "b (hn hp) (wn wp) c -> b (hn wn) c (hp wp)", hp=hp, wp=wp)
+  ctx = batch.get("image_ctx", batch.get("image", None))
+  return {"ctx": ctx, "x": x}
+
+
+def loss_fn(logits, batch, config):
+  """Compute loss for colorization task."""
+  labels = input_pp(batch, config)["x"]
+  error = logits["color"] - labels["color"]
+  loss = jnp.square(error)
+  return loss, {"loss_color": loss}
+
+
+def predict_outputs(logits, config):
+  """Make outputs for colorization task."""
+  # Map logits to (height, width, channels).
+  hp, wp = config.model.patch_size
+  hn, wn = np.array(config.model.input_size) // np.array((hp, wp))
+  assert ONE_HOT_AXIS == -2, "Rearrange below depends on this."
+  output = einops.rearrange(
+      logits["color"],
+      "b (hn wn) c (hp wp) -> b (hn hp) (wn wp) c",
+      hn=hn,
+      wn=wn,
+      hp=hp,
+      wp=wp)
+  output = jnp.clip(output, -1., 1.)
+  return {"color": output}
diff --git a/Tipsomaly/model/big_vision/trainers/proj/uvim/depth_task.py b/Tipsomaly/model/big_vision/trainers/proj/uvim/depth_task.py
new file mode 100644
index 0000000000000000000000000000000000000000..3878a782a870178a85fe17e108b46fc0425f30b1
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/uvim/depth_task.py
@@ -0,0 +1,91 @@
+# Copyright 2022 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Inputs, outputs and losses for depth prediction task."""
+import big_vision.utils as u
+import einops
+import jax
+import jax.numpy as jnp
+import numpy as np
+
+
+ONE_HOT_AXIS = -2
+
+
+def input_pp(batch, config):
+  """Makes inputs for depth prediction task."""
+  if "labels" not in batch:
+    x = None
+  else:
+    hp, wp = config.model.patch_size
+    depth = batch["labels"][..., 0]
+
+    # Discretize to [0, ..., bins - 1].
+    nbins = config.model.inputs.depth[ONE_HOT_AXIS]
+    mind = config.min_depth
+    maxd = config.max_depth
+    depth = (depth - mind) / (maxd - mind)
+    depth *= nbins
+    depth = jnp.floor(depth).astype(jnp.int32)
+    depth = jnp.minimum(depth, nbins - 1)
+    depth = jnp.maximum(depth, 0)
+
+    # Converts labels from (B, H, W, c) to (B, num_patches, c, patch_size).
+    depth = jax.nn.one_hot(
+        einops.rearrange(
+            depth, "b (hn hp) (wn wp) -> b (hn wn) (hp wp)", hp=hp, wp=wp),
+        num_classes=config.model.inputs.depth[ONE_HOT_AXIS],
+        axis=ONE_HOT_AXIS)
+    x = {"depth": depth}
+  ctx = batch.get("image_ctx", batch.get("image", None))
+  return {"ctx": ctx, "x": x}
+
+
+def loss_fn(predictions, batch, config):
+  """Computes loss for depth prediction task."""
+  labels = input_pp(batch, config)["x"]
+  losses = {}
+  loss = u.softmax_xent(
+      logits=predictions["depth"], labels=labels["depth"], reduction=False,
+      axis=ONE_HOT_AXIS)
+  # Do not train on the closest class; usually regions of the image with
+  # depth==0, which is the default for regions with no depth signal.
+  # TODO: Encode depth==0 as class==-1.
+  mask = jnp.argmax(labels["depth"], ONE_HOT_AXIS) != 0
+  loss = loss * mask
+  losses["loss_depth"] = loss
+  return sum(losses.values()), losses
+
+
+def predict_outputs(predictions, config):
+  """Makes outputs for depth predictin tasks."""
+  # Maps predictions to (height, width, channels).
+  hp, wp = config.model.patch_size
+  hn, wn = np.array(config.model.input_size) // np.array((hp, wp))
+  depth = einops.rearrange(
+      predictions["depth"],
+      "b (hn wn) c (hp wp) -> b (hn hp) (wn wp) c",
+      hn=hn, wn=wn, hp=hp, wp=wp)
+
+  depth = jnp.argmax(depth, axis=-1)  # [B, H, W]
+
+  # Revert discretization.
+  nbins = config.model.inputs.depth[ONE_HOT_AXIS]
+  mind = config.min_depth
+  maxd = config.max_depth
+  depth = depth.astype(jnp.float32) + 0.5  # Undoes floor in expectation.
+  depth /= nbins
+  depth = depth * (maxd - mind) + mind
+
+  return {"depth": depth}
diff --git a/Tipsomaly/model/big_vision/trainers/proj/uvim/panoptic_task.py b/Tipsomaly/model/big_vision/trainers/proj/uvim/panoptic_task.py
new file mode 100644
index 0000000000000000000000000000000000000000..16e690935c44532d01e3ea27df2f0e7480339094
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/uvim/panoptic_task.py
@@ -0,0 +1,87 @@
+# Copyright 2022 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Inputs, outputs and losses for panoptic task."""
+import big_vision.utils as u
+import einops
+import jax
+import jax.numpy as jnp
+import numpy as np
+
+ONE_HOT_AXIS = -2
+
+
+def input_pp(batch, config):
+  """Make inputs for panoptic segmentation task."""
+  if "labels" not in batch:
+    # During predict of phase2 there is no 'labels' field.
+    x = None
+  else:
+    hp, wp = config.model.patch_size
+    x = {
+        "semantics": batch["labels"][..., 0],
+        "instances": batch["labels"][..., 1],
+    }
+    # Convert labels from (B, H, W) to (B, num_patches, num_classes, patch_size)
+    for key in ["semantics", "instances"]:
+      x[key] = jax.nn.one_hot(
+          einops.rearrange(
+              x[key], "b (hn hp) (wn wp) -> b (hn wn) (hp wp)", hp=hp, wp=wp),
+          num_classes=config.model.inputs[key][ONE_HOT_AXIS], axis=ONE_HOT_AXIS)
+  ctx = batch.get("image_ctx", batch.get("image", None))
+  return {"ctx": ctx, "x": x}
+
+
+def loss_fn(logits, batch, config):
+  """Compute loss for panoptic task."""
+  labels = input_pp(batch, config)["x"]
+  losses = {}
+  for key in ["semantics", "instances"]:
+    losses[f"loss_{key}"] = u.softmax_xent(
+        logits=logits[key], labels=labels[key], reduction=False,
+        axis=ONE_HOT_AXIS)
+  return sum(losses.values()), losses
+
+
+def predict_outputs(logits, config, min_fraction=0.0):
+  """Make outputs for panoptic segmentation task."""
+  # Map logits to (height, width, channels).
+  hp, wp = config.model.patch_size
+  hn, wn = np.array(config.model.input_size) // np.array((hp, wp))
+  outputs = {}
+  for key in ["semantics", "instances"]:
+    assert ONE_HOT_AXIS == -2, "Rearrange below depends on this."
+    outputs[key] = einops.rearrange(
+        logits[key],
+        "b (hn wn) c (hp wp) -> b (hn hp) (wn wp) c",
+        hn=hn, wn=wn, hp=hp, wp=wp)
+  return panoptic_predictions_from_logits(
+      **outputs, min_fraction=min_fraction)
+
+
+def panoptic_predictions_from_logits(semantics, instances, min_fraction=0.0):
+  """Make panoptic prediction from logits."""
+  ins = jnp.argmax(instances, axis=-1)
+  # Note: Make sure each instance has all pixels annotated with same label.
+  # Otherwise they are further split into more instances and greatly affect
+  # the number of unmatched predicted segments (FP) and RQ.
+  masks = jax.nn.one_hot(ins, instances.shape[-1], dtype=jnp.int32)
+  label = jnp.argmax(jnp.einsum("bhwk,bhwn->bnk", semantics, masks), axis=-1)
+  sem = jnp.einsum("bhwn,bn->bhw", masks, label)
+  out = jnp.stack([sem, ins], axis=-1)
+  # Filter out small objects
+  fraction = jnp.sum(masks, axis=(1, 2), keepdims=True)/np.prod(ins.shape[1:3])
+  mask_big = (fraction > min_fraction).astype("int32")
+  mask_big_spatial = jnp.sum(masks * mask_big, axis=-1, keepdims=True) > 0
+  return out * mask_big_spatial.astype("int32")
diff --git a/Tipsomaly/model/big_vision/trainers/proj/uvim/train.py b/Tipsomaly/model/big_vision/trainers/proj/uvim/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..04ebd51cd9f18b4855f3352ceb907055f872b171
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/uvim/train.py
@@ -0,0 +1,440 @@
+# Copyright 2022 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Train loop for training the stage-II model."""
+# pylint: disable=consider-using-from-import
+import functools
+import importlib
+import multiprocessing.pool
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+from big_vision import input_pipeline
+import big_vision.datasets.core as ds_core
+import big_vision.evaluators.common as eval_common
+import big_vision.models.proj.uvim.decode as decode
+import big_vision.optax as bv_optax
+import big_vision.pp.builder as pp_builder
+import big_vision.utils as u
+from clu import parameter_overview
+import flax
+import jax
+import jax.numpy as jnp
+from ml_collections import config_flags
+import numpy as np
+import optax
+
+import tensorflow.io.gfile as gfile
+
+
+config_flags.DEFINE_config_file(
+    "config", None, "Training configuration.", lock_config=True)
+
+flags.DEFINE_string("workdir", default=None, help="Work unit directory.")
+flags.DEFINE_boolean("cleanup", default=False,
+                     help="Delete workdir (only) after successful completion.")
+
+# Adds jax flags to the program.
+jax.config.parse_flags_with_absl()
+
+
+FLAGS = flags.FLAGS
+ONE_HOT_AXIS = -2
+partial = functools.partial
+
+
+def get_model(config):
+  mod = importlib.import_module(f"big_vision.models.{config.model_name}")
+  model = mod.Model(**config.model)
+  return model, mod
+
+
+def setup_task(config):
+  """Get functions and params to encode and decode labels as token sequences."""
+  config = config.oracle
+
+  # Define task input and predict functions.
+  task_module = importlib.import_module(f"big_vision.trainers.{config.task}")
+  input_fn = partial(task_module.input_pp, config=config)
+  predict_outputs_fn = partial(task_module.predict_outputs, config=config)
+
+  oracle, mod = get_model(config)
+  if config.get("model_init", None):
+    params, state = mod.load(None, config.model_init)
+    params = {"params": params, "state": state}
+  else:
+    params = {}
+
+  def encode_labels(params, batch):
+    inputs = input_fn(batch)
+    code = oracle.apply(params, **inputs, method=oracle.encode)[1]["code"]
+    return code + 1  # To avoid padding symbol.
+
+  def decode_labels(params, code, batch, **kwargs):
+    code = code - 1
+    inputs = input_fn(batch)
+    inputs["x"] = code
+    logits, _ = oracle.apply(
+        params, **inputs, discrete_input=True, **kwargs, method=oracle.decode)
+    return logits
+
+  return encode_labels, decode_labels, predict_outputs_fn, params
+
+
+def main(argv):
+  del argv
+
+  config = FLAGS.config
+  workdir = FLAGS.workdir
+  logging.info("\u001b[33mHello from process %i holding %i/%i devices and "
+               "writing to workdir %s.\u001b[0m", jax.process_index(),
+               jax.local_device_count(), jax.device_count(), workdir)
+
+  save_ckpt_path = None
+  if workdir:  # Always create if requested, even if we may not write into it.
+    gfile.makedirs(workdir)
+    save_ckpt_path = os.path.join(workdir, "checkpoint.npz")
+
+  # The pool is used to perform misc operations such as logging in async way.
+  pool = multiprocessing.pool.ThreadPool()
+
+  # Here we register preprocessing ops from modules listed on `pp_modules`.
+  for m in config.get("pp_modules",
+                      ["ops_general", "ops_image", "proj.uvim.pp_ops"]):
+    importlib.import_module(f"big_vision.pp.{m}")
+
+  # This seed makes the Jax part of things (like model init) deterministic.
+  # However, full training still won't be deterministic, for example due to the
+  # tf.data pipeline not being deterministic even if we would set TF seed.
+  # See (internal link) for a fun read on what it takes.
+  rng = jax.random.PRNGKey(config.get("seed", 0))
+
+  # These functions do more stuff internally, for OSS release we mock them by
+  # trivial alternatives in order to minize disruptions in the code.
+  xid, wid = -1, -1
+  fillin = lambda s: s
+  def info(s, *a):
+    logging.info("\u001b[33mNOTE\u001b[0m: " + s, *a)
+  def write_note(note):
+    if jax.process_index() == 0:
+      info("%s", note)
+
+  write_note("Initializing...")
+
+  batch_size = config.input.batch_size
+  if batch_size % jax.device_count() != 0:
+    raise ValueError(f"Batch size ({batch_size}) must "
+                     f"be divisible by device number ({jax.device_count()})")
+  info("Global batch size %d on %d hosts results in %d local batch size. With "
+       "%d dev per host (%d dev total), that's a %d per-device batch size.",
+       batch_size, jax.process_count(), batch_size // jax.process_count(),
+       jax.local_device_count(), jax.device_count(),
+       batch_size // jax.device_count())
+
+  # First thing after above sanity checks, so we can log "start" ticks.
+  mw = u.BigVisionMetricWriter(xid, wid, workdir, config)
+  chrono = u.Chrono()
+
+  write_note("Initializing train dataset...")
+  train_data = ds_core.get(**config.input.data)
+  train_ds = input_pipeline.make_for_train(
+      data=train_data.get_tfdata(ordered=False),
+      batch_size=batch_size,
+      preprocess_fn=pp_builder.get_preprocess_fn(config.input.get("pp")),
+      shuffle_buffer_size=config.input.get("shuffle_buffer_size"),
+      cache_raw=config.input.get("cache_raw", False),
+      filter_fn=config.input.get("filter_fn"),
+  )
+
+  # Start prefetching already.
+  n_prefetch = config.get("prefetch_to_device", 1)
+  train_iter = input_pipeline.start_input_pipeline(train_ds, n_prefetch)
+  ntrain_img = train_data.total_examples
+
+  def get_steps(name, default=ValueError):  # partial doesn't work well here.
+    return u.steps(name, config, ntrain_img, batch_size, default)
+  total_steps = get_steps("total")
+
+  info("Running for %d steps, that means %f epochs",
+       total_steps, total_steps * batch_size / ntrain_img)
+
+  write_note(f"Initializing {config.model_name} model...")
+  model, model_mod = get_model(config)
+
+  encode_labels, decode_labels, predict_outputs_fn, task_params = (
+      setup_task(config))
+
+  # We want all parameters to be created in host RAM, not on any device, they'll
+  # be sent there later as needed, otherwise we already encountered two
+  # situations where we allocate them twice.
+  @partial(jax.jit, backend="cpu")
+  def init(rng):
+    batch = jax.tree_map(
+        lambda x: jnp.zeros(x.shape, x.dtype.as_numpy_dtype),
+        train_ds.element_spec)
+    images = batch["image"]
+    labels = encode_labels(task_params, batch)
+    variables = model.init(rng, images, labels)
+    params = flax.core.unfreeze(variables["params"])
+    return params
+
+  rng, init_rng = jax.random.split(rng)
+  params_cpu = init(init_rng)
+
+  if jax.process_index() == 0:
+    num_params = sum(p.size for p in jax.tree_leaves(params_cpu))
+    parameter_overview.log_parameter_overview(params_cpu, msg="init params")
+    mw.measure("num_params", num_params)
+
+  write_note(f"Initializing {config.optax_name} optimizer...")
+  tx, sched_fns = bv_optax.make(config, params_cpu, sched_kw=dict(
+      total_steps=total_steps, batch_size=batch_size, data_size=ntrain_img))
+
+  # We jit this, such that the arrays are created on the CPU, not device[0].
+  opt_cpu = jax.jit(tx.init, backend="cpu")(params_cpu)
+  sched_fns_cpu = [jax.jit(sched_fn, backend="cpu") for sched_fn in sched_fns]
+
+  @partial(jax.pmap, axis_name="batch", donate_argnums=(0, 1))
+  def update_fn(params, opt, batch, update_rng, task_params):
+    """Update step."""
+    images = batch["image"]
+    labels = encode_labels(task_params, batch)
+
+    measurements = {}
+
+    rng, new_rng = jax.random.split(update_rng)
+    # bind the rng key to the device id (which is unique across hosts)
+    rng_local = jax.random.fold_in(rng, jax.lax.axis_index("batch"))
+
+    def loss_fn(params, images, labels):
+      logits = model.apply({"params": params}, images, labels, train=True,
+                           rngs={"dropout": rng_local})
+      loss = u.weighted_softmax_xent(
+          logits=logits, labels=labels,
+          reduction=True, normalize=True)
+      return loss
+
+    l, grads = jax.value_and_grad(loss_fn)(params, images, labels)
+    l, grads = jax.lax.pmean((l, grads), axis_name="batch")
+    updates, opt = tx.update(grads, opt, params)
+    params = optax.apply_updates(params, updates)
+
+    gs = jax.tree_leaves(bv_optax.replace_frozen(config.schedule, grads, 0.))
+    measurements["l2_grads"] = jnp.sqrt(sum([jnp.vdot(g, g) for g in gs]))
+    ps = jax.tree_leaves(params)
+    measurements["l2_params"] = jnp.sqrt(sum([jnp.vdot(p, p) for p in ps]))
+    us = jax.tree_leaves(updates)
+    measurements["l2_updates"] = jnp.sqrt(sum([jnp.vdot(u, u) for u in us]))
+
+    return params, opt, l, new_rng, measurements
+
+  # Define evaluators.
+  def validation_fn(params, batch):
+    """Compute per-example metrics."""
+    params, task_params = params["params"], params["task_params"]
+    images = batch["image"]
+    labels = encode_labels(task_params, batch)
+    logits = model.apply({"params": params}, images, labels, train=False)
+    loss = u.weighted_softmax_xent(
+        logits=logits, labels=labels,
+        reduction=False, normalize=True)
+    losses = {"loss": loss}
+    return jax.tree_map(
+        lambda x: jnp.mean(x, axis=tuple(range(1, x.ndim))),
+        losses)
+
+  def predict_fn(params, batch, seed=0, temperature=1e-7, **extra):
+    params, task_params = params["params"], params["task_params"]
+
+    # Derive a rng key from the inputs so that all batches use different keys.
+    if "image/id" in batch:
+      key = batch["image/id"]
+    else:
+      key = batch["image"].sum(axis=[1, 2, 3]).astype(jnp.int32)
+    local_rng = jax.lax.scan(
+        lambda k, x: (jax.random.fold_in(k, x), None),
+        jax.random.PRNGKey(seed),
+        key,
+    )[0]
+
+    images = batch["image"]
+    batch_size = images.shape[0]
+    prompts = jnp.zeros((batch_size, config.model.seq_len), dtype=jnp.int32)
+    seqs, _, _ = decode.temperature_sampling(
+        params={"params": params}, model=model, seed=local_rng,
+        inputs=images, prompts=prompts,
+        num_samples=1, eos_token=-1, prefill=False,
+        temperature=temperature)
+    seqs = jnp.squeeze(seqs, 1)
+    logits = decode_labels(task_params, seqs, batch)
+    return predict_outputs_fn(logits, **extra)
+
+  # Only initialize evaluators when they are first needed.
+  @functools.lru_cache(maxsize=None)
+  def evaluators():
+    return eval_common.from_config(
+        config, {"predict": predict_fn, "validation": validation_fn},
+        lambda s: write_note(f"Initializing evaluator: {s}...\n{chrono.note}")
+    )
+
+  # Decide how to initialize training. The order is important.
+  # 1. Always resumes from the existing checkpoint, e.g. resumes a finetune job.
+  # 2. Resume from a previous checkpoint, e.g. start a cooldown training job.
+  # 3. Initialize model from something, e,g, start a fine-tuning job.
+  # 4. Initialize part of the model from something, eg. only encoder or decoder.
+  # 5. Train from scratch.
+  resume_ckpt_path = None
+  if save_ckpt_path and gfile.exists(save_ckpt_path):
+    resume_ckpt_path = save_ckpt_path
+  elif config.get("resume"):
+    resume_ckpt_path = fillin(config.resume)
+  if resume_ckpt_path:
+    write_note("Resume training from checkpoint...")
+    checkpoint = {
+        "params": params_cpu,
+        "opt": opt_cpu,
+        "chrono": chrono.save(),
+    }
+    checkpoint_tree = jax.tree_structure(checkpoint)
+    loaded = u.load_checkpoint(checkpoint_tree, resume_ckpt_path)
+    # bfloat16 type gets lost when data is saved to disk, so we recover it.
+    checkpoint = jax.tree_map(u.recover_dtype, loaded)
+    params_cpu, opt_cpu = checkpoint["params"], checkpoint["opt"]
+    chrono.load(checkpoint["chrono"])
+  elif config.get("model_init"):
+    write_note(f"Initialize model from {config.model_init}...")
+    params_cpu = model_mod.load(
+        params_cpu, config.model_init, config.model,
+        **config.get("model_load", {}))
+    if jax.process_index() == 0:
+      parameter_overview.log_parameter_overview(
+          params_cpu, msg="restored params")
+
+  write_note("Kicking off misc stuff...")
+  first_step = bv_optax.get_count(opt_cpu)
+  chrono.inform(first_step, total_steps, batch_size, ntrain_img / batch_size)
+  prof = None  # Keeps track of start/stop of profiler state.
+
+  write_note(f"Replicating...\n{chrono.note}")
+  params_repl = flax.jax_utils.replicate(params_cpu)
+  opt_repl = flax.jax_utils.replicate(opt_cpu)
+  task_params = flax.jax_utils.replicate(task_params)
+  update_rngs = flax.jax_utils.replicate(rng)
+
+  ckpt_writer = None
+
+  write_note(f"First step compilations...\n{chrono.note}")
+  error = None  # For exiting with an error after cleanup. Avoids indentation.
+
+  # Using a python integer for step here, because opt.state.step is allocated
+  # on TPU during replication.
+  for step, batch in zip(range(first_step + 1, total_steps + 1), train_iter):
+    mw.step_start(step)
+
+    with jax.profiler.StepTraceAnnotation("train_step", step_num=step):
+      params_repl, opt_repl, loss_value, update_rngs, measurements = (
+          update_fn(
+              params_repl,
+              opt_repl,
+              batch,
+              update_rng=update_rngs,
+              task_params=task_params))
+
+    # On the first host, let's always profile a handful of early steps.
+    if jax.process_index() == 0:
+      prof = u.startstop_prof(prof, step, first_step, get_steps("log_training"))
+
+    # Report training progress
+    if (u.itstime(step, get_steps("log_training"), total_steps, host=0)
+        or chrono.warmup and jax.process_index() == 0):
+      for i, sched_fn_cpu in enumerate(sched_fns_cpu):
+        mw.measure(f"global_schedule{i if i else ''}", sched_fn_cpu(step - 1))
+      l = mw.measure("training_loss", loss_value[0])
+      for name, value in measurements.items():
+        mw.measure(name, value[0])
+      chrono.tick(step, mw.measure, write_note)
+      if not np.isfinite(l):
+        error = (f"The loss became nan or inf somewhere within steps "
+                 f"[{step - get_steps('log_training')}, {step}]")
+        break
+
+    # Checkpoint saving
+    if (save_ckpt_path and
+        (u.itstime(step, get_steps("ckpt", None), total_steps, host=0) or
+         u.itstime(step, get_steps("keep_ckpt", None), total_steps, host=0))):
+      chrono.pause(wait_for=(params_repl, opt_repl))
+      u.checkpointing_timeout(ckpt_writer, config.get("ckpt_timeout", 1))
+      # We need to transfer the weights over now or else we risk keeping them
+      # alive while they'll be updated in a future step, creating hard to debug
+      # memory errors (see (internal link)). Also, takes device 0's params only.
+      opt_cpu = jax.tree_map(lambda x: np.array(x[0]), opt_repl)
+      params_cpu = jax.tree_map(lambda x: np.array(x[0]), params_repl)
+
+      # Check whether we want to keep a copy of the current checkpoint.
+      copy_step = None
+      if u.itstime(step, get_steps("keep_ckpt", None), total_steps):
+        copy_step = step
+
+      ckpt = {"params": params_cpu, "opt": opt_cpu, "chrono": chrono.save()}
+      ckpt_writer = pool.apply_async(
+          u.save_checkpoint, (ckpt, save_ckpt_path, copy_step))
+      chrono.resume()
+
+    for (name, evaluator, log_steps, prefix) in evaluators():
+      if u.itstime(step, log_steps, total_steps, first=log_steps < total_steps,
+                   last=False):
+        chrono.pause(wait_for=(params_repl, task_params))
+        write_note(f"{name} evaluation...\n{chrono.note}")
+        for key, value in evaluator.run(
+            {"params": params_repl, "task_params": task_params}):
+          mw.measure(f"{prefix}{key}", value)
+        chrono.resume()
+    mw.step_end()
+
+  # Always give a chance to stop the profiler, no matter how things ended.
+  # TODO: can we also do this when dying of an exception like OOM?
+  if jax.process_index() == 0 and prof is not None:
+    u.startstop_prof(prof)
+
+  # Run final evalution, also used for eval only jobs (when total_steps == 0).
+  for (name, evaluator, _, prefix) in evaluators():
+    write_note(f"{name} evaluation...\n{chrono.note}")
+    for key, value in evaluator.run(
+        {"params": params_repl, "task_params": task_params}):
+      mw.measure(f"{prefix}{key}", value)
+
+  # Last note needs to happen before the pool's closed =)
+  if not error:
+    write_note(f"Done!\n{chrono.note}")
+  else:
+    write_note(f"Failed!\n{error}\n{chrono.note}")
+
+  pool.close()
+  pool.join()
+  mw.close()
+
+  # Make sure all hosts stay up until the end of main.
+  u.sync()
+
+  # Before cleanup, as cleanup should only run for successful jobs.
+  if error is not None:
+    raise RuntimeError(error)
+
+  u.maybe_cleanup_workdir(workdir, flags.FLAGS.cleanup, info)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/Tipsomaly/model/big_vision/trainers/proj/uvim/vqvae.py b/Tipsomaly/model/big_vision/trainers/proj/uvim/vqvae.py
new file mode 100644
index 0000000000000000000000000000000000000000..f3638e788721d617bf93926d38514043e3e7a078
--- /dev/null
+++ b/Tipsomaly/model/big_vision/trainers/proj/uvim/vqvae.py
@@ -0,0 +1,414 @@
+# Copyright 2022 Big Vision Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Train loop for training the stage-I model."""
+# pylint: disable=consider-using-from-import
+import functools
+import importlib
+import multiprocessing.pool
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+from big_vision import input_pipeline
+import big_vision.datasets.core as ds_core
+import big_vision.evaluators.common as eval_common
+import big_vision.optax as bv_optax
+import big_vision.pp.builder as pp_builder
+import big_vision.utils as u
+from clu import parameter_overview
+import flax
+import jax
+import jax.numpy as jnp
+from ml_collections import config_flags
+import numpy as np
+import optax
+
+import tensorflow.io.gfile as gfile
+
+
+SG = jax.lax.stop_gradient
+partial = functools.partial
+
+config_flags.DEFINE_config_file(
+    "config", None, "Training configuration.", lock_config=True)
+
+flags.DEFINE_string("workdir", default=None, help="Work unit directory.")
+flags.DEFINE_boolean("cleanup", default=False,
+                     help="Delete workdir (only) after successful completion.")
+
+# Adds jax flags to the program.
+jax.config.parse_flags_with_absl()
+
+
+def main(argv):
+  del argv
+
+  config = flags.FLAGS.config
+  workdir = flags.FLAGS.workdir
+  logging.info("Workdir: %s", workdir)
+
+  logging.info("\u001b[33mHello from process %i holding %i/%i devices and "
+               "writing to workdir %s.\u001b[0m", jax.process_index(),
+               jax.local_device_count(), jax.device_count(), workdir)
+
+  # Define task input, loss and predict functions.
+  task_module = importlib.import_module(f"big_vision.trainers.{config.task}")
+  input_pp_fn = partial(task_module.input_pp, config=config)
+  task_loss_fn = partial(task_module.loss_fn, config=config)
+  predict_outputs_fn = partial(task_module.predict_outputs, config=config)
+
+  save_ckpt_path = None
+  if workdir:  # Always create if requested, even if we may not write into it.
+    gfile.makedirs(workdir)
+    save_ckpt_path = os.path.join(workdir, "checkpoint.npz")
+
+  # The pool is used to perform misc operations such as logging in async way.
+  pool = multiprocessing.pool.ThreadPool()
+
+  # Here we register preprocessing ops from modules listed on `pp_modules`.
+  for m in config.get("pp_modules",
+                      ["ops_general", "ops_image", "proj.uvim.pp_ops"]):
+    importlib.import_module(f"big_vision.pp.{m}")
+
+  # This seed makes the Jax part of things (like model init) deterministic.
+  # However, full training still won't be deterministic, for example due to the
+  # tf.data pipeline not being deterministic even if we would set TF seed.
+  # See (internal link) for a fun read on what it takes.
+  rng = jax.random.PRNGKey(config.get("seed", 0))
+
+  # These functions do more stuff internally, for OSS release we mock them by
+  # trivial alternatives in order to minize disruptions in the code.
+  xid, wid = -1, -1
+  fillin = lambda s: s
+  def info(s, *a):
+    logging.info("\u001b[33mNOTE\u001b[0m: " + s, *a)
+  def write_note(note):
+    if jax.process_index() == 0:
+      info("%s", note)
+
+  write_note("Initializing...")
+
+  batch_size = config.input.batch_size
+  if batch_size % jax.device_count() != 0:
+    raise ValueError(f"Batch size ({batch_size}) must "
+                     f"be divisible by device number ({jax.device_count()})")
+  info("Global batch size %d on %d hosts results in %d local batch size. With "
+       "%d dev per host (%d dev total), that's a %d per-device batch size.",
+       batch_size, jax.process_count(), batch_size // jax.process_count(),
+       jax.local_device_count(), jax.device_count(),
+       batch_size // jax.device_count())
+
+  # First thing after above sanity checks, so we can log "start" ticks.
+  mw = u.BigVisionMetricWriter(xid, wid, workdir, config)
+  chrono = u.Chrono()
+
+  write_note("Initializing train dataset...")
+  train_data = ds_core.get(**config.input.data)
+  train_ds = input_pipeline.make_for_train(
+      data=train_data.get_tfdata(ordered=False),
+      batch_size=batch_size,
+      preprocess_fn=pp_builder.get_preprocess_fn(config.input.get("pp")),
+      shuffle_buffer_size=config.input.get("shuffle_buffer_size"),
+      cache_raw=config.input.get("cache_raw", False),
+      filter_fn=config.input.get("filter_fn"),
+  )
+
+  # Start prefetching already.
+  n_prefetch = config.get("prefetch_to_device", 1)
+  train_iter = input_pipeline.start_input_pipeline(train_ds, n_prefetch)
+  ntrain_img = train_data.total_examples
+
+  def get_steps(name, default=ValueError):  # partial doesn't work well here.
+    return u.steps(name, config, ntrain_img, batch_size, default)
+  total_steps = get_steps("total")
+
+  info("Running for %d steps, that means %f epochs",
+       total_steps, total_steps * batch_size / ntrain_img)
+
+  write_note(f"Initializing {config.model_name} model...")
+  model_mod = importlib.import_module(f"big_vision.models.{config.model_name}")
+  model = model_mod.Model(**config.model)
+
+  # We want all parameters to be created in host RAM, not on any device, they'll
+  # be sent there later as needed, otherwise we already encountered two
+  # situations where we allocate them twice.
+  @partial(jax.jit, backend="cpu")
+  def init(rng):
+    batch = jax.tree_map(
+        lambda x: jnp.zeros(x.shape, x.dtype.as_numpy_dtype),
+        train_ds.element_spec)
+    init_res = flax.core.unfreeze(model.init(rng, **input_pp_fn(batch)))
+    params, state = init_res["params"], init_res["state"]
+
+    # Set bias in the heads to a low value, such that loss is small initially.
+    for key in config.model.outputs:
+      params[f"head_{key}"]["bias"] = jnp.full_like(
+          params[f"head_{key}"]["bias"], config.get("init_head_bias", 0))
+
+    return params, state
+
+  rng, rng_init = jax.random.split(rng)
+
+  rng_init_params, rng_init_state = jax.random.split(rng_init)
+  params_cpu, state_cpu = init({"params": rng_init_params,
+                                "state": rng_init_state})
+
+  if jax.process_index() == 0:
+    num_params = sum(p.size for p in jax.tree_leaves(params_cpu))
+    parameter_overview.log_parameter_overview(params_cpu, msg="init params")
+    mw.measure("num_params", num_params)
+
+  write_note(f"Initializing {config.optax_name} optimizer...")
+  tx, sched_fns = bv_optax.make(config, params_cpu, sched_kw=dict(
+      total_steps=total_steps, batch_size=batch_size, data_size=ntrain_img))
+
+  # We jit this, such that the arrays are created on the CPU, not device[0].
+  opt_cpu = jax.jit(tx.init, backend="cpu")(params_cpu)
+  sched_fns_cpu = [jax.jit(sched_fn, backend="cpu") for sched_fn in sched_fns]
+
+  @partial(jax.pmap, axis_name="batch", donate_argnums=(0, 1, 2),
+           static_broadcasted_argnums=(5,))
+  def update_fn(params, opt, state, batch, rng, update_dict=True):
+    """Update step."""
+    measurements = {}
+
+    # Get device-specific loss rng.
+    rng, rng_model = jax.random.split(rng, 2)
+    rng_model_local = jax.random.fold_in(rng_model, jax.lax.axis_index("batch"))
+
+    def loss_fn(params, state, batch):
+      (logits, out), mutated_col = model.apply(
+          {"params": params, "state": state},
+          **input_pp_fn(batch),
+          train=True, update_dict=update_dict,
+          rngs={"dropout": rng_model_local, "vqvae": rng_model},
+          mutable=["state"])
+      btlneck = out["bottleneck"]
+      btlneck_q = out["bottleneck_q"]
+
+      loss_rec, logs = jax.tree_map(jnp.mean, task_loss_fn(logits, batch))
+      loss_commitment = jnp.mean(jnp.square(btlneck - SG(btlneck_q)))
+      loss = loss_rec + config.get("w_commitment", 0.25) * loss_commitment
+      aux = {
+          "loss_rec": jax.lax.pmean(loss_rec, axis_name="batch"),
+          "loss_commitment": jax.lax.pmean(loss_commitment, axis_name="batch"),
+          "codebook_zeros_ratio": out["codebook_zeros_ratio"],
+          "codebook_max_ratio": out["codebook_max_ratio"],
+          "state": mutated_col["state"],
+          **jax.tree_map(partial(jax.lax.pmean, axis_name="batch"), logs),
+      }
+      return loss, aux
+
+    (l, aux), grads = jax.value_and_grad(loss_fn, has_aux=True)(
+        params, state, batch)
+    l, grads = jax.lax.pmean((l, grads), axis_name="batch")
+    updates, opt = tx.update(grads, opt, params)
+    params = optax.apply_updates(params, updates)
+    state = aux.pop("state")
+    measurements = {**measurements, **aux}
+
+    gs = jax.tree_leaves(bv_optax.replace_frozen(config.schedule, grads, 0.))
+    measurements["l2_grads"] = jnp.sqrt(sum([jnp.vdot(g, g) for g in gs]))
+    ps = jax.tree_leaves(params)
+    measurements["l2_params"] = jnp.sqrt(sum([jnp.vdot(p, p) for p in ps]))
+    us = jax.tree_leaves(updates)
+    measurements["l2_updates"] = jnp.sqrt(sum([jnp.vdot(u, u) for u in us]))
+
+    return params, opt, state, l, rng, measurements
+
+  # Define evaluators.
+  def validation_fn(params, batch):
+    """Compute per-example metrics."""
+    logits, out = model.apply(params, **input_pp_fn(batch))
+    _, losses = task_loss_fn(logits, batch)
+    btlneck = out["bottleneck"]
+    btlneck_q = out["bottleneck_q"]
+    losses["loss_commitment"] = jnp.square(btlneck - btlneck_q)
+    return jax.tree_map(
+        lambda x: jnp.mean(x, axis=tuple(range(1, x.ndim))),
+        losses)
+
+  def predict_fn(params, batch):
+    logits, _ = model.apply(params, **input_pp_fn(batch))
+    outputs = predict_outputs_fn(logits)
+    return outputs
+
+  # Only initialize evaluators when they are first needed.
+  @functools.lru_cache(maxsize=None)
+  def evaluators():
+    return eval_common.from_config(
+        config, {"predict": predict_fn, "validation": validation_fn},
+        lambda s: write_note(f"Initializing evaluator: {s}...\n{chrono.note}")
+    )
+
+  # Decide how to initialize training. The order is important.
+  # 1. Always resumes from the existing checkpoint, e.g. resumes a finetune job.
+  # 2. Resume from a previous checkpoint, e.g. start a cooldown training job.
+  # 3. Initialize model from something, e,g, start a fine-tuning job.
+  # 4. Train from scratch.
+  resume_ckpt_path = None
+  if save_ckpt_path and gfile.exists(save_ckpt_path):
+    resume_ckpt_path = save_ckpt_path
+  elif config.get("resume"):
+    resume_ckpt_path = fillin(config.resume)
+  if resume_ckpt_path:
+    write_note("Resume training from checkpoint...")
+    checkpoint = {
+        "params": params_cpu,
+        "state": state_cpu,
+        "opt": opt_cpu,
+        "chrono": chrono.save(),
+    }
+    checkpoint_tree = jax.tree_structure(checkpoint)
+    loaded = u.load_checkpoint(checkpoint_tree, resume_ckpt_path)
+    # bfloat16 type gets lost when data is saved to disk, so we recover it.
+    checkpoint = jax.tree_map(u.recover_dtype, loaded)
+    params_cpu = checkpoint["params"]
+    state_cpu = checkpoint["state"]
+    opt_cpu = checkpoint["opt"]
+    chrono.load(checkpoint["chrono"])
+  elif config.get("model_init"):
+    write_note(f"Initialize model from {config.model_init}...")
+    params_cpu, state_cpu = model_mod.load(
+        {"params": params_cpu, "state": state_cpu},
+        config.model_init, config.model,
+        **config.get("model_load", {}))
+    if jax.process_index() == 0:
+      parameter_overview.log_parameter_overview(
+          params_cpu, msg="restored params")
+
+  write_note("Kicking off misc stuff...")
+  first_step = bv_optax.get_count(opt_cpu)
+  chrono.inform(first_step, total_steps, batch_size, ntrain_img / batch_size)
+  prof = None  # Keeps track of start/stop of profiler state.
+
+  write_note(f"Replicating...\n{chrono.note}")
+  params_repl = flax.jax_utils.replicate(params_cpu)
+  opt_repl = flax.jax_utils.replicate(opt_cpu)
+  state_repl = flax.jax_utils.replicate(state_cpu)
+
+  rng, rng_loop = jax.random.split(rng, 2)
+  rngs_loop = flax.jax_utils.replicate(rng_loop)
+  ckpt_writer = None
+
+  write_note(f"First step compilations...\n{chrono.note}")
+  error = None  # For exiting with an error after cleanup. Avoids indentation.
+
+  # Using a python integer for step here, because opt.state.step is allocated
+  # on TPU during replication.
+  for step, batch in zip(range(first_step + 1, total_steps + 1), train_iter):
+    mw.step_start(step)
+
+    with jax.profiler.StepTraceAnnotation("train_step", step_num=step):
+      params_repl, opt_repl, state_repl, loss_value, rngs_loop, measurements = (
+          update_fn(
+              params_repl,
+              opt_repl,
+              state_repl,
+              batch,
+              rngs_loop,
+              not config.get("freeze_dict", True)))
+
+    # On the first host, let's always profile a handful of early steps.
+    if jax.process_index() == 0:
+      prof = u.startstop_prof(prof, step, first_step, get_steps("log_training"))
+
+    # Report training progress
+    if (u.itstime(step, get_steps("log_training"), total_steps, host=0)
+        or chrono.warmup and jax.process_index() == 0):
+      for i, sched_fn_cpu in enumerate(sched_fns_cpu):
+        mw.measure(f"global_schedule{i if i else ''}", sched_fn_cpu(step - 1))
+      l = mw.measure("training_loss", loss_value[0])
+      for name, value in measurements.items():
+        mw.measure(name, value[0])
+      chrono.tick(step, mw.measure, write_note)
+      if not np.isfinite(l):
+        error = (f"The loss became nan or inf somewhere within steps "
+                 f"[{step - get_steps('log_training')}, {step}]")
+        break
+
+    # Checkpoint saving
+    if (save_ckpt_path and
+        (u.itstime(step, get_steps("ckpt", None), total_steps, host=0) or
+         u.itstime(step, get_steps("keep_ckpt", None), total_steps, host=0))):
+      chrono.pause(wait_for=(params_repl, opt_repl, state_repl))
+      u.checkpointing_timeout(ckpt_writer, config.get("ckpt_timeout", 1))
+      # We need to transfer the weights over now or else we risk keeping them
+      # alive while they'll be updated in a future step, creating hard to debug
+      # memory errors (see (internal link)). Also, takes device 0's params only.
+      params_cpu, opt_cpu, state_cpu = jax.tree_map(
+          lambda x: np.array(x[0]), (params_repl, opt_repl, state_repl))
+
+      # Check whether we want to keep a copy of the current checkpoint.
+      copy_step = None
+      if u.itstime(step, get_steps("keep_ckpt", None), total_steps):
+        copy_step = step
+
+      ckpt = {
+          "params": params_cpu,
+          "state": state_cpu,
+          "opt": opt_cpu,
+          "chrono": chrono.save(),
+      }
+      ckpt_writer = pool.apply_async(
+          u.save_checkpoint, (ckpt, save_ckpt_path, copy_step))
+      chrono.resume()
+
+    for (name, evaluator, log_steps, prefix) in evaluators():
+      if u.itstime(step, log_steps, total_steps):
+        chrono.pause(wait_for=(params_repl, state_repl))
+        write_note(f"{name} evaluation...\n{chrono.note}")
+        for key, value in evaluator.run(
+            {"params": params_repl, "state": state_repl}):
+          mw.measure(f"{prefix}{key}", value)
+        chrono.resume()
+    mw.step_end()
+
+  # Always give a chance to stop the profiler, no matter how things ended.
+  # TODO: can we also do this when dying of an exception like OOM?
+  if jax.process_index() == 0 and prof is not None:
+    u.startstop_prof(prof)
+
+  # Support eval only runs: run evaluation if total_steps (or num_epochs) is 0.
+  if total_steps == 0:
+    for (name, evaluator, _, prefix) in evaluators():
+      write_note(f"{name} evaluation...\n{chrono.note}")
+      for key, value in evaluator.run(
+          {"params": params_repl, "state": state_repl}):
+        mw.measure(f"{prefix}{key}", value)
+
+  # Last note needs to happen before the pool's closed =)
+  if not error:
+    write_note(f"Done!\n{chrono.note}")
+  else:
+    write_note(f"Failed!\n{error}\n{chrono.note}")
+
+  pool.close()
+  pool.join()
+  mw.close()
+
+  # Make sure all hosts stay up until the end of main.
+  u.sync()
+
+  # Before cleanup, as cleanup should only run for successful jobs.
+  if error is not None:
+    raise RuntimeError(error)
+
+  u.maybe_cleanup_workdir(workdir, flags.FLAGS.cleanup, info)
+
+
+if __name__ == "__main__":
+  app.run(main)