tests/test_configuration_common.py

# Copyright 2019 HuggingFace Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import json
import os
import tempfile
from pathlib import Path

from transformers import is_torch_available
from transformers.utils import direct_transformers_import

from .utils.test_configuration_utils import config_common_kwargs


transformers_module = direct_transformers_import(Path(__file__).parent)


class ConfigTester:
    def __init__(self, parent, config_class=None, has_text_modality=True, common_properties=None, **kwargs):
        self.parent = parent
        self.config_class = config_class
        self.has_text_modality = has_text_modality
        self.inputs_dict = kwargs
        self.common_properties = common_properties

    def create_and_test_config_common_properties(self):
        config = self.config_class(**self.inputs_dict)
        common_properties = (
            ["hidden_size", "num_attention_heads", "num_hidden_layers"]
            if self.common_properties is None and not self.config_class.sub_configs
            else self.common_properties
        )
        common_properties = [] if common_properties is None else common_properties

        # Add common fields for text models
        if self.has_text_modality:
            common_properties.extend(["vocab_size"])

        # Test that config has the common properties as getters
        for prop in common_properties:
            self.parent.assertTrue(hasattr(config, prop), msg=f"`{prop}` does not exist")

        # Test that config has the common properties as setter
        for idx, name in enumerate(common_properties):
            try:
                setattr(config, name, idx)
                self.parent.assertEqual(
                    getattr(config, name), idx, msg=f"`{name} value {idx} expected, but was {getattr(config, name)}"
                )
            except NotImplementedError:
                # Some models might not be able to implement setters for common_properties
                # In that case, a NotImplementedError is raised
                pass

        # Test if config class can be called with Config(prop_name=..)
        for idx, name in enumerate(common_properties):
            try:
                config = self.config_class(**{name: idx})
                self.parent.assertEqual(
                    getattr(config, name), idx, msg=f"`{name} value {idx} expected, but was {getattr(config, name)}"
                )
            except NotImplementedError:
                # Some models might not be able to implement setters for common_properties
                # In that case, a NotImplementedError is raised
                pass

    def create_and_test_config_to_json_string(self):
        config = self.config_class(**self.inputs_dict)
        obj = json.loads(config.to_json_string())
        for key, value in self.inputs_dict.items():
            self.parent.assertEqual(obj[key], value)

    def create_and_test_config_to_json_file(self):
        config_first = self.config_class(**self.inputs_dict)

        with tempfile.TemporaryDirectory() as tmpdirname:
            json_file_path = os.path.join(tmpdirname, "config.json")
            config_first.to_json_file(json_file_path)
            config_second = self.config_class.from_json_file(json_file_path)

        self.parent.assertEqual(config_second.to_dict(), config_first.to_dict())

    def create_and_test_config_from_and_save_pretrained(self):
        config_first = self.config_class(**self.inputs_dict)

        with tempfile.TemporaryDirectory() as tmpdirname:
            config_first.save_pretrained(tmpdirname)
            config_second = self.config_class.from_pretrained(tmpdirname)

        self.parent.assertEqual(config_second.to_dict(), config_first.to_dict())

        with self.parent.assertRaises(OSError):
            self.config_class.from_pretrained(f".{tmpdirname}")

    def create_and_test_config_from_and_save_pretrained_subfolder(self):
        config_first = self.config_class(**self.inputs_dict)

        subfolder = "test"
        with tempfile.TemporaryDirectory() as tmpdirname:
            sub_tmpdirname = os.path.join(tmpdirname, subfolder)
            config_first.save_pretrained(sub_tmpdirname)
            config_second = self.config_class.from_pretrained(tmpdirname, subfolder=subfolder)

        self.parent.assertEqual(config_second.to_dict(), config_first.to_dict())

    def create_and_test_config_from_and_save_pretrained_composite(self):
        """
        Tests that composite or nested configs can be loaded and saved correctly. In case the config
        has a sub-config, we should be able to call `sub_config.from_pretrained('general_config_file')`
        and get a result same as if we loaded the whole config and obtained `config.sub_config` from it.
        """
        config = self.config_class(**self.inputs_dict)

        with tempfile.TemporaryDirectory() as tmpdirname:
            config.save_pretrained(tmpdirname)
            general_config_loaded = self.config_class.from_pretrained(tmpdirname)
            general_config_dict = config.to_dict()

            # Iterate over all sub_configs if there are any and load them with their own classes
            sub_configs = general_config_loaded.sub_configs
            for sub_config_key, sub_class in sub_configs.items():
                if general_config_dict[sub_config_key] is not None:
                    if sub_class.__name__ == "AutoConfig":
                        sub_class = sub_class.for_model(**general_config_dict[sub_config_key]).__class__
                        sub_config_loaded = sub_class.from_pretrained(tmpdirname)
                    else:
                        sub_config_loaded = sub_class.from_pretrained(tmpdirname)

                    # Pop `transformers_version`, it never exists when a config is part of a general composite config
                    # Verify that loading with subconfig class results in same dict as if we loaded with general composite config class
                    sub_config_loaded_dict = sub_config_loaded.to_dict()
                    sub_config_loaded_dict.pop("transformers_version", None)
                    general_config_dict[sub_config_key].pop("transformers_version", None)
                    self.parent.assertEqual(sub_config_loaded_dict, general_config_dict[sub_config_key])

                    # Verify that the loaded config type is same as in the general config
                    type_from_general_config = type(getattr(general_config_loaded, sub_config_key))
                    self.parent.assertTrue(isinstance(sub_config_loaded, type_from_general_config))

                    # Now save only the sub-config and load it back to make sure the whole load-save-load pipeline works
                    with tempfile.TemporaryDirectory() as tmpdirname2:
                        sub_config_loaded.save_pretrained(tmpdirname2)
                        sub_config_loaded_2 = sub_class.from_pretrained(tmpdirname2)
                        self.parent.assertEqual(sub_config_loaded.to_dict(), sub_config_loaded_2.to_dict())

    def create_and_test_config_from_pretrained_custom_kwargs(self):
        """
        Tests that passing custom kwargs to the `from_pretrained` will overwrite model's saved config values.
        for composite configs. We should overwrite only the requested keys, keeping all values of the
        subconfig that are loaded from the checkpoint.
        """
        # Check only composite configs. We can't know which attributes each type of config has so check
        # only text config because we are sure that all text configs have a `vocab_size`
        config = self.config_class(**self.inputs_dict)
        if config.get_text_config() is config or not hasattr(self.parent.model_tester, "get_config"):
            return

        # First create a config with non-default values and save it. The reload it back with a new
        # `vocab_size` and check that all values are loaded from checkpoint and not init from defaults
        non_default_inputs = self.parent.model_tester.get_config().to_dict()
        config = self.config_class(**non_default_inputs)
        original_text_config = config.get_text_config()
        text_config_key = [key for key in config if getattr(config, key) is original_text_config]

        # The heuristic is a bit brittle so let's just skip the test
        if len(text_config_key) != 1:
            return

        text_config_key = text_config_key[0]
        with tempfile.TemporaryDirectory() as tmpdirname:
            config.save_pretrained(tmpdirname)

            # Set vocab size to 20 tokens and reload from checkpoint and check if all keys/values are identical except for `vocab_size`
            config_reloaded = self.config_class.from_pretrained(tmpdirname, **{text_config_key: {"vocab_size": 20}})
            original_text_config_dict = original_text_config.to_dict()
            original_text_config_dict["vocab_size"] = 20

            text_config_reloaded_dict = config_reloaded.get_text_config().to_dict()
            self.parent.assertDictEqual(text_config_reloaded_dict, original_text_config_dict)

    def create_and_test_config_with_num_labels(self):
        config = self.config_class(**self.inputs_dict, num_labels=5)
        self.parent.assertEqual(len(config.id2label), 5)
        self.parent.assertEqual(len(config.label2id), 5)

        config.num_labels = 3
        self.parent.assertEqual(len(config.id2label), 3)
        self.parent.assertEqual(len(config.label2id), 3)

    def check_config_can_be_init_without_params(self):
        if self.config_class.has_no_defaults_at_init:
            with self.parent.assertRaises(ValueError):
                config = self.config_class()
        else:
            config = self.config_class()
            self.parent.assertIsNotNone(config)

    def check_config_arguments_init(self):
        if self.config_class.sub_configs:
            return  # TODO: @raushan composite models are not consistent in how they set general params

        kwargs = copy.deepcopy(config_common_kwargs)
        config = self.config_class(**kwargs)
        wrong_values = []
        for key, value in config_common_kwargs.items():
            if key == "dtype":
                if not is_torch_available():
                    continue
                else:
                    import torch

                    if config.dtype != torch.float16:
                        wrong_values.append(("dtype", config.dtype, torch.float16))
            elif getattr(config, key) != value:
                wrong_values.append((key, getattr(config, key), value))

        if len(wrong_values) > 0:
            errors = "\n".join([f"- {v[0]}: got {v[1]} instead of {v[2]}" for v in wrong_values])
            raise ValueError(f"The following keys were not properly set in the config:\n{errors}")

    def run_common_tests(self):
        self.create_and_test_config_common_properties()
        self.create_and_test_config_to_json_string()
        self.create_and_test_config_to_json_file()
        self.create_and_test_config_from_and_save_pretrained()
        self.create_and_test_config_from_and_save_pretrained_subfolder()
        self.create_and_test_config_from_and_save_pretrained_composite()
        self.create_and_test_config_with_num_labels()
        self.check_config_can_be_init_without_params()
        self.check_config_arguments_init()
        self.create_and_test_config_from_pretrained_custom_kwargs()