edit

omegaconf/__init__.py

@@ -0,0 +1,65 @@
+from .base import Container, DictKeyType, Node, SCMode, UnionNode
+from .dictconfig import DictConfig
+from .errors import (
+    KeyValidationError,
+    MissingMandatoryValue,
+    ReadonlyConfigError,
+    UnsupportedValueType,
+    ValidationError,
+)
+from .listconfig import ListConfig
+from .nodes import (
+    AnyNode,
+    BooleanNode,
+    BytesNode,
+    EnumNode,
+    FloatNode,
+    IntegerNode,
+    PathNode,
+    StringNode,
+    ValueNode,
+)
+from .omegaconf import (
+    II,
+    MISSING,
+    SI,
+    OmegaConf,
+    Resolver,
+    flag_override,
+    open_dict,
+    read_write,
+)
+from .version import __version__
+
+__all__ = [
+    "__version__",
+    "MissingMandatoryValue",
+    "ValidationError",
+    "ReadonlyConfigError",
+    "UnsupportedValueType",
+    "KeyValidationError",
+    "Container",
+    "UnionNode",
+    "ListConfig",
+    "DictConfig",
+    "DictKeyType",
+    "OmegaConf",
+    "Resolver",
+    "SCMode",
+    "flag_override",
+    "read_write",
+    "open_dict",
+    "Node",
+    "ValueNode",
+    "AnyNode",
+    "IntegerNode",
+    "StringNode",
+    "BytesNode",
+    "PathNode",
+    "BooleanNode",
+    "EnumNode",
+    "FloatNode",
+    "MISSING",
+    "SI",
+    "II",
+]

omegaconf/_impl.py

@@ -0,0 +1,101 @@
+from typing import Any
+
+from omegaconf import MISSING, Container, DictConfig, ListConfig, Node, ValueNode
+from omegaconf.errors import ConfigTypeError, InterpolationToMissingValueError
+
+from ._utils import _DEFAULT_MARKER_, _get_value
+
+
+def _resolve_container_value(cfg: Container, key: Any) -> None:
+    node = cfg._get_child(key)
+    assert isinstance(node, Node)
+    if node._is_interpolation():
+        try:
+            resolved = node._dereference_node()
+        except InterpolationToMissingValueError:
+            node._set_value(MISSING)
+        else:
+            if isinstance(resolved, Container):
+                _resolve(resolved)
+            if isinstance(resolved, Container) and isinstance(node, ValueNode):
+                cfg[key] = resolved
+            else:
+                node._set_value(_get_value(resolved))
+    else:
+        _resolve(node)
+
+
+def _resolve(cfg: Node) -> Node:
+    assert isinstance(cfg, Node)
+    if cfg._is_interpolation():
+        try:
+            resolved = cfg._dereference_node()
+        except InterpolationToMissingValueError:
+            cfg._set_value(MISSING)
+        else:
+            cfg._set_value(resolved._value())
+
+    if isinstance(cfg, DictConfig):
+        for k in cfg.keys():
+            _resolve_container_value(cfg, k)
+
+    elif isinstance(cfg, ListConfig):
+        for i in range(len(cfg)):
+            _resolve_container_value(cfg, i)
+
+    return cfg
+
+
+def select_value(
+    cfg: Container,
+    key: str,
+    *,
+    default: Any = _DEFAULT_MARKER_,
+    throw_on_resolution_failure: bool = True,
+    throw_on_missing: bool = False,
+    absolute_key: bool = False,
+) -> Any:
+    node = select_node(
+        cfg=cfg,
+        key=key,
+        throw_on_resolution_failure=throw_on_resolution_failure,
+        throw_on_missing=throw_on_missing,
+        absolute_key=absolute_key,
+    )
+
+    node_not_found = node is None
+    if node_not_found or node._is_missing():
+        if default is not _DEFAULT_MARKER_:
+            return default
+        else:
+            return None
+
+    return _get_value(node)
+
+
+def select_node(
+    cfg: Container,
+    key: str,
+    *,
+    throw_on_resolution_failure: bool = True,
+    throw_on_missing: bool = False,
+    absolute_key: bool = False,
+) -> Any:
+    try:
+        # for non relative keys, the interpretation can be:
+        # 1. relative to cfg
+        # 2. relative to the config root
+        # This is controlled by the absolute_key flag. By default, such keys are relative to cfg.
+        if not absolute_key and not key.startswith("."):
+            key = f".{key}"
+
+        cfg, key = cfg._resolve_key_and_root(key)
+        _root, _last_key, node = cfg._select_impl(
+            key,
+            throw_on_missing=throw_on_missing,
+            throw_on_resolution_failure=throw_on_resolution_failure,
+        )
+    except ConfigTypeError:
+        return None
+
+    return node

omegaconf/_utils.py

@@ -0,0 +1,1039 @@
+import copy
+import os
+import pathlib
+import re
+import string
+import sys
+import types
+import warnings
+from contextlib import contextmanager
+from enum import Enum
+from textwrap import dedent
+from typing import (
+    Any,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+    get_type_hints,
+)
+
+import yaml
+
+from .errors import (
+    ConfigIndexError,
+    ConfigTypeError,
+    ConfigValueError,
+    GrammarParseError,
+    OmegaConfBaseException,
+    ValidationError,
+)
+from .grammar_parser import SIMPLE_INTERPOLATION_PATTERN, parse
+
+try:
+    import dataclasses
+
+except ImportError:  # pragma: no cover
+    dataclasses = None  # type: ignore # pragma: no cover
+
+try:
+    import attr
+
+except ImportError:  # pragma: no cover
+    attr = None  # type: ignore # pragma: no cover
+
+NoneType: Type[None] = type(None)
+
+BUILTIN_VALUE_TYPES: Tuple[Type[Any], ...] = (
+    int,
+    float,
+    bool,
+    str,
+    bytes,
+    NoneType,
+)
+
+# Regexprs to match key paths like: a.b, a[b], ..a[c].d, etc.
+# We begin by matching the head (in these examples: a, a, ..a).
+# This can be read as "dots followed by any character but `.` or `[`"
+# Note that a key starting with brackets, like [a], is purposedly *not*
+# matched here and will instead be handled in the next regex below (this
+# is to keep this regex simple).
+KEY_PATH_HEAD = re.compile(r"(\.)*[^.[]*")
+# Then we match other keys. The following expression matches one key and can
+# be read as a choice between two syntaxes:
+#   - `.` followed by anything except `.` or `[` (ex: .b, .d)
+#   - `[` followed by anything then `]` (ex: [b], [c])
+KEY_PATH_OTHER = re.compile(r"\.([^.[]*)|\[(.*?)\]")
+
+
+# source: https://yaml.org/type/bool.html
+YAML_BOOL_TYPES = [
+    "y",
+    "Y",
+    "yes",
+    "Yes",
+    "YES",
+    "n",
+    "N",
+    "no",
+    "No",
+    "NO",
+    "true",
+    "True",
+    "TRUE",
+    "false",
+    "False",
+    "FALSE",
+    "on",
+    "On",
+    "ON",
+    "off",
+    "Off",
+    "OFF",
+]
+
+
+class Marker:
+    def __init__(self, desc: str):
+        self.desc = desc
+
+    def __repr__(self) -> str:
+        return self.desc
+
+
+# To be used as default value when `None` is not an option.
+_DEFAULT_MARKER_: Any = Marker("_DEFAULT_MARKER_")
+
+
+class OmegaConfDumper(yaml.Dumper):  # type: ignore
+    str_representer_added = False
+
+    @staticmethod
+    def str_representer(dumper: yaml.Dumper, data: str) -> yaml.ScalarNode:
+        with_quotes = yaml_is_bool(data) or is_int(data) or is_float(data)
+        return dumper.represent_scalar(
+            yaml.resolver.BaseResolver.DEFAULT_SCALAR_TAG,
+            data,
+            style=("'" if with_quotes else None),
+        )
+
+
+def get_omega_conf_dumper() -> Type[OmegaConfDumper]:
+    if not OmegaConfDumper.str_representer_added:
+        OmegaConfDumper.add_representer(str, OmegaConfDumper.str_representer)
+        OmegaConfDumper.str_representer_added = True
+    return OmegaConfDumper
+
+
+def yaml_is_bool(b: str) -> bool:
+    return b in YAML_BOOL_TYPES
+
+
+def get_yaml_loader() -> Any:
+    class OmegaConfLoader(yaml.SafeLoader):  # type: ignore
+        def construct_mapping(self, node: yaml.Node, deep: bool = False) -> Any:
+            keys = set()
+            for key_node, value_node in node.value:
+                if key_node.tag != yaml.resolver.BaseResolver.DEFAULT_SCALAR_TAG:
+                    continue
+                if key_node.value in keys:
+                    raise yaml.constructor.ConstructorError(
+                        "while constructing a mapping",
+                        node.start_mark,
+                        f"found duplicate key {key_node.value}",
+                        key_node.start_mark,
+                    )
+                keys.add(key_node.value)
+            return super().construct_mapping(node, deep=deep)
+
+    loader = OmegaConfLoader
+    loader.add_implicit_resolver(
+        "tag:yaml.org,2002:float",
+        re.compile(
+            """^(?:
+         [-+]?[0-9]+(?:_[0-9]+)*\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
+        |[-+]?[0-9]+(?:_[0-9]+)*(?:[eE][-+]?[0-9]+)
+        |\\.[0-9]+(?:_[0-9]+)*(?:[eE][-+][0-9]+)?
+        |[-+]?[0-9]+(?:_[0-9]+)*(?::[0-5]?[0-9])+\\.[0-9_]*
+        |[-+]?\\.(?:inf|Inf|INF)
+        |\\.(?:nan|NaN|NAN))$""",
+            re.X,
+        ),
+        list("-+0123456789."),
+    )
+    loader.yaml_implicit_resolvers = {
+        key: [
+            (tag, regexp)
+            for tag, regexp in resolvers
+            if tag != "tag:yaml.org,2002:timestamp"
+        ]
+        for key, resolvers in loader.yaml_implicit_resolvers.items()
+    }
+
+    loader.add_constructor(
+        "tag:yaml.org,2002:python/object/apply:pathlib.Path",
+        lambda loader, node: pathlib.Path(*loader.construct_sequence(node)),
+    )
+    loader.add_constructor(
+        "tag:yaml.org,2002:python/object/apply:pathlib.PosixPath",
+        lambda loader, node: pathlib.PosixPath(*loader.construct_sequence(node)),
+    )
+    loader.add_constructor(
+        "tag:yaml.org,2002:python/object/apply:pathlib.WindowsPath",
+        lambda loader, node: pathlib.WindowsPath(*loader.construct_sequence(node)),
+    )
+
+    return loader
+
+
+def _get_class(path: str) -> type:
+    from importlib import import_module
+
+    module_path, _, class_name = path.rpartition(".")
+    mod = import_module(module_path)
+    try:
+        klass: type = getattr(mod, class_name)
+    except AttributeError:
+        raise ImportError(f"Class {class_name} is not in module {module_path}")
+    return klass
+
+
+def is_union_annotation(type_: Any) -> bool:
+    if sys.version_info >= (3, 10):  # pragma: no cover
+        if isinstance(type_, types.UnionType):
+            return True
+    return getattr(type_, "__origin__", None) is Union
+
+
+def _resolve_optional(type_: Any) -> Tuple[bool, Any]:
+    """Check whether `type_` is equivalent to `typing.Optional[T]` for some T."""
+    if is_union_annotation(type_):
+        args = type_.__args__
+        if NoneType in args:
+            optional = True
+            args = tuple(a for a in args if a is not NoneType)
+        else:
+            optional = False
+        if len(args) == 1:
+            return optional, args[0]
+        elif len(args) >= 2:
+            return optional, Union[args]
+        else:
+            assert False
+
+    if type_ is Any:
+        return True, Any
+
+    if type_ in (None, NoneType):
+        return True, NoneType
+
+    return False, type_
+
+
+def _is_optional(obj: Any, key: Optional[Union[int, str]] = None) -> bool:
+    """Check `obj` metadata to see if the given node is optional."""
+    from .base import Container, Node
+
+    if key is not None:
+        assert isinstance(obj, Container)
+        obj = obj._get_node(key)
+    assert isinstance(obj, Node)
+    return obj._is_optional()
+
+
+def _resolve_forward(type_: Type[Any], module: str) -> Type[Any]:
+    import typing  # lgtm [py/import-and-import-from]
+
+    forward = typing.ForwardRef if hasattr(typing, "ForwardRef") else typing._ForwardRef  # type: ignore
+    if type(type_) is forward:
+        return _get_class(f"{module}.{type_.__forward_arg__}")
+    else:
+        if is_dict_annotation(type_):
+            kt, vt = get_dict_key_value_types(type_)
+            if kt is not None:
+                kt = _resolve_forward(kt, module=module)
+            if vt is not None:
+                vt = _resolve_forward(vt, module=module)
+            return Dict[kt, vt]  # type: ignore
+        if is_list_annotation(type_):
+            et = get_list_element_type(type_)
+            if et is not None:
+                et = _resolve_forward(et, module=module)
+            return List[et]  # type: ignore
+        if is_tuple_annotation(type_):
+            its = get_tuple_item_types(type_)
+            its = tuple(_resolve_forward(it, module=module) for it in its)
+            return Tuple[its]  # type: ignore
+
+        return type_
+
+
+def extract_dict_subclass_data(obj: Any, parent: Any) -> Optional[Dict[str, Any]]:
+    """Check if obj is an instance of a subclass of Dict. If so, extract the Dict keys/values."""
+    from omegaconf.omegaconf import _maybe_wrap
+
+    is_type = isinstance(obj, type)
+    obj_type = obj if is_type else type(obj)
+    subclasses_dict = is_dict_subclass(obj_type)
+
+    if subclasses_dict:
+        warnings.warn(
+            f"Class `{obj_type.__name__}` subclasses `Dict`."
+            + " Subclassing `Dict` in Structured Config classes is deprecated,"
+            + " see github.com/omry/omegaconf/issues/663",
+            UserWarning,
+            stacklevel=9,
+        )
+
+    if is_type:
+        return None
+    elif subclasses_dict:
+        dict_subclass_data = {}
+        key_type, element_type = get_dict_key_value_types(obj_type)
+        for name, value in obj.items():
+            is_optional, type_ = _resolve_optional(element_type)
+            type_ = _resolve_forward(type_, obj.__module__)
+            try:
+                dict_subclass_data[name] = _maybe_wrap(
+                    ref_type=type_,
+                    is_optional=is_optional,
+                    key=name,
+                    value=value,
+                    parent=parent,
+                )
+            except ValidationError as ex:
+                format_and_raise(
+                    node=None, key=name, value=value, cause=ex, msg=str(ex)
+                )
+        return dict_subclass_data
+    else:
+        return None
+
+
+def get_attr_class_fields(obj: Any) -> List["attr.Attribute[Any]"]:
+    is_type = isinstance(obj, type)
+    obj_type = obj if is_type else type(obj)
+    fields = attr.fields_dict(obj_type).values()
+    return [f for f in fields if f.metadata.get("omegaconf_ignore") is not True]
+
+
+def get_attr_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]:
+    from omegaconf.omegaconf import OmegaConf, _maybe_wrap
+
+    flags = {"allow_objects": allow_objects} if allow_objects is not None else {}
+
+    from omegaconf import MISSING
+
+    d = {}
+    is_type = isinstance(obj, type)
+    obj_type = obj if is_type else type(obj)
+    dummy_parent = OmegaConf.create({}, flags=flags)
+    dummy_parent._metadata.object_type = obj_type
+    resolved_hints = get_type_hints(obj_type)
+
+    for attrib in get_attr_class_fields(obj):
+        name = attrib.name
+        is_optional, type_ = _resolve_optional(resolved_hints[name])
+        type_ = _resolve_forward(type_, obj.__module__)
+        if not is_type:
+            value = getattr(obj, name)
+        else:
+            value = attrib.default
+            if value == attr.NOTHING:
+                value = MISSING
+        if is_union_annotation(type_) and not is_supported_union_annotation(type_):
+            e = ConfigValueError(
+                f"Unions of containers are not supported:\n{name}: {type_str(type_)}"
+            )
+            format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e))
+
+        try:
+            d[name] = _maybe_wrap(
+                ref_type=type_,
+                is_optional=is_optional,
+                key=name,
+                value=value,
+                parent=dummy_parent,
+            )
+        except (ValidationError, GrammarParseError) as ex:
+            format_and_raise(
+                node=dummy_parent, key=name, value=value, cause=ex, msg=str(ex)
+            )
+        d[name]._set_parent(None)
+    dict_subclass_data = extract_dict_subclass_data(obj=obj, parent=dummy_parent)
+    if dict_subclass_data is not None:
+        d.update(dict_subclass_data)
+    return d
+
+
+def get_dataclass_fields(obj: Any) -> List["dataclasses.Field[Any]"]:
+    fields = dataclasses.fields(obj)
+    return [f for f in fields if f.metadata.get("omegaconf_ignore") is not True]
+
+
+def get_dataclass_data(
+    obj: Any, allow_objects: Optional[bool] = None
+) -> Dict[str, Any]:
+    from omegaconf.omegaconf import MISSING, OmegaConf, _maybe_wrap
+
+    flags = {"allow_objects": allow_objects} if allow_objects is not None else {}
+    d = {}
+    is_type = isinstance(obj, type)
+    obj_type = get_type_of(obj)
+    dummy_parent = OmegaConf.create({}, flags=flags)
+    dummy_parent._metadata.object_type = obj_type
+    resolved_hints = get_type_hints(obj_type)
+    for field in get_dataclass_fields(obj):
+        name = field.name
+        is_optional, type_ = _resolve_optional(resolved_hints[field.name])
+        type_ = _resolve_forward(type_, obj.__module__)
+        has_default = field.default != dataclasses.MISSING
+        has_default_factory = field.default_factory != dataclasses.MISSING
+
+        if not is_type:
+            value = getattr(obj, name)
+        else:
+            if has_default:
+                value = field.default
+            elif has_default_factory:
+                value = field.default_factory()  # type: ignore
+            else:
+                value = MISSING
+
+        if is_union_annotation(type_) and not is_supported_union_annotation(type_):
+            e = ConfigValueError(
+                f"Unions of containers are not supported:\n{name}: {type_str(type_)}"
+            )
+            format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e))
+        try:
+            d[name] = _maybe_wrap(
+                ref_type=type_,
+                is_optional=is_optional,
+                key=name,
+                value=value,
+                parent=dummy_parent,
+            )
+        except (ValidationError, GrammarParseError) as ex:
+            format_and_raise(
+                node=dummy_parent, key=name, value=value, cause=ex, msg=str(ex)
+            )
+        d[name]._set_parent(None)
+    dict_subclass_data = extract_dict_subclass_data(obj=obj, parent=dummy_parent)
+    if dict_subclass_data is not None:
+        d.update(dict_subclass_data)
+    return d
+
+
+def is_dataclass(obj: Any) -> bool:
+    from omegaconf.base import Node
+
+    if dataclasses is None or isinstance(obj, Node):
+        return False
+    return dataclasses.is_dataclass(obj)
+
+
+def is_attr_class(obj: Any) -> bool:
+    from omegaconf.base import Node
+
+    if attr is None or isinstance(obj, Node):
+        return False
+    return attr.has(obj)
+
+
+def is_structured_config(obj: Any) -> bool:
+    return is_attr_class(obj) or is_dataclass(obj)
+
+
+def is_dataclass_frozen(type_: Any) -> bool:
+    return type_.__dataclass_params__.frozen  # type: ignore
+
+
+def is_attr_frozen(type_: type) -> bool:
+    # This is very hacky and probably fragile as well.
+    # Unfortunately currently there isn't an official API in attr that can detect that.
+    # noinspection PyProtectedMember
+    return type_.__setattr__ == attr._make._frozen_setattrs  # type: ignore
+
+
+def get_type_of(class_or_object: Any) -> Type[Any]:
+    type_ = class_or_object
+    if not isinstance(type_, type):
+        type_ = type(class_or_object)
+    assert isinstance(type_, type)
+    return type_
+
+
+def is_structured_config_frozen(obj: Any) -> bool:
+    type_ = get_type_of(obj)
+
+    if is_dataclass(type_):
+        return is_dataclass_frozen(type_)
+    if is_attr_class(type_):
+        return is_attr_frozen(type_)
+    return False
+
+
+def get_structured_config_init_field_names(obj: Any) -> List[str]:
+    fields: Union[List["dataclasses.Field[Any]"], List["attr.Attribute[Any]"]]
+    if is_dataclass(obj):
+        fields = get_dataclass_fields(obj)
+    elif is_attr_class(obj):
+        fields = get_attr_class_fields(obj)
+    else:
+        raise ValueError(f"Unsupported type: {type(obj).__name__}")
+    return [f.name for f in fields if f.init]
+
+
+def get_structured_config_data(
+    obj: Any, allow_objects: Optional[bool] = None
+) -> Dict[str, Any]:
+    if is_dataclass(obj):
+        return get_dataclass_data(obj, allow_objects=allow_objects)
+    elif is_attr_class(obj):
+        return get_attr_data(obj, allow_objects=allow_objects)
+    else:
+        raise ValueError(f"Unsupported type: {type(obj).__name__}")
+
+
+class ValueKind(Enum):
+    VALUE = 0
+    MANDATORY_MISSING = 1
+    INTERPOLATION = 2
+
+
+def _is_missing_value(value: Any) -> bool:
+    from omegaconf import Node
+
+    if isinstance(value, Node):
+        value = value._value()
+    return _is_missing_literal(value)
+
+
+def _is_missing_literal(value: Any) -> bool:
+    # Uses literal '???' instead of the MISSING const for performance reasons.
+    return isinstance(value, str) and value == "???"
+
+
+def _is_none(
+    value: Any, resolve: bool = False, throw_on_resolution_failure: bool = True
+) -> bool:
+    from omegaconf import Node
+
+    if not isinstance(value, Node):
+        return value is None
+
+    if resolve:
+        value = value._maybe_dereference_node(
+            throw_on_resolution_failure=throw_on_resolution_failure
+        )
+        if not throw_on_resolution_failure and value is None:
+            # Resolution failure: consider that it is *not* None.
+            return False
+        assert isinstance(value, Node)
+
+    return value._is_none()
+
+
+def get_value_kind(
+    value: Any, strict_interpolation_validation: bool = False
+) -> ValueKind:
+    """
+    Determine the kind of a value
+    Examples:
+    VALUE: "10", "20", True
+    MANDATORY_MISSING: "???"
+    INTERPOLATION: "${foo.bar}", "${foo.${bar}}", "${foo:bar}", "[${foo}, ${bar}]",
+                   "ftp://${host}/path", "${foo:${bar}, [true], {'baz': ${baz}}}"
+
+    :param value: Input to classify.
+    :param strict_interpolation_validation: If `True`, then when `value` is a string
+        containing "${", it is parsed to validate the interpolation syntax. If `False`,
+        this parsing step is skipped: this is more efficient, but will not detect errors.
+    """
+
+    if _is_missing_value(value):
+        return ValueKind.MANDATORY_MISSING
+
+    if _is_interpolation(value, strict_interpolation_validation):
+        return ValueKind.INTERPOLATION
+
+    return ValueKind.VALUE
+
+
+def _is_interpolation(v: Any, strict_interpolation_validation: bool = False) -> bool:
+    from omegaconf import Node
+
+    if isinstance(v, Node):
+        v = v._value()
+
+    if isinstance(v, str) and _is_interpolation_string(
+        v, strict_interpolation_validation
+    ):
+        return True
+    return False
+
+
+def _is_interpolation_string(value: str, strict_interpolation_validation: bool) -> bool:
+    # We identify potential interpolations by the presence of "${" in the string.
+    # Note that escaped interpolations (ex: "esc: \${bar}") are identified as
+    # interpolations: this is intended, since they must be processed as interpolations
+    # for the string to be properly un-escaped.
+    # Keep in mind that invalid interpolations will only be detected when
+    # `strict_interpolation_validation` is True.
+    if "${" in value:
+        if strict_interpolation_validation:
+            # First try the cheap regex matching that detects common interpolations.
+            if SIMPLE_INTERPOLATION_PATTERN.match(value) is None:
+                # If no match, do the more expensive grammar parsing to detect errors.
+                parse(value)
+        return True
+    return False
+
+
+def _is_special(value: Any) -> bool:
+    """Special values are None, MISSING, and interpolation."""
+    return _is_none(value) or get_value_kind(value) in (
+        ValueKind.MANDATORY_MISSING,
+        ValueKind.INTERPOLATION,
+    )
+
+
+def is_float(st: str) -> bool:
+    try:
+        float(st)
+        return True
+    except ValueError:
+        return False
+
+
+def is_int(st: str) -> bool:
+    try:
+        int(st)
+        return True
+    except ValueError:
+        return False
+
+
+def is_primitive_list(obj: Any) -> bool:
+    return isinstance(obj, (list, tuple))
+
+
+def is_primitive_dict(obj: Any) -> bool:
+    t = get_type_of(obj)
+    return t is dict
+
+
+def is_dict_annotation(type_: Any) -> bool:
+    if type_ in (dict, Dict):
+        return True
+    origin = getattr(type_, "__origin__", None)
+    # type_dict is a bit hard to detect.
+    # this support is tentative, if it eventually causes issues in other areas it may be dropped.
+    if sys.version_info < (3, 7, 0):  # pragma: no cover
+        typed_dict = hasattr(type_, "__base__") and type_.__base__ == Dict
+        return origin is Dict or type_ is Dict or typed_dict
+    else:  # pragma: no cover
+        typed_dict = hasattr(type_, "__base__") and type_.__base__ == dict
+        return origin is dict or typed_dict
+
+
+def is_list_annotation(type_: Any) -> bool:
+    if type_ in (list, List):
+        return True
+    origin = getattr(type_, "__origin__", None)
+    if sys.version_info < (3, 7, 0):
+        return origin is List or type_ is List  # pragma: no cover
+    else:
+        return origin is list  # pragma: no cover
+
+
+def is_tuple_annotation(type_: Any) -> bool:
+    if type_ in (tuple, Tuple):
+        return True
+    origin = getattr(type_, "__origin__", None)
+    if sys.version_info < (3, 7, 0):
+        return origin is Tuple or type_ is Tuple  # pragma: no cover
+    else:
+        return origin is tuple  # pragma: no cover
+
+
+def is_supported_union_annotation(obj: Any) -> bool:
+    """Currently only primitive types are supported in Unions, e.g. Union[int, str]"""
+    if not is_union_annotation(obj):
+        return False
+    args = obj.__args__
+    return all(is_primitive_type_annotation(arg) for arg in args)
+
+
+def is_dict_subclass(type_: Any) -> bool:
+    return type_ is not None and isinstance(type_, type) and issubclass(type_, Dict)
+
+
+def is_dict(obj: Any) -> bool:
+    return is_primitive_dict(obj) or is_dict_annotation(obj) or is_dict_subclass(obj)
+
+
+def is_primitive_container(obj: Any) -> bool:
+    return is_primitive_list(obj) or is_primitive_dict(obj)
+
+
+def get_list_element_type(ref_type: Optional[Type[Any]]) -> Any:
+    args = getattr(ref_type, "__args__", None)
+    if ref_type is not List and args is not None and args[0]:
+        element_type = args[0]
+    else:
+        element_type = Any
+    return element_type
+
+
+def get_tuple_item_types(ref_type: Type[Any]) -> Tuple[Any, ...]:
+    args = getattr(ref_type, "__args__", None)
+    if args in (None, ()):
+        args = (Any, ...)
+    assert isinstance(args, tuple)
+    return args
+
+
+def get_dict_key_value_types(ref_type: Any) -> Tuple[Any, Any]:
+    args = getattr(ref_type, "__args__", None)
+    if args is None:
+        bases = getattr(ref_type, "__orig_bases__", None)
+        if bases is not None and len(bases) > 0:
+            args = getattr(bases[0], "__args__", None)
+
+    key_type: Any
+    element_type: Any
+    if ref_type is None or ref_type == Dict:
+        key_type = Any
+        element_type = Any
+    else:
+        if args is not None:
+            key_type = args[0]
+            element_type = args[1]
+        else:
+            key_type = Any
+            element_type = Any
+
+    return key_type, element_type
+
+
+def is_valid_value_annotation(type_: Any) -> bool:
+    _, type_ = _resolve_optional(type_)
+    return (
+        type_ is Any
+        or is_primitive_type_annotation(type_)
+        or is_structured_config(type_)
+        or is_container_annotation(type_)
+        or is_supported_union_annotation(type_)
+    )
+
+
+def _valid_dict_key_annotation_type(type_: Any) -> bool:
+    from omegaconf import DictKeyType
+
+    return type_ is None or type_ is Any or issubclass(type_, DictKeyType.__args__)  # type: ignore
+
+
+def is_primitive_type_annotation(type_: Any) -> bool:
+    type_ = get_type_of(type_)
+    return issubclass(type_, (Enum, pathlib.Path)) or type_ in BUILTIN_VALUE_TYPES
+
+
+def _get_value(value: Any) -> Any:
+    from .base import Container, UnionNode
+    from .nodes import ValueNode
+
+    if isinstance(value, ValueNode):
+        return value._value()
+    elif isinstance(value, Container):
+        boxed = value._value()
+        if boxed is None or _is_missing_literal(boxed) or _is_interpolation(boxed):
+            return boxed
+    elif isinstance(value, UnionNode):
+        boxed = value._value()
+        if boxed is None or _is_missing_literal(boxed) or _is_interpolation(boxed):
+            return boxed
+        else:
+            return _get_value(boxed)  # pass through value of boxed node
+
+    # return primitives and regular OmegaConf Containers as is
+    return value
+
+
+def get_type_hint(obj: Any, key: Any = None) -> Optional[Type[Any]]:
+    from omegaconf import Container, Node
+
+    if isinstance(obj, Container):
+        if key is not None:
+            obj = obj._get_node(key)
+    else:
+        if key is not None:
+            raise ValueError("Key must only be provided when obj is a container")
+
+    if isinstance(obj, Node):
+        ref_type = obj._metadata.ref_type
+        if obj._is_optional() and ref_type is not Any:
+            return Optional[ref_type]  # type: ignore
+        else:
+            return ref_type
+    else:
+        return Any  # type: ignore
+
+
+def _raise(ex: Exception, cause: Exception) -> None:
+    # Set the environment variable OC_CAUSE=1 to get a stacktrace that includes the
+    # causing exception.
+    env_var = os.environ["OC_CAUSE"] if "OC_CAUSE" in os.environ else None
+    debugging = sys.gettrace() is not None
+    full_backtrace = (debugging and not env_var == "0") or (env_var == "1")
+    if full_backtrace:
+        ex.__cause__ = cause
+    else:
+        ex.__cause__ = None
+    raise ex.with_traceback(sys.exc_info()[2])  # set env var OC_CAUSE=1 for full trace
+
+
+def format_and_raise(
+    node: Any,
+    key: Any,
+    value: Any,
+    msg: str,
+    cause: Exception,
+    type_override: Any = None,
+) -> None:
+    from omegaconf import OmegaConf
+    from omegaconf.base import Node
+
+    if isinstance(cause, AssertionError):
+        raise
+
+    if isinstance(cause, OmegaConfBaseException) and cause._initialized:
+        ex = cause
+        if type_override is not None:
+            ex = type_override(str(cause))
+            ex.__dict__ = copy.deepcopy(cause.__dict__)
+        _raise(ex, cause)
+
+    object_type: Optional[Type[Any]]
+    object_type_str: Optional[str] = None
+    ref_type: Optional[Type[Any]]
+    ref_type_str: Optional[str]
+
+    child_node: Optional[Node] = None
+    if node is None:
+        full_key = key if key is not None else ""
+        object_type = None
+        ref_type = None
+        ref_type_str = None
+    else:
+        if key is not None and not node._is_none():
+            child_node = node._get_node(key, validate_access=False)
+
+        try:
+            full_key = node._get_full_key(key=key)
+        except Exception as exc:
+            # Since we are handling an exception, raising a different one here would
+            # be misleading. Instead, we display it in the key.
+            full_key = f"<unresolvable due to {type(exc).__name__}: {exc}>"
+
+        object_type = OmegaConf.get_type(node)
+        object_type_str = type_str(object_type)
+
+        ref_type = get_type_hint(node)
+        ref_type_str = type_str(ref_type)
+
+    msg = string.Template(msg).safe_substitute(
+        REF_TYPE=ref_type_str,
+        OBJECT_TYPE=object_type_str,
+        KEY=key,
+        FULL_KEY=full_key,
+        VALUE=value,
+        VALUE_TYPE=type_str(type(value), include_module_name=True),
+        KEY_TYPE=f"{type(key).__name__}",
+    )
+
+    if ref_type not in (None, Any):
+        template = dedent(
+            """\
+            $MSG
+                full_key: $FULL_KEY
+                reference_type=$REF_TYPE
+                object_type=$OBJECT_TYPE"""
+        )
+    else:
+        template = dedent(
+            """\
+            $MSG
+                full_key: $FULL_KEY
+                object_type=$OBJECT_TYPE"""
+        )
+    s = string.Template(template=template)
+
+    message = s.substitute(
+        REF_TYPE=ref_type_str, OBJECT_TYPE=object_type_str, MSG=msg, FULL_KEY=full_key
+    )
+    exception_type = type(cause) if type_override is None else type_override
+    if exception_type == TypeError:
+        exception_type = ConfigTypeError
+    elif exception_type == IndexError:
+        exception_type = ConfigIndexError
+
+    ex = exception_type(f"{message}")
+    if issubclass(exception_type, OmegaConfBaseException):
+        ex._initialized = True
+        ex.msg = message
+        ex.parent_node = node
+        ex.child_node = child_node
+        ex.key = key
+        ex.full_key = full_key
+        ex.value = value
+        ex.object_type = object_type
+        ex.object_type_str = object_type_str
+        ex.ref_type = ref_type
+        ex.ref_type_str = ref_type_str
+
+    _raise(ex, cause)
+
+
+def type_str(t: Any, include_module_name: bool = False) -> str:
+    is_optional, t = _resolve_optional(t)
+    if t is NoneType:
+        return str(t.__name__)
+    if t is Any:
+        return "Any"
+    if t is ...:
+        return "..."
+
+    if hasattr(t, "__name__"):
+        name = str(t.__name__)
+    elif getattr(t, "_name", None) is not None:  # pragma: no cover
+        name = str(t._name)
+    elif getattr(t, "__origin__", None) is not None:  # pragma: no cover
+        name = type_str(t.__origin__)
+    else:
+        name = str(t)
+        if name.startswith("typing."):  # pragma: no cover
+            name = name[len("typing.") :]
+
+    args = getattr(t, "__args__", None)
+    if args is not None:
+        args = ", ".join(
+            [type_str(t, include_module_name=include_module_name) for t in t.__args__]
+        )
+        ret = f"{name}[{args}]"
+    else:
+        ret = name
+    if include_module_name:
+        if (
+            hasattr(t, "__module__")
+            and t.__module__ != "builtins"
+            and t.__module__ != "typing"
+            and not t.__module__.startswith("omegaconf.")
+        ):
+            module_prefix = str(t.__module__) + "."
+        else:
+            module_prefix = ""
+        ret = module_prefix + ret
+    if is_optional:
+        return f"Optional[{ret}]"
+    else:
+        return ret
+
+
+def _ensure_container(target: Any, flags: Optional[Dict[str, bool]] = None) -> Any:
+    from omegaconf import OmegaConf
+
+    if is_primitive_container(target):
+        assert isinstance(target, (list, dict))
+        target = OmegaConf.create(target, flags=flags)
+    elif is_structured_config(target):
+        target = OmegaConf.structured(target, flags=flags)
+    elif not OmegaConf.is_config(target):
+        raise ValueError(
+            "Invalid input. Supports one of "
+            + "[dict,list,DictConfig,ListConfig,dataclass,dataclass instance,attr class,attr class instance]"
+        )
+
+    return target
+
+
+def is_generic_list(type_: Any) -> bool:
+    """
+    Checks if a type is a generic list, for example:
+    list returns False
+    typing.List returns False
+    typing.List[T] returns True
+
+    :param type_: variable type
+    :return: bool
+    """
+    return is_list_annotation(type_) and get_list_element_type(type_) is not None
+
+
+def is_generic_dict(type_: Any) -> bool:
+    """
+    Checks if a type is a generic dict, for example:
+    list returns False
+    typing.List returns False
+    typing.List[T] returns True
+
+    :param type_: variable type
+    :return: bool
+    """
+    return is_dict_annotation(type_) and len(get_dict_key_value_types(type_)) > 0
+
+
+def is_container_annotation(type_: Any) -> bool:
+    return is_list_annotation(type_) or is_dict_annotation(type_)
+
+
+def split_key(key: str) -> List[str]:
+    """
+    Split a full key path into its individual components.
+
+    This is similar to `key.split(".")` but also works with the getitem syntax:
+        "a.b"       -> ["a", "b"]
+        "a[b]"      -> ["a", "b"]
+        ".a.b[c].d" -> ["", "a", "b", "c", "d"]
+        "[a].b"     -> ["a", "b"]
+    """
+    # Obtain the first part of the key (in docstring examples: a, a, .a, '')
+    first = KEY_PATH_HEAD.match(key)
+    assert first is not None
+    first_stop = first.span()[1]
+
+    # `tokens` will contain all elements composing the key.
+    tokens = key[0:first_stop].split(".")
+
+    # Optimization in case `key` has no other component: we are done.
+    if first_stop == len(key):
+        return tokens
+
+    if key[first_stop] == "[" and not tokens[-1]:
+        # This is a special case where the first key starts with brackets, e.g.
+        # [a] or ..[a]. In that case there is an extra "" in `tokens` that we
+        # need to get rid of:
+        #   [a]   -> tokens = [""] but we would like []
+        #   ..[a] -> tokens = ["", "", ""] but we would like ["", ""]
+        tokens.pop()
+
+    # Identify other key elements (in docstring examples: b, b, b/c/d, b)
+    others = KEY_PATH_OTHER.findall(key[first_stop:])
+
+    # There are two groups in the `KEY_PATH_OTHER` regex: one for keys starting
+    # with a dot (.b, .d) and one for keys starting with a bracket ([b], [c]).
+    # Only one group can be non-empty.
+    tokens += [dot_key if dot_key else bracket_key for dot_key, bracket_key in others]
+
+    return tokens
+
+
+# Similar to Python 3.7+'s `contextlib.nullcontext` (which should be used instead,
+# once support for Python 3.6 is dropped).
+@contextmanager
+def nullcontext(enter_result: Any = None) -> Iterator[Any]:
+    yield enter_result

omegaconf/base.py

@@ -0,0 +1,962 @@
+import copy
+import sys
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, Type, Union
+
+from antlr4 import ParserRuleContext
+
+from ._utils import (
+    _DEFAULT_MARKER_,
+    NoneType,
+    ValueKind,
+    _get_value,
+    _is_interpolation,
+    _is_missing_value,
+    _is_special,
+    format_and_raise,
+    get_value_kind,
+    is_union_annotation,
+    is_valid_value_annotation,
+    split_key,
+    type_str,
+)
+from .errors import (
+    ConfigKeyError,
+    ConfigTypeError,
+    InterpolationKeyError,
+    InterpolationResolutionError,
+    InterpolationToMissingValueError,
+    InterpolationValidationError,
+    MissingMandatoryValue,
+    UnsupportedInterpolationType,
+    ValidationError,
+)
+from .grammar.gen.OmegaConfGrammarParser import OmegaConfGrammarParser
+from .grammar_parser import parse
+from .grammar_visitor import GrammarVisitor
+
+DictKeyType = Union[str, bytes, int, Enum, float, bool]
+
+
+@dataclass
+class Metadata:
+
+    ref_type: Union[Type[Any], Any]
+
+    object_type: Union[Type[Any], Any]
+
+    optional: bool
+
+    key: Any
+
+    # Flags have 3 modes:
+    #   unset : inherit from parent (None if no parent specifies)
+    #   set to true: flag is true
+    #   set to false: flag is false
+    flags: Optional[Dict[str, bool]] = None
+
+    # If True, when checking the value of a flag, if the flag is not set None is returned
+    # otherwise, the parent node is queried.
+    flags_root: bool = False
+
+    resolver_cache: Dict[str, Any] = field(default_factory=lambda: defaultdict(dict))
+
+    def __post_init__(self) -> None:
+        if self.flags is None:
+            self.flags = {}
+
+    @property
+    def type_hint(self) -> Union[Type[Any], Any]:
+        """Compute `type_hint` from `self.optional` and `self.ref_type`"""
+        # For compatibility with pickled OmegaConf objects created using older
+        # versions of OmegaConf, we store `ref_type` and `object_type`
+        # separately (rather than storing `type_hint` directly).
+        if self.optional:
+            return Optional[self.ref_type]
+        else:
+            return self.ref_type
+
+
+@dataclass
+class ContainerMetadata(Metadata):
+    key_type: Any = None
+    element_type: Any = None
+
+    def __post_init__(self) -> None:
+        if self.ref_type is None:
+            self.ref_type = Any
+        assert self.key_type is Any or isinstance(self.key_type, type)
+        if self.element_type is not None:
+            if not is_valid_value_annotation(self.element_type):
+                raise ValidationError(
+                    f"Unsupported value type: '{type_str(self.element_type, include_module_name=True)}'"
+                )
+
+        if self.flags is None:
+            self.flags = {}
+
+
+class Node(ABC):
+    _metadata: Metadata
+
+    _parent: Optional["Box"]
+    _flags_cache: Optional[Dict[str, Optional[bool]]]
+
+    def __init__(self, parent: Optional["Box"], metadata: Metadata):
+        self.__dict__["_metadata"] = metadata
+        self.__dict__["_parent"] = parent
+        self.__dict__["_flags_cache"] = None
+
+    def __getstate__(self) -> Dict[str, Any]:
+        # Overridden to ensure that the flags cache is cleared on serialization.
+        state_dict = copy.copy(self.__dict__)
+        del state_dict["_flags_cache"]
+        return state_dict
+
+    def __setstate__(self, state_dict: Dict[str, Any]) -> None:
+        self.__dict__.update(state_dict)
+        self.__dict__["_flags_cache"] = None
+
+    def _set_parent(self, parent: Optional["Box"]) -> None:
+        assert parent is None or isinstance(parent, Box)
+        self.__dict__["_parent"] = parent
+        self._invalidate_flags_cache()
+
+    def _invalidate_flags_cache(self) -> None:
+        self.__dict__["_flags_cache"] = None
+
+    def _get_parent(self) -> Optional["Box"]:
+        parent = self.__dict__["_parent"]
+        assert parent is None or isinstance(parent, Box)
+        return parent
+
+    def _get_parent_container(self) -> Optional["Container"]:
+        """
+        Like _get_parent, but returns the grandparent
+        in the case where `self` is wrapped by a UnionNode.
+        """
+        parent = self.__dict__["_parent"]
+        assert parent is None or isinstance(parent, Box)
+
+        if isinstance(parent, UnionNode):
+            grandparent = parent.__dict__["_parent"]
+            assert grandparent is None or isinstance(grandparent, Container)
+            return grandparent
+        else:
+            assert parent is None or isinstance(parent, Container)
+            return parent
+
+    def _set_flag(
+        self,
+        flags: Union[List[str], str],
+        values: Union[List[Optional[bool]], Optional[bool]],
+    ) -> "Node":
+        if isinstance(flags, str):
+            flags = [flags]
+
+        if values is None or isinstance(values, bool):
+            values = [values]
+
+        if len(values) == 1:
+            values = len(flags) * values
+
+        if len(flags) != len(values):
+            raise ValueError("Inconsistent lengths of input flag names and values")
+
+        for idx, flag in enumerate(flags):
+            value = values[idx]
+            if value is None:
+                assert self._metadata.flags is not None
+                if flag in self._metadata.flags:
+                    del self._metadata.flags[flag]
+            else:
+                assert self._metadata.flags is not None
+                self._metadata.flags[flag] = value
+        self._invalidate_flags_cache()
+        return self
+
+    def _get_node_flag(self, flag: str) -> Optional[bool]:
+        """
+        :param flag: flag to inspect
+        :return: the state of the flag on this node.
+        """
+        assert self._metadata.flags is not None
+        return self._metadata.flags.get(flag)
+
+    def _get_flag(self, flag: str) -> Optional[bool]:
+        cache = self.__dict__["_flags_cache"]
+        if cache is None:
+            cache = self.__dict__["_flags_cache"] = {}
+
+        ret = cache.get(flag, _DEFAULT_MARKER_)
+        if ret is _DEFAULT_MARKER_:
+            ret = self._get_flag_no_cache(flag)
+            cache[flag] = ret
+        assert ret is None or isinstance(ret, bool)
+        return ret
+
+    def _get_flag_no_cache(self, flag: str) -> Optional[bool]:
+        """
+        Returns True if this config node flag is set
+        A flag is set if node.set_flag(True) was called
+        or one if it's parents is flag is set
+        :return:
+        """
+        flags = self._metadata.flags
+        assert flags is not None
+        if flag in flags and flags[flag] is not None:
+            return flags[flag]
+
+        if self._is_flags_root():
+            return None
+
+        parent = self._get_parent()
+        if parent is None:
+            return None
+        else:
+            # noinspection PyProtectedMember
+            return parent._get_flag(flag)
+
+    def _format_and_raise(
+        self,
+        key: Any,
+        value: Any,
+        cause: Exception,
+        msg: Optional[str] = None,
+        type_override: Any = None,
+    ) -> None:
+        format_and_raise(
+            node=self,
+            key=key,
+            value=value,
+            msg=str(cause) if msg is None else msg,
+            cause=cause,
+            type_override=type_override,
+        )
+        assert False
+
+    @abstractmethod
+    def _get_full_key(self, key: Optional[Union[DictKeyType, int]]) -> str:
+        ...
+
+    def _dereference_node(self) -> "Node":
+        node = self._dereference_node_impl(throw_on_resolution_failure=True)
+        assert node is not None
+        return node
+
+    def _maybe_dereference_node(
+        self,
+        throw_on_resolution_failure: bool = False,
+        memo: Optional[Set[int]] = None,
+    ) -> Optional["Node"]:
+        return self._dereference_node_impl(
+            throw_on_resolution_failure=throw_on_resolution_failure,
+            memo=memo,
+        )
+
+    def _dereference_node_impl(
+        self,
+        throw_on_resolution_failure: bool,
+        memo: Optional[Set[int]] = None,
+    ) -> Optional["Node"]:
+        if not self._is_interpolation():
+            return self
+
+        parent = self._get_parent_container()
+        if parent is None:
+            if throw_on_resolution_failure:
+                raise InterpolationResolutionError(
+                    "Cannot resolve interpolation for a node without a parent"
+                )
+            return None
+        assert parent is not None
+        key = self._key()
+        return parent._resolve_interpolation_from_parse_tree(
+            parent=parent,
+            key=key,
+            value=self,
+            parse_tree=parse(_get_value(self)),
+            throw_on_resolution_failure=throw_on_resolution_failure,
+            memo=memo,
+        )
+
+    def _get_root(self) -> "Container":
+        root: Optional[Box] = self._get_parent()
+        if root is None:
+            assert isinstance(self, Container)
+            return self
+        assert root is not None and isinstance(root, Box)
+        while root._get_parent() is not None:
+            root = root._get_parent()
+            assert root is not None and isinstance(root, Box)
+        assert root is not None and isinstance(root, Container)
+        return root
+
+    def _is_missing(self) -> bool:
+        """
+        Check if the node's value is `???` (does *not* resolve interpolations).
+        """
+        return _is_missing_value(self)
+
+    def _is_none(self) -> bool:
+        """
+        Check if the node's value is `None` (does *not* resolve interpolations).
+        """
+        return self._value() is None
+
+    @abstractmethod
+    def __eq__(self, other: Any) -> bool:
+        ...
+
+    @abstractmethod
+    def __ne__(self, other: Any) -> bool:
+        ...
+
+    @abstractmethod
+    def __hash__(self) -> int:
+        ...
+
+    @abstractmethod
+    def _value(self) -> Any:
+        ...
+
+    @abstractmethod
+    def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None:
+        ...
+
+    @abstractmethod
+    def _is_optional(self) -> bool:
+        ...
+
+    @abstractmethod
+    def _is_interpolation(self) -> bool:
+        ...
+
+    def _key(self) -> Any:
+        return self._metadata.key
+
+    def _set_key(self, key: Any) -> None:
+        self._metadata.key = key
+
+    def _is_flags_root(self) -> bool:
+        return self._metadata.flags_root
+
+    def _set_flags_root(self, flags_root: bool) -> None:
+        if self._metadata.flags_root != flags_root:
+            self._metadata.flags_root = flags_root
+            self._invalidate_flags_cache()
+
+    def _has_ref_type(self) -> bool:
+        return self._metadata.ref_type is not Any
+
+
+class Box(Node):
+    """
+    Base class for nodes that can contain other nodes.
+    Concrete subclasses include DictConfig, ListConfig, and UnionNode.
+    """
+
+    _content: Any
+
+    def __init__(self, parent: Optional["Box"], metadata: Metadata):
+        super().__init__(parent=parent, metadata=metadata)
+        self.__dict__["_content"] = None
+
+    def __copy__(self) -> Any:
+        # real shallow copy is impossible because of the reference to the parent.
+        return copy.deepcopy(self)
+
+    def _re_parent(self) -> None:
+        from .dictconfig import DictConfig
+        from .listconfig import ListConfig
+
+        # update parents of first level Config nodes to self
+
+        if isinstance(self, DictConfig):
+            content = self.__dict__["_content"]
+            if isinstance(content, dict):
+                for _key, value in self.__dict__["_content"].items():
+                    if value is not None:
+                        value._set_parent(self)
+                    if isinstance(value, Box):
+                        value._re_parent()
+        elif isinstance(self, ListConfig):
+            content = self.__dict__["_content"]
+            if isinstance(content, list):
+                for item in self.__dict__["_content"]:
+                    if item is not None:
+                        item._set_parent(self)
+                    if isinstance(item, Box):
+                        item._re_parent()
+        elif isinstance(self, UnionNode):
+            content = self.__dict__["_content"]
+            if isinstance(content, Node):
+                content._set_parent(self)
+                if isinstance(content, Box):  # pragma: no cover
+                    # No coverage here as support for containers inside
+                    # UnionNode is not yet implemented
+                    content._re_parent()
+
+
+class Container(Box):
+    """
+    Container tagging interface
+    """
+
+    _metadata: ContainerMetadata
+
+    @abstractmethod
+    def _get_child(
+        self,
+        key: Any,
+        validate_access: bool = True,
+        validate_key: bool = True,
+        throw_on_missing_value: bool = False,
+        throw_on_missing_key: bool = False,
+    ) -> Union[Optional[Node], List[Optional[Node]]]:
+        ...
+
+    @abstractmethod
+    def _get_node(
+        self,
+        key: Any,
+        validate_access: bool = True,
+        validate_key: bool = True,
+        throw_on_missing_value: bool = False,
+        throw_on_missing_key: bool = False,
+    ) -> Union[Optional[Node], List[Optional[Node]]]:
+        ...
+
+    @abstractmethod
+    def __delitem__(self, key: Any) -> None:
+        ...
+
+    @abstractmethod
+    def __setitem__(self, key: Any, value: Any) -> None:
+        ...
+
+    @abstractmethod
+    def __iter__(self) -> Iterator[Any]:
+        ...
+
+    @abstractmethod
+    def __getitem__(self, key_or_index: Any) -> Any:
+        ...
+
+    def _resolve_key_and_root(self, key: str) -> Tuple["Container", str]:
+        orig = key
+        if not key.startswith("."):
+            return self._get_root(), key
+        else:
+            root: Optional[Container] = self
+            assert key.startswith(".")
+            while True:
+                assert root is not None
+                key = key[1:]
+                if not key.startswith("."):
+                    break
+                root = root._get_parent_container()
+                if root is None:
+                    raise ConfigKeyError(f"Error resolving key '{orig}'")
+
+            return root, key
+
+    def _select_impl(
+        self,
+        key: str,
+        throw_on_missing: bool,
+        throw_on_resolution_failure: bool,
+        memo: Optional[Set[int]] = None,
+    ) -> Tuple[Optional["Container"], Optional[str], Optional[Node]]:
+        """
+        Select a value using dot separated key sequence
+        """
+        from .omegaconf import _select_one
+
+        if key == "":
+            return self, "", self
+
+        split = split_key(key)
+        root: Optional[Container] = self
+        for i in range(len(split) - 1):
+            if root is None:
+                break
+
+            k = split[i]
+            ret, _ = _select_one(
+                c=root,
+                key=k,
+                throw_on_missing=throw_on_missing,
+                throw_on_type_error=throw_on_resolution_failure,
+            )
+            if isinstance(ret, Node):
+                ret = ret._maybe_dereference_node(
+                    throw_on_resolution_failure=throw_on_resolution_failure,
+                    memo=memo,
+                )
+
+            if ret is not None and not isinstance(ret, Container):
+                parent_key = ".".join(split[0 : i + 1])
+                child_key = split[i + 1]
+                raise ConfigTypeError(
+                    f"Error trying to access {key}: node `{parent_key}` "
+                    f"is not a container and thus cannot contain `{child_key}`"
+                )
+            root = ret
+
+        if root is None:
+            return None, None, None
+
+        last_key = split[-1]
+        value, _ = _select_one(
+            c=root,
+            key=last_key,
+            throw_on_missing=throw_on_missing,
+            throw_on_type_error=throw_on_resolution_failure,
+        )
+        if value is None:
+            return root, last_key, None
+
+        if memo is not None:
+            vid = id(value)
+            if vid in memo:
+                raise InterpolationResolutionError("Recursive interpolation detected")
+            # push to memo "stack"
+            memo.add(vid)
+
+        try:
+            value = root._maybe_resolve_interpolation(
+                parent=root,
+                key=last_key,
+                value=value,
+                throw_on_resolution_failure=throw_on_resolution_failure,
+                memo=memo,
+            )
+        finally:
+            if memo is not None:
+                # pop from memo "stack"
+                memo.remove(vid)
+
+        return root, last_key, value
+
+    def _resolve_interpolation_from_parse_tree(
+        self,
+        parent: Optional["Container"],
+        value: "Node",
+        key: Any,
+        parse_tree: OmegaConfGrammarParser.ConfigValueContext,
+        throw_on_resolution_failure: bool,
+        memo: Optional[Set[int]],
+    ) -> Optional["Node"]:
+        """
+        Resolve an interpolation.
+
+        This happens in two steps:
+            1. The parse tree is visited, which outputs either a `Node` (e.g.,
+               for node interpolations "${foo}"), a string (e.g., for string
+               interpolations "hello ${name}", or any other arbitrary value
+               (e.g., or custom interpolations "${foo:bar}").
+            2. This output is potentially validated and converted when the node
+               being resolved (`value`) is typed.
+
+        If an error occurs in one of the above steps, an `InterpolationResolutionError`
+        (or a subclass of it) is raised, *unless* `throw_on_resolution_failure` is set
+        to `False` (in which case the return value is `None`).
+
+        :param parent: Parent of the node being resolved.
+        :param value: Node being resolved.
+        :param key: The associated key in the parent.
+        :param parse_tree: The parse tree as obtained from `grammar_parser.parse()`.
+        :param throw_on_resolution_failure: If `False`, then exceptions raised during
+            the resolution of the interpolation are silenced, and instead `None` is
+            returned.
+
+        :return: A `Node` that contains the interpolation result. This may be an existing
+            node in the config (in the case of a node interpolation "${foo}"), or a new
+            node that is created to wrap the interpolated value. It is `None` if and only if
+            `throw_on_resolution_failure` is `False` and an error occurs during resolution.
+        """
+
+        try:
+            resolved = self.resolve_parse_tree(
+                parse_tree=parse_tree, node=value, key=key, memo=memo
+            )
+        except InterpolationResolutionError:
+            if throw_on_resolution_failure:
+                raise
+            return None
+
+        return self._validate_and_convert_interpolation_result(
+            parent=parent,
+            value=value,
+            key=key,
+            resolved=resolved,
+            throw_on_resolution_failure=throw_on_resolution_failure,
+        )
+
+    def _validate_and_convert_interpolation_result(
+        self,
+        parent: Optional["Container"],
+        value: "Node",
+        key: Any,
+        resolved: Any,
+        throw_on_resolution_failure: bool,
+    ) -> Optional["Node"]:
+        from .nodes import AnyNode, InterpolationResultNode, ValueNode
+
+        # If the output is not a Node already (e.g., because it is the output of a
+        # custom resolver), then we will need to wrap it within a Node.
+        must_wrap = not isinstance(resolved, Node)
+
+        # If the node is typed, validate (and possibly convert) the result.
+        if isinstance(value, ValueNode) and not isinstance(value, AnyNode):
+            res_value = _get_value(resolved)
+            try:
+                conv_value = value.validate_and_convert(res_value)
+            except ValidationError as e:
+                if throw_on_resolution_failure:
+                    self._format_and_raise(
+                        key=key,
+                        value=res_value,
+                        cause=e,
+                        msg=f"While dereferencing interpolation '{value}': {e}",
+                        type_override=InterpolationValidationError,
+                    )
+                return None
+
+            # If the converted value is of the same type, it means that no conversion
+            # was actually needed. As a result, we can keep the original `resolved`
+            # (and otherwise, the converted value must be wrapped into a new node).
+            if type(conv_value) != type(res_value):
+                must_wrap = True
+                resolved = conv_value
+
+        if must_wrap:
+            return InterpolationResultNode(value=resolved, key=key, parent=parent)
+        else:
+            assert isinstance(resolved, Node)
+            return resolved
+
+    def _validate_not_dereferencing_to_parent(self, node: Node, target: Node) -> None:
+        parent: Optional[Node] = node
+        while parent is not None:
+            if parent is target:
+                raise InterpolationResolutionError(
+                    "Interpolation to parent node detected"
+                )
+            parent = parent._get_parent()
+
+    def _resolve_node_interpolation(
+        self, inter_key: str, memo: Optional[Set[int]]
+    ) -> "Node":
+        """A node interpolation is of the form `${foo.bar}`"""
+        try:
+            root_node, inter_key = self._resolve_key_and_root(inter_key)
+        except ConfigKeyError as exc:
+            raise InterpolationKeyError(
+                f"ConfigKeyError while resolving interpolation: {exc}"
+            ).with_traceback(sys.exc_info()[2])
+
+        try:
+            parent, last_key, value = root_node._select_impl(
+                inter_key,
+                throw_on_missing=True,
+                throw_on_resolution_failure=True,
+                memo=memo,
+            )
+        except MissingMandatoryValue as exc:
+            raise InterpolationToMissingValueError(
+                f"MissingMandatoryValue while resolving interpolation: {exc}"
+            ).with_traceback(sys.exc_info()[2])
+
+        if parent is None or value is None:
+            raise InterpolationKeyError(f"Interpolation key '{inter_key}' not found")
+        else:
+            self._validate_not_dereferencing_to_parent(node=self, target=value)
+            return value
+
+    def _evaluate_custom_resolver(
+        self,
+        key: Any,
+        node: Node,
+        inter_type: str,
+        inter_args: Tuple[Any, ...],
+        inter_args_str: Tuple[str, ...],
+    ) -> Any:
+        from omegaconf import OmegaConf
+
+        resolver = OmegaConf._get_resolver(inter_type)
+        if resolver is not None:
+            root_node = self._get_root()
+            return resolver(
+                root_node,
+                self,
+                node,
+                inter_args,
+                inter_args_str,
+            )
+        else:
+            raise UnsupportedInterpolationType(
+                f"Unsupported interpolation type {inter_type}"
+            )
+
+    def _maybe_resolve_interpolation(
+        self,
+        parent: Optional["Container"],
+        key: Any,
+        value: Node,
+        throw_on_resolution_failure: bool,
+        memo: Optional[Set[int]] = None,
+    ) -> Optional[Node]:
+        value_kind = get_value_kind(value)
+        if value_kind != ValueKind.INTERPOLATION:
+            return value
+
+        parse_tree = parse(_get_value(value))
+        return self._resolve_interpolation_from_parse_tree(
+            parent=parent,
+            value=value,
+            key=key,
+            parse_tree=parse_tree,
+            throw_on_resolution_failure=throw_on_resolution_failure,
+            memo=memo if memo is not None else set(),
+        )
+
+    def resolve_parse_tree(
+        self,
+        parse_tree: ParserRuleContext,
+        node: Node,
+        memo: Optional[Set[int]] = None,
+        key: Optional[Any] = None,
+    ) -> Any:
+        """
+        Resolve a given parse tree into its value.
+
+        We make no assumption here on the type of the tree's root, so that the
+        return value may be of any type.
+        """
+
+        def node_interpolation_callback(
+            inter_key: str, memo: Optional[Set[int]]
+        ) -> Optional["Node"]:
+            return self._resolve_node_interpolation(inter_key=inter_key, memo=memo)
+
+        def resolver_interpolation_callback(
+            name: str, args: Tuple[Any, ...], args_str: Tuple[str, ...]
+        ) -> Any:
+            return self._evaluate_custom_resolver(
+                key=key,
+                node=node,
+                inter_type=name,
+                inter_args=args,
+                inter_args_str=args_str,
+            )
+
+        visitor = GrammarVisitor(
+            node_interpolation_callback=node_interpolation_callback,
+            resolver_interpolation_callback=resolver_interpolation_callback,
+            memo=memo,
+        )
+        try:
+            return visitor.visit(parse_tree)
+        except InterpolationResolutionError:
+            raise
+        except Exception as exc:
+            # Other kinds of exceptions are wrapped in an `InterpolationResolutionError`.
+            raise InterpolationResolutionError(
+                f"{type(exc).__name__} raised while resolving interpolation: {exc}"
+            ).with_traceback(sys.exc_info()[2])
+
+    def _invalidate_flags_cache(self) -> None:
+        from .dictconfig import DictConfig
+        from .listconfig import ListConfig
+
+        # invalidate subtree cache only if the cache is initialized in this node.
+
+        if self.__dict__["_flags_cache"] is not None:
+            self.__dict__["_flags_cache"] = None
+            if isinstance(self, DictConfig):
+                content = self.__dict__["_content"]
+                if isinstance(content, dict):
+                    for value in self.__dict__["_content"].values():
+                        value._invalidate_flags_cache()
+            elif isinstance(self, ListConfig):
+                content = self.__dict__["_content"]
+                if isinstance(content, list):
+                    for item in self.__dict__["_content"]:
+                        item._invalidate_flags_cache()
+
+
+class SCMode(Enum):
+    DICT = 1  # Convert to plain dict
+    DICT_CONFIG = 2  # Keep as OmegaConf DictConfig
+    INSTANTIATE = 3  # Create a dataclass or attrs class instance
+
+
+class UnionNode(Box):
+    """
+    This class handles Union type hints. The `_content` attribute is either a
+    child node that is compatible with the given Union ref_type, or it is a
+    special value (None or MISSING or interpolation).
+
+    Much of the logic for e.g. value assignment and type validation is
+    delegated to the child node. As such, UnionNode functions as a
+    "pass-through" node. User apps and downstream libraries should not need to
+    know about UnionNode (assuming they only use OmegaConf's public API).
+    """
+
+    _parent: Optional[Container]
+    _content: Union[Node, None, str]
+
+    def __init__(
+        self,
+        content: Any,
+        ref_type: Any,
+        is_optional: bool = True,
+        key: Any = None,
+        parent: Optional[Box] = None,
+    ) -> None:
+        try:
+            if not is_union_annotation(ref_type):  # pragma: no cover
+                msg = (
+                    f"UnionNode got unexpected ref_type {ref_type}. Please file a bug"
+                    + " report at https://github.com/omry/omegaconf/issues"
+                )
+                raise AssertionError(msg)
+            if not isinstance(parent, (Container, NoneType)):
+                raise ConfigTypeError("Parent type is not omegaconf.Container")
+            super().__init__(
+                parent=parent,
+                metadata=Metadata(
+                    ref_type=ref_type,
+                    object_type=None,
+                    optional=is_optional,
+                    key=key,
+                    flags={"convert": False},
+                ),
+            )
+            self._set_value(content)
+        except Exception as ex:
+            format_and_raise(node=None, key=key, value=content, msg=str(ex), cause=ex)
+
+    def _get_full_key(self, key: Optional[Union[DictKeyType, int]]) -> str:
+        parent = self._get_parent()
+        if parent is None:
+            if self._metadata.key is None:
+                return ""
+            else:
+                return str(self._metadata.key)
+        else:
+            return parent._get_full_key(self._metadata.key)
+
+    def __eq__(self, other: Any) -> bool:
+        content = self.__dict__["_content"]
+        if isinstance(content, Node):
+            ret = content.__eq__(other)
+        elif isinstance(other, Node):
+            ret = other.__eq__(content)
+        else:
+            ret = content.__eq__(other)
+        assert isinstance(ret, (bool, type(NotImplemented)))
+        return ret
+
+    def __ne__(self, other: Any) -> bool:
+        x = self.__eq__(other)
+        if x is NotImplemented:
+            return NotImplemented
+        return not x
+
+    def __hash__(self) -> int:
+        return hash(self.__dict__["_content"])
+
+    def _value(self) -> Union[Node, None, str]:
+        content = self.__dict__["_content"]
+        assert isinstance(content, (Node, NoneType, str))
+        return content
+
+    def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None:
+        previous_content = self.__dict__["_content"]
+        previous_metadata = self.__dict__["_metadata"]
+        try:
+            self._set_value_impl(value, flags)
+        except Exception as e:
+            self.__dict__["_content"] = previous_content
+            self.__dict__["_metadata"] = previous_metadata
+            raise e
+
+    def _set_value_impl(
+        self, value: Any, flags: Optional[Dict[str, bool]] = None
+    ) -> None:
+        from omegaconf.omegaconf import _node_wrap
+
+        ref_type = self._metadata.ref_type
+        type_hint = self._metadata.type_hint
+
+        value = _get_value(value)
+        if _is_special(value):
+            assert isinstance(value, (str, NoneType))
+            if value is None:
+                if not self._is_optional():
+                    raise ValidationError(
+                        f"Value '$VALUE' is incompatible with type hint '{type_str(type_hint)}'"
+                    )
+            self.__dict__["_content"] = value
+        elif isinstance(value, Container):
+            raise ValidationError(
+                f"Cannot assign container '$VALUE' of type '$VALUE_TYPE' to {type_str(type_hint)}"
+            )
+        else:
+            for candidate_ref_type in ref_type.__args__:
+                try:
+                    self.__dict__["_content"] = _node_wrap(
+                        value=value,
+                        ref_type=candidate_ref_type,
+                        is_optional=False,
+                        key=None,
+                        parent=self,
+                    )
+                    break
+                except ValidationError:
+                    continue
+            else:
+                raise ValidationError(
+                    f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_str(type_hint)}'"
+                )
+
+    def _is_optional(self) -> bool:
+        return self.__dict__["_metadata"].optional is True
+
+    def _is_interpolation(self) -> bool:
+        return _is_interpolation(self.__dict__["_content"])
+
+    def __str__(self) -> str:
+        return str(self.__dict__["_content"])
+
+    def __repr__(self) -> str:
+        return repr(self.__dict__["_content"])
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "UnionNode":
+        res = object.__new__(type(self))
+        for key, value in self.__dict__.items():
+            if key not in ("_content", "_parent"):
+                res.__dict__[key] = copy.deepcopy(value, memo=memo)
+
+        src_content = self.__dict__["_content"]
+        if isinstance(src_content, Node):
+            old_parent = src_content.__dict__["_parent"]
+            try:
+                src_content.__dict__["_parent"] = None
+                content_copy = copy.deepcopy(src_content, memo=memo)
+                content_copy.__dict__["_parent"] = res
+            finally:
+                src_content.__dict__["_parent"] = old_parent
+        else:
+            # None and strings can be assigned as is
+            content_copy = src_content
+
+        res.__dict__["_content"] = content_copy
+        res.__dict__["_parent"] = self.__dict__["_parent"]
+        return res

omegaconf/basecontainer.py

@@ -0,0 +1,916 @@
+import copy
+import sys
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple, Union
+
+import yaml
+
+from ._utils import (
+    _DEFAULT_MARKER_,
+    ValueKind,
+    _ensure_container,
+    _get_value,
+    _is_interpolation,
+    _is_missing_value,
+    _is_none,
+    _is_special,
+    _resolve_optional,
+    get_structured_config_data,
+    get_type_hint,
+    get_value_kind,
+    get_yaml_loader,
+    is_container_annotation,
+    is_dict_annotation,
+    is_list_annotation,
+    is_primitive_dict,
+    is_primitive_type_annotation,
+    is_structured_config,
+    is_tuple_annotation,
+    is_union_annotation,
+)
+from .base import (
+    Box,
+    Container,
+    ContainerMetadata,
+    DictKeyType,
+    Node,
+    SCMode,
+    UnionNode,
+)
+from .errors import (
+    ConfigCycleDetectedException,
+    ConfigTypeError,
+    InterpolationResolutionError,
+    KeyValidationError,
+    MissingMandatoryValue,
+    OmegaConfBaseException,
+    ReadonlyConfigError,
+    ValidationError,
+)
+
+if TYPE_CHECKING:
+    from .dictconfig import DictConfig  # pragma: no cover
+
+
+class BaseContainer(Container, ABC):
+    _resolvers: ClassVar[Dict[str, Any]] = {}
+
+    def __init__(self, parent: Optional[Box], metadata: ContainerMetadata):
+        if not (parent is None or isinstance(parent, Box)):
+            raise ConfigTypeError("Parent type is not omegaconf.Box")
+        super().__init__(parent=parent, metadata=metadata)
+
+    def _get_child(
+        self,
+        key: Any,
+        validate_access: bool = True,
+        validate_key: bool = True,
+        throw_on_missing_value: bool = False,
+        throw_on_missing_key: bool = False,
+    ) -> Union[Optional[Node], List[Optional[Node]]]:
+        """Like _get_node, passing through to the nearest concrete Node."""
+        child = self._get_node(
+            key=key,
+            validate_access=validate_access,
+            validate_key=validate_key,
+            throw_on_missing_value=throw_on_missing_value,
+            throw_on_missing_key=throw_on_missing_key,
+        )
+        if isinstance(child, UnionNode) and not _is_special(child):
+            value = child._value()
+            assert isinstance(value, Node) and not isinstance(value, UnionNode)
+            child = value
+        return child
+
+    def _resolve_with_default(
+        self,
+        key: Union[DictKeyType, int],
+        value: Node,
+        default_value: Any = _DEFAULT_MARKER_,
+    ) -> Any:
+        """returns the value with the specified key, like obj.key and obj['key']"""
+        if _is_missing_value(value):
+            if default_value is not _DEFAULT_MARKER_:
+                return default_value
+            raise MissingMandatoryValue("Missing mandatory value: $FULL_KEY")
+
+        resolved_node = self._maybe_resolve_interpolation(
+            parent=self,
+            key=key,
+            value=value,
+            throw_on_resolution_failure=True,
+        )
+
+        return _get_value(resolved_node)
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
+    def __repr__(self) -> str:
+        if self.__dict__["_content"] is None:
+            return "None"
+        elif self._is_interpolation() or self._is_missing():
+            v = self.__dict__["_content"]
+            return f"'{v}'"
+        else:
+            return self.__dict__["_content"].__repr__()  # type: ignore
+
+    # Support pickle
+    def __getstate__(self) -> Dict[str, Any]:
+        dict_copy = copy.copy(self.__dict__)
+
+        # no need to serialize the flags cache, it can be re-constructed later
+        dict_copy.pop("_flags_cache", None)
+
+        dict_copy["_metadata"] = copy.copy(dict_copy["_metadata"])
+        ref_type = self._metadata.ref_type
+        if is_container_annotation(ref_type):
+            if is_dict_annotation(ref_type):
+                dict_copy["_metadata"].ref_type = Dict
+            elif is_list_annotation(ref_type):
+                dict_copy["_metadata"].ref_type = List
+            else:
+                assert False
+        if sys.version_info < (3, 7):  # pragma: no cover
+            element_type = self._metadata.element_type
+            if is_union_annotation(element_type):
+                raise OmegaConfBaseException(
+                    "Serializing structured configs with `Union` element type requires python >= 3.7"
+                )
+        return dict_copy
+
+    # Support pickle
+    def __setstate__(self, d: Dict[str, Any]) -> None:
+        from omegaconf import DictConfig
+        from omegaconf._utils import is_generic_dict, is_generic_list
+
+        if isinstance(self, DictConfig):
+            key_type = d["_metadata"].key_type
+
+            # backward compatibility to load OmegaConf 2.0 configs
+            if key_type is None:
+                key_type = Any
+                d["_metadata"].key_type = key_type
+
+        element_type = d["_metadata"].element_type
+
+        # backward compatibility to load OmegaConf 2.0 configs
+        if element_type is None:
+            element_type = Any
+            d["_metadata"].element_type = element_type
+
+        ref_type = d["_metadata"].ref_type
+        if is_container_annotation(ref_type):
+            if is_generic_dict(ref_type):
+                d["_metadata"].ref_type = Dict[key_type, element_type]  # type: ignore
+            elif is_generic_list(ref_type):
+                d["_metadata"].ref_type = List[element_type]  # type: ignore
+            else:
+                assert False
+
+        d["_flags_cache"] = None
+        self.__dict__.update(d)
+
+    @abstractmethod
+    def __delitem__(self, key: Any) -> None:
+        ...
+
+    def __len__(self) -> int:
+        if self._is_none() or self._is_missing() or self._is_interpolation():
+            return 0
+        content = self.__dict__["_content"]
+        return len(content)
+
+    def merge_with_cli(self) -> None:
+        args_list = sys.argv[1:]
+        self.merge_with_dotlist(args_list)
+
+    def merge_with_dotlist(self, dotlist: List[str]) -> None:
+        from omegaconf import OmegaConf
+
+        def fail() -> None:
+            raise ValueError("Input list must be a list or a tuple of strings")
+
+        if not isinstance(dotlist, (list, tuple)):
+            fail()
+
+        for arg in dotlist:
+            if not isinstance(arg, str):
+                fail()
+
+            idx = arg.find("=")
+            if idx == -1:
+                key = arg
+                value = None
+            else:
+                key = arg[0:idx]
+                value = arg[idx + 1 :]
+                value = yaml.load(value, Loader=get_yaml_loader())
+
+            OmegaConf.update(self, key, value)
+
+    def is_empty(self) -> bool:
+        """return true if config is empty"""
+        return len(self.__dict__["_content"]) == 0
+
+    @staticmethod
+    def _to_content(
+        conf: Container,
+        resolve: bool,
+        throw_on_missing: bool,
+        enum_to_str: bool = False,
+        structured_config_mode: SCMode = SCMode.DICT,
+    ) -> Union[None, Any, str, Dict[DictKeyType, Any], List[Any]]:
+        from omegaconf import MISSING, DictConfig, ListConfig
+
+        def convert(val: Node) -> Any:
+            value = val._value()
+            if enum_to_str and isinstance(value, Enum):
+                value = f"{value.name}"
+
+            return value
+
+        def get_node_value(key: Union[DictKeyType, int]) -> Any:
+            try:
+                node = conf._get_child(key, throw_on_missing_value=throw_on_missing)
+            except MissingMandatoryValue as e:
+                conf._format_and_raise(key=key, value=None, cause=e)
+            assert isinstance(node, Node)
+            if resolve:
+                try:
+                    node = node._dereference_node()
+                except InterpolationResolutionError as e:
+                    conf._format_and_raise(key=key, value=None, cause=e)
+
+            if isinstance(node, Container):
+                value = BaseContainer._to_content(
+                    node,
+                    resolve=resolve,
+                    throw_on_missing=throw_on_missing,
+                    enum_to_str=enum_to_str,
+                    structured_config_mode=structured_config_mode,
+                )
+            else:
+                value = convert(node)
+            return value
+
+        if conf._is_none():
+            return None
+        elif conf._is_missing():
+            if throw_on_missing:
+                conf._format_and_raise(
+                    key=None,
+                    value=None,
+                    cause=MissingMandatoryValue("Missing mandatory value"),
+                )
+            else:
+                return MISSING
+        elif not resolve and conf._is_interpolation():
+            inter = conf._value()
+            assert isinstance(inter, str)
+            return inter
+
+        if resolve:
+            _conf = conf._dereference_node()
+            assert isinstance(_conf, Container)
+            conf = _conf
+
+        if isinstance(conf, DictConfig):
+            if (
+                conf._metadata.object_type not in (dict, None)
+                and structured_config_mode == SCMode.DICT_CONFIG
+            ):
+                return conf
+            if structured_config_mode == SCMode.INSTANTIATE and is_structured_config(
+                conf._metadata.object_type
+            ):
+                return conf._to_object()
+
+            retdict: Dict[DictKeyType, Any] = {}
+            for key in conf.keys():
+                value = get_node_value(key)
+                if enum_to_str and isinstance(key, Enum):
+                    key = f"{key.name}"
+                retdict[key] = value
+            return retdict
+        elif isinstance(conf, ListConfig):
+            retlist: List[Any] = []
+            for index in range(len(conf)):
+                item = get_node_value(index)
+                retlist.append(item)
+
+            return retlist
+        assert False
+
+    @staticmethod
+    def _map_merge(dest: "BaseContainer", src: "BaseContainer") -> None:
+        """merge src into dest and return a new copy, does not modified input"""
+        from omegaconf import AnyNode, DictConfig, ValueNode
+
+        assert isinstance(dest, DictConfig)
+        assert isinstance(src, DictConfig)
+        src_type = src._metadata.object_type
+        src_ref_type = get_type_hint(src)
+        assert src_ref_type is not None
+
+        # If source DictConfig is:
+        #  - None => set the destination DictConfig to None
+        #  - an interpolation => set the destination DictConfig to be the same interpolation
+        if src._is_none() or src._is_interpolation():
+            dest._set_value(src._value())
+            _update_types(node=dest, ref_type=src_ref_type, object_type=src_type)
+            return
+
+        dest._validate_merge(value=src)
+
+        def expand(node: Container) -> None:
+            rt = node._metadata.ref_type
+            val: Any
+            if rt is not Any:
+                if is_dict_annotation(rt):
+                    val = {}
+                elif is_list_annotation(rt) or is_tuple_annotation(rt):
+                    val = []
+                else:
+                    val = rt
+            elif isinstance(node, DictConfig):
+                val = {}
+            else:
+                assert False
+
+            node._set_value(val)
+
+        if (
+            src._is_missing()
+            and not dest._is_missing()
+            and is_structured_config(src_ref_type)
+        ):
+            # Replace `src` with a prototype of its corresponding structured config
+            # whose fields are all missing (to avoid overwriting fields in `dest`).
+            assert src_type is None  # src missing, so src's object_type should be None
+            src_type = src_ref_type
+            src = _create_structured_with_missing_fields(
+                ref_type=src_ref_type, object_type=src_type
+            )
+
+        if (dest._is_interpolation() or dest._is_missing()) and not src._is_missing():
+            expand(dest)
+
+        src_items = list(src) if not src._is_missing() else []
+        for key in src_items:
+            src_node = src._get_node(key, validate_access=False)
+            dest_node = dest._get_node(key, validate_access=False)
+            assert isinstance(src_node, Node)
+            assert dest_node is None or isinstance(dest_node, Node)
+            src_value = _get_value(src_node)
+
+            src_vk = get_value_kind(src_node)
+            src_node_missing = src_vk is ValueKind.MANDATORY_MISSING
+
+            if isinstance(dest_node, DictConfig):
+                dest_node._validate_merge(value=src_node)
+
+            if (
+                isinstance(dest_node, Container)
+                and dest_node._is_none()
+                and not src_node_missing
+                and not _is_none(src_node, resolve=True)
+            ):
+                expand(dest_node)
+
+            if dest_node is not None and dest_node._is_interpolation():
+                target_node = dest_node._maybe_dereference_node()
+                if isinstance(target_node, Container):
+                    dest[key] = target_node
+                    dest_node = dest._get_node(key)
+
+            is_optional, et = _resolve_optional(dest._metadata.element_type)
+            if dest_node is None and is_structured_config(et) and not src_node_missing:
+                # merging into a new node. Use element_type as a base
+                dest[key] = DictConfig(
+                    et, parent=dest, ref_type=et, is_optional=is_optional
+                )
+                dest_node = dest._get_node(key)
+
+            if dest_node is not None:
+                if isinstance(dest_node, BaseContainer):
+                    if isinstance(src_node, BaseContainer):
+                        dest_node._merge_with(src_node)
+                    elif not src_node_missing:
+                        dest.__setitem__(key, src_node)
+                else:
+                    if isinstance(src_node, BaseContainer):
+                        dest.__setitem__(key, src_node)
+                    else:
+                        assert isinstance(dest_node, (ValueNode, UnionNode))
+                        assert isinstance(src_node, (ValueNode, UnionNode))
+                        try:
+                            if isinstance(dest_node, AnyNode):
+                                if src_node_missing:
+                                    node = copy.copy(src_node)
+                                    # if src node is missing, use the value from the dest_node,
+                                    # but validate it against the type of the src node before assigment
+                                    node._set_value(dest_node._value())
+                                else:
+                                    node = src_node
+                                dest.__setitem__(key, node)
+                            else:
+                                if not src_node_missing:
+                                    dest_node._set_value(src_value)
+
+                        except (ValidationError, ReadonlyConfigError) as e:
+                            dest._format_and_raise(key=key, value=src_value, cause=e)
+            else:
+                from omegaconf import open_dict
+
+                if is_structured_config(src_type):
+                    # verified to be compatible above in _validate_merge
+                    with open_dict(dest):
+                        dest[key] = src._get_node(key)
+                else:
+                    dest[key] = src._get_node(key)
+
+        _update_types(node=dest, ref_type=src_ref_type, object_type=src_type)
+
+        # explicit flags on the source config are replacing the flag values in the destination
+        flags = src._metadata.flags
+        assert flags is not None
+        for flag, value in flags.items():
+            if value is not None:
+                dest._set_flag(flag, value)
+
+    @staticmethod
+    def _list_merge(dest: Any, src: Any) -> None:
+        from omegaconf import DictConfig, ListConfig, OmegaConf
+
+        assert isinstance(dest, ListConfig)
+        assert isinstance(src, ListConfig)
+
+        if src._is_none():
+            dest._set_value(None)
+        elif src._is_missing():
+            # do not change dest if src is MISSING.
+            if dest._metadata.element_type is Any:
+                dest._metadata.element_type = src._metadata.element_type
+        elif src._is_interpolation():
+            dest._set_value(src._value())
+        else:
+            temp_target = ListConfig(content=[], parent=dest._get_parent())
+            temp_target.__dict__["_metadata"] = copy.deepcopy(
+                dest.__dict__["_metadata"]
+            )
+            is_optional, et = _resolve_optional(dest._metadata.element_type)
+            if is_structured_config(et):
+                prototype = DictConfig(et, ref_type=et, is_optional=is_optional)
+                for item in src._iter_ex(resolve=False):
+                    if isinstance(item, DictConfig):
+                        item = OmegaConf.merge(prototype, item)
+                    temp_target.append(item)
+            else:
+                for item in src._iter_ex(resolve=False):
+                    temp_target.append(item)
+
+            dest.__dict__["_content"] = temp_target.__dict__["_content"]
+
+        # explicit flags on the source config are replacing the flag values in the destination
+        flags = src._metadata.flags
+        assert flags is not None
+        for flag, value in flags.items():
+            if value is not None:
+                dest._set_flag(flag, value)
+
+    def merge_with(
+        self,
+        *others: Union[
+            "BaseContainer", Dict[str, Any], List[Any], Tuple[Any, ...], Any
+        ],
+    ) -> None:
+        try:
+            self._merge_with(*others)
+        except Exception as e:
+            self._format_and_raise(key=None, value=None, cause=e)
+
+    def _merge_with(
+        self,
+        *others: Union[
+            "BaseContainer", Dict[str, Any], List[Any], Tuple[Any, ...], Any
+        ],
+    ) -> None:
+        from .dictconfig import DictConfig
+        from .listconfig import ListConfig
+
+        """merge a list of other Config objects into this one, overriding as needed"""
+        for other in others:
+            if other is None:
+                raise ValueError("Cannot merge with a None config")
+
+            my_flags = {}
+            if self._get_flag("allow_objects") is True:
+                my_flags = {"allow_objects": True}
+            other = _ensure_container(other, flags=my_flags)
+
+            if isinstance(self, DictConfig) and isinstance(other, DictConfig):
+                BaseContainer._map_merge(self, other)
+            elif isinstance(self, ListConfig) and isinstance(other, ListConfig):
+                BaseContainer._list_merge(self, other)
+            else:
+                raise TypeError("Cannot merge DictConfig with ListConfig")
+
+        # recursively correct the parent hierarchy after the merge
+        self._re_parent()
+
+    # noinspection PyProtectedMember
+    def _set_item_impl(self, key: Any, value: Any) -> None:
+        """
+        Changes the value of the node key with the desired value. If the node key doesn't
+        exist it creates a new one.
+        """
+        from .nodes import AnyNode, ValueNode
+
+        if isinstance(value, Node):
+            do_deepcopy = not self._get_flag("no_deepcopy_set_nodes")
+            if not do_deepcopy and isinstance(value, Box):
+                # if value is from the same config, perform a deepcopy no matter what.
+                if self._get_root() is value._get_root():
+                    do_deepcopy = True
+
+            if do_deepcopy:
+                value = copy.deepcopy(value)
+            value._set_parent(None)
+
+            try:
+                old = value._key()
+                value._set_key(key)
+                self._validate_set(key, value)
+            finally:
+                value._set_key(old)
+        else:
+            self._validate_set(key, value)
+
+        if self._get_flag("readonly"):
+            raise ReadonlyConfigError("Cannot change read-only config container")
+
+        input_is_node = isinstance(value, Node)
+        target_node_ref = self._get_node(key)
+        assert target_node_ref is None or isinstance(target_node_ref, Node)
+
+        input_is_typed_vnode = isinstance(value, ValueNode) and not isinstance(
+            value, AnyNode
+        )
+
+        def get_target_type_hint(val: Any) -> Any:
+            if not is_structured_config(val):
+                type_hint = self._metadata.element_type
+            else:
+                target = self._get_node(key)
+                if target is None:
+                    type_hint = self._metadata.element_type
+                else:
+                    assert isinstance(target, Node)
+                    type_hint = target._metadata.type_hint
+            return type_hint
+
+        target_type_hint = get_target_type_hint(value)
+        _, target_ref_type = _resolve_optional(target_type_hint)
+
+        def assign(value_key: Any, val: Node) -> None:
+            assert val._get_parent() is None
+            v = val
+            v._set_parent(self)
+            v._set_key(value_key)
+            _deep_update_type_hint(node=v, type_hint=self._metadata.element_type)
+            self.__dict__["_content"][value_key] = v
+
+        if input_is_typed_vnode and not is_union_annotation(target_ref_type):
+            assign(key, value)
+        else:
+            # input is not a ValueNode, can be primitive or box
+
+            special_value = _is_special(value)
+            # We use the `Node._set_value` method if the target node exists and:
+            # 1. the target has an explicit ref_type, or
+            # 2. the target is an AnyNode and the input is a primitive type.
+            should_set_value = target_node_ref is not None and (
+                target_node_ref._has_ref_type()
+                or (
+                    isinstance(target_node_ref, AnyNode)
+                    and is_primitive_type_annotation(value)
+                )
+            )
+            if should_set_value:
+                if special_value and isinstance(value, Node):
+                    value = value._value()
+                self.__dict__["_content"][key]._set_value(value)
+            elif input_is_node:
+                if (
+                    special_value
+                    and (
+                        is_container_annotation(target_ref_type)
+                        or is_structured_config(target_ref_type)
+                    )
+                    or is_primitive_type_annotation(target_ref_type)
+                    or is_union_annotation(target_ref_type)
+                ):
+                    value = _get_value(value)
+                    self._wrap_value_and_set(key, value, target_type_hint)
+                else:
+                    assign(key, value)
+            else:
+                self._wrap_value_and_set(key, value, target_type_hint)
+
+    def _wrap_value_and_set(self, key: Any, val: Any, type_hint: Any) -> None:
+        from omegaconf.omegaconf import _maybe_wrap
+
+        is_optional, ref_type = _resolve_optional(type_hint)
+
+        try:
+            wrapped = _maybe_wrap(
+                ref_type=ref_type,
+                key=key,
+                value=val,
+                is_optional=is_optional,
+                parent=self,
+            )
+        except ValidationError as e:
+            self._format_and_raise(key=key, value=val, cause=e)
+        self.__dict__["_content"][key] = wrapped
+
+    @staticmethod
+    def _item_eq(
+        c1: Container,
+        k1: Union[DictKeyType, int],
+        c2: Container,
+        k2: Union[DictKeyType, int],
+    ) -> bool:
+        v1 = c1._get_child(k1)
+        v2 = c2._get_child(k2)
+        assert v1 is not None and v2 is not None
+
+        assert isinstance(v1, Node)
+        assert isinstance(v2, Node)
+
+        if v1._is_none() and v2._is_none():
+            return True
+
+        if v1._is_missing() and v2._is_missing():
+            return True
+
+        v1_inter = v1._is_interpolation()
+        v2_inter = v2._is_interpolation()
+        dv1: Optional[Node] = v1
+        dv2: Optional[Node] = v2
+
+        if v1_inter:
+            dv1 = v1._maybe_dereference_node()
+        if v2_inter:
+            dv2 = v2._maybe_dereference_node()
+
+        if v1_inter and v2_inter:
+            if dv1 is None or dv2 is None:
+                return v1 == v2
+            else:
+                # both are not none, if both are containers compare as container
+                if isinstance(dv1, Container) and isinstance(dv2, Container):
+                    if dv1 != dv2:
+                        return False
+                dv1 = _get_value(dv1)
+                dv2 = _get_value(dv2)
+                return dv1 == dv2
+        elif not v1_inter and not v2_inter:
+            v1 = _get_value(v1)
+            v2 = _get_value(v2)
+            ret = v1 == v2
+            assert isinstance(ret, bool)
+            return ret
+        else:
+            dv1 = _get_value(dv1)
+            dv2 = _get_value(dv2)
+            ret = dv1 == dv2
+            assert isinstance(ret, bool)
+            return ret
+
+    def _is_optional(self) -> bool:
+        return self.__dict__["_metadata"].optional is True
+
+    def _is_interpolation(self) -> bool:
+        return _is_interpolation(self.__dict__["_content"])
+
+    @abstractmethod
+    def _validate_get(self, key: Any, value: Any = None) -> None:
+        ...
+
+    @abstractmethod
+    def _validate_set(self, key: Any, value: Any) -> None:
+        ...
+
+    def _value(self) -> Any:
+        return self.__dict__["_content"]
+
+    def _get_full_key(self, key: Union[DictKeyType, int, slice, None]) -> str:
+        from .listconfig import ListConfig
+        from .omegaconf import _select_one
+
+        if not isinstance(key, (int, str, Enum, float, bool, slice, bytes, type(None))):
+            return ""
+
+        def _slice_to_str(x: slice) -> str:
+            if x.step is not None:
+                return f"{x.start}:{x.stop}:{x.step}"
+            else:
+                return f"{x.start}:{x.stop}"
+
+        def prepand(
+            full_key: str,
+            parent_type: Any,
+            cur_type: Any,
+            key: Optional[Union[DictKeyType, int, slice]],
+        ) -> str:
+            if key is None:
+                return full_key
+
+            if isinstance(key, slice):
+                key = _slice_to_str(key)
+            elif isinstance(key, Enum):
+                key = key.name
+            else:
+                key = str(key)
+
+            assert isinstance(key, str)
+
+            if issubclass(parent_type, ListConfig):
+                if full_key != "":
+                    if issubclass(cur_type, ListConfig):
+                        full_key = f"[{key}]{full_key}"
+                    else:
+                        full_key = f"[{key}].{full_key}"
+                else:
+                    full_key = f"[{key}]"
+            else:
+                if full_key == "":
+                    full_key = key
+                else:
+                    if issubclass(cur_type, ListConfig):
+                        full_key = f"{key}{full_key}"
+                    else:
+                        full_key = f"{key}.{full_key}"
+            return full_key
+
+        if key is not None and key != "":
+            assert isinstance(self, Container)
+            cur, _ = _select_one(
+                c=self, key=str(key), throw_on_missing=False, throw_on_type_error=False
+            )
+            if cur is None:
+                cur = self
+                full_key = prepand("", type(cur), None, key)
+                if cur._key() is not None:
+                    full_key = prepand(
+                        full_key, type(cur._get_parent()), type(cur), cur._key()
+                    )
+            else:
+                full_key = prepand("", type(cur._get_parent()), type(cur), cur._key())
+        else:
+            cur = self
+            if cur._key() is None:
+                return ""
+            full_key = self._key()
+
+        assert cur is not None
+        memo = {id(cur)}  # remember already visited nodes so as to detect cycles
+        while cur._get_parent() is not None:
+            cur = cur._get_parent()
+            if id(cur) in memo:
+                raise ConfigCycleDetectedException(
+                    f"Cycle when iterating over parents of key `{key!s}`"
+                )
+            memo.add(id(cur))
+            assert cur is not None
+            if cur._key() is not None:
+                full_key = prepand(
+                    full_key, type(cur._get_parent()), type(cur), cur._key()
+                )
+
+        return full_key
+
+
+def _create_structured_with_missing_fields(
+    ref_type: type, object_type: Optional[type] = None
+) -> "DictConfig":
+    from . import MISSING, DictConfig
+
+    cfg_data = get_structured_config_data(ref_type)
+    for v in cfg_data.values():
+        v._set_value(MISSING)
+
+    cfg = DictConfig(cfg_data)
+    cfg._metadata.optional, cfg._metadata.ref_type = _resolve_optional(ref_type)
+    cfg._metadata.object_type = object_type
+
+    return cfg
+
+
+def _update_types(node: Node, ref_type: Any, object_type: Optional[type]) -> None:
+    if object_type is not None and not is_primitive_dict(object_type):
+        node._metadata.object_type = object_type
+
+    if node._metadata.ref_type is Any:
+        _deep_update_type_hint(node, ref_type)
+
+
+def _deep_update_type_hint(node: Node, type_hint: Any) -> None:
+    """Ensure node is compatible with type_hint, mutating if necessary."""
+    from omegaconf import DictConfig, ListConfig
+
+    from ._utils import get_dict_key_value_types, get_list_element_type
+
+    if type_hint is Any:
+        return
+
+    _shallow_validate_type_hint(node, type_hint)
+
+    new_is_optional, new_ref_type = _resolve_optional(type_hint)
+    node._metadata.ref_type = new_ref_type
+    node._metadata.optional = new_is_optional
+
+    if is_list_annotation(new_ref_type) and isinstance(node, ListConfig):
+        new_element_type = get_list_element_type(new_ref_type)
+        node._metadata.element_type = new_element_type
+        if not _is_special(node):
+            for i in range(len(node)):
+                _deep_update_subnode(node, i, new_element_type)
+
+    if is_dict_annotation(new_ref_type) and isinstance(node, DictConfig):
+        new_key_type, new_element_type = get_dict_key_value_types(new_ref_type)
+        node._metadata.key_type = new_key_type
+        node._metadata.element_type = new_element_type
+        if not _is_special(node):
+            for key in node:
+                if new_key_type is not Any and not isinstance(key, new_key_type):
+                    raise KeyValidationError(
+                        f"Key {key!r} ({type(key).__name__}) is incompatible"
+                        + f" with key type hint '{new_key_type.__name__}'"
+                    )
+                _deep_update_subnode(node, key, new_element_type)
+
+
+def _deep_update_subnode(node: BaseContainer, key: Any, value_type_hint: Any) -> None:
+    """Get node[key] and ensure it is compatible with value_type_hint, mutating if necessary."""
+    subnode = node._get_node(key)
+    assert isinstance(subnode, Node)
+    if _is_special(subnode):
+        # Ensure special values are wrapped in a Node subclass that
+        # is compatible with the type hint.
+        node._wrap_value_and_set(key, subnode._value(), value_type_hint)
+        subnode = node._get_node(key)
+        assert isinstance(subnode, Node)
+    _deep_update_type_hint(subnode, value_type_hint)
+
+
+def _shallow_validate_type_hint(node: Node, type_hint: Any) -> None:
+    """Error if node's type, content and metadata are not compatible with type_hint."""
+    from omegaconf import DictConfig, ListConfig, ValueNode
+
+    is_optional, ref_type = _resolve_optional(type_hint)
+
+    vk = get_value_kind(node)
+
+    if node._is_none():
+        if not is_optional:
+            value = _get_value(node)
+            raise ValidationError(
+                f"Value {value!r} ({type(value).__name__})"
+                + f" is incompatible with type hint '{ref_type.__name__}'"
+            )
+        return
+    elif vk in (ValueKind.MANDATORY_MISSING, ValueKind.INTERPOLATION):
+        return
+    elif vk == ValueKind.VALUE:
+        if is_primitive_type_annotation(ref_type) and isinstance(node, ValueNode):
+            value = node._value()
+            if not isinstance(value, ref_type):
+                raise ValidationError(
+                    f"Value {value!r} ({type(value).__name__})"
+                    + f" is incompatible with type hint '{ref_type.__name__}'"
+                )
+        elif is_structured_config(ref_type) and isinstance(node, DictConfig):
+            return
+        elif is_dict_annotation(ref_type) and isinstance(node, DictConfig):
+            return
+        elif is_list_annotation(ref_type) and isinstance(node, ListConfig):
+            return
+        else:
+            if isinstance(node, ValueNode):
+                value = node._value()
+                raise ValidationError(
+                    f"Value {value!r} ({type(value).__name__})"
+                    + f" is incompatible with type hint '{ref_type}'"
+                )
+            else:
+                raise ValidationError(
+                    f"'{type(node).__name__}' is incompatible"
+                    + f" with type hint '{ref_type}'"
+                )
+
+    else:
+        assert False

omegaconf/dictconfig.py

@@ -0,0 +1,776 @@
+import copy
+from enum import Enum
+from typing import (
+    Any,
+    Dict,
+    ItemsView,
+    Iterable,
+    Iterator,
+    KeysView,
+    List,
+    MutableMapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+)
+
+from ._utils import (
+    _DEFAULT_MARKER_,
+    ValueKind,
+    _get_value,
+    _is_interpolation,
+    _is_missing_literal,
+    _is_missing_value,
+    _is_none,
+    _resolve_optional,
+    _valid_dict_key_annotation_type,
+    format_and_raise,
+    get_structured_config_data,
+    get_structured_config_init_field_names,
+    get_type_of,
+    get_value_kind,
+    is_container_annotation,
+    is_dict,
+    is_primitive_dict,
+    is_structured_config,
+    is_structured_config_frozen,
+    type_str,
+)
+from .base import Box, Container, ContainerMetadata, DictKeyType, Node
+from .basecontainer import BaseContainer
+from .errors import (
+    ConfigAttributeError,
+    ConfigKeyError,
+    ConfigTypeError,
+    InterpolationResolutionError,
+    KeyValidationError,
+    MissingMandatoryValue,
+    OmegaConfBaseException,
+    ReadonlyConfigError,
+    ValidationError,
+)
+from .nodes import EnumNode, ValueNode
+
+
+class DictConfig(BaseContainer, MutableMapping[Any, Any]):
+
+    _metadata: ContainerMetadata
+    _content: Union[Dict[DictKeyType, Node], None, str]
+
+    def __init__(
+        self,
+        content: Union[Dict[DictKeyType, Any], "DictConfig", Any],
+        key: Any = None,
+        parent: Optional[Box] = None,
+        ref_type: Union[Any, Type[Any]] = Any,
+        key_type: Union[Any, Type[Any]] = Any,
+        element_type: Union[Any, Type[Any]] = Any,
+        is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
+    ) -> None:
+        try:
+            if isinstance(content, DictConfig):
+                if flags is None:
+                    flags = content._metadata.flags
+            super().__init__(
+                parent=parent,
+                metadata=ContainerMetadata(
+                    key=key,
+                    optional=is_optional,
+                    ref_type=ref_type,
+                    object_type=dict,
+                    key_type=key_type,
+                    element_type=element_type,
+                    flags=flags,
+                ),
+            )
+
+            if not _valid_dict_key_annotation_type(key_type):
+                raise KeyValidationError(f"Unsupported key type {key_type}")
+
+            if is_structured_config(content) or is_structured_config(ref_type):
+                self._set_value(content, flags=flags)
+                if is_structured_config_frozen(content) or is_structured_config_frozen(
+                    ref_type
+                ):
+                    self._set_flag("readonly", True)
+
+            else:
+                if isinstance(content, DictConfig):
+                    metadata = copy.deepcopy(content._metadata)
+                    metadata.key = key
+                    metadata.ref_type = ref_type
+                    metadata.optional = is_optional
+                    metadata.element_type = element_type
+                    metadata.key_type = key_type
+                    self.__dict__["_metadata"] = metadata
+                self._set_value(content, flags=flags)
+        except Exception as ex:
+            format_and_raise(node=None, key=key, value=None, cause=ex, msg=str(ex))
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "DictConfig":
+        res = DictConfig(None)
+        res.__dict__["_metadata"] = copy.deepcopy(self.__dict__["_metadata"], memo=memo)
+        res.__dict__["_flags_cache"] = copy.deepcopy(
+            self.__dict__["_flags_cache"], memo=memo
+        )
+
+        src_content = self.__dict__["_content"]
+        if isinstance(src_content, dict):
+            content_copy = {}
+            for k, v in src_content.items():
+                old_parent = v.__dict__["_parent"]
+                try:
+                    v.__dict__["_parent"] = None
+                    vc = copy.deepcopy(v, memo=memo)
+                    vc.__dict__["_parent"] = res
+                    content_copy[k] = vc
+                finally:
+                    v.__dict__["_parent"] = old_parent
+        else:
+            # None and strings can be assigned as is
+            content_copy = src_content
+
+        res.__dict__["_content"] = content_copy
+        # parent is retained, but not copied
+        res.__dict__["_parent"] = self.__dict__["_parent"]
+        return res
+
+    def copy(self) -> "DictConfig":
+        return copy.copy(self)
+
+    def _is_typed(self) -> bool:
+        return self._metadata.object_type not in (Any, None) and not is_dict(
+            self._metadata.object_type
+        )
+
+    def _validate_get(self, key: Any, value: Any = None) -> None:
+        is_typed = self._is_typed()
+
+        is_struct = self._get_flag("struct") is True
+        if key not in self.__dict__["_content"]:
+            if is_typed:
+                # do not raise an exception if struct is explicitly set to False
+                if self._get_node_flag("struct") is False:
+                    return
+            if is_typed or is_struct:
+                if is_typed:
+                    assert self._metadata.object_type not in (dict, None)
+                    msg = f"Key '{key}' not in '{self._metadata.object_type.__name__}'"
+                else:
+                    msg = f"Key '{key}' is not in struct"
+                self._format_and_raise(
+                    key=key, value=value, cause=ConfigAttributeError(msg)
+                )
+
+    def _validate_set(self, key: Any, value: Any) -> None:
+        from omegaconf import OmegaConf
+
+        vk = get_value_kind(value)
+        if vk == ValueKind.INTERPOLATION:
+            return
+        if _is_none(value):
+            self._validate_non_optional(key, value)
+            return
+        if vk == ValueKind.MANDATORY_MISSING or value is None:
+            return
+
+        target = self._get_node(key) if key is not None else self
+
+        target_has_ref_type = isinstance(
+            target, DictConfig
+        ) and target._metadata.ref_type not in (Any, dict)
+        is_valid_target = target is None or not target_has_ref_type
+
+        if is_valid_target:
+            return
+
+        assert isinstance(target, Node)
+
+        target_type = target._metadata.ref_type
+        value_type = OmegaConf.get_type(value)
+
+        if is_dict(value_type) and is_dict(target_type):
+            return
+        if is_container_annotation(target_type) and not is_container_annotation(
+            value_type
+        ):
+            raise ValidationError(
+                f"Cannot assign {type_str(value_type)} to {type_str(target_type)}"
+            )
+
+        if target_type is not None and value_type is not None:
+            origin = getattr(target_type, "__origin__", target_type)
+            if not issubclass(value_type, origin):
+                self._raise_invalid_value(value, value_type, target_type)
+
+    def _validate_merge(self, value: Any) -> None:
+        from omegaconf import OmegaConf
+
+        dest = self
+        src = value
+
+        self._validate_non_optional(None, src)
+
+        dest_obj_type = OmegaConf.get_type(dest)
+        src_obj_type = OmegaConf.get_type(src)
+
+        if dest._is_missing() and src._metadata.object_type not in (dict, None):
+            self._validate_set(key=None, value=_get_value(src))
+
+        if src._is_missing():
+            return
+
+        validation_error = (
+            dest_obj_type is not None
+            and src_obj_type is not None
+            and is_structured_config(dest_obj_type)
+            and not src._is_none()
+            and not is_dict(src_obj_type)
+            and not issubclass(src_obj_type, dest_obj_type)
+        )
+        if validation_error:
+            msg = (
+                f"Merge error: {type_str(src_obj_type)} is not a "
+                f"subclass of {type_str(dest_obj_type)}. value: {src}"
+            )
+            raise ValidationError(msg)
+
+    def _validate_non_optional(self, key: Optional[DictKeyType], value: Any) -> None:
+        if _is_none(value, resolve=True, throw_on_resolution_failure=False):
+
+            if key is not None:
+                child = self._get_node(key)
+                if child is not None:
+                    assert isinstance(child, Node)
+                    field_is_optional = child._is_optional()
+                else:
+                    field_is_optional, _ = _resolve_optional(
+                        self._metadata.element_type
+                    )
+            else:
+                field_is_optional = self._is_optional()
+
+            if not field_is_optional:
+                self._format_and_raise(
+                    key=key,
+                    value=value,
+                    cause=ValidationError("field '$FULL_KEY' is not Optional"),
+                )
+
+    def _raise_invalid_value(
+        self, value: Any, value_type: Any, target_type: Any
+    ) -> None:
+        assert value_type is not None
+        assert target_type is not None
+        msg = (
+            f"Invalid type assigned: {type_str(value_type)} is not a "
+            f"subclass of {type_str(target_type)}. value: {value}"
+        )
+        raise ValidationError(msg)
+
+    def _validate_and_normalize_key(self, key: Any) -> DictKeyType:
+        return self._s_validate_and_normalize_key(self._metadata.key_type, key)
+
+    def _s_validate_and_normalize_key(self, key_type: Any, key: Any) -> DictKeyType:
+        if key_type is Any:
+            for t in DictKeyType.__args__:  # type: ignore
+                if isinstance(key, t):
+                    return key  # type: ignore
+            raise KeyValidationError("Incompatible key type '$KEY_TYPE'")
+        elif key_type is bool and key in [0, 1]:
+            # Python treats True as 1 and False as 0 when used as dict keys
+            #   assert hash(0) == hash(False)
+            #   assert hash(1) == hash(True)
+            return bool(key)
+        elif key_type in (str, bytes, int, float, bool):  # primitive type
+            if not isinstance(key, key_type):
+                raise KeyValidationError(
+                    f"Key $KEY ($KEY_TYPE) is incompatible with ({key_type.__name__})"
+                )
+
+            return key  # type: ignore
+        elif issubclass(key_type, Enum):
+            try:
+                return EnumNode.validate_and_convert_to_enum(key_type, key)
+            except ValidationError:
+                valid = ", ".join([x for x in key_type.__members__.keys()])
+                raise KeyValidationError(
+                    f"Key '$KEY' is incompatible with the enum type '{key_type.__name__}', valid: [{valid}]"
+                )
+        else:
+            assert False, f"Unsupported key type {key_type}"
+
+    def __setitem__(self, key: DictKeyType, value: Any) -> None:
+        try:
+            self.__set_impl(key=key, value=value)
+        except AttributeError as e:
+            self._format_and_raise(
+                key=key, value=value, type_override=ConfigKeyError, cause=e
+            )
+        except Exception as e:
+            self._format_and_raise(key=key, value=value, cause=e)
+
+    def __set_impl(self, key: DictKeyType, value: Any) -> None:
+        key = self._validate_and_normalize_key(key)
+        self._set_item_impl(key, value)
+
+    # hide content while inspecting in debugger
+    def __dir__(self) -> Iterable[str]:
+        if self._is_missing() or self._is_none():
+            return []
+        return self.__dict__["_content"].keys()  # type: ignore
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        """
+        Allow assigning attributes to DictConfig
+        :param key:
+        :param value:
+        :return:
+        """
+        try:
+            self.__set_impl(key, value)
+        except Exception as e:
+            if isinstance(e, OmegaConfBaseException) and e._initialized:
+                raise e
+            self._format_and_raise(key=key, value=value, cause=e)
+            assert False
+
+    def __getattr__(self, key: str) -> Any:
+        """
+        Allow accessing dictionary values as attributes
+        :param key:
+        :return:
+        """
+        if key == "__name__":
+            raise AttributeError()
+
+        try:
+            return self._get_impl(
+                key=key, default_value=_DEFAULT_MARKER_, validate_key=False
+            )
+        except ConfigKeyError as e:
+            self._format_and_raise(
+                key=key, value=None, cause=e, type_override=ConfigAttributeError
+            )
+        except Exception as e:
+            self._format_and_raise(key=key, value=None, cause=e)
+
+    def __getitem__(self, key: DictKeyType) -> Any:
+        """
+        Allow map style access
+        :param key:
+        :return:
+        """
+
+        try:
+            return self._get_impl(key=key, default_value=_DEFAULT_MARKER_)
+        except AttributeError as e:
+            self._format_and_raise(
+                key=key, value=None, cause=e, type_override=ConfigKeyError
+            )
+        except Exception as e:
+            self._format_and_raise(key=key, value=None, cause=e)
+
+    def __delattr__(self, key: str) -> None:
+        """
+        Allow deleting dictionary values as attributes
+        :param key:
+        :return:
+        """
+        if self._get_flag("readonly"):
+            self._format_and_raise(
+                key=key,
+                value=None,
+                cause=ReadonlyConfigError(
+                    "DictConfig in read-only mode does not support deletion"
+                ),
+            )
+        try:
+            del self.__dict__["_content"][key]
+        except KeyError:
+            msg = "Attribute not found: '$KEY'"
+            self._format_and_raise(key=key, value=None, cause=ConfigAttributeError(msg))
+
+    def __delitem__(self, key: DictKeyType) -> None:
+        key = self._validate_and_normalize_key(key)
+        if self._get_flag("readonly"):
+            self._format_and_raise(
+                key=key,
+                value=None,
+                cause=ReadonlyConfigError(
+                    "DictConfig in read-only mode does not support deletion"
+                ),
+            )
+        if self._get_flag("struct"):
+            self._format_and_raise(
+                key=key,
+                value=None,
+                cause=ConfigTypeError(
+                    "DictConfig in struct mode does not support deletion"
+                ),
+            )
+        if self._is_typed() and self._get_node_flag("struct") is not False:
+            self._format_and_raise(
+                key=key,
+                value=None,
+                cause=ConfigTypeError(
+                    f"{type_str(self._metadata.object_type)} (DictConfig) does not support deletion"
+                ),
+            )
+
+        try:
+            del self.__dict__["_content"][key]
+        except KeyError:
+            msg = "Key not found: '$KEY'"
+            self._format_and_raise(key=key, value=None, cause=ConfigKeyError(msg))
+
+    def get(self, key: DictKeyType, default_value: Any = None) -> Any:
+        """Return the value for `key` if `key` is in the dictionary, else
+        `default_value` (defaulting to `None`)."""
+        try:
+            return self._get_impl(key=key, default_value=default_value)
+        except KeyValidationError as e:
+            self._format_and_raise(key=key, value=None, cause=e)
+
+    def _get_impl(
+        self, key: DictKeyType, default_value: Any, validate_key: bool = True
+    ) -> Any:
+        try:
+            node = self._get_child(
+                key=key, throw_on_missing_key=True, validate_key=validate_key
+            )
+        except (ConfigAttributeError, ConfigKeyError):
+            if default_value is not _DEFAULT_MARKER_:
+                return default_value
+            else:
+                raise
+        assert isinstance(node, Node)
+        return self._resolve_with_default(
+            key=key, value=node, default_value=default_value
+        )
+
+    def _get_node(
+        self,
+        key: DictKeyType,
+        validate_access: bool = True,
+        validate_key: bool = True,
+        throw_on_missing_value: bool = False,
+        throw_on_missing_key: bool = False,
+    ) -> Optional[Node]:
+        try:
+            key = self._validate_and_normalize_key(key)
+        except KeyValidationError:
+            if validate_access and validate_key:
+                raise
+            else:
+                if throw_on_missing_key:
+                    raise ConfigAttributeError
+                else:
+                    return None
+
+        if validate_access:
+            self._validate_get(key)
+
+        value: Optional[Node] = self.__dict__["_content"].get(key)
+        if value is None:
+            if throw_on_missing_key:
+                raise ConfigKeyError(f"Missing key {key!s}")
+        elif throw_on_missing_value and value._is_missing():
+            raise MissingMandatoryValue("Missing mandatory value: $KEY")
+        return value
+
+    def pop(self, key: DictKeyType, default: Any = _DEFAULT_MARKER_) -> Any:
+        try:
+            if self._get_flag("readonly"):
+                raise ReadonlyConfigError("Cannot pop from read-only node")
+            if self._get_flag("struct"):
+                raise ConfigTypeError("DictConfig in struct mode does not support pop")
+            if self._is_typed() and self._get_node_flag("struct") is not False:
+                raise ConfigTypeError(
+                    f"{type_str(self._metadata.object_type)} (DictConfig) does not support pop"
+                )
+            key = self._validate_and_normalize_key(key)
+            node = self._get_child(key=key, validate_access=False)
+            if node is not None:
+                assert isinstance(node, Node)
+                value = self._resolve_with_default(
+                    key=key, value=node, default_value=default
+                )
+
+                del self[key]
+                return value
+            else:
+                if default is not _DEFAULT_MARKER_:
+                    return default
+                else:
+                    full = self._get_full_key(key=key)
+                    if full != key:
+                        raise ConfigKeyError(
+                            f"Key not found: '{key!s}' (path: '{full}')"
+                        )
+                    else:
+                        raise ConfigKeyError(f"Key not found: '{key!s}'")
+        except Exception as e:
+            self._format_and_raise(key=key, value=None, cause=e)
+
+    def keys(self) -> KeysView[DictKeyType]:
+        if self._is_missing() or self._is_interpolation() or self._is_none():
+            return {}.keys()
+        ret = self.__dict__["_content"].keys()
+        assert isinstance(ret, KeysView)
+        return ret
+
+    def __contains__(self, key: object) -> bool:
+        """
+        A key is contained in a DictConfig if there is an associated value and
+        it is not a mandatory missing value ('???').
+        :param key:
+        :return:
+        """
+
+        try:
+            key = self._validate_and_normalize_key(key)
+        except KeyValidationError:
+            return False
+
+        try:
+            node = self._get_child(key)
+            assert node is None or isinstance(node, Node)
+        except (KeyError, AttributeError):
+            node = None
+
+        if node is None:
+            return False
+        else:
+            try:
+                self._resolve_with_default(key=key, value=node)
+                return True
+            except InterpolationResolutionError:
+                # Interpolations that fail count as existing.
+                return True
+            except MissingMandatoryValue:
+                # Missing values count as *not* existing.
+                return False
+
+    def __iter__(self) -> Iterator[DictKeyType]:
+        return iter(self.keys())
+
+    def items(self) -> ItemsView[DictKeyType, Any]:
+        return dict(self.items_ex(resolve=True, keys=None)).items()
+
+    def setdefault(self, key: DictKeyType, default: Any = None) -> Any:
+        if key in self:
+            ret = self.__getitem__(key)
+        else:
+            ret = default
+            self.__setitem__(key, default)
+        return ret
+
+    def items_ex(
+        self, resolve: bool = True, keys: Optional[Sequence[DictKeyType]] = None
+    ) -> List[Tuple[DictKeyType, Any]]:
+        items: List[Tuple[DictKeyType, Any]] = []
+
+        if self._is_none():
+            self._format_and_raise(
+                key=None,
+                value=None,
+                cause=TypeError("Cannot iterate a DictConfig object representing None"),
+            )
+        if self._is_missing():
+            raise MissingMandatoryValue("Cannot iterate a missing DictConfig")
+
+        for key in self.keys():
+            if resolve:
+                value = self[key]
+            else:
+                value = self.__dict__["_content"][key]
+                if isinstance(value, ValueNode):
+                    value = value._value()
+            if keys is None or key in keys:
+                items.append((key, value))
+
+        return items
+
+    def __eq__(self, other: Any) -> bool:
+        if other is None:
+            return self.__dict__["_content"] is None
+        if is_primitive_dict(other) or is_structured_config(other):
+            other = DictConfig(other, flags={"allow_objects": True})
+            return DictConfig._dict_conf_eq(self, other)
+        if isinstance(other, DictConfig):
+            return DictConfig._dict_conf_eq(self, other)
+        if self._is_missing():
+            return _is_missing_literal(other)
+        return NotImplemented
+
+    def __ne__(self, other: Any) -> bool:
+        x = self.__eq__(other)
+        if x is not NotImplemented:
+            return not x
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(str(self))
+
+    def _promote(self, type_or_prototype: Optional[Type[Any]]) -> None:
+        """
+        Retypes a node.
+        This should only be used in rare circumstances, where you want to dynamically change
+        the runtime structured-type of a DictConfig.
+        It will change the type and add the additional fields based on the input class or object
+        """
+        if type_or_prototype is None:
+            return
+        if not is_structured_config(type_or_prototype):
+            raise ValueError(f"Expected structured config class: {type_or_prototype}")
+
+        from omegaconf import OmegaConf
+
+        proto: DictConfig = OmegaConf.structured(type_or_prototype)
+        object_type = proto._metadata.object_type
+        # remove the type to prevent assignment validation from rejecting the promotion.
+        proto._metadata.object_type = None
+        self.merge_with(proto)
+        # restore the type.
+        self._metadata.object_type = object_type
+
+    def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None:
+        try:
+            previous_content = self.__dict__["_content"]
+            self._set_value_impl(value, flags)
+        except Exception as e:
+            self.__dict__["_content"] = previous_content
+            raise e
+
+    def _set_value_impl(
+        self, value: Any, flags: Optional[Dict[str, bool]] = None
+    ) -> None:
+        from omegaconf import MISSING, flag_override
+
+        if flags is None:
+            flags = {}
+
+        assert not isinstance(value, ValueNode)
+        self._validate_set(key=None, value=value)
+
+        if _is_none(value, resolve=True):
+            self.__dict__["_content"] = None
+            self._metadata.object_type = None
+        elif _is_interpolation(value, strict_interpolation_validation=True):
+            self.__dict__["_content"] = value
+            self._metadata.object_type = None
+        elif _is_missing_value(value):
+            self.__dict__["_content"] = MISSING
+            self._metadata.object_type = None
+        else:
+            self.__dict__["_content"] = {}
+            if is_structured_config(value):
+                self._metadata.object_type = None
+                ao = self._get_flag("allow_objects")
+                data = get_structured_config_data(value, allow_objects=ao)
+                with flag_override(self, ["struct", "readonly"], False):
+                    for k, v in data.items():
+                        self.__setitem__(k, v)
+                self._metadata.object_type = get_type_of(value)
+
+            elif isinstance(value, DictConfig):
+                self._metadata.flags = copy.deepcopy(flags)
+                with flag_override(self, ["struct", "readonly"], False):
+                    for k, v in value.__dict__["_content"].items():
+                        self.__setitem__(k, v)
+                self._metadata.object_type = value._metadata.object_type
+
+            elif isinstance(value, dict):
+                with flag_override(self, ["struct", "readonly"], False):
+                    for k, v in value.items():
+                        self.__setitem__(k, v)
+                self._metadata.object_type = dict
+
+            else:  # pragma: no cover
+                msg = f"Unsupported value type: {value}"
+                raise ValidationError(msg)
+
+    @staticmethod
+    def _dict_conf_eq(d1: "DictConfig", d2: "DictConfig") -> bool:
+
+        d1_none = d1.__dict__["_content"] is None
+        d2_none = d2.__dict__["_content"] is None
+        if d1_none and d2_none:
+            return True
+        if d1_none != d2_none:
+            return False
+
+        assert isinstance(d1, DictConfig)
+        assert isinstance(d2, DictConfig)
+        if len(d1) != len(d2):
+            return False
+        if d1._is_missing() or d2._is_missing():
+            return d1._is_missing() is d2._is_missing()
+
+        for k, v in d1.items_ex(resolve=False):
+            if k not in d2.__dict__["_content"]:
+                return False
+            if not BaseContainer._item_eq(d1, k, d2, k):
+                return False
+
+        return True
+
+    def _to_object(self) -> Any:
+        """
+        Instantiate an instance of `self._metadata.object_type`.
+        This requires `self` to be a structured config.
+        Nested subconfigs are converted by calling `OmegaConf.to_object`.
+        """
+        from omegaconf import OmegaConf
+
+        object_type = self._metadata.object_type
+        assert is_structured_config(object_type)
+        init_field_names = set(get_structured_config_init_field_names(object_type))
+
+        init_field_items: Dict[str, Any] = {}
+        non_init_field_items: Dict[str, Any] = {}
+        for k in self.keys():
+            assert isinstance(k, str)
+            node = self._get_child(k)
+            assert isinstance(node, Node)
+            try:
+                node = node._dereference_node()
+            except InterpolationResolutionError as e:
+                self._format_and_raise(key=k, value=None, cause=e)
+            if node._is_missing():
+                if k not in init_field_names:
+                    continue  # MISSING is ignored for init=False fields
+                self._format_and_raise(
+                    key=k,
+                    value=None,
+                    cause=MissingMandatoryValue(
+                        "Structured config of type `$OBJECT_TYPE` has missing mandatory value: $KEY"
+                    ),
+                )
+            if isinstance(node, Container):
+                v = OmegaConf.to_object(node)
+            else:
+                v = node._value()
+
+            if k in init_field_names:
+                init_field_items[k] = v
+            else:
+                non_init_field_items[k] = v
+
+        try:
+            result = object_type(**init_field_items)
+        except TypeError as exc:
+            self._format_and_raise(
+                key=None,
+                value=None,
+                cause=exc,
+                msg="Could not create instance of `$OBJECT_TYPE`: " + str(exc),
+            )
+
+        for k, v in non_init_field_items.items():
+            setattr(result, k, v)
+        return result

omegaconf/errors.py

@@ -0,0 +1,141 @@
+from typing import Any, Optional, Type
+
+
+class OmegaConfBaseException(Exception):
+    # would ideally be typed Optional[Node]
+    parent_node: Any
+    child_node: Any
+    key: Any
+    full_key: Optional[str]
+    value: Any
+    msg: Optional[str]
+    cause: Optional[Exception]
+    object_type: Optional[Type[Any]]
+    object_type_str: Optional[str]
+    ref_type: Optional[Type[Any]]
+    ref_type_str: Optional[str]
+
+    _initialized: bool = False
+
+    def __init__(self, *_args: Any, **_kwargs: Any) -> None:
+        self.parent_node = None
+        self.child_node = None
+        self.key = None
+        self.full_key = None
+        self.value = None
+        self.msg = None
+        self.object_type = None
+        self.ref_type = None
+
+
+class MissingMandatoryValue(OmegaConfBaseException):
+    """Thrown when a variable flagged with '???' value is accessed to
+    indicate that the value was not set"""
+
+
+class KeyValidationError(OmegaConfBaseException, ValueError):
+    """
+    Thrown when an a key of invalid type is used
+    """
+
+
+class ValidationError(OmegaConfBaseException, ValueError):
+    """
+    Thrown when a value fails validation
+    """
+
+
+class UnsupportedValueType(ValidationError, ValueError):
+    """
+    Thrown when an input value is not of supported type
+    """
+
+
+class ReadonlyConfigError(OmegaConfBaseException):
+    """
+    Thrown when someone tries to modify a frozen config
+    """
+
+
+class InterpolationResolutionError(OmegaConfBaseException, ValueError):
+    """
+    Base class for exceptions raised when resolving an interpolation.
+    """
+
+
+class UnsupportedInterpolationType(InterpolationResolutionError):
+    """
+    Thrown when an attempt to use an unregistered interpolation is made
+    """
+
+
+class InterpolationKeyError(InterpolationResolutionError):
+    """
+    Thrown when a node does not exist when resolving an interpolation.
+    """
+
+
+class InterpolationToMissingValueError(InterpolationResolutionError):
+    """
+    Thrown when a node interpolation points to a node that is set to ???.
+    """
+
+
+class InterpolationValidationError(InterpolationResolutionError, ValidationError):
+    """
+    Thrown when the result of an interpolation fails the validation step.
+    """
+
+
+class ConfigKeyError(OmegaConfBaseException, KeyError):
+    """
+    Thrown from DictConfig when a regular dict access would have caused a KeyError.
+    """
+
+    msg: str
+
+    def __init__(self, msg: str) -> None:
+        super().__init__(msg)
+        self.msg = msg
+
+    def __str__(self) -> str:
+        """
+        Workaround to nasty KeyError quirk: https://bugs.python.org/issue2651
+        """
+        return self.msg
+
+
+class ConfigAttributeError(OmegaConfBaseException, AttributeError):
+    """
+    Thrown from a config object when a regular access would have caused an AttributeError.
+    """
+
+
+class ConfigTypeError(OmegaConfBaseException, TypeError):
+    """
+    Thrown from a config object when a regular access would have caused a TypeError.
+    """
+
+
+class ConfigIndexError(OmegaConfBaseException, IndexError):
+    """
+    Thrown from a config object when a regular access would have caused an IndexError.
+    """
+
+
+class ConfigValueError(OmegaConfBaseException, ValueError):
+    """
+    Thrown from a config object when a regular access would have caused a ValueError.
+    """
+
+
+class ConfigCycleDetectedException(OmegaConfBaseException):
+    """
+    Thrown when a cycle is detected in the graph made by config nodes.
+    """
+
+
+class GrammarParseError(OmegaConfBaseException):
+    """
+    Thrown when failing to parse an expression according to the ANTLR grammar.
+    """

omegaconf/grammar/OmegaConfGrammarLexer.g4

@@ -0,0 +1,137 @@
+// Regenerate lexer and parser by running 'python setup.py antlr' at project root.
+// See `OmegaConfGrammarParser.g4` for some important information regarding how to
+// properly maintain this grammar.
+
+lexer grammar OmegaConfGrammarLexer;
+
+// Re-usable fragments.
+fragment CHAR: [a-zA-Z];
+fragment DIGIT: [0-9];
+fragment INT_UNSIGNED: '0' | [1-9] (('_')? DIGIT)*;
+fragment ESC_BACKSLASH: '\\\\';  // escaped backslash
+
+/////////////////////////////
+// DEFAULT_MODE (TOPLEVEL) //
+/////////////////////////////
+
+TOP_INTER_OPEN: INTER_OPEN -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE);
+
+// Regular string: anything that does not contain any $ and does not end with \
+// (this ensures this rule will not consume characters required to recognize other tokens).
+ANY_STR: ~[$]* ~[\\$];
+
+// Escaped interpolation: '\${', optionally preceded by an even number of \
+ESC_INTER: ESC_BACKSLASH* '\\${';
+
+// Backslashes that *may* be escaped (even number).
+TOP_ESC: ESC_BACKSLASH+;
+
+// Other backslashes that will not need escaping (odd number due to not matching the previous rule).
+BACKSLASHES: '\\'+ -> type(ANY_STR);
+
+// The dollar sign must be singled out so that we can recognize interpolations.
+DOLLAR: '$' -> type(ANY_STR);
+
+
+////////////////
+// VALUE_MODE //
+////////////////
+
+mode VALUE_MODE;
+
+INTER_OPEN: '${' WS? -> pushMode(INTERPOLATION_MODE);
+BRACE_OPEN: '{' WS? -> pushMode(VALUE_MODE);  // must keep track of braces to detect end of interpolation
+BRACE_CLOSE: WS? '}' -> popMode;
+QUOTE_OPEN_SINGLE: '\'' -> pushMode(QUOTED_SINGLE_MODE);
+QUOTE_OPEN_DOUBLE: '"' -> pushMode(QUOTED_DOUBLE_MODE);
+
+COMMA: WS? ',' WS?;
+BRACKET_OPEN: '[' WS?;
+BRACKET_CLOSE: WS? ']';
+COLON: WS? ':' WS?;
+
+// Numbers.
+
+fragment POINT_FLOAT: INT_UNSIGNED '.' | INT_UNSIGNED? '.' DIGIT (('_')? DIGIT)*;
+fragment EXPONENT_FLOAT: (INT_UNSIGNED | POINT_FLOAT) [eE] [+-]? DIGIT (('_')? DIGIT)*;
+FLOAT: [+-]? (POINT_FLOAT | EXPONENT_FLOAT | [Ii][Nn][Ff] | [Nn][Aa][Nn]);
+INT: [+-]? INT_UNSIGNED;
+
+// Other reserved keywords.
+
+BOOL:
+      [Tt][Rr][Uu][Ee]      // TRUE
+    | [Ff][Aa][Ll][Ss][Ee]; // FALSE
+
+NULL: [Nn][Uu][Ll][Ll];
+
+UNQUOTED_CHAR: [/\-\\+.$%*@?|];  // other characters allowed in unquoted strings
+ID: (CHAR|'_') (CHAR|DIGIT|'_'|'-')*;
+ESC: (ESC_BACKSLASH | '\\(' | '\\)' | '\\[' | '\\]' | '\\{' | '\\}' |
+      '\\:' | '\\=' | '\\,' | '\\ ' | '\\\t')+;
+WS: [ \t]+;
+
+
+////////////////////////
+// INTERPOLATION_MODE //
+////////////////////////
+
+mode INTERPOLATION_MODE;
+
+NESTED_INTER_OPEN: INTER_OPEN WS? -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE);
+INTER_COLON: WS? ':' WS? -> type(COLON), mode(VALUE_MODE);
+INTER_CLOSE: WS? '}' -> popMode;
+
+DOT: '.';
+INTER_BRACKET_OPEN: '[' -> type(BRACKET_OPEN);
+INTER_BRACKET_CLOSE: ']' -> type(BRACKET_CLOSE);
+INTER_ID: ID -> type(ID);
+
+// Interpolation key, may contain any non special character.
+// Note that we can allow '$' because the parser does not support interpolations that
+// are only part of a key name, i.e., "${foo${bar}}" is not allowed. As a result, it
+// is ok to "consume" all '$' characters within the `INTER_KEY` token.
+INTER_KEY: ~[\\{}()[\]:. \t'"]+;
+
+
+////////////////////////
+// QUOTED_SINGLE_MODE //
+////////////////////////
+
+mode QUOTED_SINGLE_MODE;
+
+// This mode is very similar to `DEFAULT_MODE` except for the handling of quotes.
+
+QSINGLE_INTER_OPEN: INTER_OPEN -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE);
+MATCHING_QUOTE_CLOSE: '\'' -> popMode;
+
+// Regular string: anything that does not contain any $ *or quote* and does not end with \
+QSINGLE_STR: ~['$]* ~['\\$] -> type(ANY_STR);
+
+QSINGLE_ESC_INTER: ESC_INTER -> type(ESC_INTER);
+
+// Escaped quote (optionally preceded by an even number of backslashes).
+QSINGLE_ESC_QUOTE: ESC_BACKSLASH* '\\\'' -> type(ESC);
+
+QUOTED_ESC: ESC_BACKSLASH+;
+QSINGLE_BACKSLASHES: '\\'+ -> type(ANY_STR);
+QSINGLE_DOLLAR: '$' -> type(ANY_STR);
+
+
+////////////////////////
+// QUOTED_DOUBLE_MODE //
+////////////////////////
+
+mode QUOTED_DOUBLE_MODE;
+
+// Same as `QUOTED_SINGLE_MODE` but for double quotes.
+
+QDOUBLE_INTER_OPEN: INTER_OPEN -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE);
+QDOUBLE_CLOSE: '"' -> type(MATCHING_QUOTE_CLOSE), popMode;
+
+QDOUBLE_STR: ~["$]* ~["\\$] -> type(ANY_STR);
+QDOUBLE_ESC_INTER: ESC_INTER -> type(ESC_INTER);
+QDOUBLE_ESC_QUOTE: ESC_BACKSLASH* '\\"' -> type(ESC);
+QDOUBLE_ESC: ESC_BACKSLASH+ -> type(QUOTED_ESC);
+QDOUBLE_BACKSLASHES: '\\'+ -> type(ANY_STR);
+QDOUBLE_DOLLAR: '$' -> type(ANY_STR);

omegaconf/grammar/OmegaConfGrammarParser.g4

@@ -0,0 +1,91 @@
+// Regenerate parser by running 'python setup.py antlr' at project root.
+
+// Maintenance guidelines when modifying this grammar:
+//
+// - Consider whether the regex pattern `SIMPLE_INTERPOLATION_PATTERN` found in
+//   `grammar_parser.py` should be updated as well.
+//
+// - Update Hydra's grammar accordingly.
+//
+// - Keep up-to-date the comments in the visitor (in `grammar_visitor.py`)
+//   that contain grammar excerpts (within each `visit...()` method).
+//
+// - Remember to update the documentation (including the tutorial notebook as
+//   well as grammar.rst)
+
+parser grammar OmegaConfGrammarParser;
+options {tokenVocab = OmegaConfGrammarLexer;}
+
+// Main rules used to parse OmegaConf strings.
+
+configValue: text EOF;
+singleElement: element EOF;
+
+
+// Composite text expression (may contain interpolations).
+
+text: (interpolation | ANY_STR | ESC | ESC_INTER | TOP_ESC | QUOTED_ESC)+;
+
+
+// Elements.
+
+element:
+      primitive
+    | quotedValue
+    | listContainer
+    | dictContainer
+;
+
+
+// Data structures.
+
+listContainer: BRACKET_OPEN sequence? BRACKET_CLOSE;                         // [], [1,2,3], [a,b,[1,2]]
+dictContainer: BRACE_OPEN (dictKeyValuePair (COMMA dictKeyValuePair)*)? BRACE_CLOSE;  // {}, {a:10,b:20}
+dictKeyValuePair: dictKey COLON element;
+sequence: (element (COMMA element?)*) | (COMMA element?)+;
+
+
+// Interpolations.
+
+interpolation: interpolationNode | interpolationResolver;
+
+interpolationNode:
+      INTER_OPEN
+      DOT*                                                     // relative interpolation?
+      (configKey | BRACKET_OPEN configKey BRACKET_CLOSE)       // foo, [foo]
+      (DOT configKey | BRACKET_OPEN configKey BRACKET_CLOSE)*  // .foo, [foo], .foo[bar], [foo].bar[baz]
+      INTER_CLOSE;
+interpolationResolver: INTER_OPEN resolverName COLON sequence? BRACE_CLOSE;
+configKey: interpolation | ID | INTER_KEY;
+resolverName: (interpolation | ID) (DOT (interpolation | ID))* ;  // oc.env, myfunc, ns.${x}, ns1.ns2.f
+
+
+// Primitive types.
+
+// Ex: "hello world", 'hello ${world}'
+quotedValue: (QUOTE_OPEN_SINGLE | QUOTE_OPEN_DOUBLE) text? MATCHING_QUOTE_CLOSE;
+
+primitive:
+    (   ID                                     // foo_10
+      | NULL                                   // null, NULL
+      | INT                                    // 0, 10, -20, 1_000_000
+      | FLOAT                                  // 3.14, -20.0, 1e-1, -10e3
+      | BOOL                                   // true, TrUe, false, False
+      | UNQUOTED_CHAR                          // /, -, \, +, ., $, %, *, @, ?, |
+      | COLON                                  // :
+      | ESC                                    // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \,
+      | WS                                     // whitespaces
+      | interpolation
+    )+;
+
+// Same as `primitive` except that `COLON` and interpolations are not allowed.
+dictKey:
+    (   ID                                     // foo_10
+      | NULL                                   // null, NULL
+      | INT                                    // 0, 10, -20, 1_000_000
+      | FLOAT                                  // 3.14, -20.0, 1e-1, -10e3
+      | BOOL                                   // true, TrUe, false, False
+      | UNQUOTED_CHAR                          // /, -, \, +, ., $, %, *, @, ?, |
+      | ESC                                    // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \,
+      | WS                                     // whitespaces
+    )+;

omegaconf/grammar_parser.py

@@ -0,0 +1,144 @@
+import re
+import threading
+from typing import Any
+
+from antlr4 import CommonTokenStream, InputStream, ParserRuleContext
+from antlr4.error.ErrorListener import ErrorListener
+
+from .errors import GrammarParseError
+
+# Import from visitor in order to check the presence of generated grammar files
+# files in a single place.
+from .grammar_visitor import (  # type: ignore
+    OmegaConfGrammarLexer,
+    OmegaConfGrammarParser,
+)
+
+# Used to cache grammar objects to avoid re-creating them on each call to `parse()`.
+# We use a per-thread cache to make it thread-safe.
+_grammar_cache = threading.local()
+
+# Build regex pattern to efficiently identify typical interpolations.
+# See test `test_match_simple_interpolation_pattern` for examples.
+_config_key = r"[$\w]+"  # foo, $0, $bar, $foo_$bar123$
+_key_maybe_brackets = f"{_config_key}|\\[{_config_key}\\]"  # foo, [foo], [$bar]
+_node_access = f"\\.{_key_maybe_brackets}"  # .foo, [foo], [$bar]
+_node_path = f"(\\.)*({_key_maybe_brackets})({_node_access})*"  # [foo].bar, .foo[bar]
+_node_inter = f"\\${{\\s*{_node_path}\\s*}}"  # node interpolation ${foo.bar}
+_id = "[a-zA-Z_][\\w\\-]*"  # foo, foo_bar, foo-bar, abc123
+_resolver_name = f"({_id}(\\.{_id})*)?"  # foo, ns.bar3, ns_1.ns_2.b0z
+_arg = r"[a-zA-Z_0-9/\-\+.$%*@?|]+"  # string representing a resolver argument
+_args = f"{_arg}(\\s*,\\s*{_arg})*"  # list of resolver arguments
+_resolver_inter = f"\\${{\\s*{_resolver_name}\\s*:\\s*{_args}?\\s*}}"  # ${foo:bar}
+_inter = f"({_node_inter}|{_resolver_inter})"  # any kind of interpolation
+_outer = "([^$]|\\$(?!{))+"  # any character except $ (unless not followed by {)
+SIMPLE_INTERPOLATION_PATTERN = re.compile(
+    f"({_outer})?({_inter}({_outer})?)+$", flags=re.ASCII
+)
+# NOTE: SIMPLE_INTERPOLATION_PATTERN must not generate false positive matches:
+# it must not accept anything that isn't a valid interpolation (per the
+# interpolation grammar defined in `omegaconf/grammar/*.g4`).
+
+
+class OmegaConfErrorListener(ErrorListener):  # type: ignore
+    def syntaxError(
+        self,
+        recognizer: Any,
+        offending_symbol: Any,
+        line: Any,
+        column: Any,
+        msg: Any,
+        e: Any,
+    ) -> None:
+        raise GrammarParseError(str(e) if msg is None else msg) from e
+
+    def reportAmbiguity(
+        self,
+        recognizer: Any,
+        dfa: Any,
+        startIndex: Any,
+        stopIndex: Any,
+        exact: Any,
+        ambigAlts: Any,
+        configs: Any,
+    ) -> None:
+        raise GrammarParseError("ANTLR error: Ambiguity")  # pragma: no cover
+
+    def reportAttemptingFullContext(
+        self,
+        recognizer: Any,
+        dfa: Any,
+        startIndex: Any,
+        stopIndex: Any,
+        conflictingAlts: Any,
+        configs: Any,
+    ) -> None:
+        # Note: for now we raise an error to be safe. However this is mostly a
+        # performance warning, so in the future this may be relaxed if we need
+        # to change the grammar in such a way that this warning cannot be
+        # avoided (another option would be to switch to SLL parsing mode).
+        raise GrammarParseError(
+            "ANTLR error: Attempting Full Context"
+        )  # pragma: no cover
+
+    def reportContextSensitivity(
+        self,
+        recognizer: Any,
+        dfa: Any,
+        startIndex: Any,
+        stopIndex: Any,
+        prediction: Any,
+        configs: Any,
+    ) -> None:
+        raise GrammarParseError("ANTLR error: ContextSensitivity")  # pragma: no cover
+
+
+def parse(
+    value: str, parser_rule: str = "configValue", lexer_mode: str = "DEFAULT_MODE"
+) -> ParserRuleContext:
+    """
+    Parse interpolated string `value` (and return the parse tree).
+    """
+    l_mode = getattr(OmegaConfGrammarLexer, lexer_mode)
+    istream = InputStream(value)
+
+    cached = getattr(_grammar_cache, "data", None)
+    if cached is None:
+        error_listener = OmegaConfErrorListener()
+        lexer = OmegaConfGrammarLexer(istream)
+        lexer.removeErrorListeners()
+        lexer.addErrorListener(error_listener)
+        lexer.mode(l_mode)
+        token_stream = CommonTokenStream(lexer)
+        parser = OmegaConfGrammarParser(token_stream)
+        parser.removeErrorListeners()
+        parser.addErrorListener(error_listener)
+
+        # The two lines below could be enabled in the future if we decide to switch
+        # to SLL prediction mode. Warning though, it has not been fully tested yet!
+        # from antlr4 import PredictionMode
+        # parser._interp.predictionMode = PredictionMode.SLL
+
+        # Note that although the input stream `istream` is implicitly cached within
+        # the lexer, it will be replaced by a new input next time the lexer is re-used.
+        _grammar_cache.data = lexer, token_stream, parser
+
+    else:
+        lexer, token_stream, parser = cached
+        # Replace the old input stream with the new one.
+        lexer.inputStream = istream
+        # Initialize the lexer / token stream / parser to process the new input.
+        lexer.mode(l_mode)
+        token_stream.setTokenSource(lexer)
+        parser.reset()
+
+    try:
+        return getattr(parser, parser_rule)()
+    except Exception as exc:
+        if type(exc) is Exception and str(exc) == "Empty Stack":
+            # This exception is raised by antlr when trying to pop a mode while
+            # no mode has been pushed. We convert it into an `GrammarParseError`
+            # to facilitate exception handling from the caller.
+            raise GrammarParseError("Empty Stack")
+        else:
+            raise

omegaconf/grammar_visitor.py

@@ -0,0 +1,392 @@
+import sys
+import warnings
+from itertools import zip_longest
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
+
+from antlr4 import TerminalNode
+
+from .errors import InterpolationResolutionError
+
+if TYPE_CHECKING:
+    from .base import Node  # noqa F401
+
+try:
+    from omegaconf.grammar.gen.OmegaConfGrammarLexer import OmegaConfGrammarLexer
+    from omegaconf.grammar.gen.OmegaConfGrammarParser import OmegaConfGrammarParser
+    from omegaconf.grammar.gen.OmegaConfGrammarParserVisitor import (
+        OmegaConfGrammarParserVisitor,
+    )
+
+except ModuleNotFoundError:  # pragma: no cover
+    print(
+        "Error importing OmegaConf's generated parsers, run `python setup.py antlr` to regenerate.",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+
+class GrammarVisitor(OmegaConfGrammarParserVisitor):
+    def __init__(
+        self,
+        node_interpolation_callback: Callable[
+            [str, Optional[Set[int]]],
+            Optional["Node"],
+        ],
+        resolver_interpolation_callback: Callable[..., Any],
+        memo: Optional[Set[int]],
+        **kw: Dict[Any, Any],
+    ):
+        """
+        Constructor.
+
+        :param node_interpolation_callback: Callback function that is called when
+            needing to resolve a node interpolation. This function should take a single
+            string input which is the key's dot path (ex: `"foo.bar"`).
+
+        :param resolver_interpolation_callback: Callback function that is called when
+            needing to resolve a resolver interpolation. This function should accept
+            three keyword arguments: `name` (str, the name of the resolver),
+            `args` (tuple, the inputs to the resolver), and `args_str` (tuple,
+            the string representation of the inputs to the resolver).
+
+        :param kw: Additional keyword arguments to be forwarded to parent class.
+        """
+        super().__init__(**kw)
+        self.node_interpolation_callback = node_interpolation_callback
+        self.resolver_interpolation_callback = resolver_interpolation_callback
+        self.memo = memo
+
+    def aggregateResult(self, aggregate: List[Any], nextResult: Any) -> List[Any]:
+        raise NotImplementedError
+
+    def defaultResult(self) -> List[Any]:
+        # Raising an exception because not currently used (like `aggregateResult()`).
+        raise NotImplementedError
+
+    def visitConfigKey(self, ctx: OmegaConfGrammarParser.ConfigKeyContext) -> str:
+        from ._utils import _get_value
+
+        # interpolation | ID | INTER_KEY
+        assert ctx.getChildCount() == 1
+        child = ctx.getChild(0)
+        if isinstance(child, OmegaConfGrammarParser.InterpolationContext):
+            res = _get_value(self.visitInterpolation(child))
+            if not isinstance(res, str):
+                raise InterpolationResolutionError(
+                    f"The following interpolation is used to denote a config key and "
+                    f"thus should return a string, but instead returned `{res}` of "
+                    f"type `{type(res)}`: {ctx.getChild(0).getText()}"
+                )
+            return res
+        else:
+            assert isinstance(child, TerminalNode) and isinstance(
+                child.symbol.text, str
+            )
+            return child.symbol.text
+
+    def visitConfigValue(self, ctx: OmegaConfGrammarParser.ConfigValueContext) -> Any:
+        # text EOF
+        assert ctx.getChildCount() == 2
+        return self.visit(ctx.getChild(0))
+
+    def visitDictKey(self, ctx: OmegaConfGrammarParser.DictKeyContext) -> Any:
+        return self._createPrimitive(ctx)
+
+    def visitDictContainer(
+        self, ctx: OmegaConfGrammarParser.DictContainerContext
+    ) -> Dict[Any, Any]:
+        # BRACE_OPEN (dictKeyValuePair (COMMA dictKeyValuePair)*)? BRACE_CLOSE
+        assert ctx.getChildCount() >= 2
+        return dict(
+            self.visitDictKeyValuePair(ctx.getChild(i))
+            for i in range(1, ctx.getChildCount() - 1, 2)
+        )
+
+    def visitElement(self, ctx: OmegaConfGrammarParser.ElementContext) -> Any:
+        # primitive | quotedValue | listContainer | dictContainer
+        assert ctx.getChildCount() == 1
+        return self.visit(ctx.getChild(0))
+
+    def visitInterpolation(
+        self, ctx: OmegaConfGrammarParser.InterpolationContext
+    ) -> Any:
+        assert ctx.getChildCount() == 1  # interpolationNode | interpolationResolver
+        return self.visit(ctx.getChild(0))
+
+    def visitInterpolationNode(
+        self, ctx: OmegaConfGrammarParser.InterpolationNodeContext
+    ) -> Optional["Node"]:
+        # INTER_OPEN
+        # DOT*                                                     // relative interpolation?
+        # (configKey | BRACKET_OPEN configKey BRACKET_CLOSE)       // foo, [foo]
+        # (DOT configKey | BRACKET_OPEN configKey BRACKET_CLOSE)*  // .foo, [foo], .foo[bar], [foo].bar[baz]
+        # INTER_CLOSE;
+
+        assert ctx.getChildCount() >= 3
+
+        inter_key_tokens = []  # parsed elements of the dot path
+        for child in ctx.getChildren():
+            if isinstance(child, TerminalNode):
+                s = child.symbol
+                if s.type in [
+                    OmegaConfGrammarLexer.DOT,
+                    OmegaConfGrammarLexer.BRACKET_OPEN,
+                    OmegaConfGrammarLexer.BRACKET_CLOSE,
+                ]:
+                    inter_key_tokens.append(s.text)
+                else:
+                    assert s.type in (
+                        OmegaConfGrammarLexer.INTER_OPEN,
+                        OmegaConfGrammarLexer.INTER_CLOSE,
+                    )
+            else:
+                assert isinstance(child, OmegaConfGrammarParser.ConfigKeyContext)
+                inter_key_tokens.append(self.visitConfigKey(child))
+
+        inter_key = "".join(inter_key_tokens)
+        return self.node_interpolation_callback(inter_key, self.memo)
+
+    def visitInterpolationResolver(
+        self, ctx: OmegaConfGrammarParser.InterpolationResolverContext
+    ) -> Any:
+
+        # INTER_OPEN resolverName COLON sequence? BRACE_CLOSE
+        assert 4 <= ctx.getChildCount() <= 5
+
+        resolver_name = self.visit(ctx.getChild(1))
+        maybe_seq = ctx.getChild(3)
+        args = []
+        args_str = []
+        if isinstance(maybe_seq, TerminalNode):  # means there are no args
+            assert maybe_seq.symbol.type == OmegaConfGrammarLexer.BRACE_CLOSE
+        else:
+            assert isinstance(maybe_seq, OmegaConfGrammarParser.SequenceContext)
+            for val, txt in self.visitSequence(maybe_seq):
+                args.append(val)
+                args_str.append(txt)
+
+        return self.resolver_interpolation_callback(
+            name=resolver_name,
+            args=tuple(args),
+            args_str=tuple(args_str),
+        )
+
+    def visitDictKeyValuePair(
+        self, ctx: OmegaConfGrammarParser.DictKeyValuePairContext
+    ) -> Tuple[Any, Any]:
+        from ._utils import _get_value
+
+        assert ctx.getChildCount() == 3  # dictKey COLON element
+        key = self.visit(ctx.getChild(0))
+        colon = ctx.getChild(1)
+        assert (
+            isinstance(colon, TerminalNode)
+            and colon.symbol.type == OmegaConfGrammarLexer.COLON
+        )
+        value = _get_value(self.visitElement(ctx.getChild(2)))
+        return key, value
+
+    def visitListContainer(
+        self, ctx: OmegaConfGrammarParser.ListContainerContext
+    ) -> List[Any]:
+        # BRACKET_OPEN sequence? BRACKET_CLOSE;
+        assert ctx.getChildCount() in (2, 3)
+        if ctx.getChildCount() == 2:
+            return []
+        sequence = ctx.getChild(1)
+        assert isinstance(sequence, OmegaConfGrammarParser.SequenceContext)
+        return list(val for val, _ in self.visitSequence(sequence))  # ignore raw text
+
+    def visitPrimitive(self, ctx: OmegaConfGrammarParser.PrimitiveContext) -> Any:
+        return self._createPrimitive(ctx)
+
+    def visitQuotedValue(self, ctx: OmegaConfGrammarParser.QuotedValueContext) -> str:
+        # (QUOTE_OPEN_SINGLE | QUOTE_OPEN_DOUBLE) text? MATCHING_QUOTE_CLOSE
+        n = ctx.getChildCount()
+        assert n in [2, 3]
+        return str(self.visit(ctx.getChild(1))) if n == 3 else ""
+
+    def visitResolverName(self, ctx: OmegaConfGrammarParser.ResolverNameContext) -> str:
+        from ._utils import _get_value
+
+        # (interpolation | ID) (DOT (interpolation | ID))*
+        assert ctx.getChildCount() >= 1
+        items = []
+        for child in list(ctx.getChildren())[::2]:
+            if isinstance(child, TerminalNode):
+                assert child.symbol.type == OmegaConfGrammarLexer.ID
+                items.append(child.symbol.text)
+            else:
+                assert isinstance(child, OmegaConfGrammarParser.InterpolationContext)
+                item = _get_value(self.visitInterpolation(child))
+                if not isinstance(item, str):
+                    raise InterpolationResolutionError(
+                        f"The name of a resolver must be a string, but the interpolation "
+                        f"{child.getText()} resolved to `{item}` which is of type "
+                        f"{type(item)}"
+                    )
+                items.append(item)
+        return ".".join(items)
+
+    def visitSequence(
+        self, ctx: OmegaConfGrammarParser.SequenceContext
+    ) -> Generator[Any, None, None]:
+        from ._utils import _get_value
+
+        # (element (COMMA element?)*) | (COMMA element?)+
+        assert ctx.getChildCount() >= 1
+
+        # DEPRECATED: remove in 2.2 (revert #571)
+        def empty_str_warning() -> None:
+            txt = ctx.getText()
+            warnings.warn(
+                f"In the sequence `{txt}` some elements are missing: please replace "
+                f"them with empty quoted strings. "
+                f"See https://github.com/omry/omegaconf/issues/572 for details.",
+                category=UserWarning,
+            )
+
+        is_previous_comma = True  # whether previous child was a comma (init to True)
+        for child in ctx.getChildren():
+            if isinstance(child, OmegaConfGrammarParser.ElementContext):
+                # Also preserve the original text representation of `child` so
+                # as to allow backward compatibility with old resolvers (registered
+                # with `legacy_register_resolver()`). Note that we cannot just cast
+                # the value to string later as for instance `null` would become "None".
+                yield _get_value(self.visitElement(child)), child.getText()
+                is_previous_comma = False
+            else:
+                assert (
+                    isinstance(child, TerminalNode)
+                    and child.symbol.type == OmegaConfGrammarLexer.COMMA
+                )
+                if is_previous_comma:
+                    empty_str_warning()
+                    yield "", ""
+                else:
+                    is_previous_comma = True
+        if is_previous_comma:
+            # Trailing comma.
+            empty_str_warning()
+            yield "", ""
+
+    def visitSingleElement(
+        self, ctx: OmegaConfGrammarParser.SingleElementContext
+    ) -> Any:
+        # element EOF
+        assert ctx.getChildCount() == 2
+        return self.visit(ctx.getChild(0))
+
+    def visitText(self, ctx: OmegaConfGrammarParser.TextContext) -> Any:
+        # (interpolation | ANY_STR | ESC | ESC_INTER | TOP_ESC | QUOTED_ESC)+
+
+        # Single interpolation? If yes, return its resolved value "as is".
+        if ctx.getChildCount() == 1:
+            c = ctx.getChild(0)
+            if isinstance(c, OmegaConfGrammarParser.InterpolationContext):
+                return self.visitInterpolation(c)
+
+        # Otherwise, concatenate string representations together.
+        return self._unescape(list(ctx.getChildren()))
+
+    def _createPrimitive(
+        self,
+        ctx: Union[
+            OmegaConfGrammarParser.PrimitiveContext,
+            OmegaConfGrammarParser.DictKeyContext,
+        ],
+    ) -> Any:
+        # (ID | NULL | INT | FLOAT | BOOL | UNQUOTED_CHAR | COLON | ESC | WS | interpolation)+
+        if ctx.getChildCount() == 1:
+            child = ctx.getChild(0)
+            if isinstance(child, OmegaConfGrammarParser.InterpolationContext):
+                return self.visitInterpolation(child)
+            assert isinstance(child, TerminalNode)
+            symbol = child.symbol
+            # Parse primitive types.
+            if symbol.type in (
+                OmegaConfGrammarLexer.ID,
+                OmegaConfGrammarLexer.UNQUOTED_CHAR,
+                OmegaConfGrammarLexer.COLON,
+            ):
+                return symbol.text
+            elif symbol.type == OmegaConfGrammarLexer.NULL:
+                return None
+            elif symbol.type == OmegaConfGrammarLexer.INT:
+                return int(symbol.text)
+            elif symbol.type == OmegaConfGrammarLexer.FLOAT:
+                return float(symbol.text)
+            elif symbol.type == OmegaConfGrammarLexer.BOOL:
+                return symbol.text.lower() == "true"
+            elif symbol.type == OmegaConfGrammarLexer.ESC:
+                return self._unescape([child])
+            elif symbol.type == OmegaConfGrammarLexer.WS:  # pragma: no cover
+                # A single WS should have been "consumed" by another token.
+                raise AssertionError("WS should never be reached")
+            assert False, symbol.type
+        # Concatenation of multiple items ==> un-escape the concatenation.
+        return self._unescape(list(ctx.getChildren()))
+
+    def _unescape(
+        self,
+        seq: List[Union[TerminalNode, OmegaConfGrammarParser.InterpolationContext]],
+    ) -> str:
+        """
+        Concatenate all symbols / interpolations in `seq`, unescaping symbols as needed.
+
+        Interpolations are resolved and cast to string *WITHOUT* escaping their result
+        (it is assumed that whatever escaping is required was already handled during the
+        resolving of the interpolation).
+        """
+        chrs = []
+        for node, next_node in zip_longest(seq, seq[1:]):
+            if isinstance(node, TerminalNode):
+                s = node.symbol
+                if s.type == OmegaConfGrammarLexer.ESC_INTER:
+                    # `ESC_INTER` is of the form `\\...\${`: the formula below computes
+                    # the number of characters to keep at the end of the string to remove
+                    # the correct number of backslashes.
+                    text = s.text[-(len(s.text) // 2 + 1) :]
+                elif (
+                    # Character sequence identified as requiring un-escaping.
+                    s.type == OmegaConfGrammarLexer.ESC
+                    or (
+                        # At top level, we need to un-escape backslashes that precede
+                        # an interpolation.
+                        s.type == OmegaConfGrammarLexer.TOP_ESC
+                        and isinstance(
+                            next_node, OmegaConfGrammarParser.InterpolationContext
+                        )
+                    )
+                    or (
+                        # In a quoted sring, we need to un-escape backslashes that
+                        # either end the string, or are followed by an interpolation.
+                        s.type == OmegaConfGrammarLexer.QUOTED_ESC
+                        and (
+                            next_node is None
+                            or isinstance(
+                                next_node, OmegaConfGrammarParser.InterpolationContext
+                            )
+                        )
+                    )
+                ):
+                    text = s.text[1::2]  # un-escape the sequence
+                else:
+                    text = s.text  # keep the original text
+            else:
+                assert isinstance(node, OmegaConfGrammarParser.InterpolationContext)
+                text = str(self.visitInterpolation(node))
+            chrs.append(text)
+
+        return "".join(chrs)

omegaconf/listconfig.py

@@ -0,0 +1,679 @@
+import copy
+import itertools
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    MutableSequence,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+)
+
+from ._utils import (
+    ValueKind,
+    _is_missing_literal,
+    _is_none,
+    _resolve_optional,
+    format_and_raise,
+    get_value_kind,
+    is_int,
+    is_primitive_list,
+    is_structured_config,
+    type_str,
+)
+from .base import Box, ContainerMetadata, Node
+from .basecontainer import BaseContainer
+from .errors import (
+    ConfigAttributeError,
+    ConfigTypeError,
+    ConfigValueError,
+    KeyValidationError,
+    MissingMandatoryValue,
+    ReadonlyConfigError,
+    ValidationError,
+)
+
+
+class ListConfig(BaseContainer, MutableSequence[Any]):
+
+    _content: Union[List[Node], None, str]
+
+    def __init__(
+        self,
+        content: Union[List[Any], Tuple[Any, ...], "ListConfig", str, None],
+        key: Any = None,
+        parent: Optional[Box] = None,
+        element_type: Union[Type[Any], Any] = Any,
+        is_optional: bool = True,
+        ref_type: Union[Type[Any], Any] = Any,
+        flags: Optional[Dict[str, bool]] = None,
+    ) -> None:
+        try:
+            if isinstance(content, ListConfig):
+                if flags is None:
+                    flags = content._metadata.flags
+            super().__init__(
+                parent=parent,
+                metadata=ContainerMetadata(
+                    ref_type=ref_type,
+                    object_type=list,
+                    key=key,
+                    optional=is_optional,
+                    element_type=element_type,
+                    key_type=int,
+                    flags=flags,
+                ),
+            )
+
+            if isinstance(content, ListConfig):
+                metadata = copy.deepcopy(content._metadata)
+                metadata.key = key
+                metadata.ref_type = ref_type
+                metadata.optional = is_optional
+                metadata.element_type = element_type
+                self.__dict__["_metadata"] = metadata
+            self._set_value(value=content, flags=flags)
+        except Exception as ex:
+            format_and_raise(node=None, key=key, value=None, cause=ex, msg=str(ex))
+
+    def _validate_get(self, key: Any, value: Any = None) -> None:
+        if not isinstance(key, (int, slice)):
+            raise KeyValidationError(
+                "ListConfig indices must be integers or slices, not $KEY_TYPE"
+            )
+
+    def _validate_set(self, key: Any, value: Any) -> None:
+        from omegaconf import OmegaConf
+
+        self._validate_get(key, value)
+
+        if self._get_flag("readonly"):
+            raise ReadonlyConfigError("ListConfig is read-only")
+
+        if 0 <= key < self.__len__():
+            target = self._get_node(key)
+            if target is not None:
+                assert isinstance(target, Node)
+                if value is None and not target._is_optional():
+                    raise ValidationError(
+                        "$FULL_KEY is not optional and cannot be assigned None"
+                    )
+
+        vk = get_value_kind(value)
+        if vk == ValueKind.MANDATORY_MISSING:
+            return
+        else:
+            is_optional, target_type = _resolve_optional(self._metadata.element_type)
+            value_type = OmegaConf.get_type(value)
+
+            if (value_type is None and not is_optional) or (
+                is_structured_config(target_type)
+                and value_type is not None
+                and not issubclass(value_type, target_type)
+            ):
+                msg = (
+                    f"Invalid type assigned: {type_str(value_type)} is not a "
+                    f"subclass of {type_str(target_type)}. value: {value}"
+                )
+                raise ValidationError(msg)
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "ListConfig":
+        res = ListConfig(None)
+        res.__dict__["_metadata"] = copy.deepcopy(self.__dict__["_metadata"], memo=memo)
+        res.__dict__["_flags_cache"] = copy.deepcopy(
+            self.__dict__["_flags_cache"], memo=memo
+        )
+
+        src_content = self.__dict__["_content"]
+        if isinstance(src_content, list):
+            content_copy: List[Optional[Node]] = []
+            for v in src_content:
+                old_parent = v.__dict__["_parent"]
+                try:
+                    v.__dict__["_parent"] = None
+                    vc = copy.deepcopy(v, memo=memo)
+                    vc.__dict__["_parent"] = res
+                    content_copy.append(vc)
+                finally:
+                    v.__dict__["_parent"] = old_parent
+        else:
+            # None and strings can be assigned as is
+            content_copy = src_content
+
+        res.__dict__["_content"] = content_copy
+        res.__dict__["_parent"] = self.__dict__["_parent"]
+
+        return res
+
+    def copy(self) -> "ListConfig":
+        return copy.copy(self)
+
+    # hide content while inspecting in debugger
+    def __dir__(self) -> Iterable[str]:
+        if self._is_missing() or self._is_none():
+            return []
+        return [str(x) for x in range(0, len(self))]
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        self._format_and_raise(
+            key=key,
+            value=value,
+            cause=ConfigAttributeError("ListConfig does not support attribute access"),
+        )
+        assert False
+
+    def __getattr__(self, key: str) -> Any:
+        # PyCharm is sometimes inspecting __members__, be sure to tell it we don't have that.
+        if key == "__members__":
+            raise AttributeError()
+
+        if key == "__name__":
+            raise AttributeError()
+
+        if is_int(key):
+            return self.__getitem__(int(key))
+        else:
+            self._format_and_raise(
+                key=key,
+                value=None,
+                cause=ConfigAttributeError(
+                    "ListConfig does not support attribute access"
+                ),
+            )
+
+    def __getitem__(self, index: Union[int, slice]) -> Any:
+        try:
+            if self._is_missing():
+                raise MissingMandatoryValue("ListConfig is missing")
+            self._validate_get(index, None)
+            if self._is_none():
+                raise TypeError(
+                    "ListConfig object representing None is not subscriptable"
+                )
+
+            assert isinstance(self.__dict__["_content"], list)
+            if isinstance(index, slice):
+                result = []
+                start, stop, step = self._correct_index_params(index)
+                for slice_idx in itertools.islice(
+                    range(0, len(self)), start, stop, step
+                ):
+                    val = self._resolve_with_default(
+                        key=slice_idx, value=self.__dict__["_content"][slice_idx]
+                    )
+                    result.append(val)
+                if index.step and index.step < 0:
+                    result.reverse()
+                return result
+            else:
+                return self._resolve_with_default(
+                    key=index, value=self.__dict__["_content"][index]
+                )
+        except Exception as e:
+            self._format_and_raise(key=index, value=None, cause=e)
+
+    def _correct_index_params(self, index: slice) -> Tuple[int, int, int]:
+        start = index.start
+        stop = index.stop
+        step = index.step
+        if index.start and index.start < 0:
+            start = self.__len__() + index.start
+        if index.stop and index.stop < 0:
+            stop = self.__len__() + index.stop
+        if index.step and index.step < 0:
+            step = abs(step)
+            if start and stop:
+                if start > stop:
+                    start, stop = stop + 1, start + 1
+                else:
+                    start = stop = 0
+            elif not start and stop:
+                start = list(range(self.__len__() - 1, stop, -step))[0]
+                stop = None
+            elif start and not stop:
+                stop = start + 1
+                start = (stop - 1) % step
+            else:
+                start = (self.__len__() - 1) % step
+        return start, stop, step
+
+    def _set_at_index(self, index: Union[int, slice], value: Any) -> None:
+        self._set_item_impl(index, value)
+
+    def __setitem__(self, index: Union[int, slice], value: Any) -> None:
+        try:
+            if isinstance(index, slice):
+                _ = iter(value)  # check iterable
+                self_indices = index.indices(len(self))
+                indexes = range(*self_indices)
+
+                # Ensure lengths match for extended slice assignment
+                if index.step not in (None, 1):
+                    if len(indexes) != len(value):
+                        raise ValueError(
+                            f"attempt to assign sequence of size {len(value)}"
+                            f" to extended slice of size {len(indexes)}"
+                        )
+
+                # Initialize insertion offsets for empty slices
+                if len(indexes) == 0:
+                    curr_index = self_indices[0] - 1
+                    val_i = -1
+
+                work_copy = self.copy()  # For atomicity manipulate a copy
+
+                # Delete and optionally replace non empty slices
+                only_removed = 0
+                for val_i, i in enumerate(indexes):
+                    curr_index = i - only_removed
+                    del work_copy[curr_index]
+                    if val_i < len(value):
+                        work_copy.insert(curr_index, value[val_i])
+                    else:
+                        only_removed += 1
+
+                # Insert any remaining input items
+                for val_i in range(val_i + 1, len(value)):
+                    curr_index += 1
+                    work_copy.insert(curr_index, value[val_i])
+
+                # Reinitialize self with work_copy
+                self.clear()
+                self.extend(work_copy)
+            else:
+                self._set_at_index(index, value)
+        except Exception as e:
+            self._format_and_raise(key=index, value=value, cause=e)
+
+    def append(self, item: Any) -> None:
+        content = self.__dict__["_content"]
+        index = len(content)
+        content.append(None)
+        try:
+            self._set_item_impl(index, item)
+        except Exception as e:
+            del content[index]
+            self._format_and_raise(key=index, value=item, cause=e)
+            assert False
+
+    def _update_keys(self) -> None:
+        for i in range(len(self)):
+            node = self._get_node(i)
+            if node is not None:
+                assert isinstance(node, Node)
+                node._metadata.key = i
+
+    def insert(self, index: int, item: Any) -> None:
+        from omegaconf.omegaconf import _maybe_wrap
+
+        try:
+            if self._get_flag("readonly"):
+                raise ReadonlyConfigError("Cannot insert into a read-only ListConfig")
+            if self._is_none():
+                raise TypeError(
+                    "Cannot insert into ListConfig object representing None"
+                )
+            if self._is_missing():
+                raise MissingMandatoryValue("Cannot insert into missing ListConfig")
+
+            try:
+                assert isinstance(self.__dict__["_content"], list)
+                # insert place holder
+                self.__dict__["_content"].insert(index, None)
+                is_optional, ref_type = _resolve_optional(self._metadata.element_type)
+                node = _maybe_wrap(
+                    ref_type=ref_type,
+                    key=index,
+                    value=item,
+                    is_optional=is_optional,
+                    parent=self,
+                )
+                self._validate_set(key=index, value=node)
+                self._set_at_index(index, node)
+                self._update_keys()
+            except Exception:
+                del self.__dict__["_content"][index]
+                self._update_keys()
+                raise
+        except Exception as e:
+            self._format_and_raise(key=index, value=item, cause=e)
+            assert False
+
+    def extend(self, lst: Iterable[Any]) -> None:
+        assert isinstance(lst, (tuple, list, ListConfig))
+        for x in lst:
+            self.append(x)
+
+    def remove(self, x: Any) -> None:
+        del self[self.index(x)]
+
+    def __delitem__(self, key: Union[int, slice]) -> None:
+        if self._get_flag("readonly"):
+            self._format_and_raise(
+                key=key,
+                value=None,
+                cause=ReadonlyConfigError(
+                    "Cannot delete item from read-only ListConfig"
+                ),
+            )
+        del self.__dict__["_content"][key]
+        self._update_keys()
+
+    def clear(self) -> None:
+        del self[:]
+
+    def index(
+        self, x: Any, start: Optional[int] = None, end: Optional[int] = None
+    ) -> int:
+        if start is None:
+            start = 0
+        if end is None:
+            end = len(self)
+        assert start >= 0
+        assert end <= len(self)
+        found_idx = -1
+        for idx in range(start, end):
+            item = self[idx]
+            if x == item:
+                found_idx = idx
+                break
+        if found_idx != -1:
+            return found_idx
+        else:
+            self._format_and_raise(
+                key=None,
+                value=None,
+                cause=ConfigValueError("Item not found in ListConfig"),
+            )
+            assert False
+
+    def count(self, x: Any) -> int:
+        c = 0
+        for item in self:
+            if item == x:
+                c = c + 1
+        return c
+
+    def _get_node(
+        self,
+        key: Union[int, slice],
+        validate_access: bool = True,
+        validate_key: bool = True,
+        throw_on_missing_value: bool = False,
+        throw_on_missing_key: bool = False,
+    ) -> Union[Optional[Node], List[Optional[Node]]]:
+        try:
+            if self._is_none():
+                raise TypeError(
+                    "Cannot get_node from a ListConfig object representing None"
+                )
+            if self._is_missing():
+                raise MissingMandatoryValue("Cannot get_node from a missing ListConfig")
+            assert isinstance(self.__dict__["_content"], list)
+            if validate_access:
+                self._validate_get(key)
+
+            value = self.__dict__["_content"][key]
+            if value is not None:
+                if isinstance(key, slice):
+                    assert isinstance(value, list)
+                    for v in value:
+                        if throw_on_missing_value and v._is_missing():
+                            raise MissingMandatoryValue("Missing mandatory value")
+                else:
+                    assert isinstance(value, Node)
+                    if throw_on_missing_value and value._is_missing():
+                        raise MissingMandatoryValue("Missing mandatory value: $KEY")
+            return value
+        except (IndexError, TypeError, MissingMandatoryValue, KeyValidationError) as e:
+            if isinstance(e, MissingMandatoryValue) and throw_on_missing_value:
+                raise
+            if validate_access:
+                self._format_and_raise(key=key, value=None, cause=e)
+                assert False
+            else:
+                return None
+
+    def get(self, index: int, default_value: Any = None) -> Any:
+        try:
+            if self._is_none():
+                raise TypeError("Cannot get from a ListConfig object representing None")
+            if self._is_missing():
+                raise MissingMandatoryValue("Cannot get from a missing ListConfig")
+            self._validate_get(index, None)
+            assert isinstance(self.__dict__["_content"], list)
+            return self._resolve_with_default(
+                key=index,
+                value=self.__dict__["_content"][index],
+                default_value=default_value,
+            )
+        except Exception as e:
+            self._format_and_raise(key=index, value=None, cause=e)
+            assert False
+
+    def pop(self, index: int = -1) -> Any:
+        try:
+            if self._get_flag("readonly"):
+                raise ReadonlyConfigError("Cannot pop from read-only ListConfig")
+            if self._is_none():
+                raise TypeError("Cannot pop from a ListConfig object representing None")
+            if self._is_missing():
+                raise MissingMandatoryValue("Cannot pop from a missing ListConfig")
+
+            assert isinstance(self.__dict__["_content"], list)
+            node = self._get_child(index)
+            assert isinstance(node, Node)
+            ret = self._resolve_with_default(key=index, value=node, default_value=None)
+            del self.__dict__["_content"][index]
+            self._update_keys()
+            return ret
+        except KeyValidationError as e:
+            self._format_and_raise(
+                key=index, value=None, cause=e, type_override=ConfigTypeError
+            )
+            assert False
+        except Exception as e:
+            self._format_and_raise(key=index, value=None, cause=e)
+            assert False
+
+    def sort(
+        self, key: Optional[Callable[[Any], Any]] = None, reverse: bool = False
+    ) -> None:
+        try:
+            if self._get_flag("readonly"):
+                raise ReadonlyConfigError("Cannot sort a read-only ListConfig")
+            if self._is_none():
+                raise TypeError("Cannot sort a ListConfig object representing None")
+            if self._is_missing():
+                raise MissingMandatoryValue("Cannot sort a missing ListConfig")
+
+            if key is None:
+
+                def key1(x: Any) -> Any:
+                    return x._value()
+
+            else:
+
+                def key1(x: Any) -> Any:
+                    return key(x._value())  # type: ignore
+
+            assert isinstance(self.__dict__["_content"], list)
+            self.__dict__["_content"].sort(key=key1, reverse=reverse)
+
+        except Exception as e:
+            self._format_and_raise(key=None, value=None, cause=e)
+            assert False
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, (list, tuple)) or other is None:
+            other = ListConfig(other, flags={"allow_objects": True})
+            return ListConfig._list_eq(self, other)
+        if other is None or isinstance(other, ListConfig):
+            return ListConfig._list_eq(self, other)
+        if self._is_missing():
+            return _is_missing_literal(other)
+        return NotImplemented
+
+    def __ne__(self, other: Any) -> bool:
+        x = self.__eq__(other)
+        if x is not NotImplemented:
+            return not x
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(str(self))
+
+    def __iter__(self) -> Iterator[Any]:
+        return self._iter_ex(resolve=True)
+
+    class ListIterator(Iterator[Any]):
+        def __init__(self, lst: Any, resolve: bool) -> None:
+            self.resolve = resolve
+            self.iterator = iter(lst.__dict__["_content"])
+            self.index = 0
+            from .nodes import ValueNode
+
+            self.ValueNode = ValueNode
+
+        def __next__(self) -> Any:
+
+            x = next(self.iterator)
+            if self.resolve:
+                x = x._dereference_node()
+                if x._is_missing():
+                    raise MissingMandatoryValue(f"Missing value at index {self.index}")
+
+            self.index = self.index + 1
+            if isinstance(x, self.ValueNode):
+                return x._value()
+            else:
+                # Must be omegaconf.Container. not checking for perf reasons.
+                if x._is_none():
+                    return None
+                return x
+
+        def __repr__(self) -> str:  # pragma: no cover
+            return f"ListConfig.ListIterator(resolve={self.resolve})"
+
+    def _iter_ex(self, resolve: bool) -> Iterator[Any]:
+        try:
+            if self._is_none():
+                raise TypeError("Cannot iterate a ListConfig object representing None")
+            if self._is_missing():
+                raise MissingMandatoryValue("Cannot iterate a missing ListConfig")
+
+            return ListConfig.ListIterator(self, resolve)
+        except (TypeError, MissingMandatoryValue) as e:
+            self._format_and_raise(key=None, value=None, cause=e)
+            assert False
+
+    def __add__(self, other: Union[List[Any], "ListConfig"]) -> "ListConfig":
+        # res is sharing this list's parent to allow interpolation to work as expected
+        res = ListConfig(parent=self._get_parent(), content=[])
+        res.extend(self)
+        res.extend(other)
+        return res
+
+    def __radd__(self, other: Union[List[Any], "ListConfig"]) -> "ListConfig":
+        # res is sharing this list's parent to allow interpolation to work as expected
+        res = ListConfig(parent=self._get_parent(), content=[])
+        res.extend(other)
+        res.extend(self)
+        return res
+
+    def __iadd__(self, other: Iterable[Any]) -> "ListConfig":
+        self.extend(other)
+        return self
+
+    def __contains__(self, item: Any) -> bool:
+        if self._is_none():
+            raise TypeError(
+                "Cannot check if an item is in a ListConfig object representing None"
+            )
+        if self._is_missing():
+            raise MissingMandatoryValue(
+                "Cannot check if an item is in missing ListConfig"
+            )
+
+        lst = self.__dict__["_content"]
+        for x in lst:
+            x = x._dereference_node()
+            if x == item:
+                return True
+        return False
+
+    def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None:
+        try:
+            previous_content = self.__dict__["_content"]
+            previous_metadata = self.__dict__["_metadata"]
+            self._set_value_impl(value, flags)
+        except Exception as e:
+            self.__dict__["_content"] = previous_content
+            self.__dict__["_metadata"] = previous_metadata
+            raise e
+
+    def _set_value_impl(
+        self, value: Any, flags: Optional[Dict[str, bool]] = None
+    ) -> None:
+        from omegaconf import MISSING, flag_override
+
+        if flags is None:
+            flags = {}
+
+        vk = get_value_kind(value, strict_interpolation_validation=True)
+        if _is_none(value):
+            if not self._is_optional():
+                raise ValidationError(
+                    "Non optional ListConfig cannot be constructed from None"
+                )
+            self.__dict__["_content"] = None
+            self._metadata.object_type = None
+        elif vk is ValueKind.MANDATORY_MISSING:
+            self.__dict__["_content"] = MISSING
+            self._metadata.object_type = None
+        elif vk == ValueKind.INTERPOLATION:
+            self.__dict__["_content"] = value
+            self._metadata.object_type = None
+        else:
+            if not (is_primitive_list(value) or isinstance(value, ListConfig)):
+                type_ = type(value)
+                msg = f"Invalid value assigned: {type_.__name__} is not a ListConfig, list or tuple."
+                raise ValidationError(msg)
+
+            self.__dict__["_content"] = []
+            if isinstance(value, ListConfig):
+                self._metadata.flags = copy.deepcopy(flags)
+                # disable struct and readonly for the construction phase
+                # retaining other flags like allow_objects. The real flags are restored at the end of this function
+                with flag_override(self, ["struct", "readonly"], False):
+                    for item in value._iter_ex(resolve=False):
+                        self.append(item)
+            elif is_primitive_list(value):
+                with flag_override(self, ["struct", "readonly"], False):
+                    for item in value:
+                        self.append(item)
+            self._metadata.object_type = list
+
+    @staticmethod
+    def _list_eq(l1: Optional["ListConfig"], l2: Optional["ListConfig"]) -> bool:
+        l1_none = l1.__dict__["_content"] is None
+        l2_none = l2.__dict__["_content"] is None
+        if l1_none and l2_none:
+            return True
+        if l1_none != l2_none:
+            return False
+
+        assert isinstance(l1, ListConfig)
+        assert isinstance(l2, ListConfig)
+        if len(l1) != len(l2):
+            return False
+        for i in range(len(l1)):
+            if not BaseContainer._item_eq(l1, i, l2, i):
+                return False
+
+        return True

omegaconf/nodes.py

@@ -0,0 +1,545 @@
+import copy
+import math
+import sys
+from abc import abstractmethod
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, Optional, Type, Union
+
+from omegaconf._utils import (
+    ValueKind,
+    _is_interpolation,
+    get_type_of,
+    get_value_kind,
+    is_primitive_container,
+    type_str,
+)
+from omegaconf.base import Box, DictKeyType, Metadata, Node
+from omegaconf.errors import ReadonlyConfigError, UnsupportedValueType, ValidationError
+
+
+class ValueNode(Node):
+    _val: Any
+
+    def __init__(self, parent: Optional[Box], value: Any, metadata: Metadata):
+        from omegaconf import read_write
+
+        super().__init__(parent=parent, metadata=metadata)
+        with read_write(self):
+            self._set_value(value)  # lgtm [py/init-calls-subclass]
+
+    def _value(self) -> Any:
+        return self._val
+
+    def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None:
+        if self._get_flag("readonly"):
+            raise ReadonlyConfigError("Cannot set value of read-only config node")
+
+        if isinstance(value, str) and get_value_kind(
+            value, strict_interpolation_validation=True
+        ) in (
+            ValueKind.INTERPOLATION,
+            ValueKind.MANDATORY_MISSING,
+        ):
+            self._val = value
+        else:
+            self._val = self.validate_and_convert(value)
+
+    def _strict_validate_type(self, value: Any) -> None:
+        ref_type = self._metadata.ref_type
+        if isinstance(ref_type, type) and type(value) is not ref_type:
+            type_hint = type_str(self._metadata.type_hint)
+            raise ValidationError(
+                f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_hint}'"
+            )
+
+    def validate_and_convert(self, value: Any) -> Any:
+        """
+        Validates input and converts to canonical form
+        :param value: input value
+        :return: converted value ("100" may be converted to 100 for example)
+        """
+        if value is None:
+            if self._is_optional():
+                return None
+            ref_type_str = type_str(self._metadata.ref_type)
+            raise ValidationError(
+                f"Incompatible value '{value}' for field of type '{ref_type_str}'"
+            )
+
+        # Subclasses can assume that `value` is not None in
+        # `_validate_and_convert_impl()` and in `_strict_validate_type()`.
+        if self._get_flag("convert") is False:
+            self._strict_validate_type(value)
+            return value
+        else:
+            return self._validate_and_convert_impl(value)
+
+    @abstractmethod
+    def _validate_and_convert_impl(self, value: Any) -> Any:
+        ...
+
+    def __str__(self) -> str:
+        return str(self._val)
+
+    def __repr__(self) -> str:
+        return repr(self._val) if hasattr(self, "_val") else "__INVALID__"
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, AnyNode):
+            return self._val == other._val  # type: ignore
+        else:
+            return self._val == other  # type: ignore
+
+    def __ne__(self, other: Any) -> bool:
+        x = self.__eq__(other)
+        assert x is not NotImplemented
+        return not x
+
+    def __hash__(self) -> int:
+        return hash(self._val)
+
+    def _deepcopy_impl(self, res: Any, memo: Dict[int, Any]) -> None:
+        res.__dict__["_metadata"] = copy.deepcopy(self._metadata, memo=memo)
+        # shallow copy for value to support non-copyable value
+        res.__dict__["_val"] = self._val
+
+        # parent is retained, but not copied
+        res.__dict__["_parent"] = self._parent
+
+    def _is_optional(self) -> bool:
+        return self._metadata.optional
+
+    def _is_interpolation(self) -> bool:
+        return _is_interpolation(self._value())
+
+    def _get_full_key(self, key: Optional[Union[DictKeyType, int]]) -> str:
+        parent = self._get_parent()
+        if parent is None:
+            if self._metadata.key is None:
+                return ""
+            else:
+                return str(self._metadata.key)
+        else:
+            return parent._get_full_key(self._metadata.key)
+
+
+class AnyNode(ValueNode):
+    def __init__(
+        self,
+        value: Any = None,
+        key: Any = None,
+        parent: Optional[Box] = None,
+        flags: Optional[Dict[str, bool]] = None,
+    ):
+        super().__init__(
+            parent=parent,
+            value=value,
+            metadata=Metadata(
+                ref_type=Any, object_type=None, key=key, optional=True, flags=flags
+            ),
+        )
+
+    def _validate_and_convert_impl(self, value: Any) -> Any:
+        from ._utils import is_primitive_type_annotation
+
+        # allow_objects is internal and not an official API. use at your own risk.
+        # Please be aware that this support is subject to change without notice.
+        # If this is deemed useful and supportable it may become an official API.
+
+        if self._get_flag(
+            "allow_objects"
+        ) is not True and not is_primitive_type_annotation(value):
+            t = get_type_of(value)
+            raise UnsupportedValueType(
+                f"Value '{t.__name__}' is not a supported primitive type"
+            )
+        return value
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "AnyNode":
+        res = AnyNode()
+        self._deepcopy_impl(res, memo)
+        return res
+
+
+class StringNode(ValueNode):
+    def __init__(
+        self,
+        value: Any = None,
+        key: Any = None,
+        parent: Optional[Box] = None,
+        is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
+    ):
+        super().__init__(
+            parent=parent,
+            value=value,
+            metadata=Metadata(
+                key=key,
+                optional=is_optional,
+                ref_type=str,
+                object_type=str,
+                flags=flags,
+            ),
+        )
+
+    def _validate_and_convert_impl(self, value: Any) -> str:
+        from omegaconf import OmegaConf
+
+        if (
+            OmegaConf.is_config(value)
+            or is_primitive_container(value)
+            or isinstance(value, bytes)
+        ):
+            raise ValidationError("Cannot convert '$VALUE_TYPE' to string: '$VALUE'")
+        return str(value)
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "StringNode":
+        res = StringNode()
+        self._deepcopy_impl(res, memo)
+        return res
+
+
+class PathNode(ValueNode):
+    def __init__(
+        self,
+        value: Any = None,
+        key: Any = None,
+        parent: Optional[Box] = None,
+        is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
+    ):
+        super().__init__(
+            parent=parent,
+            value=value,
+            metadata=Metadata(
+                key=key,
+                optional=is_optional,
+                ref_type=Path,
+                object_type=Path,
+                flags=flags,
+            ),
+        )
+
+    def _strict_validate_type(self, value: Any) -> None:
+        if not isinstance(value, Path):
+            raise ValidationError(
+                "Value '$VALUE' of type '$VALUE_TYPE' is not an instance of 'pathlib.Path'"
+            )
+
+    def _validate_and_convert_impl(self, value: Any) -> Path:
+        if not isinstance(value, (str, Path)):
+            raise ValidationError(
+                "Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Path"
+            )
+
+        return Path(value)
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "PathNode":
+        res = PathNode()
+        self._deepcopy_impl(res, memo)
+        return res
+
+
+class IntegerNode(ValueNode):
+    def __init__(
+        self,
+        value: Any = None,
+        key: Any = None,
+        parent: Optional[Box] = None,
+        is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
+    ):
+        super().__init__(
+            parent=parent,
+            value=value,
+            metadata=Metadata(
+                key=key,
+                optional=is_optional,
+                ref_type=int,
+                object_type=int,
+                flags=flags,
+            ),
+        )
+
+    def _validate_and_convert_impl(self, value: Any) -> int:
+        try:
+            if type(value) in (str, int):
+                val = int(value)
+            else:
+                raise ValueError()
+        except ValueError:
+            raise ValidationError(
+                "Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Integer"
+            )
+        return val
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "IntegerNode":
+        res = IntegerNode()
+        self._deepcopy_impl(res, memo)
+        return res
+
+
+class BytesNode(ValueNode):
+    def __init__(
+        self,
+        value: Any = None,
+        key: Any = None,
+        parent: Optional[Box] = None,
+        is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
+    ):
+        super().__init__(
+            parent=parent,
+            value=value,
+            metadata=Metadata(
+                key=key,
+                optional=is_optional,
+                ref_type=bytes,
+                object_type=bytes,
+                flags=flags,
+            ),
+        )
+
+    def _validate_and_convert_impl(self, value: Any) -> bytes:
+        if not isinstance(value, bytes):
+            raise ValidationError(
+                "Value '$VALUE' of type '$VALUE_TYPE' is not of type 'bytes'"
+            )
+        return value
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "BytesNode":
+        res = BytesNode()
+        self._deepcopy_impl(res, memo)
+        return res
+
+
+class FloatNode(ValueNode):
+    def __init__(
+        self,
+        value: Any = None,
+        key: Any = None,
+        parent: Optional[Box] = None,
+        is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
+    ):
+        super().__init__(
+            parent=parent,
+            value=value,
+            metadata=Metadata(
+                key=key,
+                optional=is_optional,
+                ref_type=float,
+                object_type=float,
+                flags=flags,
+            ),
+        )
+
+    def _validate_and_convert_impl(self, value: Any) -> float:
+        try:
+            if type(value) in (float, str, int):
+                return float(value)
+            else:
+                raise ValueError()
+        except ValueError:
+            raise ValidationError(
+                "Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Float"
+            )
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, ValueNode):
+            other_val = other._val
+        else:
+            other_val = other
+        if self._val is None and other is None:
+            return True
+        if self._val is None and other is not None:
+            return False
+        if self._val is not None and other is None:
+            return False
+        nan1 = math.isnan(self._val) if isinstance(self._val, float) else False
+        nan2 = math.isnan(other_val) if isinstance(other_val, float) else False
+        return self._val == other_val or (nan1 and nan2)
+
+    def __hash__(self) -> int:
+        return hash(self._val)
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "FloatNode":
+        res = FloatNode()
+        self._deepcopy_impl(res, memo)
+        return res
+
+
+class BooleanNode(ValueNode):
+    def __init__(
+        self,
+        value: Any = None,
+        key: Any = None,
+        parent: Optional[Box] = None,
+        is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
+    ):
+        super().__init__(
+            parent=parent,
+            value=value,
+            metadata=Metadata(
+                key=key,
+                optional=is_optional,
+                ref_type=bool,
+                object_type=bool,
+                flags=flags,
+            ),
+        )
+
+    def _validate_and_convert_impl(self, value: Any) -> bool:
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, int):
+            return value != 0
+        elif isinstance(value, str):
+            try:
+                return self._validate_and_convert_impl(int(value))
+            except ValueError as e:
+                if value.lower() in ("yes", "y", "on", "true"):
+                    return True
+                elif value.lower() in ("no", "n", "off", "false"):
+                    return False
+                else:
+                    raise ValidationError(
+                        "Value '$VALUE' is not a valid bool (type $VALUE_TYPE)"
+                    ).with_traceback(sys.exc_info()[2]) from e
+        else:
+            raise ValidationError(
+                "Value '$VALUE' is not a valid bool (type $VALUE_TYPE)"
+            )
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "BooleanNode":
+        res = BooleanNode()
+        self._deepcopy_impl(res, memo)
+        return res
+
+
+class EnumNode(ValueNode):  # lgtm [py/missing-equals] : Intentional.
+    """
+    NOTE: EnumNode is serialized to yaml as a string ("Color.BLUE"), not as a fully qualified yaml type.
+    this means serialization to YAML of a typed config (with EnumNode) will not retain the type of the Enum
+    when loaded.
+    This is intentional, Please open an issue against OmegaConf if you wish to discuss this decision.
+    """
+
+    def __init__(
+        self,
+        enum_type: Type[Enum],
+        value: Optional[Union[Enum, str]] = None,
+        key: Any = None,
+        parent: Optional[Box] = None,
+        is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
+    ):
+        if not isinstance(enum_type, type) or not issubclass(enum_type, Enum):
+            raise ValidationError(
+                f"EnumNode can only operate on Enum subclasses ({enum_type})"
+            )
+        self.fields: Dict[str, str] = {}
+        self.enum_type: Type[Enum] = enum_type
+        for name, constant in enum_type.__members__.items():
+            self.fields[name] = constant.value
+        super().__init__(
+            parent=parent,
+            value=value,
+            metadata=Metadata(
+                key=key,
+                optional=is_optional,
+                ref_type=enum_type,
+                object_type=enum_type,
+                flags=flags,
+            ),
+        )
+
+    def _strict_validate_type(self, value: Any) -> None:
+        ref_type = self._metadata.ref_type
+        if not isinstance(value, ref_type):
+            type_hint = type_str(self._metadata.type_hint)
+            raise ValidationError(
+                f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_hint}'"
+            )
+
+    def _validate_and_convert_impl(self, value: Any) -> Enum:
+        return self.validate_and_convert_to_enum(enum_type=self.enum_type, value=value)
+
+    @staticmethod
+    def validate_and_convert_to_enum(enum_type: Type[Enum], value: Any) -> Enum:
+        if not isinstance(value, (str, int)) and not isinstance(value, enum_type):
+            raise ValidationError(
+                f"Value $VALUE ($VALUE_TYPE) is not a valid input for {enum_type}"
+            )
+
+        if isinstance(value, enum_type):
+            return value
+
+        try:
+            if isinstance(value, (float, bool)):
+                raise ValueError
+
+            if isinstance(value, int):
+                return enum_type(value)
+
+            if isinstance(value, str):
+                prefix = f"{enum_type.__name__}."
+                if value.startswith(prefix):
+                    value = value[len(prefix) :]
+                return enum_type[value]
+
+            assert False
+
+        except (ValueError, KeyError) as e:
+            valid = ", ".join([x for x in enum_type.__members__.keys()])
+            raise ValidationError(
+                f"Invalid value '$VALUE', expected one of [{valid}]"
+            ).with_traceback(sys.exc_info()[2]) from e
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "EnumNode":
+        res = EnumNode(enum_type=self.enum_type)
+        self._deepcopy_impl(res, memo)
+        return res
+
+
+class InterpolationResultNode(ValueNode):
+    """
+    Special node type, used to wrap interpolation results.
+    """
+
+    def __init__(
+        self,
+        value: Any,
+        key: Any = None,
+        parent: Optional[Box] = None,
+        flags: Optional[Dict[str, bool]] = None,
+    ):
+        super().__init__(
+            parent=parent,
+            value=value,
+            metadata=Metadata(
+                ref_type=Any, object_type=None, key=key, optional=True, flags=flags
+            ),
+        )
+        # In general we should not try to write into interpolation results.
+        if flags is None or "readonly" not in flags:
+            self._set_flag("readonly", True)
+
+    def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None:
+        if self._get_flag("readonly"):
+            raise ReadonlyConfigError("Cannot set value of read-only config node")
+        self._val = self.validate_and_convert(value)
+
+    def _validate_and_convert_impl(self, value: Any) -> Any:
+        # Interpolation results may be anything.
+        return value
+
+    def __deepcopy__(self, memo: Dict[int, Any]) -> "InterpolationResultNode":
+        # Currently there should be no need to deep-copy such nodes.
+        raise NotImplementedError
+
+    def _is_interpolation(self) -> bool:
+        # The result of an interpolation cannot be itself an interpolation.
+        return False

omegaconf/omegaconf.py

@@ -0,0 +1,1157 @@
+"""OmegaConf module"""
+import copy
+import inspect
+import io
+import os
+import pathlib
+import sys
+import warnings
+from collections import defaultdict
+from contextlib import contextmanager
+from enum import Enum
+from textwrap import dedent
+from typing import (
+    IO,
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+    Union,
+    overload,
+)
+
+import yaml
+
+from . import DictConfig, DictKeyType, ListConfig
+from ._utils import (
+    _DEFAULT_MARKER_,
+    _ensure_container,
+    _get_value,
+    format_and_raise,
+    get_dict_key_value_types,
+    get_list_element_type,
+    get_omega_conf_dumper,
+    get_type_of,
+    is_attr_class,
+    is_dataclass,
+    is_dict_annotation,
+    is_int,
+    is_list_annotation,
+    is_primitive_container,
+    is_primitive_dict,
+    is_primitive_list,
+    is_structured_config,
+    is_tuple_annotation,
+    is_union_annotation,
+    nullcontext,
+    split_key,
+    type_str,
+)
+from .base import Box, Container, Node, SCMode, UnionNode
+from .basecontainer import BaseContainer
+from .errors import (
+    MissingMandatoryValue,
+    OmegaConfBaseException,
+    UnsupportedInterpolationType,
+    ValidationError,
+)
+from .nodes import (
+    AnyNode,
+    BooleanNode,
+    BytesNode,
+    EnumNode,
+    FloatNode,
+    IntegerNode,
+    PathNode,
+    StringNode,
+    ValueNode,
+)
+
+MISSING: Any = "???"
+
+Resolver = Callable[..., Any]
+
+
+def II(interpolation: str) -> Any:
+    """
+    Equivalent to ``${interpolation}``
+
+    :param interpolation:
+    :return: input ``${node}`` with type Any
+    """
+    return "${" + interpolation + "}"
+
+
+def SI(interpolation: str) -> Any:
+    """
+    Use this for String interpolation, for example ``"http://${host}:${port}"``
+
+    :param interpolation: interpolation string
+    :return: input interpolation with type ``Any``
+    """
+    return interpolation
+
+
+def register_default_resolvers() -> None:
+    from omegaconf.resolvers import oc
+
+    OmegaConf.register_new_resolver("oc.create", oc.create)
+    OmegaConf.register_new_resolver("oc.decode", oc.decode)
+    OmegaConf.register_new_resolver("oc.deprecated", oc.deprecated)
+    OmegaConf.register_new_resolver("oc.env", oc.env)
+    OmegaConf.register_new_resolver("oc.select", oc.select)
+    OmegaConf.register_new_resolver("oc.dict.keys", oc.dict.keys)
+    OmegaConf.register_new_resolver("oc.dict.values", oc.dict.values)
+
+
+class OmegaConf:
+    """OmegaConf primary class"""
+
+    def __init__(self) -> None:
+        raise NotImplementedError("Use one of the static construction functions")
+
+    @staticmethod
+    def structured(
+        obj: Any,
+        parent: Optional[BaseContainer] = None,
+        flags: Optional[Dict[str, bool]] = None,
+    ) -> Any:
+        return OmegaConf.create(obj, parent, flags)
+
+    @staticmethod
+    @overload
+    def create(
+        obj: str,
+        parent: Optional[BaseContainer] = None,
+        flags: Optional[Dict[str, bool]] = None,
+    ) -> Union[DictConfig, ListConfig]:
+        ...
+
+    @staticmethod
+    @overload
+    def create(
+        obj: Union[List[Any], Tuple[Any, ...]],
+        parent: Optional[BaseContainer] = None,
+        flags: Optional[Dict[str, bool]] = None,
+    ) -> ListConfig:
+        ...
+
+    @staticmethod
+    @overload
+    def create(
+        obj: DictConfig,
+        parent: Optional[BaseContainer] = None,
+        flags: Optional[Dict[str, bool]] = None,
+    ) -> DictConfig:
+        ...
+
+    @staticmethod
+    @overload
+    def create(
+        obj: ListConfig,
+        parent: Optional[BaseContainer] = None,
+        flags: Optional[Dict[str, bool]] = None,
+    ) -> ListConfig:
+        ...
+
+    @staticmethod
+    @overload
+    def create(
+        obj: Optional[Dict[Any, Any]] = None,
+        parent: Optional[BaseContainer] = None,
+        flags: Optional[Dict[str, bool]] = None,
+    ) -> DictConfig:
+        ...
+
+    @staticmethod
+    def create(  # noqa F811
+        obj: Any = _DEFAULT_MARKER_,
+        parent: Optional[BaseContainer] = None,
+        flags: Optional[Dict[str, bool]] = None,
+    ) -> Union[DictConfig, ListConfig]:
+        return OmegaConf._create_impl(
+            obj=obj,
+            parent=parent,
+            flags=flags,
+        )
+
+    @staticmethod
+    def load(file_: Union[str, pathlib.Path, IO[Any]]) -> Union[DictConfig, ListConfig]:
+        from ._utils import get_yaml_loader
+
+        if isinstance(file_, (str, pathlib.Path)):
+            with io.open(os.path.abspath(file_), "r", encoding="utf-8") as f:
+                obj = yaml.load(f, Loader=get_yaml_loader())
+        elif getattr(file_, "read", None):
+            obj = yaml.load(file_, Loader=get_yaml_loader())
+        else:
+            raise TypeError("Unexpected file type")
+
+        if obj is not None and not isinstance(obj, (list, dict, str)):
+            raise IOError(  # pragma: no cover
+                f"Invalid loaded object type: {type(obj).__name__}"
+            )
+
+        ret: Union[DictConfig, ListConfig]
+        if obj is None:
+            ret = OmegaConf.create()
+        else:
+            ret = OmegaConf.create(obj)
+        return ret
+
+    @staticmethod
+    def save(
+        config: Any, f: Union[str, pathlib.Path, IO[Any]], resolve: bool = False
+    ) -> None:
+        """
+        Save as configuration object to a file
+
+        :param config: omegaconf.Config object (DictConfig or ListConfig).
+        :param f: filename or file object
+        :param resolve: True to save a resolved config (defaults to False)
+        """
+        if is_dataclass(config) or is_attr_class(config):
+            config = OmegaConf.create(config)
+        data = OmegaConf.to_yaml(config, resolve=resolve)
+        if isinstance(f, (str, pathlib.Path)):
+            with io.open(os.path.abspath(f), "w", encoding="utf-8") as file:
+                file.write(data)
+        elif hasattr(f, "write"):
+            f.write(data)
+            f.flush()
+        else:
+            raise TypeError("Unexpected file type")
+
+    @staticmethod
+    def from_cli(args_list: Optional[List[str]] = None) -> DictConfig:
+        if args_list is None:
+            # Skip program name
+            args_list = sys.argv[1:]
+        return OmegaConf.from_dotlist(args_list)
+
+    @staticmethod
+    def from_dotlist(dotlist: List[str]) -> DictConfig:
+        """
+        Creates config from the content sys.argv or from the specified args list of not None
+
+        :param dotlist: A list of dotlist-style strings, e.g. ``["foo.bar=1", "baz=qux"]``.
+        :return: A ``DictConfig`` object created from the dotlist.
+        """
+        conf = OmegaConf.create()
+        conf.merge_with_dotlist(dotlist)
+        return conf
+
+    @staticmethod
+    def merge(
+        *configs: Union[
+            DictConfig,
+            ListConfig,
+            Dict[DictKeyType, Any],
+            List[Any],
+            Tuple[Any, ...],
+            Any,
+        ],
+    ) -> Union[ListConfig, DictConfig]:
+        """
+        Merge a list of previously created configs into a single one
+
+        :param configs: Input configs
+        :return: the merged config object.
+        """
+        assert len(configs) > 0
+        target = copy.deepcopy(configs[0])
+        target = _ensure_container(target)
+        assert isinstance(target, (DictConfig, ListConfig))
+
+        with flag_override(target, "readonly", False):
+            target.merge_with(*configs[1:])
+            turned_readonly = target._get_flag("readonly") is True
+
+        if turned_readonly:
+            OmegaConf.set_readonly(target, True)
+
+        return target
+
+    @staticmethod
+    def unsafe_merge(
+        *configs: Union[
+            DictConfig,
+            ListConfig,
+            Dict[DictKeyType, Any],
+            List[Any],
+            Tuple[Any, ...],
+            Any,
+        ],
+    ) -> Union[ListConfig, DictConfig]:
+        """
+        Merge a list of previously created configs into a single one
+        This is much faster than OmegaConf.merge() as the input configs are not copied.
+        However, the input configs must not be used after this operation as will become inconsistent.
+
+        :param configs: Input configs
+        :return: the merged config object.
+        """
+        assert len(configs) > 0
+        target = configs[0]
+        target = _ensure_container(target)
+        assert isinstance(target, (DictConfig, ListConfig))
+
+        with flag_override(
+            target, ["readonly", "no_deepcopy_set_nodes"], [False, True]
+        ):
+            target.merge_with(*configs[1:])
+            turned_readonly = target._get_flag("readonly") is True
+
+        if turned_readonly:
+            OmegaConf.set_readonly(target, True)
+
+        return target
+
+    @staticmethod
+    def register_resolver(name: str, resolver: Resolver) -> None:
+        warnings.warn(
+            dedent(
+                """\
+            register_resolver() is deprecated.
+            See https://github.com/omry/omegaconf/issues/426 for migration instructions.
+            """
+            ),
+            stacklevel=2,
+        )
+        return OmegaConf.legacy_register_resolver(name, resolver)
+
+    # This function will eventually be deprecated and removed.
+    @staticmethod
+    def legacy_register_resolver(name: str, resolver: Resolver) -> None:
+        assert callable(resolver), "resolver must be callable"
+        # noinspection PyProtectedMember
+        assert (
+            name not in BaseContainer._resolvers
+        ), f"resolver '{name}' is already registered"
+
+        def resolver_wrapper(
+            config: BaseContainer,
+            parent: BaseContainer,
+            node: Node,
+            args: Tuple[Any, ...],
+            args_str: Tuple[str, ...],
+        ) -> Any:
+            cache = OmegaConf.get_cache(config)[name]
+            # "Un-escape " spaces and commas.
+            args_unesc = [x.replace(r"\ ", " ").replace(r"\,", ",") for x in args_str]
+
+            # Nested interpolations behave in a potentially surprising way with
+            # legacy resolvers (they remain as strings, e.g., "${foo}"). If any
+            # input looks like an interpolation we thus raise an exception.
+            try:
+                bad_arg = next(i for i in args_unesc if "${" in i)
+            except StopIteration:
+                pass
+            else:
+                raise ValueError(
+                    f"Resolver '{name}' was called with argument '{bad_arg}' that appears "
+                    f"to be an interpolation. Nested interpolations are not supported for "
+                    f"resolvers registered with `[legacy_]register_resolver()`, please use "
+                    f"`register_new_resolver()` instead (see "
+                    f"https://github.com/omry/omegaconf/issues/426 for migration instructions)."
+                )
+            key = args_str
+            val = cache[key] if key in cache else resolver(*args_unesc)
+            cache[key] = val
+            return val
+
+        # noinspection PyProtectedMember
+        BaseContainer._resolvers[name] = resolver_wrapper
+
+    @staticmethod
+    def register_new_resolver(
+        name: str,
+        resolver: Resolver,
+        *,
+        replace: bool = False,
+        use_cache: bool = False,
+    ) -> None:
+        """
+        Register a resolver.
+
+        :param name: Name of the resolver.
+        :param resolver: Callable whose arguments are provided in the interpolation,
+            e.g., with ${foo:x,0,${y.z}} these arguments are respectively "x" (str),
+            0 (int) and the value of ``y.z``.
+        :param replace: If set to ``False`` (default), then a ``ValueError`` is raised if
+            an existing resolver has already been registered with the same name.
+            If set to ``True``, then the new resolver replaces the previous one.
+            NOTE: The cache on existing config objects is not affected, use
+            ``OmegaConf.clear_cache(cfg)`` to clear it.
+        :param use_cache: Whether the resolver's outputs should be cached. The cache is
+            based only on the string literals representing the resolver arguments, e.g.,
+            ${foo:${bar}} will always return the same value regardless of the value of
+            ``bar`` if the cache is enabled for ``foo``.
+        """
+        if not callable(resolver):
+            raise TypeError("resolver must be callable")
+        if not name:
+            raise ValueError("cannot use an empty resolver name")
+
+        if not replace and OmegaConf.has_resolver(name):
+            raise ValueError(f"resolver '{name}' is already registered")
+
+        try:
+            sig: Optional[inspect.Signature] = inspect.signature(resolver)
+        except ValueError:
+            sig = None
+
+        def _should_pass(special: str) -> bool:
+            ret = sig is not None and special in sig.parameters
+            if ret and use_cache:
+                raise ValueError(
+                    f"use_cache=True is incompatible with functions that receive the {special}"
+                )
+            return ret
+
+        pass_parent = _should_pass("_parent_")
+        pass_node = _should_pass("_node_")
+        pass_root = _should_pass("_root_")
+
+        def resolver_wrapper(
+            config: BaseContainer,
+            parent: Container,
+            node: Node,
+            args: Tuple[Any, ...],
+            args_str: Tuple[str, ...],
+        ) -> Any:
+            if use_cache:
+                cache = OmegaConf.get_cache(config)[name]
+                try:
+                    return cache[args_str]
+                except KeyError:
+                    pass
+
+            # Call resolver.
+            kwargs: Dict[str, Node] = {}
+            if pass_parent:
+                kwargs["_parent_"] = parent
+            if pass_node:
+                kwargs["_node_"] = node
+            if pass_root:
+                kwargs["_root_"] = config
+
+            ret = resolver(*args, **kwargs)
+
+            if use_cache:
+                cache[args_str] = ret
+            return ret
+
+        # noinspection PyProtectedMember
+        BaseContainer._resolvers[name] = resolver_wrapper
+
+    @classmethod
+    def has_resolver(cls, name: str) -> bool:
+        return cls._get_resolver(name) is not None
+
+    # noinspection PyProtectedMember
+    @staticmethod
+    def clear_resolvers() -> None:
+        """
+        Clear(remove) all OmegaConf resolvers, then re-register OmegaConf's default resolvers.
+        """
+        BaseContainer._resolvers = {}
+        register_default_resolvers()
+
+    @classmethod
+    def clear_resolver(cls, name: str) -> bool:
+        """
+        Clear(remove) any resolver only if it exists.
+
+        Returns a bool: True if resolver is removed and False if not removed.
+
+        .. warning:
+            This method can remove deafult resolvers as well.
+
+        :param name: Name of the resolver.
+        :return: A bool (``True`` if resolver is removed, ``False`` if not found before removing).
+        """
+        if cls.has_resolver(name):
+            BaseContainer._resolvers.pop(name)
+            return True
+        else:
+            # return False if resolver does not exist
+            return False
+
+    @staticmethod
+    def get_cache(conf: BaseContainer) -> Dict[str, Any]:
+        return conf._metadata.resolver_cache
+
+    @staticmethod
+    def set_cache(conf: BaseContainer, cache: Dict[str, Any]) -> None:
+        conf._metadata.resolver_cache = copy.deepcopy(cache)
+
+    @staticmethod
+    def clear_cache(conf: BaseContainer) -> None:
+        OmegaConf.set_cache(conf, defaultdict(dict, {}))
+
+    @staticmethod
+    def copy_cache(from_config: BaseContainer, to_config: BaseContainer) -> None:
+        OmegaConf.set_cache(to_config, OmegaConf.get_cache(from_config))
+
+    @staticmethod
+    def set_readonly(conf: Node, value: Optional[bool]) -> None:
+        # noinspection PyProtectedMember
+        conf._set_flag("readonly", value)
+
+    @staticmethod
+    def is_readonly(conf: Node) -> Optional[bool]:
+        # noinspection PyProtectedMember
+        return conf._get_flag("readonly")
+
+    @staticmethod
+    def set_struct(conf: Container, value: Optional[bool]) -> None:
+        # noinspection PyProtectedMember
+        conf._set_flag("struct", value)
+
+    @staticmethod
+    def is_struct(conf: Container) -> Optional[bool]:
+        # noinspection PyProtectedMember
+        return conf._get_flag("struct")
+
+    @staticmethod
+    def masked_copy(conf: DictConfig, keys: Union[str, List[str]]) -> DictConfig:
+        """
+        Create a masked copy of of this config that contains a subset of the keys
+
+        :param conf: DictConfig object
+        :param keys: keys to preserve in the copy
+        :return: The masked ``DictConfig`` object.
+        """
+        from .dictconfig import DictConfig
+
+        if not isinstance(conf, DictConfig):
+            raise ValueError("masked_copy is only supported for DictConfig")
+
+        if isinstance(keys, str):
+            keys = [keys]
+        content = {key: value for key, value in conf.items_ex(resolve=False, keys=keys)}
+        return DictConfig(content=content)
+
+    @staticmethod
+    def to_container(
+        cfg: Any,
+        *,
+        resolve: bool = False,
+        throw_on_missing: bool = False,
+        enum_to_str: bool = False,
+        structured_config_mode: SCMode = SCMode.DICT,
+    ) -> Union[Dict[DictKeyType, Any], List[Any], None, str, Any]:
+        """
+        Resursively converts an OmegaConf config to a primitive container (dict or list).
+
+        :param cfg: the config to convert
+        :param resolve: True to resolve all values
+        :param throw_on_missing: When True, raise MissingMandatoryValue if any missing values are present.
+            When False (the default), replace missing values with the string "???" in the output container.
+        :param enum_to_str: True to convert Enum keys and values to strings
+        :param structured_config_mode: Specify how Structured Configs (DictConfigs backed by a dataclass) are handled.
+            - By default (``structured_config_mode=SCMode.DICT``) structured configs are converted to plain dicts.
+            - If ``structured_config_mode=SCMode.DICT_CONFIG``, structured config nodes will remain as DictConfig.
+            - If ``structured_config_mode=SCMode.INSTANTIATE``, this function will instantiate structured configs
+              (DictConfigs backed by a dataclass), by creating an instance of the underlying dataclass.
+
+          See also OmegaConf.to_object.
+        :return: A dict or a list representing this config as a primitive container.
+        """
+        if not OmegaConf.is_config(cfg):
+            raise ValueError(
+                f"Input cfg is not an OmegaConf config object ({type_str(type(cfg))})"
+            )
+
+        return BaseContainer._to_content(
+            cfg,
+            resolve=resolve,
+            throw_on_missing=throw_on_missing,
+            enum_to_str=enum_to_str,
+            structured_config_mode=structured_config_mode,
+        )
+
+    @staticmethod
+    def to_object(cfg: Any) -> Union[Dict[DictKeyType, Any], List[Any], None, str, Any]:
+        """
+        Resursively converts an OmegaConf config to a primitive container (dict or list).
+        Any DictConfig objects backed by dataclasses or attrs classes are instantiated
+        as instances of those backing classes.
+
+        This is an alias for OmegaConf.to_container(..., resolve=True, throw_on_missing=True,
+                                                    structured_config_mode=SCMode.INSTANTIATE)
+
+        :param cfg: the config to convert
+        :return: A dict or a list or dataclass representing this config.
+        """
+        return OmegaConf.to_container(
+            cfg=cfg,
+            resolve=True,
+            throw_on_missing=True,
+            enum_to_str=False,
+            structured_config_mode=SCMode.INSTANTIATE,
+        )
+
+    @staticmethod
+    def is_missing(cfg: Any, key: DictKeyType) -> bool:
+        assert isinstance(cfg, Container)
+        try:
+            node = cfg._get_child(key)
+            if node is None:
+                return False
+            assert isinstance(node, Node)
+            return node._is_missing()
+        except (UnsupportedInterpolationType, KeyError, AttributeError):
+            return False
+
+    @staticmethod
+    def is_interpolation(node: Any, key: Optional[Union[int, str]] = None) -> bool:
+        if key is not None:
+            assert isinstance(node, Container)
+            target = node._get_child(key)
+        else:
+            target = node
+        if target is not None:
+            assert isinstance(target, Node)
+            return target._is_interpolation()
+        return False
+
+    @staticmethod
+    def is_list(obj: Any) -> bool:
+        from . import ListConfig
+
+        return isinstance(obj, ListConfig)
+
+    @staticmethod
+    def is_dict(obj: Any) -> bool:
+        from . import DictConfig
+
+        return isinstance(obj, DictConfig)
+
+    @staticmethod
+    def is_config(obj: Any) -> bool:
+        from . import Container
+
+        return isinstance(obj, Container)
+
+    @staticmethod
+    def get_type(obj: Any, key: Optional[str] = None) -> Optional[Type[Any]]:
+        if key is not None:
+            c = obj._get_child(key)
+        else:
+            c = obj
+        return OmegaConf._get_obj_type(c)
+
+    @staticmethod
+    def select(
+        cfg: Container,
+        key: str,
+        *,
+        default: Any = _DEFAULT_MARKER_,
+        throw_on_resolution_failure: bool = True,
+        throw_on_missing: bool = False,
+    ) -> Any:
+        """
+        :param cfg: Config node to select from
+        :param key: Key to select
+        :param default: Default value to return if key is not found
+        :param throw_on_resolution_failure: Raise an exception if an interpolation
+               resolution error occurs, otherwise return None
+        :param throw_on_missing: Raise an exception if an attempt to select a missing key (with the value '???')
+               is made, otherwise return None
+        :return: selected value or None if not found.
+        """
+        from ._impl import select_value
+
+        try:
+            return select_value(
+                cfg=cfg,
+                key=key,
+                default=default,
+                throw_on_resolution_failure=throw_on_resolution_failure,
+                throw_on_missing=throw_on_missing,
+            )
+        except Exception as e:
+            format_and_raise(node=cfg, key=key, value=None, cause=e, msg=str(e))
+
+    @staticmethod
+    def update(
+        cfg: Container,
+        key: str,
+        value: Any = None,
+        *,
+        merge: bool = True,
+        force_add: bool = False,
+    ) -> None:
+        """
+        Updates a dot separated key sequence to a value
+
+        :param cfg: input config to update
+        :param key: key to update (can be a dot separated path)
+        :param value: value to set, if value if a list or a dict it will be merged or set
+            depending on merge_config_values
+        :param merge: If value is a dict or a list, True (default) to merge
+                      into the destination, False to replace the destination.
+        :param force_add: insert the entire path regardless of Struct flag or Structured Config nodes.
+        """
+
+        split = split_key(key)
+        root = cfg
+        for i in range(len(split) - 1):
+            k = split[i]
+            # if next_root is a primitive (string, int etc) replace it with an empty map
+            next_root, key_ = _select_one(root, k, throw_on_missing=False)
+            if not isinstance(next_root, Container):
+                if force_add:
+                    with flag_override(root, "struct", False):
+                        root[key_] = {}
+                else:
+                    root[key_] = {}
+            root = root[key_]
+
+        last = split[-1]
+
+        assert isinstance(
+            root, Container
+        ), f"Unexpected type for root: {type(root).__name__}"
+
+        last_key: Union[str, int] = last
+        if isinstance(root, ListConfig):
+            last_key = int(last)
+
+        ctx = flag_override(root, "struct", False) if force_add else nullcontext()
+        with ctx:
+            if merge and (OmegaConf.is_config(value) or is_primitive_container(value)):
+                assert isinstance(root, BaseContainer)
+                node = root._get_child(last_key)
+                if OmegaConf.is_config(node):
+                    assert isinstance(node, BaseContainer)
+                    node.merge_with(value)
+                    return
+
+            if OmegaConf.is_dict(root):
+                assert isinstance(last_key, str)
+                root.__setattr__(last_key, value)
+            elif OmegaConf.is_list(root):
+                assert isinstance(last_key, int)
+                root.__setitem__(last_key, value)
+            else:
+                assert False
+
+    @staticmethod
+    def to_yaml(cfg: Any, *, resolve: bool = False, sort_keys: bool = False) -> str:
+        """
+        returns a yaml dump of this config object.
+
+        :param cfg: Config object, Structured Config type or instance
+        :param resolve: if True, will return a string with the interpolations resolved, otherwise
+            interpolations are preserved
+        :param sort_keys: If True, will print dict keys in sorted order. default False.
+        :return: A string containing the yaml representation.
+        """
+        cfg = _ensure_container(cfg)
+        container = OmegaConf.to_container(cfg, resolve=resolve, enum_to_str=True)
+        return yaml.dump(  # type: ignore
+            container,
+            default_flow_style=False,
+            allow_unicode=True,
+            sort_keys=sort_keys,
+            Dumper=get_omega_conf_dumper(),
+        )
+
+    @staticmethod
+    def resolve(cfg: Container) -> None:
+        """
+        Resolves all interpolations in the given config object in-place.
+
+        :param cfg: An OmegaConf container (DictConfig, ListConfig)
+                    Raises a ValueError if the input object is not an OmegaConf container.
+        """
+        import omegaconf._impl
+
+        if not OmegaConf.is_config(cfg):
+            # Since this function is mutating the input object in-place, it doesn't make sense to
+            # auto-convert the input object to an OmegaConf container
+            raise ValueError(
+                f"Invalid config type ({type(cfg).__name__}), expected an OmegaConf Container"
+            )
+        omegaconf._impl._resolve(cfg)
+
+    @staticmethod
+    def missing_keys(cfg: Any) -> Set[str]:
+        """
+        Returns a set of missing keys in a dotlist style.
+
+        :param cfg: An ``OmegaConf.Container``,
+                    or a convertible object via ``OmegaConf.create`` (dict, list, ...).
+        :return: set of strings of the missing keys.
+        :raises ValueError: On input not representing a config.
+        """
+        cfg = _ensure_container(cfg)
+        missings: Set[str] = set()
+
+        def gather(_cfg: Container) -> None:
+            itr: Iterable[Any]
+            if isinstance(_cfg, ListConfig):
+                itr = range(len(_cfg))
+            else:
+                itr = _cfg
+
+            for key in itr:
+                if OmegaConf.is_missing(_cfg, key):
+                    missings.add(_cfg._get_full_key(key))
+                elif OmegaConf.is_config(_cfg[key]):
+                    gather(_cfg[key])
+
+        gather(cfg)
+        return missings
+
+    # === private === #
+
+    @staticmethod
+    def _create_impl(  # noqa F811
+        obj: Any = _DEFAULT_MARKER_,
+        parent: Optional[BaseContainer] = None,
+        flags: Optional[Dict[str, bool]] = None,
+    ) -> Union[DictConfig, ListConfig]:
+        try:
+            from ._utils import get_yaml_loader
+            from .dictconfig import DictConfig
+            from .listconfig import ListConfig
+
+            if obj is _DEFAULT_MARKER_:
+                obj = {}
+            if isinstance(obj, str):
+                obj = yaml.load(obj, Loader=get_yaml_loader())
+                if obj is None:
+                    return OmegaConf.create({}, parent=parent, flags=flags)
+                elif isinstance(obj, str):
+                    return OmegaConf.create({obj: None}, parent=parent, flags=flags)
+                else:
+                    assert isinstance(obj, (list, dict))
+                    return OmegaConf.create(obj, parent=parent, flags=flags)
+
+            else:
+                if (
+                    is_primitive_dict(obj)
+                    or OmegaConf.is_dict(obj)
+                    or is_structured_config(obj)
+                    or obj is None
+                ):
+                    if isinstance(obj, DictConfig):
+                        return DictConfig(
+                            content=obj,
+                            parent=parent,
+                            ref_type=obj._metadata.ref_type,
+                            is_optional=obj._metadata.optional,
+                            key_type=obj._metadata.key_type,
+                            element_type=obj._metadata.element_type,
+                            flags=flags,
+                        )
+                    else:
+                        obj_type = OmegaConf.get_type(obj)
+                        key_type, element_type = get_dict_key_value_types(obj_type)
+                        return DictConfig(
+                            content=obj,
+                            parent=parent,
+                            key_type=key_type,
+                            element_type=element_type,
+                            flags=flags,
+                        )
+                elif is_primitive_list(obj) or OmegaConf.is_list(obj):
+                    if isinstance(obj, ListConfig):
+                        return ListConfig(
+                            content=obj,
+                            parent=parent,
+                            element_type=obj._metadata.element_type,
+                            ref_type=obj._metadata.ref_type,
+                            is_optional=obj._metadata.optional,
+                            flags=flags,
+                        )
+                    else:
+                        obj_type = OmegaConf.get_type(obj)
+                        element_type = get_list_element_type(obj_type)
+                        return ListConfig(
+                            content=obj,
+                            parent=parent,
+                            element_type=element_type,
+                            ref_type=Any,
+                            is_optional=True,
+                            flags=flags,
+                        )
+                else:
+                    if isinstance(obj, type):
+                        raise ValidationError(
+                            f"Input class '{obj.__name__}' is not a structured config. "
+                            "did you forget to decorate it as a dataclass?"
+                        )
+                    else:
+                        raise ValidationError(
+                            f"Object of unsupported type: '{type(obj).__name__}'"
+                        )
+        except OmegaConfBaseException as e:
+            format_and_raise(node=None, key=None, value=None, msg=str(e), cause=e)
+            assert False
+
+    @staticmethod
+    def _get_obj_type(c: Any) -> Optional[Type[Any]]:
+        if is_structured_config(c):
+            return get_type_of(c)
+        elif c is None:
+            return None
+        elif isinstance(c, DictConfig):
+            if c._is_none():
+                return None
+            elif c._is_missing():
+                return None
+            else:
+                if is_structured_config(c._metadata.object_type):
+                    return c._metadata.object_type
+                else:
+                    return dict
+        elif isinstance(c, ListConfig):
+            return list
+        elif isinstance(c, ValueNode):
+            return type(c._value())
+        elif isinstance(c, UnionNode):
+            return type(_get_value(c))
+        elif isinstance(c, dict):
+            return dict
+        elif isinstance(c, (list, tuple)):
+            return list
+        else:
+            return get_type_of(c)
+
+    @staticmethod
+    def _get_resolver(
+        name: str,
+    ) -> Optional[
+        Callable[
+            [Container, Container, Node, Tuple[Any, ...], Tuple[str, ...]],
+            Any,
+        ]
+    ]:
+        # noinspection PyProtectedMember
+        return (
+            BaseContainer._resolvers[name] if name in BaseContainer._resolvers else None
+        )
+
+
+# register all default resolvers
+register_default_resolvers()
+
+
+@contextmanager
+def flag_override(
+    config: Node,
+    names: Union[List[str], str],
+    values: Union[List[Optional[bool]], Optional[bool]],
+) -> Generator[Node, None, None]:
+    if isinstance(names, str):
+        names = [names]
+    if values is None or isinstance(values, bool):
+        values = [values]
+
+    prev_states = [config._get_node_flag(name) for name in names]
+
+    try:
+        config._set_flag(names, values)
+        yield config
+    finally:
+        config._set_flag(names, prev_states)
+
+
+@contextmanager
+def read_write(config: Node) -> Generator[Node, None, None]:
+    prev_state = config._get_node_flag("readonly")
+    try:
+        OmegaConf.set_readonly(config, False)
+        yield config
+    finally:
+        OmegaConf.set_readonly(config, prev_state)
+
+
+@contextmanager
+def open_dict(config: Container) -> Generator[Container, None, None]:
+    prev_state = config._get_node_flag("struct")
+    try:
+        OmegaConf.set_struct(config, False)
+        yield config
+    finally:
+        OmegaConf.set_struct(config, prev_state)
+
+
+# === private === #
+
+
+def _node_wrap(
+    parent: Optional[Box],
+    is_optional: bool,
+    value: Any,
+    key: Any,
+    ref_type: Any = Any,
+) -> Node:
+    node: Node
+    if is_dict_annotation(ref_type) or (is_primitive_dict(value) and ref_type is Any):
+        key_type, element_type = get_dict_key_value_types(ref_type)
+        node = DictConfig(
+            content=value,
+            key=key,
+            parent=parent,
+            ref_type=ref_type,
+            is_optional=is_optional,
+            key_type=key_type,
+            element_type=element_type,
+        )
+    elif (is_list_annotation(ref_type) or is_tuple_annotation(ref_type)) or (
+        type(value) in (list, tuple) and ref_type is Any
+    ):
+        element_type = get_list_element_type(ref_type)
+        node = ListConfig(
+            content=value,
+            key=key,
+            parent=parent,
+            is_optional=is_optional,
+            element_type=element_type,
+            ref_type=ref_type,
+        )
+    elif is_structured_config(ref_type) or is_structured_config(value):
+        key_type, element_type = get_dict_key_value_types(value)
+        node = DictConfig(
+            ref_type=ref_type,
+            is_optional=is_optional,
+            content=value,
+            key=key,
+            parent=parent,
+            key_type=key_type,
+            element_type=element_type,
+        )
+    elif is_union_annotation(ref_type):
+        node = UnionNode(
+            content=value,
+            ref_type=ref_type,
+            is_optional=is_optional,
+            key=key,
+            parent=parent,
+        )
+    elif ref_type == Any or ref_type is None:
+        node = AnyNode(value=value, key=key, parent=parent)
+    elif isinstance(ref_type, type) and issubclass(ref_type, Enum):
+        node = EnumNode(
+            enum_type=ref_type,
+            value=value,
+            key=key,
+            parent=parent,
+            is_optional=is_optional,
+        )
+    elif ref_type == int:
+        node = IntegerNode(value=value, key=key, parent=parent, is_optional=is_optional)
+    elif ref_type == float:
+        node = FloatNode(value=value, key=key, parent=parent, is_optional=is_optional)
+    elif ref_type == bool:
+        node = BooleanNode(value=value, key=key, parent=parent, is_optional=is_optional)
+    elif ref_type == str:
+        node = StringNode(value=value, key=key, parent=parent, is_optional=is_optional)
+    elif ref_type == bytes:
+        node = BytesNode(value=value, key=key, parent=parent, is_optional=is_optional)
+    elif ref_type == pathlib.Path:
+        node = PathNode(value=value, key=key, parent=parent, is_optional=is_optional)
+    else:
+        if parent is not None and parent._get_flag("allow_objects") is True:
+            if type(value) in (list, tuple):
+                node = ListConfig(
+                    content=value,
+                    key=key,
+                    parent=parent,
+                    ref_type=ref_type,
+                    is_optional=is_optional,
+                )
+            elif is_primitive_dict(value):
+                node = DictConfig(
+                    content=value,
+                    key=key,
+                    parent=parent,
+                    ref_type=ref_type,
+                    is_optional=is_optional,
+                )
+            else:
+                node = AnyNode(value=value, key=key, parent=parent)
+        else:
+            raise ValidationError(f"Unexpected type annotation: {type_str(ref_type)}")
+    return node
+
+
+def _maybe_wrap(
+    ref_type: Any,
+    key: Any,
+    value: Any,
+    is_optional: bool,
+    parent: Optional[BaseContainer],
+) -> Node:
+    # if already a node, update key and parent and return as is.
+    # NOTE: that this mutate the input node!
+    if isinstance(value, Node):
+        value._set_key(key)
+        value._set_parent(parent)
+        return value
+    else:
+        return _node_wrap(
+            ref_type=ref_type,
+            parent=parent,
+            is_optional=is_optional,
+            value=value,
+            key=key,
+        )
+
+
+def _select_one(
+    c: Container, key: str, throw_on_missing: bool, throw_on_type_error: bool = True
+) -> Tuple[Optional[Node], Union[str, int]]:
+    from .dictconfig import DictConfig
+    from .listconfig import ListConfig
+
+    ret_key: Union[str, int] = key
+    assert isinstance(c, Container), f"Unexpected type: {c}"
+    if c._is_none():
+        return None, ret_key
+
+    if isinstance(c, DictConfig):
+        assert isinstance(ret_key, str)
+        val = c._get_child(ret_key, validate_access=False)
+    elif isinstance(c, ListConfig):
+        assert isinstance(ret_key, str)
+        if not is_int(ret_key):
+            if throw_on_type_error:
+                raise TypeError(
+                    f"Index '{ret_key}' ({type(ret_key).__name__}) is not an int"
+                )
+            else:
+                val = None
+        else:
+            ret_key = int(ret_key)
+            if ret_key < 0 or ret_key + 1 > len(c):
+                val = None
+            else:
+                val = c._get_child(ret_key)
+    else:
+        assert False
+
+    if val is not None:
+        assert isinstance(val, Node)
+        if val._is_missing():
+            if throw_on_missing:
+                raise MissingMandatoryValue(
+                    f"Missing mandatory value: {c._get_full_key(ret_key)}"
+                )
+            else:
+                return val, ret_key
+
+    assert val is None or isinstance(val, Node)
+    return val, ret_key

omegaconf/resolvers/__init__.py

@@ -0,0 +1,5 @@
+from omegaconf.resolvers import oc
+
+__all__ = [
+    "oc",
+]

omegaconf/resolvers/oc/__init__.py

@@ -0,0 +1,113 @@
+import os
+import string
+import warnings
+from typing import Any, Optional
+
+from omegaconf import Container, Node
+from omegaconf._utils import _DEFAULT_MARKER_, _get_value
+from omegaconf.basecontainer import BaseContainer
+from omegaconf.errors import ConfigKeyError
+from omegaconf.grammar_parser import parse
+from omegaconf.resolvers.oc import dict
+
+
+def create(obj: Any, _parent_: Container) -> Any:
+    """Create a config object from `obj`, similar to `OmegaConf.create`"""
+    from omegaconf import OmegaConf
+
+    assert isinstance(_parent_, BaseContainer)
+    return OmegaConf.create(obj, parent=_parent_)
+
+
+def env(key: str, default: Any = _DEFAULT_MARKER_) -> Optional[str]:
+    """
+    :param key: Environment variable key
+    :param default: Optional default value to use in case the key environment variable is not set.
+                    If default is not a string, it is converted with str(default).
+                    None default is returned as is.
+    :return: The environment variable 'key'. If the environment variable is not set and a default is
+            provided, the default is used. If used, the default is converted to a string with str(default).
+            If the default is None, None is returned (without a string conversion).
+    """
+    try:
+        return os.environ[key]
+    except KeyError:
+        if default is not _DEFAULT_MARKER_:
+            return str(default) if default is not None else None
+        else:
+            raise KeyError(f"Environment variable '{key}' not found")
+
+
+def decode(expr: Optional[str], _parent_: Container, _node_: Node) -> Any:
+    """
+    Parse and evaluate `expr` according to the `singleElement` rule of the grammar.
+
+    If `expr` is `None`, then return `None`.
+    """
+    if expr is None:
+        return None
+
+    if not isinstance(expr, str):
+        raise TypeError(
+            f"`oc.decode` can only take strings or None as input, "
+            f"but `{expr}` is of type {type(expr).__name__}"
+        )
+
+    parse_tree = parse(expr, parser_rule="singleElement", lexer_mode="VALUE_MODE")
+    val = _parent_.resolve_parse_tree(parse_tree, node=_node_)
+    return _get_value(val)
+
+
+def deprecated(
+    key: str,
+    message: str = "'$OLD_KEY' is deprecated. Change your code and config to use '$NEW_KEY'",
+    *,
+    _parent_: Container,
+    _node_: Node,
+) -> Any:
+    from omegaconf._impl import select_node
+
+    if not isinstance(key, str):
+        raise TypeError(
+            f"oc.deprecated: interpolation key type is not a string ({type(key).__name__})"
+        )
+
+    if not isinstance(message, str):
+        raise TypeError(
+            f"oc.deprecated: interpolation message type is not a string ({type(message).__name__})"
+        )
+
+    full_key = _node_._get_full_key(key=None)
+    target_node = select_node(_parent_, key, absolute_key=True)
+    if target_node is None:
+        raise ConfigKeyError(
+            f"In oc.deprecated resolver at '{full_key}': Key not found: '{key}'"
+        )
+    new_key = target_node._get_full_key(key=None)
+    msg = string.Template(message).safe_substitute(
+        OLD_KEY=full_key,
+        NEW_KEY=new_key,
+    )
+    warnings.warn(category=UserWarning, message=msg)
+    return target_node
+
+
+def select(
+    key: str,
+    default: Any = _DEFAULT_MARKER_,
+    *,
+    _parent_: Container,
+) -> Any:
+    from omegaconf._impl import select_value
+
+    return select_value(cfg=_parent_, key=key, absolute_key=True, default=default)
+
+
+__all__ = [
+    "create",
+    "decode",
+    "deprecated",
+    "dict",
+    "env",
+    "select",
+]

omegaconf/resolvers/oc/dict.py

@@ -0,0 +1,83 @@
+from typing import Any, List
+
+from omegaconf import AnyNode, Container, DictConfig, ListConfig
+from omegaconf._utils import Marker
+from omegaconf.basecontainer import BaseContainer
+from omegaconf.errors import ConfigKeyError
+
+_DEFAULT_SELECT_MARKER_: Any = Marker("_DEFAULT_SELECT_MARKER_")
+
+
+def keys(
+    key: str,
+    _parent_: Container,
+) -> ListConfig:
+    from omegaconf import OmegaConf
+
+    assert isinstance(_parent_, BaseContainer)
+
+    in_dict = _get_and_validate_dict_input(
+        key, parent=_parent_, resolver_name="oc.dict.keys"
+    )
+
+    ret = OmegaConf.create(list(in_dict.keys()), parent=_parent_)
+    assert isinstance(ret, ListConfig)
+    return ret
+
+
+def values(key: str, _root_: BaseContainer, _parent_: Container) -> ListConfig:
+    assert isinstance(_parent_, BaseContainer)
+    in_dict = _get_and_validate_dict_input(
+        key, parent=_parent_, resolver_name="oc.dict.values"
+    )
+
+    content = in_dict._content
+    assert isinstance(content, dict)
+
+    ret = ListConfig([])
+    if key.startswith("."):
+        key = f".{key}"  # extra dot to compensate for extra level of nesting within ret ListConfig
+    for k in content:
+        ref_node = AnyNode(f"${{{key}.{k!s}}}")
+        ret.append(ref_node)
+
+    # Finalize result by setting proper type and parent.
+    element_type: Any = in_dict._metadata.element_type
+    ret._metadata.element_type = element_type
+    ret._metadata.ref_type = List[element_type]
+    ret._set_parent(_parent_)
+
+    return ret
+
+
+def _get_and_validate_dict_input(
+    key: str,
+    parent: BaseContainer,
+    resolver_name: str,
+) -> DictConfig:
+    from omegaconf._impl import select_value
+
+    if not isinstance(key, str):
+        raise TypeError(
+            f"`{resolver_name}` requires a string as input, but obtained `{key}` "
+            f"of type: {type(key).__name__}"
+        )
+
+    in_dict = select_value(
+        parent,
+        key,
+        throw_on_missing=True,
+        absolute_key=True,
+        default=_DEFAULT_SELECT_MARKER_,
+    )
+
+    if in_dict is _DEFAULT_SELECT_MARKER_:
+        raise ConfigKeyError(f"Key not found: '{key}'")
+
+    if not isinstance(in_dict, DictConfig):
+        raise TypeError(
+            f"`{resolver_name}` cannot be applied to objects of type: "
+            f"{type(in_dict).__name__}"
+        )
+
+    return in_dict

omegaconf/version.py

@@ -0,0 +1,13 @@
+import sys  # pragma: no cover
+
+__version__ = "2.3.0"
+
+msg = """OmegaConf 2.0 and above is compatible with Python 3.6 and newer.
+You have the following options:
+1. Upgrade to Python 3.6 or newer.
+   This is highly recommended. new features will not be added to OmegaConf 1.4.
+2. Continue using OmegaConf 1.4:
+    You can pip install 'OmegaConf<1.5' to do that.
+"""
+if sys.version_info < (3, 6):
+    raise ImportError(msg)  # pragma: no cover