diff --git a/mmengine/config/__init__.py b/mmengine/config/__init__.py index 9a1bc47db4..71292d274c 100644 --- a/mmengine/config/__init__.py +++ b/mmengine/config/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. -from .config import Config, ConfigDict, DictAction, read_base +from .config import Config, ConfigDict, DictAction +from .new_config import read_base __all__ = ['Config', 'ConfigDict', 'DictAction', 'read_base'] diff --git a/mmengine/config/config.py b/mmengine/config/config.py index f85795066a..f227124473 100644 --- a/mmengine/config/config.py +++ b/mmengine/config/config.py @@ -3,17 +3,12 @@ import copy import difflib import os -import os.path as osp import platform -import shutil -import sys import tempfile -import types -import uuid import warnings +from abc import ABCMeta, abstractmethod from argparse import Action, ArgumentParser, Namespace from collections import OrderedDict, abc -from contextlib import contextmanager from pathlib import Path from typing import Any, Optional, Sequence, Tuple, Union @@ -23,26 +18,14 @@ from rich.text import Text from yapf.yapflib.yapf_api import FormatCode -from mmengine.fileio import dump, load +from mmengine.fileio import dump from mmengine.logging import print_log -from mmengine.utils import (check_file_exist, digit_version, - get_installed_path, import_modules_from_strings, - is_installed) -from .lazy import LazyAttr, LazyObject -from .utils import (ConfigParsingError, ImportTransformer, RemoveAssignFromAST, - _gather_abs_import_lazyobj, _get_external_cfg_base_path, - _get_external_cfg_path, _get_package_and_cfg_path, - _is_builtin_module) +from mmengine.utils import digit_version +from .lazy import LazyObject +from .utils import ConfigParsingError, _is_builtin_module BASE_KEY = '_base_' DELETE_KEY = '_delete_' -DEPRECATION_KEY = '_deprecation_' -RESERVED_KEYS = ['filename', 'text', 'pretty_text', 'env_variables'] - -if platform.system() == 'Windows': - import regex as re -else: - import re # type: ignore def _lazy2string(cfg_dict, dict_type=None): @@ -53,8 +36,8 @@ def _lazy2string(cfg_dict, dict_type=None): for k, v in dict.items(cfg_dict)}) elif isinstance(cfg_dict, (tuple, list)): return type(cfg_dict)(_lazy2string(v, dict_type) for v in cfg_dict) - elif isinstance(cfg_dict, (LazyAttr, LazyObject)): - return f'{cfg_dict.module}.{str(cfg_dict)}' + elif isinstance(cfg_dict, LazyObject): + return cfg_dict.dump_str else: return cfg_dict @@ -107,7 +90,7 @@ def __missing__(self, name): def __getattr__(self, name): try: value = super().__getattr__(name) - if isinstance(value, (LazyAttr, LazyObject)) and not self.lazy: + if isinstance(value, LazyObject) and not self.lazy: value = value.build() except KeyError: raise AttributeError(f"'{self.__class__.__name__}' object has no " @@ -213,7 +196,7 @@ def build_lazy(self, value: Any) -> Any: Returns: Any: The built value. """ - if isinstance(value, (LazyAttr, LazyObject)) and not self.lazy: + if isinstance(value, LazyObject) and not self.lazy: value = value.build() return value @@ -349,7 +332,7 @@ def add_args(parser: ArgumentParser, return parser -class Config: +class Config(metaclass=ABCMeta): """A facility for config and config files. It supports common file formats as configs: python/json/yaml. @@ -391,51 +374,18 @@ class Config: .. _config tutorial: https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html """ # noqa: E501 - def __init__( - self, - cfg_dict: dict = None, - cfg_text: Optional[str] = None, - filename: Optional[Union[str, Path]] = None, - env_variables: Optional[dict] = None, - format_python_code: bool = True, - ): - filename = str(filename) if isinstance(filename, Path) else filename - if cfg_dict is None: - cfg_dict = dict() - elif not isinstance(cfg_dict, dict): - raise TypeError('cfg_dict must be a dict, but ' - f'got {type(cfg_dict)}') - for key in cfg_dict: - if key in RESERVED_KEYS: - raise KeyError(f'{key} is reserved for config file') - - if not isinstance(cfg_dict, ConfigDict): - cfg_dict = ConfigDict(cfg_dict) - super().__setattr__('_cfg_dict', cfg_dict) - super().__setattr__('_filename', filename) - super().__setattr__('_format_python_code', format_python_code) - if not hasattr(self, '_imported_names'): - super().__setattr__('_imported_names', set()) - - if cfg_text: - text = cfg_text - elif filename: - with open(filename, encoding='utf-8') as f: - text = f.read() - else: - text = '' - super().__setattr__('_text', text) - if env_variables is None: - env_variables = dict() - super().__setattr__('_env_variables', env_variables) + @classmethod + def __new__(cls, *args, **kwargs): + if cls is Config: + from .new_config import ConfigV2 + cls = ConfigV2 + return super().__new__(cls) @staticmethod def fromfile(filename: Union[str, Path], - use_predefined_variables: bool = True, - import_custom_modules: bool = True, - use_environment_variables: bool = True, lazy_import: Optional[bool] = None, - format_python_code: bool = True) -> 'Config': + format_python_code: bool = True, + **kwargs) -> 'Config': """Build a Config instance from config file. Args: @@ -456,53 +406,21 @@ def fromfile(filename: Union[str, Path], Config: Config instance built from config file. """ filename = str(filename) if isinstance(filename, Path) else filename - if lazy_import is False or \ - lazy_import is None and not Config._is_lazy_import(filename): - cfg_dict, cfg_text, env_variables = Config._file2dict( - filename, use_predefined_variables, use_environment_variables, - lazy_import) - if import_custom_modules and cfg_dict.get('custom_imports', None): - try: - import_modules_from_strings(**cfg_dict['custom_imports']) - except ImportError as e: - err_msg = ( - 'Failed to import custom modules from ' - f"{cfg_dict['custom_imports']}, the current sys.path " - 'is: ') - for p in sys.path: - err_msg += f'\n {p}' - err_msg += ( - '\nYou should set `PYTHONPATH` to make `sys.path` ' - 'include the directory which contains your custom ' - 'module') - raise ImportError(err_msg) from e - return Config( - cfg_dict, - cfg_text=cfg_text, + if lazy_import is None: + lazy_import = Config._is_lazy_import(filename) + + if not lazy_import: + from .old_config import ConfigV1 + return ConfigV1.fromfile( filename=filename, - env_variables=env_variables, - ) + format_python_code=format_python_code, + **kwargs) else: - # Enable lazy import when parsing the config. - # Using try-except to make sure ``ConfigDict.lazy`` will be reset - # to False. See more details about lazy in the docstring of - # ConfigDict - ConfigDict.lazy = True - try: - cfg_dict, imported_names = Config._parse_lazy_import(filename) - except Exception as e: - raise e - finally: - # disable lazy import to get the real type. See more details - # about lazy in the docstring of ConfigDict - ConfigDict.lazy = False - - cfg = Config( - cfg_dict, + from .new_config import ConfigV2 + return ConfigV2.fromfile( filename=filename, - format_python_code=format_python_code) - object.__setattr__(cfg, '_imported_names', imported_names) - return cfg + format_python_code=format_python_code, + **kwargs) @staticmethod def fromstring(cfg_str: str, file_format: str) -> 'Config': @@ -540,80 +458,6 @@ def fromstring(cfg_str: str, file_format: str) -> 'Config': os.remove(temp_file.name) # manually delete the temporary file return cfg - @staticmethod - def _get_base_modules(nodes: list) -> list: - """Get base module name from parsed code. - - Args: - nodes (list): Parsed code of the config file. - - Returns: - list: Name of base modules. - """ - - def _get_base_module_from_with(with_nodes: list) -> list: - """Get base module name from if statement in python file. - - Args: - with_nodes (list): List of if statement. - - Returns: - list: Name of base modules. - """ - base_modules = [] - for node in with_nodes: - assert isinstance(node, ast.ImportFrom), ( - 'Illegal syntax in config file! Only ' - '`from ... import ...` could be implemented` in ' - 'with read_base()`') - assert node.module is not None, ( - 'Illegal syntax in config file! Syntax like ' - '`from . import xxx` is not allowed in `with read_base()`') - base_modules.append(node.level * '.' + node.module) - return base_modules - - for idx, node in enumerate(nodes): - if (isinstance(node, ast.Assign) - and isinstance(node.targets[0], ast.Name) - and node.targets[0].id == BASE_KEY): - raise ConfigParsingError( - 'The configuration file type in the inheritance chain ' - 'must match the current configuration file type, either ' - '"lazy_import" or non-"lazy_import". You got this error ' - f'since you use the syntax like `_base_ = "{node.targets[0].id}"` ' # noqa: E501 - 'in your config. You should use `with read_base(): ... to` ' # noqa: E501 - 'mark the inherited config file. See more information ' - 'in https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html' # noqa: E501 - ) - - if not isinstance(node, ast.With): - continue - - expr = node.items[0].context_expr - if (not isinstance(expr, ast.Call) - or not expr.func.id == 'read_base' or # type: ignore - len(node.items) > 1): - raise ConfigParsingError( - 'Only `read_base` context manager can be used in the ' - 'config') - - # The original code: - # ``` - # with read_base(): - # from .._base_.default_runtime import * - # ``` - # The processed code: - # ``` - # from .._base_.default_runtime import * - # ``` - # As you can see, the if statement is removed and the - # from ... import statement will be unindent - for nested_idx, nested_node in enumerate(node.body): - nodes.insert(idx + nested_idx + 1, nested_node) - nodes.pop(idx) - return _get_base_module_from_with(node.body) - return [] - @staticmethod def _validate_py_syntax(filename: str): """Validate syntax of python config. @@ -629,661 +473,6 @@ def _validate_py_syntax(filename: str): raise SyntaxError('There are syntax errors in config ' f'file {filename}: {e}') - @staticmethod - def _substitute_predefined_vars(filename: str, temp_config_name: str): - """Substitute predefined variables in config with actual values. - - Sometimes we want some variables in the config to be related to the - current path or file name, etc. - - Here is an example of a typical usage scenario. When training a model, - we define a working directory in the config that save the models and - logs. For different configs, we expect to define different working - directories. A common way for users is to use the config file name - directly as part of the working directory name, e.g. for the config - ``config_setting1.py``, the working directory is - ``. /work_dir/config_setting1``. - - This can be easily achieved using predefined variables, which can be - written in the config `config_setting1.py` as follows - - .. code-block:: python - - work_dir = '. /work_dir/{{ fileBasenameNoExtension }}' - - - Here `{{ fileBasenameNoExtension }}` indicates the file name of the - config (without the extension), and when the config class reads the - config file, it will automatically parse this double-bracketed string - to the corresponding actual value. - - .. code-block:: python - - cfg = Config.fromfile('. /config_setting1.py') - cfg.work_dir # ". /work_dir/config_setting1" - - - For details, Please refer to docs/zh_cn/advanced_tutorials/config.md . - - Args: - filename (str): Filename of config. - temp_config_name (str): Temporary filename to save substituted - config. - """ - file_dirname = osp.dirname(filename) - file_basename = osp.basename(filename) - file_basename_no_extension = osp.splitext(file_basename)[0] - file_extname = osp.splitext(filename)[1] - support_templates = dict( - fileDirname=file_dirname, - fileBasename=file_basename, - fileBasenameNoExtension=file_basename_no_extension, - fileExtname=file_extname) - with open(filename, encoding='utf-8') as f: - config_file = f.read() - for key, value in support_templates.items(): - regexp = r'\{\{\s*' + str(key) + r'\s*\}\}' - value = value.replace('\\', '/') - config_file = re.sub(regexp, value, config_file) - with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: - tmp_config_file.write(config_file) - - @staticmethod - def _substitute_env_variables(filename: str, temp_config_name: str): - """Substitute environment variables in config with actual values. - - Sometimes, we want to change some items in the config with environment - variables. For examples, we expect to change dataset root by setting - ``DATASET_ROOT=/dataset/root/path`` in the command line. This can be - easily achieved by writing lines in the config as follows - - .. code-block:: python - - data_root = '{{$DATASET_ROOT:/default/dataset}}/images' - - - Here, ``{{$DATASET_ROOT:/default/dataset}}`` indicates using the - environment variable ``DATASET_ROOT`` to replace the part between - ``{{}}``. If the ``DATASET_ROOT`` is not set, the default value - ``/default/dataset`` will be used. - - Environment variables not only can replace items in the string, they - can also substitute other types of data in config. In this situation, - we can write the config as below - - .. code-block:: python - - model = dict( - bbox_head = dict(num_classes={{'$NUM_CLASSES:80'}})) - - - For details, Please refer to docs/zh_cn/tutorials/config.md . - - Args: - filename (str): Filename of config. - temp_config_name (str): Temporary filename to save substituted - config. - """ - with open(filename, encoding='utf-8') as f: - config_file = f.read() - regexp = r'\{\{[\'\"]?\s*\$(\w+)\s*\:\s*(\S*?)\s*[\'\"]?\}\}' - keys = re.findall(regexp, config_file) - env_variables = dict() - for var_name, value in keys: - regexp = r'\{\{[\'\"]?\s*\$' + var_name + r'\s*\:\s*' \ - + value + r'\s*[\'\"]?\}\}' - if var_name in os.environ: - value = os.environ[var_name] - env_variables[var_name] = value - print_log( - f'Using env variable `{var_name}` with value of ' - f'{value} to replace item in config.', - logger='current') - if not value: - raise KeyError(f'`{var_name}` cannot be found in `os.environ`.' - f' Please set `{var_name}` in environment or ' - 'give a default value.') - config_file = re.sub(regexp, value, config_file) - - with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: - tmp_config_file.write(config_file) - return env_variables - - @staticmethod - def _pre_substitute_base_vars(filename: str, - temp_config_name: str) -> dict: - """Preceding step for substituting variables in base config with actual - value. - - Args: - filename (str): Filename of config. - temp_config_name (str): Temporary filename to save substituted - config. - - Returns: - dict: A dictionary contains variables in base config. - """ - with open(filename, encoding='utf-8') as f: - config_file = f.read() - base_var_dict = {} - regexp = r'\{\{\s*' + BASE_KEY + r'\.([\w\.]+)\s*\}\}' - base_vars = set(re.findall(regexp, config_file)) - for base_var in base_vars: - randstr = f'_{base_var}_{uuid.uuid4().hex.lower()[:6]}' - base_var_dict[randstr] = base_var - regexp = r'\{\{\s*' + BASE_KEY + r'\.' + base_var + r'\s*\}\}' - config_file = re.sub(regexp, f'"{randstr}"', config_file) - with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: - tmp_config_file.write(config_file) - return base_var_dict - - @staticmethod - def _substitute_base_vars(cfg: Any, base_var_dict: dict, - base_cfg: dict) -> Any: - """Substitute base variables from strings to their actual values. - - Args: - Any : Config dictionary. - base_var_dict (dict): A dictionary contains variables in base - config. - base_cfg (dict): Base config dictionary. - - Returns: - Any : A dictionary with origin base variables - substituted with actual values. - """ - cfg = copy.deepcopy(cfg) - - if isinstance(cfg, dict): - for k, v in cfg.items(): - if isinstance(v, str) and v in base_var_dict: - new_v = base_cfg - for new_k in base_var_dict[v].split('.'): - new_v = new_v[new_k] - cfg[k] = new_v - elif isinstance(v, (list, tuple, dict)): - cfg[k] = Config._substitute_base_vars( - v, base_var_dict, base_cfg) - elif isinstance(cfg, tuple): - cfg = tuple( - Config._substitute_base_vars(c, base_var_dict, base_cfg) - for c in cfg) - elif isinstance(cfg, list): - cfg = [ - Config._substitute_base_vars(c, base_var_dict, base_cfg) - for c in cfg - ] - elif isinstance(cfg, str) and cfg in base_var_dict: - new_v = base_cfg - for new_k in base_var_dict[cfg].split('.'): - new_v = new_v[new_k] - cfg = new_v - - return cfg - - @staticmethod - def _file2dict( - filename: str, - use_predefined_variables: bool = True, - use_environment_variables: bool = True, - lazy_import: Optional[bool] = None) -> Tuple[dict, str, dict]: - """Transform file to variables dictionary. - - Args: - filename (str): Name of config file. - use_predefined_variables (bool, optional): Whether to use - predefined variables. Defaults to True. - use_environment_variables (bool, optional): Whether to use - environment variables. Defaults to True. - lazy_import (bool): Whether to load config in `lazy_import` mode. - If it is `None`, it will be deduced by the content of the - config file. Defaults to None. - - Returns: - Tuple[dict, str]: Variables dictionary and text of Config. - """ - if lazy_import is None and Config._is_lazy_import(filename): - raise RuntimeError( - 'The configuration file type in the inheritance chain ' - 'must match the current configuration file type, either ' - '"lazy_import" or non-"lazy_import". You got this error ' - 'since you use the syntax like `with read_base(): ...` ' - f'or import non-builtin module in {filename}. See more ' - 'information in https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html' # noqa: E501 - ) - - filename = osp.abspath(osp.expanduser(filename)) - check_file_exist(filename) - fileExtname = osp.splitext(filename)[1] - if fileExtname not in ['.py', '.json', '.yaml', '.yml']: - raise OSError('Only py/yml/yaml/json type are supported now!') - try: - with tempfile.TemporaryDirectory() as temp_config_dir: - temp_config_file = tempfile.NamedTemporaryFile( - dir=temp_config_dir, suffix=fileExtname, delete=False) - if platform.system() == 'Windows': - temp_config_file.close() - - # Substitute predefined variables - if use_predefined_variables: - Config._substitute_predefined_vars(filename, - temp_config_file.name) - else: - shutil.copyfile(filename, temp_config_file.name) - # Substitute environment variables - env_variables = dict() - if use_environment_variables: - env_variables = Config._substitute_env_variables( - temp_config_file.name, temp_config_file.name) - # Substitute base variables from placeholders to strings - base_var_dict = Config._pre_substitute_base_vars( - temp_config_file.name, temp_config_file.name) - - # Handle base files - base_cfg_dict = ConfigDict() - cfg_text_list = list() - for base_cfg_path in Config._get_base_files( - temp_config_file.name): - base_cfg_path, scope = Config._get_cfg_path( - base_cfg_path, filename) - _cfg_dict, _cfg_text, _env_variables = Config._file2dict( - filename=base_cfg_path, - use_predefined_variables=use_predefined_variables, - use_environment_variables=use_environment_variables, - lazy_import=lazy_import, - ) - cfg_text_list.append(_cfg_text) - env_variables.update(_env_variables) - duplicate_keys = base_cfg_dict.keys() & _cfg_dict.keys() - if len(duplicate_keys) > 0: - raise KeyError( - 'Duplicate key is not allowed among bases. ' - f'Duplicate keys: {duplicate_keys}') - - # _dict_to_config_dict will do the following things: - # 1. Recursively converts ``dict`` to :obj:`ConfigDict`. - # 2. Set `_scope_` for the outer dict variable for the base - # config. - # 3. Set `scope` attribute for each base variable. - # Different from `_scope_`, `scope` is not a key of base - # dict, `scope` attribute will be parsed to key `_scope_` - # by function `_parse_scope` only if the base variable is - # accessed by the current config. - _cfg_dict = Config._dict_to_config_dict(_cfg_dict, scope) - base_cfg_dict.update(_cfg_dict) - - if filename.endswith('.py'): - with open(temp_config_file.name, encoding='utf-8') as f: - parsed_codes = ast.parse(f.read()) - parsed_codes = RemoveAssignFromAST(BASE_KEY).visit( - parsed_codes) - codeobj = compile(parsed_codes, filename, mode='exec') - # Support load global variable in nested function of the - # config. - global_locals_var = {BASE_KEY: base_cfg_dict} - ori_keys = set(global_locals_var.keys()) - eval(codeobj, global_locals_var, global_locals_var) - cfg_dict = { - key: value - for key, value in global_locals_var.items() - if (key not in ori_keys and not key.startswith('__')) - } - elif filename.endswith(('.yml', '.yaml', '.json')): - cfg_dict = load(temp_config_file.name) - # close temp file - for key, value in list(cfg_dict.items()): - if isinstance(value, - (types.FunctionType, types.ModuleType)): - cfg_dict.pop(key) - temp_config_file.close() - - # If the current config accesses a base variable of base - # configs, The ``scope`` attribute of corresponding variable - # will be converted to the `_scope_`. - Config._parse_scope(cfg_dict) - except Exception as e: - if osp.exists(temp_config_dir): - shutil.rmtree(temp_config_dir) - raise e - - # check deprecation information - if DEPRECATION_KEY in cfg_dict: - deprecation_info = cfg_dict.pop(DEPRECATION_KEY) - warning_msg = f'The config file {filename} will be deprecated ' \ - 'in the future.' - if 'expected' in deprecation_info: - warning_msg += f' Please use {deprecation_info["expected"]} ' \ - 'instead.' - if 'reference' in deprecation_info: - warning_msg += ' More information can be found at ' \ - f'{deprecation_info["reference"]}' - warnings.warn(warning_msg, DeprecationWarning) - - cfg_text = filename + '\n' - with open(filename, encoding='utf-8') as f: - # Setting encoding explicitly to resolve coding issue on windows - cfg_text += f.read() - - # Substitute base variables from strings to their actual values - cfg_dict = Config._substitute_base_vars(cfg_dict, base_var_dict, - base_cfg_dict) - cfg_dict.pop(BASE_KEY, None) - - cfg_dict = Config._merge_a_into_b(cfg_dict, base_cfg_dict) - cfg_dict = { - k: v - for k, v in cfg_dict.items() if not k.startswith('__') - } - - # merge cfg_text - cfg_text_list.append(cfg_text) - cfg_text = '\n'.join(cfg_text_list) - - return cfg_dict, cfg_text, env_variables - - @staticmethod - def _parse_lazy_import(filename: str) -> Tuple[ConfigDict, set]: - """Transform file to variables dictionary. - - Args: - filename (str): Name of config file. - - Returns: - Tuple[dict, dict]: ``cfg_dict`` and ``imported_names``. - - - cfg_dict (dict): Variables dictionary of parsed config. - - imported_names (set): Used to mark the names of - imported object. - """ - # In lazy import mode, users can use the Python syntax `import` to - # implement inheritance between configuration files, which is easier - # for users to understand the hierarchical relationships between - # different configuration files. - - # Besides, users can also using `import` syntax to import corresponding - # module which will be filled in the `type` field. It means users - # can directly navigate to the source of the module in the - # configuration file by clicking the `type` field. - - # To avoid really importing the third party package like `torch` - # during import `type` object, we use `_parse_lazy_import` to parse the - # configuration file, which will not actually trigger the import - # process, but simply parse the imported `type`s as LazyObject objects. - - # The overall pipeline of _parse_lazy_import is: - # 1. Parse the base module from the config file. - # || - # \/ - # base_module = ['mmdet.configs.default_runtime'] - # || - # \/ - # 2. recursively parse the base module and gather imported objects to - # a dict. - # || - # \/ - # The base_dict will be: - # { - # 'mmdet.configs.default_runtime': {...} - # 'mmdet.configs.retinanet_r50_fpn_1x_coco': {...} - # ... - # }, each item in base_dict is a dict of `LazyObject` - # 3. parse the current config file filling the imported variable - # with the base_dict. - # - # 4. During the parsing process, all imported variable will be - # recorded in the `imported_names` set. These variables can be - # accessed, but will not be dumped by default. - - with open(filename, encoding='utf-8') as f: - global_dict = {'LazyObject': LazyObject, '__file__': filename} - base_dict = {} - - parsed_codes = ast.parse(f.read()) - # get the names of base modules, and remove the - # `with read_base():'` statement - base_modules = Config._get_base_modules(parsed_codes.body) - base_imported_names = set() - for base_module in base_modules: - # If base_module means a relative import, assuming the level is - # 2, which means the module is imported like - # "from ..a.b import c". we must ensure that c is an - # object `defined` in module b, and module b should not be a - # package including `__init__` file but a single python file. - level = len(re.match(r'\.*', base_module).group()) - if level > 0: - # Relative import - base_dir = osp.dirname(filename) - module_path = osp.join( - base_dir, *(['..'] * (level - 1)), - f'{base_module[level:].replace(".", "/")}.py') - else: - # Absolute import - module_list = base_module.split('.') - if len(module_list) == 1: - raise ConfigParsingError( - 'The imported configuration file should not be ' - f'an independent package {module_list[0]}. Here ' - 'is an example: ' - '`with read_base(): from mmdet.configs.retinanet_r50_fpn_1x_coco import *`' # noqa: E501 - ) - else: - package = module_list[0] - root_path = get_installed_path(package) - module_path = f'{osp.join(root_path, *module_list[1:])}.py' # noqa: E501 - if not osp.isfile(module_path): - raise ConfigParsingError( - f'{module_path} not found! It means that incorrect ' - 'module is defined in ' - f'`with read_base(): = from {base_module} import ...`, please ' # noqa: E501 - 'make sure the base config module is valid ' - 'and is consistent with the prior import ' - 'logic') - _base_cfg_dict, _base_imported_names = Config._parse_lazy_import( # noqa: E501 - module_path) - base_imported_names |= _base_imported_names - # The base_dict will be: - # { - # 'mmdet.configs.default_runtime': {...} - # 'mmdet.configs.retinanet_r50_fpn_1x_coco': {...} - # ... - # } - base_dict[base_module] = _base_cfg_dict - - # `base_dict` contains all the imported modules from `base_cfg`. - # In order to collect the specific imported module from `base_cfg` - # before parse the current file, we using AST Transform to - # transverse the imported module from base_cfg and merge then into - # the global dict. After the ast transformation, most of import - # syntax will be removed (except for the builtin import) and - # replaced with the `LazyObject` - transform = ImportTransformer( - global_dict=global_dict, - base_dict=base_dict, - filename=filename) - modified_code = transform.visit(parsed_codes) - modified_code, abs_imported = _gather_abs_import_lazyobj( - modified_code, filename=filename) - imported_names = transform.imported_obj | abs_imported - imported_names |= base_imported_names - modified_code = ast.fix_missing_locations(modified_code) - exec( - compile(modified_code, filename, mode='exec'), global_dict, - global_dict) - - ret: dict = {} - for key, value in global_dict.items(): - if key.startswith('__') or key in ['LazyObject']: - continue - ret[key] = value - # convert dict to ConfigDict - cfg_dict = Config._dict_to_config_dict_lazy(ret) - - return cfg_dict, imported_names - - @staticmethod - def _dict_to_config_dict_lazy(cfg: dict): - """Recursively converts ``dict`` to :obj:`ConfigDict`. The only - difference between ``_dict_to_config_dict_lazy`` and - ``_dict_to_config_dict_lazy`` is that the former one does not consider - the scope, and will not trigger the building of ``LazyObject``. - - Args: - cfg (dict): Config dict. - - Returns: - ConfigDict: Converted dict. - """ - # Only the outer dict with key `type` should have the key `_scope_`. - if isinstance(cfg, dict): - cfg_dict = ConfigDict() - for key, value in cfg.items(): - cfg_dict[key] = Config._dict_to_config_dict_lazy(value) - return cfg_dict - if isinstance(cfg, (tuple, list)): - return type(cfg)( - Config._dict_to_config_dict_lazy(_cfg) for _cfg in cfg) - return cfg - - @staticmethod - def _dict_to_config_dict(cfg: dict, - scope: Optional[str] = None, - has_scope=True): - """Recursively converts ``dict`` to :obj:`ConfigDict`. - - Args: - cfg (dict): Config dict. - scope (str, optional): Scope of instance. - has_scope (bool): Whether to add `_scope_` key to config dict. - - Returns: - ConfigDict: Converted dict. - """ - # Only the outer dict with key `type` should have the key `_scope_`. - if isinstance(cfg, dict): - if has_scope and 'type' in cfg: - has_scope = False - if scope is not None and cfg.get('_scope_', None) is None: - cfg._scope_ = scope # type: ignore - cfg = ConfigDict(cfg) - dict.__setattr__(cfg, 'scope', scope) - for key, value in cfg.items(): - cfg[key] = Config._dict_to_config_dict( - value, scope=scope, has_scope=has_scope) - elif isinstance(cfg, tuple): - cfg = tuple( - Config._dict_to_config_dict(_cfg, scope, has_scope=has_scope) - for _cfg in cfg) - elif isinstance(cfg, list): - cfg = [ - Config._dict_to_config_dict(_cfg, scope, has_scope=has_scope) - for _cfg in cfg - ] - return cfg - - @staticmethod - def _parse_scope(cfg: dict) -> None: - """Adds ``_scope_`` to :obj:`ConfigDict` instance, which means a base - variable. - - If the config dict already has the scope, scope will not be - overwritten. - - Args: - cfg (dict): Config needs to be parsed with scope. - """ - if isinstance(cfg, ConfigDict): - cfg._scope_ = cfg.scope - elif isinstance(cfg, (tuple, list)): - [Config._parse_scope(value) for value in cfg] - else: - return - - @staticmethod - def _get_base_files(filename: str) -> list: - """Get the base config file. - - Args: - filename (str): The config file. - - Raises: - TypeError: Name of config file. - - Returns: - list: A list of base config. - """ - file_format = osp.splitext(filename)[1] - if file_format == '.py': - Config._validate_py_syntax(filename) - with open(filename, encoding='utf-8') as f: - parsed_codes = ast.parse(f.read()).body - - def is_base_line(c): - return (isinstance(c, ast.Assign) - and isinstance(c.targets[0], ast.Name) - and c.targets[0].id == BASE_KEY) - - base_code = next((c for c in parsed_codes if is_base_line(c)), - None) - if base_code is not None: - base_code = ast.Expression( # type: ignore - body=base_code.value) # type: ignore - base_files = eval(compile(base_code, '', mode='eval')) - else: - base_files = [] - elif file_format in ('.yml', '.yaml', '.json'): - import mmengine - cfg_dict = mmengine.load(filename) - base_files = cfg_dict.get(BASE_KEY, []) - else: - raise ConfigParsingError( - 'The config type should be py, json, yaml or ' - f'yml, but got {file_format}') - base_files = base_files if isinstance(base_files, - list) else [base_files] - return base_files - - @staticmethod - def _get_cfg_path(cfg_path: str, - filename: str) -> Tuple[str, Optional[str]]: - """Get the config path from the current or external package. - - Args: - cfg_path (str): Relative path of config. - filename (str): The config file being parsed. - - Returns: - Tuple[str, str or None]: Path and scope of config. If the config - is not an external config, the scope will be `None`. - """ - if '::' in cfg_path: - # `cfg_path` startswith '::' means an external config path. - # Get package name and relative config path. - scope = cfg_path.partition('::')[0] - package, cfg_path = _get_package_and_cfg_path(cfg_path) - - if not is_installed(package): - raise ModuleNotFoundError( - f'{package} is not installed, please install {package} ' - f'manually') - - # Get installed package path. - package_path = get_installed_path(package) - try: - # Get config path from meta file. - cfg_path = _get_external_cfg_path(package_path, cfg_path) - except ValueError: - # Since base config does not have a metafile, it should be - # concatenated with package path and relative config path. - cfg_path = _get_external_cfg_base_path(package_path, cfg_path) - except FileNotFoundError as e: - raise e - return cfg_path, scope - else: - # Get local config path. - cfg_dir = osp.dirname(filename) - cfg_path = osp.join(cfg_dir, cfg_path) - return cfg_path, None - @staticmethod def _merge_a_into_b(a: dict, b: dict, @@ -1357,19 +546,14 @@ def auto_argparser(description=None): return parser, cfg @property + @abstractmethod def filename(self) -> str: """get file name of config.""" - return self._filename @property + @abstractmethod def text(self) -> str: """get config text.""" - return self._text - - @property - def env_variables(self) -> dict: - """get used environment variables.""" - return self._env_variables @property def pretty_text(self) -> str: @@ -1514,13 +698,9 @@ def __setitem__(self, name, value): def __iter__(self): return iter(self._cfg_dict) - def __getstate__( - self - ) -> Tuple[dict, Optional[str], Optional[str], dict, bool, set]: - state = (self._cfg_dict, self._filename, self._text, - self._env_variables, self._format_python_code, - self._imported_names) - return state + @abstractmethod + def __getstate__(self) -> Tuple: + pass def __deepcopy__(self, memo): cls = self.__class__ @@ -1542,14 +722,9 @@ def __copy__(self): copy = __copy__ - def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str], - dict, bool, set]): - super().__setattr__('_cfg_dict', state[0]) - super().__setattr__('_filename', state[1]) - super().__setattr__('_text', state[2]) - super().__setattr__('_env_variables', state[3]) - super().__setattr__('_format_python_code', state[4]) - super().__setattr__('_imported_names', state[5]) + @abstractmethod + def __setstate__(self, state: Tuple): + pass def dump(self, file: Optional[Union[str, Path]] = None): """Dump config to file or return config text. @@ -1694,36 +869,9 @@ def _is_lazy_import(filename: str) -> bool: return True return False - def _to_lazy_dict(self, keep_imported: bool = False) -> dict: - """Convert config object to dictionary with lazy object, and filter the - imported object.""" - res = self._cfg_dict._to_lazy_dict() - if hasattr(self, '_imported_names') and not keep_imported: - res = { - key: value - for key, value in res.items() - if key not in self._imported_names - } - return res - - def to_dict(self, keep_imported: bool = False): - """Convert all data in the config to a builtin ``dict``. - - Args: - keep_imported (bool): Whether to keep the imported field. - Defaults to False - - If you import third-party objects in the config file, all imported - objects will be converted to a string like ``torch.optim.SGD`` - """ - cfg_dict = self._cfg_dict.to_dict() - if hasattr(self, '_imported_names') and not keep_imported: - cfg_dict = { - key: value - for key, value in cfg_dict.items() - if key not in self._imported_names - } - return cfg_dict + def to_dict(self): + """Convert all data in the config to a builtin ``dict``.""" + return self._cfg_dict.to_dict() class DictAction(Action): @@ -1839,19 +987,3 @@ def __call__(self, key, val = kv.split('=', maxsplit=1) options[key] = self._parse_iterable(val) setattr(namespace, self.dest, options) - - -@contextmanager -def read_base(): - """Context manager to mark the base config. - - The pure Python-style configuration file allows you to use the import - syntax. However, it is important to note that you need to import the base - configuration file within the context of ``read_base``, and import other - dependencies outside of it. - - You can see more usage of Python-style configuration in the `tutorial`_ - - .. _tutorial: https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html#a-pure-python-style-configuration-file-beta - """ # noqa: E501 - yield diff --git a/mmengine/config/lazy.py b/mmengine/config/lazy.py index e83cce7c89..f96e8cdf1b 100644 --- a/mmengine/config/lazy.py +++ b/mmengine/config/lazy.py @@ -1,8 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. import importlib -from typing import Any, Optional, Union - -from mmengine.utils import is_seq_of +import re +import sys +from importlib.util import find_spec, spec_from_loader +from typing import Any, Optional class LazyObject: @@ -15,47 +16,29 @@ class LazyObject: >>> import torch.nn as nn >>> from mmdet.models import RetinaNet >>> import mmcls.models - >>> import mmcls.datasets - >>> import mmcls Will be parsed as: Examples: >>> # import torch.nn as nn - >>> nn = lazyObject('torch.nn') + >>> nn = LazyObject('torch.nn') >>> # from mmdet.models import RetinaNet - >>> RetinaNet = lazyObject('mmdet.models', 'RetinaNet') - >>> # import mmcls.models; import mmcls.datasets; import mmcls - >>> mmcls = lazyObject(['mmcls', 'mmcls.datasets', 'mmcls.models']) + >>> RetinaNet = LazyObject('RetinaNet', LazyObject('mmdet.models')) + >>> # import mmcls.models + >>> mmcls = LazyObject('mmcls.models') ``LazyObject`` records all module information and will be further referenced by the configuration file. Args: - module (str or list or tuple): The module name to be imported. - imported (str, optional): The imported module name. Defaults to None. - location (str, optional): The filename and line number of the imported - module statement happened. + name (str): The name of a module or attribution. + source (LazyObject, optional): The source of the lazy object. + Defaults to None. """ - def __init__(self, - module: Union[str, list, tuple], - imported: Optional[str] = None, - location: Optional[str] = None): - if not isinstance(module, str) and not is_seq_of(module, str): - raise TypeError('module should be `str`, `list`, or `tuple`' - f'but got {type(module)}, this might be ' - 'a bug of MMEngine, please report it to ' - 'https://github.com/open-mmlab/mmengine/issues') - self._module: Union[str, list, tuple] = module - - if not isinstance(imported, str) and imported is not None: - raise TypeError('imported should be `str` or None, but got ' - f'{type(imported)}, this might be ' - 'a bug of MMEngine, please report it to ' - 'https://github.com/open-mmlab/mmengine/issues') - self._imported = imported - self.location = location + def __init__(self, name: str, source: Optional['LazyObject'] = None): + self.name = name + self.source = source def build(self) -> Any: """Return imported object. @@ -63,63 +46,60 @@ def build(self) -> Any: Returns: Any: Imported object """ - if isinstance(self._module, str): + if self.source is not None: + module = self.source.build() try: - module = importlib.import_module(self._module) - except Exception as e: - raise type(e)(f'Failed to import {self._module} ' - f'in {self.location} for {e}') - - if self._imported is not None: - if hasattr(module, self._imported): - module = getattr(module, self._imported) - else: - raise ImportError( - f'Failed to import {self._imported} ' - f'from {self._module} in {self.location}') - - return module + return getattr(module, self.name) + except AttributeError: + raise ImportError( + f'Failed to import {self.name} from {self.source}') else: - # import xxx.xxx - # import xxx.yyy - # import xxx.zzz - # return imported xxx try: - for module in self._module: - importlib.import_module(module) # type: ignore - module_name = self._module[0].split('.')[0] - return importlib.import_module(module_name) - except Exception as e: - raise type(e)(f'Failed to import {self.module} ' - f'in {self.location} for {e}') - - @property - def module(self): - if isinstance(self._module, str): - return self._module - return self._module[0].split('.')[0] - - def __call__(self, *args, **kwargs): - raise RuntimeError() + for idx in range(self.name.count('.') + 1): + module, *attrs = self.name.rsplit('.', idx) + try: + spec = find_spec(module) + except ImportError: + spec = None + if spec is not None: + res = importlib.import_module(module) + for attr in attrs: + res = getattr(res, attr) + return res + raise ImportError(f'No module named `{module}`.') + except (ImportError, AttributeError) as e: + raise ImportError(f'Failed to import {self.name} for {e}') def __deepcopy__(self, memo): - return LazyObject(self._module, self._imported, self.location) + return LazyObject(self.name, self.source) def __getattr__(self, name): - # Cannot locate the line number of the getting attribute. - # Therefore only record the filename. - if self.location is not None: - location = self.location.split(', line')[0] - else: - location = self.location - return LazyAttr(name, self, location) + return LazyObject(name, self) def __str__(self) -> str: - if self._imported is not None: - return self._imported - return self.module + if self.source is not None: + return str(self.source) + '.' + self.name + return self.name - __repr__ = __str__ + def __repr__(self) -> str: + arg = f'name={repr(self.name)}' + if self.source is not None: + arg += f', source={repr(self.source)}' + return f'LazyObject({arg})' + + @property + def dump_str(self) -> str: + return f'' + + @classmethod + def from_str(cls, string): + match_ = re.match(r'^$', string) + if match_ and '.' in match_.group(1): + source, _, name = match_.group(1).rpartition('.') + return cls(name, cls(source)) + elif match_: + return cls(match_.group(1)) + return None # `pickle.dump` will try to get the `__getstate__` and `__setstate__` # methods of the dumped object. If these two methods are not defined, @@ -132,110 +112,58 @@ def __setstate__(self, state): self.__dict__ = state -class LazyAttr: - """The attribute of the LazyObject. - - When parsing the configuration file, the imported syntax will be - parsed as the assignment ``LazyObject``. During the subsequent parsing - process, users may reference the attributes of the LazyObject. - To ensure that these attributes also contain information needed to - reconstruct the attribute itself, LazyAttr was introduced. - - Examples: - >>> models = LazyObject(['mmdet.models']) - >>> model = dict(type=models.RetinaNet) - >>> print(type(model['type'])) # - >>> print(model['type'].build()) # - """ # noqa: E501 - - def __init__(self, - name: str, - source: Union['LazyObject', 'LazyAttr'], - location=None): - self.name = name - self.source: Union[LazyAttr, LazyObject] = source - - if isinstance(self.source, LazyObject): - if isinstance(self.source._module, str): - if self.source._imported is None: - # source code: - # from xxx.yyy import zzz - # equivalent code: - # zzz = LazyObject('xxx.yyy', 'zzz') - # The source code of get attribute: - # eee = zzz.eee - # Then, `eee._module` should be "xxx.yyy.zzz" - self._module = self.source._module - else: - # source code: - # import xxx.yyy as zzz - # equivalent code: - # zzz = LazyObject('xxx.yyy') - # The source code of get attribute: - # eee = zzz.eee - # Then, `eee._module` should be "xxx.yyy" - self._module = f'{self.source._module}.{self.source}' - else: - # The source code of LazyObject should be - # 1. import xxx.yyy - # 2. import xxx.zzz - # Equivalent to - # xxx = LazyObject(['xxx.yyy', 'xxx.zzz']) - - # The source code of LazyAttr should be - # eee = xxx.eee - # Then, eee._module = xxx - self._module = str(self.source) - elif isinstance(self.source, LazyAttr): - # 1. import xxx - # 2. zzz = xxx.yyy.zzz - - # Equivalent to: - # xxx = LazyObject('xxx') - # zzz = xxx.yyy.zzz - # zzz._module = xxx.yyy._module + zzz.name - self._module = f'{self.source._module}.{self.source.name}' - self.location = location - - @property - def module(self): - return self._module - - def __call__(self, *args, **kwargs: Any) -> Any: - raise RuntimeError() - - def __getattr__(self, name: str) -> 'LazyAttr': - return LazyAttr(name, self) - - def __deepcopy__(self, memo): - return LazyAttr(self.name, self.source) - - def build(self) -> Any: - """Return the attribute of the imported object. - - Returns: - Any: attribute of the imported object. - """ - obj = self.source.build() - try: - return getattr(obj, self.name) - except AttributeError: - raise ImportError(f'Failed to import {self.module}.{self.name} in ' - f'{self.location}') - except ImportError as e: - raise e - - def __str__(self) -> str: - return self.name - - __repr__ = __str__ - - # `pickle.dump` will try to get the `__getstate__` and `__setstate__` - # methods of the dumped object. If these two methods are not defined, - # LazyAttr will return a `__getstate__` LazyAttr` or `__setstate__` - # LazyAttr. - def __getstate__(self): - return self.__dict__ - - def __setstate__(self, state): - self.__dict__ = state +class LazyImportContext: + + def __init__(self, enable=True): + self.enable = enable + + def find_spec(self, fullname, path=None, target=None): + if not self.enable or 'mmengine.config' in fullname: + # avoid lazy import mmengine functions + return None + spec = spec_from_loader(fullname, self) + return spec + + def create_module(self, spec): + self.lazy_modules.append(spec.name) + return LazyObject(spec.name) + + @classmethod + def exec_module(self, module): + pass + + def __enter__(self): + # insert after FrozenImporter + index = sys.meta_path.index(importlib.machinery.FrozenImporter) + sys.meta_path.insert(index + 1, self) + self.lazy_modules = [] + + def __exit__(self, exc_type, exc_val, exc_tb): + sys.meta_path.remove(self) + for name in self.lazy_modules: + if '.' in name: + parent_module, _, child_name = name.rpartition('.') + if parent_module in sys.modules: + delattr(sys.modules[parent_module], child_name) + sys.modules.pop(name, None) + + def __repr__(self): + return f'' + + +def recover_lazy_field(cfg): + + if isinstance(cfg, dict): + for k, v in cfg.items(): + cfg[k] = recover_lazy_field(v) + return cfg + elif isinstance(cfg, (tuple, list)): + container_type = type(cfg) + cfg = list(cfg) + for i, v in enumerate(cfg): + cfg[i] = recover_lazy_field(v) + return container_type(cfg) + elif isinstance(cfg, str): + recover = LazyObject.from_str(cfg) + return recover if recover is not None else cfg + return cfg diff --git a/mmengine/config/new_config.py b/mmengine/config/new_config.py new file mode 100644 index 0000000000..61c086f828 --- /dev/null +++ b/mmengine/config/new_config.py @@ -0,0 +1,471 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import builtins +import importlib +import inspect +import os +import platform +import sys +from importlib.machinery import PathFinder +from pathlib import Path +from types import BuiltinFunctionType, FunctionType, ModuleType +from typing import Optional, Tuple, Union + +import yapf +from yapf.yapflib.yapf_api import FormatCode + +from mmengine.utils import digit_version +from .config import Config, ConfigDict +from .lazy import LazyImportContext, LazyObject, recover_lazy_field + +RESERVED_KEYS = ['filename', 'text', 'pretty_text'] +_CFG_UID = 0 + +if platform.system() == 'Windows': + import regex as re +else: + import re # type: ignore + + +def format_inpsect(obj): + file = inspect.getsourcefile(obj) + lines, lineno = inspect.getsourcelines(obj) + msg = f'File "{file}", line {lineno}\n--> {lines[0]}' + return msg + + +def dump_extra_type(value): + if isinstance(value, LazyObject): + return value.dump_str + if isinstance(value, (type, FunctionType, BuiltinFunctionType)): + return LazyObject(value.__name__, value.__module__).dump_str + if isinstance(value, ModuleType): + return LazyObject(value.__name__).dump_str + + typename = type(value).__module__ + '.' + type(value).__name__ + if typename == 'torch.dtype': + return LazyObject(str(value)).dump_str + + return None + + +def filter_imports(item): + k, v = item + # If the name is the same as the function/type name, + # It should come from import instead of a field + if isinstance(v, (FunctionType, type)): + return v.__name__ != k + elif isinstance(v, LazyObject): + return v.name != k + elif isinstance(v, ModuleType): + return False + return True + + +class ConfigV2(Config): + """A facility for config and config files. + + It supports common file formats as configs: python/json/yaml. + ``Config.fromfile`` can parse a dictionary from a config file, then + build a ``Config`` instance with the dictionary. + The interface is the same as a dict object and also allows access config + values as attributes. + + Args: + cfg_dict (dict, optional): A config dictionary. Defaults to None. + cfg_text (str, optional): Text of config. Defaults to None. + filename (str or Path, optional): Name of config file. + Defaults to None. + + Here is a simple example: + + Examples: + >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + >>> cfg.a + 1 + >>> cfg.b + {'b1': [0, 1]} + >>> cfg.b.b1 + [0, 1] + >>> cfg = Config.fromfile('tests/data/config/a.py') + >>> cfg.filename + "/home/username/projects/mmengine/tests/data/config/a.py" + >>> cfg.item4 + 'test' + >>> cfg + "Config [path: /home/username/projects/mmengine/tests/data/config/a.py] + :" + "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}" + + You can find more advance usage in the `config tutorial`_. + + .. _config tutorial: https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html + """ # noqa: E501 + _pkg_prefix = '_mmengine_cfg' + + def __init__(self, + cfg_dict: dict = None, + cfg_text: Optional[str] = None, + filename: Optional[Union[str, Path]] = None, + format_python_code: bool = True): + filename = str(filename) if isinstance(filename, Path) else filename + if cfg_dict is None: + cfg_dict = dict() + elif not isinstance(cfg_dict, dict): + raise TypeError('cfg_dict must be a dict, but ' + f'got {type(cfg_dict)}') + + for key in cfg_dict: + if key in RESERVED_KEYS: + raise KeyError(f'{key} is reserved for config file') + + if not isinstance(cfg_dict, ConfigDict): + cfg_dict = ConfigDict(cfg_dict) + # Recover dumped lazy object like '' from string + cfg_dict = recover_lazy_field(cfg_dict) + + super(Config, self).__setattr__('_cfg_dict', cfg_dict) + super(Config, self).__setattr__('_filename', filename) + super(Config, self).__setattr__('_format_python_code', + format_python_code) + if cfg_text: + text = cfg_text + elif filename: + with open(filename, encoding='utf-8') as f: + text = f.read() + else: + text = '' + + super(Config, self).__setattr__('_text', text) + + self._sanity_check(self._to_lazy_dict()) + + @staticmethod + def _sanity_check(cfg): + if isinstance(cfg, dict): + for v in cfg.values(): + ConfigV2._sanity_check(v) + elif isinstance(cfg, (tuple, list, set)): + for v in cfg: + ConfigV2._sanity_check(v) + elif isinstance(cfg, (type, FunctionType)): + if (ConfigV2._pkg_prefix in cfg.__module__ + or '__main__' in cfg.__module__): + msg = ('You cannot use temporary functions ' + 'as the value of a field.\n\n') + msg += format_inpsect(cfg) + raise ValueError(msg) + + @staticmethod + def fromfile( # type: ignore + filename: Union[str, Path], + keep_imported: bool = False, + format_python_code: bool = True) -> 'ConfigV2': + """Build a Config instance from config file. + + Args: + filename (str or Path): Name of config file. + use_predefined_variables (bool, optional): Whether to use + predefined variables. Defaults to True. + import_custom_modules (bool, optional): Whether to support + importing custom modules in config. Defaults to None. + lazy_import (bool): Whether to load config in `lazy_import` mode. + If it is `None`, it will be deduced by the content of the + config file. Defaults to None. + + Returns: + Config: Config instance built from config file. + """ + # Enable lazy import when parsing the config. + # Using try-except to make sure ``ConfigDict.lazy`` will be reset + # to False. See more details about lazy in the docstring of + # ConfigDict + ConfigDict.lazy = True + try: + module = ConfigV2._get_config_module(filename) + module_dict = { + k: getattr(module, k) + for k in dir(module) if not k.startswith('__') + } + if not keep_imported: + module_dict = dict(filter(filter_imports, module_dict.items())) + + cfg_dict = ConfigDict(module_dict) + + cfg = ConfigV2( + cfg_dict, + filename=filename, + format_python_code=format_python_code) + finally: + ConfigDict.lazy = False + global _CFG_UID + _CFG_UID = 0 + for mod in list(sys.modules): + if mod.startswith(ConfigV2._pkg_prefix): + del sys.modules[mod] + + return cfg + + @staticmethod + def _get_config_module(filename: Union[str, Path]): + file = Path(filename).absolute() + module_name = re.sub(r'\W|^(?=\d)', '_', file.stem) + global _CFG_UID + # Build a unique module name to avoid conflict. + fullname = f'{ConfigV2._pkg_prefix}{_CFG_UID}_{module_name}' + _CFG_UID += 1 + + # import config file as a module + with LazyImportContext(): + spec = importlib.util.spec_from_file_location(fullname, file) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore + sys.modules[fullname] = module + + return module + + @staticmethod + def _dict_to_config_dict_lazy(cfg: dict): + """Recursively converts ``dict`` to :obj:`ConfigDict`. The only + difference between ``_dict_to_config_dict_lazy`` and + ``_dict_to_config_dict_lazy`` is that the former one does not consider + the scope, and will not trigger the building of ``LazyObject``. + + Args: + cfg (dict): Config dict. + + Returns: + ConfigDict: Converted dict. + """ + # Only the outer dict with key `type` should have the key `_scope_`. + if isinstance(cfg, dict): + cfg_dict = ConfigDict() + for key, value in cfg.items(): + cfg_dict[key] = ConfigV2._dict_to_config_dict_lazy(value) + return cfg_dict + if isinstance(cfg, (tuple, list)): + return type(cfg)( + ConfigV2._dict_to_config_dict_lazy(_cfg) for _cfg in cfg) + return cfg + + @property + def filename(self) -> str: + """get file name of config.""" + return self._filename + + @property + def text(self) -> str: + """get config text.""" + return self._text + + @property + def pretty_text(self) -> str: + """get formatted python config text.""" + + def _format_dict(input_dict): + use_mapping = not all(str(k).isidentifier() for k in input_dict) + + if use_mapping: + item_tmpl = '{k}: {v}' + else: + item_tmpl = '{k}={v}' + + items = [] + for k, v in input_dict.items(): + v_str = _format_basic_types(v) + k_str = _format_basic_types(k) if use_mapping else k + items.append(item_tmpl.format(k=k_str, v=v_str)) + items = ','.join(items) + + if use_mapping: + return '{' + items + '}' + else: + return f'dict({items})' + + def _format_list_tuple_set(input_container): + items = [] + + for item in input_container: + items.append(_format_basic_types(item)) + + if isinstance(input_container, tuple): + items = items + [''] if len(items) == 1 else items + return '(' + ','.join(items) + ')' + elif isinstance(input_container, list): + return '[' + ','.join(items) + ']' + elif isinstance(input_container, set): + return '{' + ','.join(items) + '}' + + def _format_basic_types(input_): + if isinstance(input_, str): + return repr(input_) + elif isinstance(input_, dict): + return _format_dict(input_) + elif isinstance(input_, (list, set, tuple)): + return _format_list_tuple_set(input_) + else: + dump_str = dump_extra_type(input_) + if dump_str is not None: + return repr(dump_str) + else: + return str(input_) + + cfg_dict = self._to_lazy_dict() + + items = [] + for k, v in cfg_dict.items(): + items.append(f'{k} = {_format_basic_types(v)}') + + text = '\n'.join(items) + if self._format_python_code: + # copied from setup.cfg + yapf_style = dict( + based_on_style='pep8', + blank_line_before_nested_class_or_def=True, + split_before_expression_after_opening_paren=True) + try: + if digit_version(yapf.__version__) >= digit_version('0.40.2'): + text, _ = FormatCode(text, style_config=yapf_style) + else: + text, _ = FormatCode( + text, style_config=yapf_style, verify=True) + except: # noqa: E722 + raise SyntaxError('Failed to format the config file, please ' + f'check the syntax of: \n{text}') + + return text + + def __getstate__( + self + ) -> Tuple[dict, Optional[str], Optional[str], bool]: # type: ignore + return (self._cfg_dict, self._filename, self._text, + self._format_python_code) + + def __setstate__( # type: ignore + self, + state: Tuple[dict, Optional[str], Optional[str], bool], + ): + super(Config, self).__setattr__('_cfg_dict', state[0]) + super(Config, self).__setattr__('_filename', state[1]) + super(Config, self).__setattr__('_text', state[2]) + super(Config, self).__setattr__('_format_python_code', state[3]) + + def _to_lazy_dict(self, keep_imported: bool = False) -> dict: + """Convert config object to dictionary and filter the imported + object.""" + res = self._cfg_dict._to_lazy_dict() + + if keep_imported: + return res + else: + return dict(filter(filter_imports, res.items())) + + def to_dict(self, keep_imported: bool = False): + """Convert all data in the config to a builtin ``dict``. + + Args: + keep_imported (bool): Whether to keep the imported field. + Defaults to False + + If you import third-party objects in the config file, all imported + objects will be converted to a string like ``torch.optim.SGD`` + """ + _cfg_dict = self._to_lazy_dict(keep_imported=keep_imported) + + def lazy2string(cfg_dict): + if isinstance(cfg_dict, dict): + return type(cfg_dict)( + {k: lazy2string(v) + for k, v in cfg_dict.items()}) + elif isinstance(cfg_dict, (tuple, list)): + return type(cfg_dict)(lazy2string(v) for v in cfg_dict) + else: + dump_str = dump_extra_type(cfg_dict) + return dump_str if dump_str is not None else cfg_dict + + return lazy2string(_cfg_dict) + + +class BaseImportContext(): + + def __enter__(self): + # Disable enabled lazy loader during parsing base + self.lazy_importers = [] + for p in sys.meta_path: + if isinstance(p, LazyImportContext) and p.enable: + self.lazy_importers.append(p) + p.enable = False + + old_import = builtins.__import__ + + def new_import(name, globals=None, locals=None, fromlist=(), level=0): + # For relative import, the new import allows import from files + # which are not in a package. + # For absolute import, the new import will try to find the python + # file according to the module name literally, it's used to handle + # importing from installed packages, like + # `mmpretrain.configs.resnet.resnet18_8xb32_in1k`. + + cur_file = None + + # Try to import the base config source file + if level != 0 and globals is not None: + # For relative import path + if '__file__' in globals: + loc = Path(globals['__file__']).parent + else: + loc = Path(os.getcwd()) + cur_file = self.find_relative_file(loc, name, level - 1) + if not cur_file.exists(): + raise ImportError( + f'Cannot find the base config "{name}" from ' + f'{loc}: {cur_file} does not exist.') + elif level == 0: + # For absolute import path + pkg, _, mod = name.partition('.') + pkg = PathFinder.find_spec(pkg) + if mod and pkg.submodule_search_locations: + loc = Path(pkg.submodule_search_locations[0]) + cur_file = self.find_relative_file(loc, mod) + if not cur_file.exists(): + raise ImportError( + f'Cannot find the base config "{name}": ' + f'{cur_file} does not exist.') + + # Recover the original import during handle the base config file. + builtins.__import__ = old_import + + if cur_file is not None: + mod = ConfigV2._get_config_module(cur_file) + + for k in dir(mod): + mod.__dict__[k] = ConfigV2._dict_to_config_dict_lazy( + getattr(mod, k)) + else: + mod = old_import( + name, globals, locals, fromlist=fromlist, level=level) + + builtins.__import__ = new_import + + return mod + + self.old_import = old_import + builtins.__import__ = new_import + + def __exit__(self, exc_type, exc_val, exc_tb): + builtins.__import__ = self.old_import + for p in self.lazy_importers: + p.enable = True + + @staticmethod + def find_relative_file(loc: Path, relative_import_path, level=0): + if level > 0: + loc = loc.parents[level - 1] + names = relative_import_path.lstrip('.').split('.') + + for name in names: + loc = loc / name + + return loc.with_suffix('.py') + + +read_base = BaseImportContext diff --git a/mmengine/config/old_config.py b/mmengine/config/old_config.py new file mode 100644 index 0000000000..9c26619228 --- /dev/null +++ b/mmengine/config/old_config.py @@ -0,0 +1,691 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import ast +import copy +import os +import os.path as osp +import platform +import shutil +import sys +import tempfile +import types +import uuid +import warnings +from pathlib import Path +from typing import Any, Optional, Tuple, Union + +from mmengine.fileio import load +from mmengine.logging import print_log +from mmengine.utils import (check_file_exist, get_installed_path, + import_modules_from_strings, is_installed) +from .config import BASE_KEY, Config, ConfigDict +from .lazy import recover_lazy_field +from .utils import (ConfigParsingError, RemoveAssignFromAST, + _get_external_cfg_base_path, _get_external_cfg_path, + _get_package_and_cfg_path) + +DEPRECATION_KEY = '_deprecation_' +RESERVED_KEYS = ['filename', 'text', 'pretty_text', 'env_variables'] + +if platform.system() == 'Windows': + import regex as re +else: + import re # type: ignore + + +class ConfigV1(Config): + """A facility for config and config files. + + It supports common file formats as configs: python/json/yaml. + ``ConfigV1.fromfile`` can parse a dictionary from a config file, then + build a ``ConfigV1`` instance with the dictionary. + The interface is the same as a dict object and also allows access config + values as attributes. + + Args: + cfg_dict (dict, optional): A config dictionary. Defaults to None. + cfg_text (str, optional): Text of config. Defaults to None. + filename (str or Path, optional): Name of config file. + Defaults to None. + format_python_code (bool): Whether to format Python code by yapf. + Defaults to True. + + Here is a simple example: + + Examples: + >>> cfg = ConfigV1(dict(a=1, b=dict(b1=[0, 1]))) + >>> cfg.a + 1 + >>> cfg.b + {'b1': [0, 1]} + >>> cfg.b.b1 + [0, 1] + >>> cfg = ConfigV1.fromfile('tests/data/config/a.py') + >>> cfg.filename + "/home/username/projects/mmengine/tests/data/config/a.py" + >>> cfg.item4 + 'test' + >>> cfg + "Config [path: /home/username/projects/mmengine/tests/data/config/a.py] + :" + "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}" + + You can find more advance usage in the `config tutorial`_. + + .. _config tutorial: https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html + """ # noqa: E501 + + def __init__(self, + cfg_dict: dict = None, + cfg_text: Optional[str] = None, + filename: Optional[Union[str, Path]] = None, + env_variables: Optional[dict] = None, + format_python_code: bool = True): + filename = str(filename) if isinstance(filename, Path) else filename + if cfg_dict is None: + cfg_dict = dict() + elif not isinstance(cfg_dict, dict): + raise TypeError('cfg_dict must be a dict, but ' + f'got {type(cfg_dict)}') + for key in cfg_dict: + if key in RESERVED_KEYS: + raise KeyError(f'{key} is reserved for config file') + + if not isinstance(cfg_dict, ConfigDict): + cfg_dict = ConfigDict(cfg_dict) + # Recover dumped lazy object like '' from string + cfg_dict = recover_lazy_field(cfg_dict) + + super(Config, self).__setattr__('_cfg_dict', cfg_dict) + super(Config, self).__setattr__('_filename', filename) + super(Config, self).__setattr__('_format_python_code', + format_python_code) + if cfg_text: + text = cfg_text + elif filename: + with open(filename, encoding='utf-8') as f: + text = f.read() + else: + text = '' + super(Config, self).__setattr__('_text', text) + if env_variables is None: + env_variables = dict() + super(Config, self).__setattr__('_env_variables', env_variables) + + @staticmethod + def fromfile( # type: ignore + filename: Union[str, Path], + use_predefined_variables: bool = True, + import_custom_modules: bool = True, + use_environment_variables: bool = True, + format_python_code: bool = True, + ) -> 'ConfigV1': + """Build a Config instance from config file. + + Args: + filename (str or Path): Name of config file. + use_predefined_variables (bool, optional): Whether to use + predefined variables. Defaults to True. + import_custom_modules (bool, optional): Whether to support + importing custom modules in config. Defaults to None. + format_python_code (bool): Whether to format Python code by yapf. + Defaults to True. + + Returns: + ConfigV1: Config instance built from config file. + """ + filename = str(filename) if isinstance(filename, Path) else filename + cfg_dict, cfg_text, env_variables = ConfigV1._file2dict( + filename, use_predefined_variables, use_environment_variables) + if import_custom_modules and cfg_dict.get('custom_imports', None): + try: + import_modules_from_strings(**cfg_dict['custom_imports']) + except ImportError as e: + err_msg = ( + 'Failed to import custom modules from ' + f"{cfg_dict['custom_imports']}, the current sys.path " + 'is: ') + for p in sys.path: + err_msg += f'\n {p}' + err_msg += ('\nYou should set `PYTHONPATH` to make `sys.path` ' + 'include the directory which contains your custom ' + 'module') + raise ImportError(err_msg) from e + return ConfigV1( + cfg_dict, + cfg_text=cfg_text, + filename=filename, + env_variables=env_variables, + format_python_code=format_python_code) + + @staticmethod + def _validate_py_syntax(filename: str): + """Validate syntax of python config. + + Args: + filename (str): Filename of python config file. + """ + with open(filename, encoding='utf-8') as f: + content = f.read() + try: + ast.parse(content) + except SyntaxError as e: + raise SyntaxError('There are syntax errors in config ' + f'file {filename}: {e}') + + @staticmethod + def _substitute_predefined_vars(filename: str, temp_config_name: str): + """Substitute predefined variables in config with actual values. + + Sometimes we want some variables in the config to be related to the + current path or file name, etc. + + Here is an example of a typical usage scenario. When training a model, + we define a working directory in the config that save the models and + logs. For different configs, we expect to define different working + directories. A common way for users is to use the config file name + directly as part of the working directory name, e.g. for the config + ``config_setting1.py``, the working directory is + ``. /work_dir/config_setting1``. + + This can be easily achieved using predefined variables, which can be + written in the config `config_setting1.py` as follows + + .. code-block:: python + + work_dir = '. /work_dir/{{ fileBasenameNoExtension }}' + + + Here `{{ fileBasenameNoExtension }}` indicates the file name of the + config (without the extension), and when the config class reads the + config file, it will automatically parse this double-bracketed string + to the corresponding actual value. + + .. code-block:: python + + cfg = ConfigV1.fromfile('. /config_setting1.py') + cfg.work_dir # ". /work_dir/config_setting1" + + + For details, Please refer to docs/zh_cn/advanced_tutorials/config.md . + + Args: + filename (str): Filename of config. + temp_config_name (str): Temporary filename to save substituted + config. + """ + file_dirname = osp.dirname(filename) + file_basename = osp.basename(filename) + file_basename_no_extension = osp.splitext(file_basename)[0] + file_extname = osp.splitext(filename)[1] + support_templates = dict( + fileDirname=file_dirname, + fileBasename=file_basename, + fileBasenameNoExtension=file_basename_no_extension, + fileExtname=file_extname) + with open(filename, encoding='utf-8') as f: + config_file = f.read() + for key, value in support_templates.items(): + regexp = r'\{\{\s*' + str(key) + r'\s*\}\}' + value = value.replace('\\', '/') + config_file = re.sub(regexp, value, config_file) + with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: + tmp_config_file.write(config_file) + + @staticmethod + def _substitute_env_variables(filename: str, temp_config_name: str): + """Substitute environment variables in config with actual values. + + Sometimes, we want to change some items in the config with environment + variables. For examples, we expect to change dataset root by setting + ``DATASET_ROOT=/dataset/root/path`` in the command line. This can be + easily achieved by writing lines in the config as follows + + .. code-block:: python + + data_root = '{{$DATASET_ROOT:/default/dataset}}/images' + + + Here, ``{{$DATASET_ROOT:/default/dataset}}`` indicates using the + environment variable ``DATASET_ROOT`` to replace the part between + ``{{}}``. If the ``DATASET_ROOT`` is not set, the default value + ``/default/dataset`` will be used. + + Environment variables not only can replace items in the string, they + can also substitute other types of data in config. In this situation, + we can write the config as below + + .. code-block:: python + + model = dict( + bbox_head = dict(num_classes={{'$NUM_CLASSES:80'}})) + + + For details, Please refer to docs/zh_cn/tutorials/config.md . + + Args: + filename (str): Filename of config. + temp_config_name (str): Temporary filename to save substituted + config. + """ + with open(filename, encoding='utf-8') as f: + config_file = f.read() + regexp = r'\{\{[\'\"]?\s*\$(\w+)\s*\:\s*(\S*?)\s*[\'\"]?\}\}' + keys = re.findall(regexp, config_file) + env_variables = dict() + for var_name, value in keys: + regexp = r'\{\{[\'\"]?\s*\$' + var_name + r'\s*\:\s*' \ + + value + r'\s*[\'\"]?\}\}' + if var_name in os.environ: + value = os.environ[var_name] + env_variables[var_name] = value + print_log( + f'Using env variable `{var_name}` with value of ' + f'{value} to replace item in config.', + logger='current') + if not value: + raise KeyError(f'`{var_name}` cannot be found in `os.environ`.' + f' Please set `{var_name}` in environment or ' + 'give a default value.') + config_file = re.sub(regexp, value, config_file) + + with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: + tmp_config_file.write(config_file) + return env_variables + + @staticmethod + def _pre_substitute_base_vars(filename: str, + temp_config_name: str) -> dict: + """Preceding step for substituting variables in base config with actual + value. + + Args: + filename (str): Filename of config. + temp_config_name (str): Temporary filename to save substituted + config. + + Returns: + dict: A dictionary contains variables in base config. + """ + with open(filename, encoding='utf-8') as f: + config_file = f.read() + base_var_dict = {} + regexp = r'\{\{\s*' + BASE_KEY + r'\.([\w\.]+)\s*\}\}' + base_vars = set(re.findall(regexp, config_file)) + for base_var in base_vars: + randstr = f'_{base_var}_{uuid.uuid4().hex.lower()[:6]}' + base_var_dict[randstr] = base_var + regexp = r'\{\{\s*' + BASE_KEY + r'\.' + base_var + r'\s*\}\}' + config_file = re.sub(regexp, f'"{randstr}"', config_file) + with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: + tmp_config_file.write(config_file) + return base_var_dict + + @staticmethod + def _substitute_base_vars(cfg: Any, base_var_dict: dict, + base_cfg: dict) -> Any: + """Substitute base variables from strings to their actual values. + + Args: + Any : Config dictionary. + base_var_dict (dict): A dictionary contains variables in base + config. + base_cfg (dict): Base config dictionary. + + Returns: + Any : A dictionary with origin base variables + substituted with actual values. + """ + cfg = copy.deepcopy(cfg) + + if isinstance(cfg, dict): + for k, v in cfg.items(): + if isinstance(v, str) and v in base_var_dict: + new_v = base_cfg + for new_k in base_var_dict[v].split('.'): + new_v = new_v[new_k] + cfg[k] = new_v + elif isinstance(v, (list, tuple, dict)): + cfg[k] = ConfigV1._substitute_base_vars( + v, base_var_dict, base_cfg) + elif isinstance(cfg, tuple): + cfg = tuple( + ConfigV1._substitute_base_vars(c, base_var_dict, base_cfg) + for c in cfg) + elif isinstance(cfg, list): + cfg = [ + ConfigV1._substitute_base_vars(c, base_var_dict, base_cfg) + for c in cfg + ] + elif isinstance(cfg, str) and cfg in base_var_dict: + new_v = base_cfg + for new_k in base_var_dict[cfg].split('.'): + new_v = new_v[new_k] + cfg = new_v + + return cfg + + @staticmethod + def _file2dict( + filename: str, + use_predefined_variables: bool = True, + use_environment_variables: bool = True) -> Tuple[dict, str, dict]: + """Transform file to variables dictionary. + + Args: + filename (str): Name of config file. + use_predefined_variables (bool, optional): Whether to use + predefined variables. Defaults to True. + + Returns: + Tuple[dict, str]: Variables dictionary and text of config. + """ + filename = osp.abspath(osp.expanduser(filename)) + lazy_import = Config._is_lazy_import(filename) + if lazy_import: + raise ConfigParsingError( + 'The configuration file type in the inheritance chain ' + 'must match the current configuration file type, either ' + '"lazy_import" or non-"lazy_import". You got this error ' + 'since you use the syntax like `_base_ = ..."` ' # noqa: E501 + 'in your config. You should use `with read_base(): ... to` ' # noqa: E501 + 'mark the inherited config file. See more information ' + 'in https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html' # noqa: E501 + ) + + check_file_exist(filename) + fileExtname = osp.splitext(filename)[1] + if fileExtname not in ['.py', '.json', '.yaml', '.yml']: + raise OSError('Only py/yml/yaml/json type are supported now!') + try: + with tempfile.TemporaryDirectory() as temp_config_dir: + temp_config_file = tempfile.NamedTemporaryFile( + dir=temp_config_dir, suffix=fileExtname, delete=False) + if platform.system() == 'Windows': + temp_config_file.close() + + # Substitute predefined variables + if use_predefined_variables: + ConfigV1._substitute_predefined_vars( + filename, temp_config_file.name) + else: + shutil.copyfile(filename, temp_config_file.name) + # Substitute environment variables + env_variables = dict() + if use_environment_variables: + env_variables = ConfigV1._substitute_env_variables( + temp_config_file.name, temp_config_file.name) + # Substitute base variables from placeholders to strings + base_var_dict = ConfigV1._pre_substitute_base_vars( + temp_config_file.name, temp_config_file.name) + + # Handle base files + base_cfg_dict = ConfigDict() + cfg_text_list = list() + for base_cfg_path in ConfigV1._get_base_files( + temp_config_file.name): + base_cfg_path, scope = ConfigV1._get_cfg_path( + base_cfg_path, filename) + _cfg_dict, _cfg_text, _env_variables = ConfigV1._file2dict( + filename=base_cfg_path, + use_predefined_variables=use_predefined_variables, + use_environment_variables=use_environment_variables) + cfg_text_list.append(_cfg_text) + env_variables.update(_env_variables) + duplicate_keys = base_cfg_dict.keys() & _cfg_dict.keys() + if len(duplicate_keys) > 0: + raise KeyError( + 'Duplicate key is not allowed among bases. ' + f'Duplicate keys: {duplicate_keys}') + + # _dict_to_config_dict will do the following things: + # 1. Recursively converts ``dict`` to :obj:`ConfigDict`. + # 2. Set `_scope_` for the outer dict variable for the base + # config. + # 3. Set `scope` attribute for each base variable. + # Different from `_scope_`， `scope` is not a key of base + # dict, `scope` attribute will be parsed to key `_scope_` + # by function `_parse_scope` only if the base variable is + # accessed by the current config. + _cfg_dict = ConfigV1._dict_to_config_dict(_cfg_dict, scope) + base_cfg_dict.update(_cfg_dict) + + if filename.endswith('.py'): + with open(temp_config_file.name, encoding='utf-8') as f: + parsed_codes = ast.parse(f.read()) + parsed_codes = RemoveAssignFromAST(BASE_KEY).visit( + parsed_codes) + codeobj = compile(parsed_codes, filename, mode='exec') + # Support load global variable in nested function of the + # config. + global_locals_var = {BASE_KEY: base_cfg_dict} + ori_keys = set(global_locals_var.keys()) + eval(codeobj, global_locals_var, global_locals_var) + cfg_dict = { + key: value + for key, value in global_locals_var.items() + if (key not in ori_keys and not key.startswith('__')) + } + elif filename.endswith(('.yml', '.yaml', '.json')): + cfg_dict = load(temp_config_file.name) + # close temp file + for key, value in list(cfg_dict.items()): + if isinstance(value, + (types.FunctionType, types.ModuleType)): + cfg_dict.pop(key) + temp_config_file.close() + + # If the current config accesses a base variable of base + # configs, The ``scope`` attribute of corresponding variable + # will be converted to the `_scope_`. + ConfigV1._parse_scope(cfg_dict) + except Exception as e: + if osp.exists(temp_config_dir): + shutil.rmtree(temp_config_dir) + raise e + + # check deprecation information + if DEPRECATION_KEY in cfg_dict: + deprecation_info = cfg_dict.pop(DEPRECATION_KEY) + warning_msg = f'The config file {filename} will be deprecated ' \ + 'in the future.' + if 'expected' in deprecation_info: + warning_msg += f' Please use {deprecation_info["expected"]} ' \ + 'instead.' + if 'reference' in deprecation_info: + warning_msg += ' More information can be found at ' \ + f'{deprecation_info["reference"]}' + warnings.warn(warning_msg, DeprecationWarning) + + cfg_text = filename + '\n' + with open(filename, encoding='utf-8') as f: + # Setting encoding explicitly to resolve coding issue on windows + cfg_text += f.read() + + # Substitute base variables from strings to their actual values + cfg_dict = ConfigV1._substitute_base_vars(cfg_dict, base_var_dict, + base_cfg_dict) + cfg_dict.pop(BASE_KEY, None) + + cfg_dict = ConfigV1._merge_a_into_b(cfg_dict, base_cfg_dict) + cfg_dict = { + k: v + for k, v in cfg_dict.items() if not k.startswith('__') + } + + # merge cfg_text + cfg_text_list.append(cfg_text) + cfg_text = '\n'.join(cfg_text_list) + + return cfg_dict, cfg_text, env_variables + + @staticmethod + def _dict_to_config_dict(cfg: dict, + scope: Optional[str] = None, + has_scope=True): + """Recursively converts ``dict`` to :obj:`ConfigDict`. + + Args: + cfg (dict): Config dict. + scope (str, optional): Scope of instance. + has_scope (bool): Whether to add `_scope_` key to config dict. + + Returns: + ConfigDict: Converted dict. + """ + # Only the outer dict with key `type` should have the key `_scope_`. + if isinstance(cfg, dict): + if has_scope and 'type' in cfg: + has_scope = False + if scope is not None and cfg.get('_scope_', None) is None: + cfg._scope_ = scope # type: ignore + cfg = ConfigDict(cfg) + dict.__setattr__(cfg, 'scope', scope) + for key, value in cfg.items(): + cfg[key] = ConfigV1._dict_to_config_dict( + value, scope=scope, has_scope=has_scope) + elif isinstance(cfg, tuple): + cfg = tuple( + ConfigV1._dict_to_config_dict( + _cfg, scope, has_scope=has_scope) for _cfg in cfg) + elif isinstance(cfg, list): + cfg = [ + ConfigV1._dict_to_config_dict( + _cfg, scope, has_scope=has_scope) for _cfg in cfg + ] + return cfg + + @staticmethod + def _parse_scope(cfg: dict) -> None: + """Adds ``_scope_`` to :obj:`ConfigDict` instance, which means a base + variable. + + If the config dict already has the scope, scope will not be + overwritten. + + Args: + cfg (dict): Config needs to be parsed with scope. + """ + if isinstance(cfg, ConfigDict): + cfg._scope_ = cfg.scope + elif isinstance(cfg, (tuple, list)): + [ConfigV1._parse_scope(value) for value in cfg] + else: + return + + @staticmethod + def _get_base_files(filename: str) -> list: + """Get the base config file. + + Args: + filename (str): The config file. + + Raises: + TypeError: Name of config file. + + Returns: + list: A list of base config. + """ + file_format = osp.splitext(filename)[1] + if file_format == '.py': + ConfigV1._validate_py_syntax(filename) + with open(filename, encoding='utf-8') as f: + parsed_codes = ast.parse(f.read()).body + + def is_base_line(c): + return (isinstance(c, ast.Assign) + and isinstance(c.targets[0], ast.Name) + and c.targets[0].id == BASE_KEY) + + base_code = next((c for c in parsed_codes if is_base_line(c)), + None) + if base_code is not None: + base_code = ast.Expression( # type: ignore + body=base_code.value) # type: ignore + base_files = eval(compile(base_code, '', mode='eval')) + else: + base_files = [] + elif file_format in ('.yml', '.yaml', '.json'): + import mmengine + cfg_dict = mmengine.load(filename) + base_files = cfg_dict.get(BASE_KEY, []) + else: + raise ConfigParsingError( + 'The config type should be py, json, yaml or ' + f'yml, but got {file_format}') + base_files = base_files if isinstance(base_files, + list) else [base_files] + return base_files + + @staticmethod + def _get_cfg_path(cfg_path: str, + filename: str) -> Tuple[str, Optional[str]]: + """Get the config path from the current or external package. + + Args: + cfg_path (str): Relative path of config. + filename (str): The config file being parsed. + + Returns: + Tuple[str, str or None]: Path and scope of config. If the config + is not an external config, the scope will be `None`. + """ + if '::' in cfg_path: + # `cfg_path` startswith '::' means an external config path. + # Get package name and relative config path. + scope = cfg_path.partition('::')[0] + package, cfg_path = _get_package_and_cfg_path(cfg_path) + + if not is_installed(package): + raise ModuleNotFoundError( + f'{package} is not installed, please install {package} ' + f'manually') + + # Get installed package path. + package_path = get_installed_path(package) + try: + # Get config path from meta file. + cfg_path = _get_external_cfg_path(package_path, cfg_path) + except ValueError: + # Since base config does not have a metafile, it should be + # concatenated with package path and relative config path. + cfg_path = _get_external_cfg_base_path(package_path, cfg_path) + except FileNotFoundError as e: + raise e + return cfg_path, scope + else: + # Get local config path. + cfg_dir = osp.dirname(filename) + cfg_path = osp.join(cfg_dir, cfg_path) + return cfg_path, None + + @property + def filename(self) -> str: + """get file name of config.""" + return self._filename + + @property + def text(self) -> str: + """get config text.""" + return self._text + + @property + def env_variables(self) -> dict: + """get used environment variables.""" + return self._env_variables + + def __getstate__( + self + ) -> Tuple[dict, Optional[str], Optional[str], dict, bool]: # type: ignore + state = (self._cfg_dict, self._filename, self._text, + self._env_variables, self._format_python_code) + return state + + def __setstate__( # type: ignore + self, + state: Tuple[dict, Optional[str], Optional[str], dict, bool], + ): + super(Config, self).__setattr__('_cfg_dict', state[0]) + super(Config, self).__setattr__('_filename', state[1]) + super(Config, self).__setattr__('_text', state[2]) + super(Config, self).__setattr__('_env_variables', state[3]) + super(Config, self).__setattr__('_format_python_code', state[4]) diff --git a/mmengine/config/utils.py b/mmengine/config/utils.py index 81b58fb49a..78e6bb69fa 100644 --- a/mmengine/config/utils.py +++ b/mmengine/config/utils.py @@ -4,9 +4,8 @@ import re import sys import warnings -from collections import defaultdict from importlib.util import find_spec -from typing import List, Optional, Tuple, Union +from typing import Tuple from mmengine.fileio import load from mmengine.utils import check_file_exist @@ -182,288 +181,3 @@ def _is_builtin_module(module_name: str) -> bool: return False else: return True - - -class ImportTransformer(ast.NodeTransformer): - """Convert the import syntax to the assignment of - :class:`mmengine.config.LazyObject` and preload the base variable before - parsing the configuration file. - - Since you are already looking at this part of the code, I believe you must - be interested in the mechanism of the ``lazy_import`` feature of - :class:`Config`. In this docstring, we will dive deeper into its - principles. - - Most of OpenMMLab users maybe bothered with that: - - * In most of popular IDEs, they cannot navigate to the source code in - configuration file - * In most of popular IDEs, they cannot jump to the base file in current - configuration file, which is much painful when the inheritance - relationship is complex. - - In order to solve this problem, we introduce the ``lazy_import`` mode. - - A very intuitive idea for solving this problem is to import the module - corresponding to the "type" field using the ``import`` syntax. Similarly, - we can also ``import`` base file. - - However, this approach has a significant drawback. It requires triggering - the import logic to parse the configuration file, which can be - time-consuming. Additionally, it implies downloading numerous dependencies - solely for the purpose of parsing the configuration file. - However, it's possible that only a portion of the config will actually be - used. For instance, the package used in the ``train_pipeline`` may not - be necessary for an evaluation task. Forcing users to download these - unused packages is not a desirable solution. - - To avoid this problem, we introduce :class:`mmengine.config.LazyObject` and - :class:`mmengine.config.LazyAttr`. Before we proceed with further - explanations, you may refer to the documentation of these two modules to - gain an understanding of their functionalities. - - Actually, one of the functions of ``ImportTransformer`` is to hack the - ``import`` syntax. It will replace the import syntax - (exclude import the base files) with the assignment of ``LazyObject``. - - As for the import syntax of the base file, we cannot lazy import it since - we're eager to merge the fields of current file and base files. Therefore, - another function of the ``ImportTransformer`` is to collaborate with - ``Config._parse_lazy_import`` to parse the base files. - - Args: - global_dict (dict): The global dict of the current configuration file. - If we divide ordinary Python syntax into two parts, namely the - import section and the non-import section (assuming a simple case - with imports at the beginning and the rest of the code following), - the variables generated by the import statements are stored in - global variables for subsequent code use. In this context, - the ``global_dict`` represents the global variables required when - executing the non-import code. ``global_dict`` will be filled - during visiting the parsed code. - base_dict (dict): All variables defined in base files. - - Examples: - >>> from mmengine.config import read_base - >>> - >>> - >>> with read_base(): - >>> from .._base_.default_runtime import * - >>> from .._base_.datasets.coco_detection import dataset - - In this case, the base_dict will be: - - Examples: - >>> base_dict = { - >>> '.._base_.default_runtime': ... - >>> '.._base_.datasets.coco_detection': dataset} - - and `global_dict` will be updated like this: - - Examples: - >>> global_dict.update(base_dict['.._base_.default_runtime']) # `import *` means update all data - >>> global_dict.update(dataset=base_dict['.._base_.datasets.coco_detection']['dataset']) # only update `dataset` - """ # noqa: E501 - - def __init__(self, - global_dict: dict, - base_dict: Optional[dict] = None, - filename: Optional[str] = None): - self.base_dict = base_dict if base_dict is not None else {} - self.global_dict = global_dict - # In Windows, the filename could be like this: - # "C:\\Users\\runneradmin\\AppData\\Local\\" - # Although it has been an raw string, ast.parse will firstly escape - # it as the executed code: - # "C:\Users\runneradmin\AppData\Local\\\" - # As you see, the `\U` will be treated as a part of - # the escape sequence during code parsing, leading to an - # parsing error - # Here we use `encode('unicode_escape').decode()` for double escaping - if isinstance(filename, str): - filename = filename.encode('unicode_escape').decode() - self.filename = filename - self.imported_obj: set = set() - super().__init__() - - def visit_ImportFrom( - self, node: ast.ImportFrom - ) -> Optional[Union[List[ast.Assign], ast.ImportFrom]]: - """Hack the ``from ... import ...`` syntax and update the global_dict. - - Examples: - >>> from mmdet.models import RetinaNet - - Will be parsed as: - - Examples: - >>> RetinaNet = lazyObject('mmdet.models', 'RetinaNet') - - ``global_dict`` will also be updated by ``base_dict`` as the - class docstring says. - - Args: - node (ast.AST): The node of the current import statement. - - Returns: - Optional[List[ast.Assign]]: There three cases: - - * If the node is a statement of importing base files. - None will be returned. - * If the node is a statement of importing a builtin module, - node will be directly returned - * Otherwise, it will return the assignment statements of - ``LazyObject``. - """ - # Built-in modules will not be parsed as LazyObject - module = f'{node.level*"."}{node.module}' - if _is_builtin_module(module): - # Make sure builtin module will be added into `self.imported_obj` - for alias in node.names: - if alias.asname is not None: - self.imported_obj.add(alias.asname) - elif alias.name == '*': - raise ConfigParsingError( - 'Cannot import * from non-base config') - else: - self.imported_obj.add(alias.name) - return node - - if module in self.base_dict: - for alias_node in node.names: - if alias_node.name == '*': - self.global_dict.update(self.base_dict[module]) - return None - if alias_node.asname is not None: - base_key = alias_node.asname - else: - base_key = alias_node.name - self.global_dict[base_key] = self.base_dict[module][ - alias_node.name] - return None - - nodes: List[ast.Assign] = [] - for alias_node in node.names: - # `ast.alias` has lineno attr after Python 3.10, - if hasattr(alias_node, 'lineno'): - lineno = alias_node.lineno - else: - lineno = node.lineno - if alias_node.name == '*': - # TODO: If users import * from a non-config module, it should - # fallback to import the real module and raise a warning to - # remind users the real module will be imported which will slow - # down the parsing speed. - raise ConfigParsingError( - 'Illegal syntax in config! `from xxx import *` is not ' - 'allowed to appear outside the `if base:` statement') - elif alias_node.asname is not None: - # case1: - # from mmengine.dataset import BaseDataset as Dataset -> - # Dataset = LazyObject('mmengine.dataset', 'BaseDataset') - code = f'{alias_node.asname} = LazyObject("{module}", "{alias_node.name}", "{self.filename}, line {lineno}")' # noqa: E501 - self.imported_obj.add(alias_node.asname) - else: - # case2: - # from mmengine.model import BaseModel - # BaseModel = LazyObject('mmengine.model', 'BaseModel') - code = f'{alias_node.name} = LazyObject("{module}", "{alias_node.name}", "{self.filename}, line {lineno}")' # noqa: E501 - self.imported_obj.add(alias_node.name) - try: - nodes.append(ast.parse(code).body[0]) # type: ignore - except Exception as e: - raise ConfigParsingError( - f'Cannot import {alias_node} from {module}' - '1. Cannot import * from 3rd party lib in the config ' - 'file\n' - '2. Please check if the module is a base config which ' - 'should be added to `_base_`\n') from e - return nodes - - def visit_Import(self, node) -> Union[ast.Assign, ast.Import]: - """Work with ``_gather_abs_import_lazyobj`` to hack the ``import ...`` - syntax. - - Examples: - >>> import mmcls.models - >>> import mmcls.datasets - >>> import mmcls - - Will be parsed as: - - Examples: - >>> # import mmcls.models; import mmcls.datasets; import mmcls - >>> mmcls = lazyObject(['mmcls', 'mmcls.datasets', 'mmcls.models']) - - Args: - node (ast.AST): The node of the current import statement. - - Returns: - ast.Assign: If the import statement is ``import ... as ...``, - ast.Assign will be returned, otherwise node will be directly - returned. - """ - # For absolute import like: `import mmdet.configs as configs`. - # It will be parsed as: - # configs = LazyObject('mmdet.configs') - # For absolute import like: - # `import mmdet.configs` - # `import mmdet.configs.default_runtime` - # This will be parsed as - # mmdet = LazyObject(['mmdet.configs.default_runtime', 'mmdet.configs]) - # However, visit_Import cannot gather other import information, so - # `_gather_abs_import_LazyObject` will gather all import information - # from the same module and construct the LazyObject. - alias_list = node.names - assert len(alias_list) == 1, ( - 'Illegal syntax in config! import multiple modules in one line is ' - 'not supported') - # TODO Support multiline import - alias = alias_list[0] - if alias.asname is not None: - self.imported_obj.add(alias.asname) - if _is_builtin_module(alias.name.split('.')[0]): - return node - return ast.parse( # type: ignore - f'{alias.asname} = LazyObject(' - f'"{alias.name}",' - f'location="{self.filename}, line {node.lineno}")').body[0] - return node - - -def _gather_abs_import_lazyobj(tree: ast.Module, - filename: Optional[str] = None): - """Experimental implementation of gathering absolute import information.""" - if isinstance(filename, str): - filename = filename.encode('unicode_escape').decode() - imported = defaultdict(list) - abs_imported = set() - new_body: List[ast.stmt] = [] - # module2node is used to get lineno when Python < 3.10 - module2node: dict = dict() - for node in tree.body: - if isinstance(node, ast.Import): - for alias in node.names: - # Skip converting built-in module to LazyObject - if _is_builtin_module(alias.name): - new_body.append(node) - continue - module = alias.name.split('.')[0] - module2node.setdefault(module, node) - imported[module].append(alias) - continue - new_body.append(node) - - for key, value in imported.items(): - names = [_value.name for _value in value] - if hasattr(value[0], 'lineno'): - lineno = value[0].lineno - else: - lineno = module2node[key].lineno - lazy_module_assign = ast.parse( - f'{key} = LazyObject({names}, location="{filename}, line {lineno}")' # noqa: E501 - ) # noqa: E501 - abs_imported.add(key) - new_body.insert(0, lazy_module_assign.body[0]) - tree.body = new_body - return tree, abs_imported diff --git a/mmengine/registry/registry.py b/mmengine/registry/registry.py index 31fd44d827..a389c23e0c 100644 --- a/mmengine/registry/registry.py +++ b/mmengine/registry/registry.py @@ -10,8 +10,9 @@ from rich.console import Console from rich.table import Table +from mmengine.config.lazy import LazyObject from mmengine.config.utils import MODULE2PACKAGE -from mmengine.utils import get_object_from_string, is_seq_of +from mmengine.utils import is_seq_of from .default_scope import DefaultScope @@ -442,6 +443,13 @@ def get(self, key: str) -> Optional[Type]: 'The key argument of `Registry.get` must be a str, ' f'got {type(key)}') + try: + obj_cls = LazyObject.from_str(key) + if obj_cls is not None: + return obj_cls.build() + except Exception: + raise RuntimeError(f'Failed to get {key}') + scope, real_key = self.split_scope_key(key) obj_cls = None registry_name = self.name @@ -495,18 +503,6 @@ def get(self, key: str) -> Optional[Type]: else: obj_cls = root.get(key) - if obj_cls is None: - # Actually, it's strange to implement this `try ... except` to - # get the object by its name in `Registry.get`. However, If we - # want to build the model using a configuration like - # `dict(type='mmengine.model.BaseModel')`, which can - # be dumped by lazy import config, we need this code snippet - # for `Registry.get` to work. - try: - obj_cls = get_object_from_string(key) - except Exception: - raise RuntimeError(f'Failed to get {key}') - if obj_cls is not None: # For some rare cases (e.g. obj_cls is a partial function), obj_cls # doesn't have `__name__`. Use default value to prevent error @@ -516,7 +512,6 @@ def get(self, key: str) -> Optional[Type]: f' registry in "{scope_name}"', logger='current', level=logging.DEBUG) - return obj_cls def _search_child(self, scope: str) -> Optional['Registry']: diff --git a/mmengine/runner/_flexible_runner.py b/mmengine/runner/_flexible_runner.py index 6d727fb4d5..834cbea070 100644 --- a/mmengine/runner/_flexible_runner.py +++ b/mmengine/runner/_flexible_runner.py @@ -285,9 +285,9 @@ def __init__( if isinstance(cfg, Config): self.cfg = copy.deepcopy(cfg) elif isinstance(cfg, dict): - self.cfg = Config(cfg) + self.cfg = Config(cfg) # type: ignore else: - self.cfg = Config(dict()) + self.cfg = Config(dict()) # type: ignore # lazy initialization training_related = [train_dataloader, train_cfg, optim_wrapper] diff --git a/mmengine/runner/runner.py b/mmengine/runner/runner.py index 68716ab253..3be30cd67b 100644 --- a/mmengine/runner/runner.py +++ b/mmengine/runner/runner.py @@ -298,9 +298,9 @@ def __init__( if isinstance(cfg, Config): self.cfg = copy.deepcopy(cfg) elif isinstance(cfg, dict): - self.cfg = Config(cfg) + self.cfg = Config(cfg) # type: ignore else: - self.cfg = Config(dict()) + self.cfg = Config(dict()) # type: ignore # lazy initialization training_related = [train_dataloader, train_cfg, optim_wrapper] diff --git a/mmengine/testing/runner_test_case.py b/mmengine/testing/runner_test_case.py index f64594acef..1b77b4b948 100644 --- a/mmengine/testing/runner_test_case.py +++ b/mmengine/testing/runner_test_case.py @@ -133,7 +133,7 @@ def setUp(self) -> None: custom_hooks=[], env_cfg=dict(dist_cfg=dict(backend='nccl')), experiment_name='test1') - self.epoch_based_cfg = Config(epoch_based_cfg) + self.epoch_based_cfg = Config(epoch_based_cfg) # type: ignore # prepare iter based cfg. self.iter_based_cfg: Config = copy.deepcopy(self.epoch_based_cfg) diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index 905485c16a..2fbe3d1eff 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -17,6 +17,7 @@ import mmengine from mmengine import Config, ConfigDict, DictAction from mmengine.config.lazy import LazyObject +from mmengine.config.old_config import ConfigV1 from mmengine.fileio import dump, load from mmengine.registry import MODELS, DefaultScope, Registry from mmengine.utils import is_installed @@ -220,7 +221,7 @@ def test_auto_argparser(self): def test_dict_to_config_dict(self): cfg_dict = dict( a=1, b=dict(c=dict()), d=[dict(e=dict(f=(dict(g=1), [])))]) - cfg_dict = Config._dict_to_config_dict(cfg_dict) + cfg_dict = ConfigV1._dict_to_config_dict(cfg_dict) assert isinstance(cfg_dict, ConfigDict) assert isinstance(cfg_dict.a, int) assert isinstance(cfg_dict.b, ConfigDict) @@ -398,7 +399,7 @@ def test_substitute_predefined_vars(self, tmp_path): expected_text = expected_text.replace('\\', '/') with open(cfg, 'w') as f: f.write(cfg_text) - Config._substitute_predefined_vars(cfg, substituted_cfg) + ConfigV1._substitute_predefined_vars(cfg, substituted_cfg) with open(substituted_cfg) as f: assert f.read() == expected_text @@ -411,10 +412,10 @@ def test_substitute_environment_vars(self, tmp_path): with open(cfg, 'w') as f: f.write(cfg_text) with pytest.raises(KeyError): - Config._substitute_env_variables(cfg, substituted_cfg) + ConfigV1._substitute_env_variables(cfg, substituted_cfg) os.environ['A'] = 'text_A' - Config._substitute_env_variables(cfg, substituted_cfg) + ConfigV1._substitute_env_variables(cfg, substituted_cfg) with open(substituted_cfg) as f: assert f.read() == 'a=text_A\n' os.environ.pop('A') @@ -422,12 +423,12 @@ def test_substitute_environment_vars(self, tmp_path): cfg_text = 'b={{$B:80}}\n' with open(cfg, 'w') as f: f.write(cfg_text) - Config._substitute_env_variables(cfg, substituted_cfg) + ConfigV1._substitute_env_variables(cfg, substituted_cfg) with open(substituted_cfg) as f: assert f.read() == 'b=80\n' os.environ['B'] = '100' - Config._substitute_env_variables(cfg, substituted_cfg) + ConfigV1._substitute_env_variables(cfg, substituted_cfg) with open(substituted_cfg) as f: assert f.read() == 'b=100\n' os.environ.pop('B') @@ -435,7 +436,7 @@ def test_substitute_environment_vars(self, tmp_path): cfg_text = 'c={{"$C:80"}}\n' with open(cfg, 'w') as f: f.write(cfg_text) - Config._substitute_env_variables(cfg, substituted_cfg) + ConfigV1._substitute_env_variables(cfg, substituted_cfg) with open(substituted_cfg) as f: assert f.read() == 'c=80\n' @@ -443,7 +444,7 @@ def test_pre_substitute_base_vars(self, tmp_path): cfg_path = osp.join(self.data_path, 'config', 'py_config/test_pre_substitute_base_vars.py') tmp_cfg = tmp_path / 'tmp_cfg.py' - base_var_dict = Config._pre_substitute_base_vars(cfg_path, tmp_cfg) + base_var_dict = ConfigV1._pre_substitute_base_vars(cfg_path, tmp_cfg) assert 'item6' in base_var_dict.values() assert 'item10' in base_var_dict.values() assert 'item11' in base_var_dict.values() @@ -457,7 +458,7 @@ def test_pre_substitute_base_vars(self, tmp_path): cfg_path = osp.join(self.data_path, 'config', 'json_config/test_base.json') tmp_cfg = tmp_path / 'tmp_cfg.json' - Config._pre_substitute_base_vars(cfg_path, tmp_cfg) + ConfigV1._pre_substitute_base_vars(cfg_path, tmp_cfg) cfg_module_dict = load(tmp_cfg) assert cfg_module_dict['item9'].startswith('_item2') assert cfg_module_dict['item10'].startswith('_item7') @@ -465,7 +466,7 @@ def test_pre_substitute_base_vars(self, tmp_path): cfg_path = osp.join(self.data_path, 'config', 'yaml_config/test_base.yaml') tmp_cfg = tmp_path / 'tmp_cfg.yaml' - Config._pre_substitute_base_vars(cfg_path, tmp_cfg) + ConfigV1._pre_substitute_base_vars(cfg_path, tmp_cfg) cfg_module_dict = load(tmp_cfg) assert cfg_module_dict['item9'].startswith('_item2') assert cfg_module_dict['item10'].startswith('_item7') @@ -481,7 +482,7 @@ def test_substitute_base_vars(self): '_item2_.fswf': 'item2', '_item0_.12ed21wq': 'item0' } - cfg = Config._substitute_base_vars(cfg, base_var_dict, cfg_base) + cfg = ConfigV1._substitute_base_vars(cfg, base_var_dict, cfg_base) assert cfg['item4'] == cfg_base['item1'] assert cfg['item5']['item2'] == cfg_base['item2'] @@ -513,7 +514,7 @@ def test_get_cfg_path_local(self): filename = 'py_config/simple_config.py' filename = osp.join(self.data_path, 'config', filename) cfg_name = './base.py' - cfg_path, scope = Config._get_cfg_path(cfg_name, filename) + cfg_path, scope = ConfigV1._get_cfg_path(cfg_name, filename) assert scope is None osp.isfile(cfg_path) @@ -525,12 +526,12 @@ def test_get_cfg_path_external(self): filename = osp.join(self.data_path, 'config', filename) cfg_name = 'mmdet::faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py' - cfg_path, scope = Config._get_cfg_path(cfg_name, filename) + cfg_path, scope = ConfigV1._get_cfg_path(cfg_name, filename) assert scope == 'mmdet' osp.isfile(cfg_path) cfg_name = 'mmcls::cspnet/cspresnet50_8xb32_in1k.py' - cfg_path, scope = Config._get_cfg_path(cfg_name, filename) + cfg_path, scope = ConfigV1._get_cfg_path(cfg_name, filename) assert scope == 'mmcls' osp.isfile(cfg_path) @@ -541,7 +542,8 @@ def _simple_load(self): filename = f'{file_format}_config/{name}.{file_format}' cfg_file = osp.join(self.data_path, 'config', filename) - cfg_dict, cfg_text, env_variables = Config._file2dict(cfg_file) + cfg_dict, cfg_text, env_variables = ConfigV1._file2dict( + cfg_file) assert isinstance(cfg_text, str) assert isinstance(cfg_dict, dict) assert isinstance(env_variables, dict) @@ -564,9 +566,12 @@ def _predefined_vars(self): item2=path, item3='abc_test_predefined_var') - assert Config._file2dict(cfg_file)[0]['item1'] == cfg_dict_dst['item1'] - assert Config._file2dict(cfg_file)[0]['item2'] == cfg_dict_dst['item2'] - assert Config._file2dict(cfg_file)[0]['item3'] == cfg_dict_dst['item3'] + assert ConfigV1._file2dict( + cfg_file)[0]['item1'] == cfg_dict_dst['item1'] + assert ConfigV1._file2dict( + cfg_file)[0]['item2'] == cfg_dict_dst['item2'] + assert ConfigV1._file2dict( + cfg_file)[0]['item3'] == cfg_dict_dst['item3'] # test `use_predefined_variable=False` cfg_dict_ori = dict( @@ -574,28 +579,31 @@ def _predefined_vars(self): item2='{{ fileDirname}}', item3='abc_{{ fileBasenameNoExtension }}') - assert Config._file2dict(cfg_file, - False)[0]['item1'] == cfg_dict_ori['item1'] - assert Config._file2dict(cfg_file, - False)[0]['item2'] == cfg_dict_ori['item2'] - assert Config._file2dict(cfg_file, - False)[0]['item3'] == cfg_dict_ori['item3'] + assert ConfigV1._file2dict(cfg_file, + False)[0]['item1'] == cfg_dict_ori['item1'] + assert ConfigV1._file2dict(cfg_file, + False)[0]['item2'] == cfg_dict_ori['item2'] + assert ConfigV1._file2dict(cfg_file, + False)[0]['item3'] == cfg_dict_ori['item3'] # test test_predefined_var.yaml cfg_file = osp.join(self.data_path, 'config/yaml_config/test_predefined_var.yaml') # test `use_predefined_variable=False` - assert Config._file2dict(cfg_file, - False)[0]['item1'] == '{{ fileDirname }}' - assert Config._file2dict(cfg_file)[0]['item1'] == self._get_file_path( - osp.dirname(cfg_file)) + assert ConfigV1._file2dict(cfg_file, + False)[0]['item1'] == '{{ fileDirname }}' + assert ConfigV1._file2dict( + cfg_file)[0]['item1'] == self._get_file_path( + osp.dirname(cfg_file)) # test test_predefined_var.json cfg_file = osp.join(self.data_path, 'config/json_config/test_predefined_var.json') - assert Config.fromfile(cfg_file, False)['item1'] == '{{ fileDirname }}' + assert Config.fromfile( + cfg_file, + use_predefined_variables=False)['item1'] == '{{ fileDirname }}' assert Config.fromfile(cfg_file)['item1'] == self._get_file_path( osp.dirname(cfg_file)) @@ -605,20 +613,26 @@ def _environment_vars(self): 'config/py_config/test_environment_var.py') with pytest.raises(KeyError): - Config._file2dict(cfg_file) + ConfigV1._file2dict(cfg_file) os.environ['ITEM1'] = '60' cfg_dict_dst = dict(item1='60', item2='default_value', item3=80) - assert Config._file2dict(cfg_file)[0]['item1'] == cfg_dict_dst['item1'] - assert Config._file2dict(cfg_file)[0]['item2'] == cfg_dict_dst['item2'] - assert Config._file2dict(cfg_file)[0]['item3'] == cfg_dict_dst['item3'] + assert ConfigV1._file2dict( + cfg_file)[0]['item1'] == cfg_dict_dst['item1'] + assert ConfigV1._file2dict( + cfg_file)[0]['item2'] == cfg_dict_dst['item2'] + assert ConfigV1._file2dict( + cfg_file)[0]['item3'] == cfg_dict_dst['item3'] os.environ['ITEM2'] = 'new_value' os.environ['ITEM3'] = '50' cfg_dict_dst = dict(item1='60', item2='new_value', item3=50) - assert Config._file2dict(cfg_file)[0]['item1'] == cfg_dict_dst['item1'] - assert Config._file2dict(cfg_file)[0]['item2'] == cfg_dict_dst['item2'] - assert Config._file2dict(cfg_file)[0]['item3'] == cfg_dict_dst['item3'] + assert ConfigV1._file2dict( + cfg_file)[0]['item1'] == cfg_dict_dst['item1'] + assert ConfigV1._file2dict( + cfg_file)[0]['item2'] == cfg_dict_dst['item2'] + assert ConfigV1._file2dict( + cfg_file)[0]['item3'] == cfg_dict_dst['item3'] os.environ.pop('ITEM1') os.environ.pop('ITEM2') @@ -627,7 +641,7 @@ def _environment_vars(self): def _merge_from_base(self): cfg_file = osp.join(self.data_path, 'config/py_config/test_merge_from_base_single.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] assert cfg_dict['item1'] == [2, 3] assert cfg_dict['item2']['a'] == 1 @@ -643,7 +657,7 @@ def _merge_from_multiple_bases(self): cfg_file = osp.join( self.data_path, 'config/py_config/test_merge_from_multiple_bases.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] # cfg.fcfg_dictd assert cfg_dict['item1'] == [1, 2] @@ -666,7 +680,7 @@ def _base_variables(self): 'json_config/test_base.json', 'yaml_config/test_base.yaml' ]: cfg_file = osp.join(self.data_path, 'config', file) - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] assert cfg_dict['item1'] == [1, 2] assert cfg_dict['item2']['a'] == 0 @@ -687,7 +701,7 @@ def _base_variables(self): 'yaml_config/test_base_variables_nested.yaml' ]: cfg_file = osp.join(self.data_path, 'config', file) - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] assert cfg_dict['base'] == '_base_.item8' assert cfg_dict['item1'] == [1, 2] @@ -722,7 +736,7 @@ def _base_variables(self): cfg_file = osp.join( self.data_path, 'config/py_config/test_pre_substitute_base_vars.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] assert cfg_dict['item21'] == 'test_base_variables.py' assert cfg_dict['item22'] == 'test_base_variables.py' @@ -796,7 +810,7 @@ def _base_variables(self): # Test use global variable in config function cfg_file = osp.join(self.data_path, 'config/py_config/test_py_function_global_var.py') - cfg = Config._file2dict(cfg_file)[0] + cfg = ConfigV1._file2dict(cfg_file)[0] assert cfg['item1'] == 1 assert cfg['item2'] == 2 @@ -804,7 +818,7 @@ def _base_variables(self): # config. cfg_file = osp.join(self.data_path, 'config/py_config/test_py_modify_key.py') - cfg = Config._file2dict(cfg_file)[0] + cfg = ConfigV1._file2dict(cfg_file)[0] assert cfg == dict(item1=dict(a=1)) # Simulate the case that the temporary directory includes `.`, etc. @@ -819,13 +833,13 @@ def __init__(self, *args, prefix='test.', **kwargs): PatchedTempDirectory): cfg_file = osp.join(self.data_path, 'config/py_config/test_py_modify_key.py') - cfg = Config._file2dict(cfg_file)[0] + cfg = ConfigV1._file2dict(cfg_file)[0] assert cfg == dict(item1=dict(a=1)) def _merge_recursive_bases(self): cfg_file = osp.join(self.data_path, 'config/py_config/test_merge_recursive_bases.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] assert cfg_dict['item1'] == [2, 3] assert cfg_dict['item2']['a'] == 1 @@ -835,7 +849,7 @@ def _merge_recursive_bases(self): def _merge_delete(self): cfg_file = osp.join(self.data_path, 'config/py_config/test_merge_delete.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] # cfg.field assert cfg_dict['item1'] == dict(a=0) assert cfg_dict['item2'] == dict(a=0, b=0) @@ -851,7 +865,7 @@ def _merge_intermediate_variable(self): cfg_file = osp.join( self.data_path, 'config/py_config/test_merge_intermediate_variable_child.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] # cfg.field assert cfg_dict['item1'] == [1, 2] assert cfg_dict['item2'] == dict(a=0) @@ -988,9 +1002,10 @@ def test_lazy_import(self, tmp_path): cfg = Config.fromfile(lazy_import_cfg_path) cfg_dict = cfg.to_dict() assert (cfg_dict['train_dataloader']['dataset']['type'] == - 'mmengine.testing.runner_test_case.ToyDataset') - assert ( - cfg_dict['custom_hooks'][0]['type'] == 'mmengine.hooks.EMAHook') + '') + assert (cfg_dict['custom_hooks'][0]['type'] + in ('', + '')) # Dumped config dumped_cfg_path = tmp_path / 'test_dump_lazy.py' cfg.dump(dumped_cfg_path) @@ -1010,6 +1025,8 @@ def _compare_dict(a, b): assert len(a) == len(b) for item_a, item_b in zip(a, b): _compare_dict(item_a, item_b) + elif isinstance(a, type): + assert a.__module__ + a.__name__ == str(b) else: assert str(a) == str(b) @@ -1032,7 +1049,7 @@ def _compare_dict(a, b): error_obj = tmp_path / 'error_obj.py' error_obj.write_text("""from mmengine.fileio import error_obj""") # match pattern should be double escaped - match = str(error_obj).encode('unicode_escape').decode() + match = 'Failed to import mmengine.fileio.error_obj' with pytest.raises(ImportError, match=match): cfg = Config.fromfile(str(error_obj)) cfg.error_obj @@ -1042,17 +1059,17 @@ def _compare_dict(a, b): import mmengine error_attr = mmengine.error_attr """) # noqa: E122 - match = str(error_attr).encode('unicode_escape').decode() - with pytest.raises(ImportError, match=match): + match = "module 'mmengine' has no attribute 'error_attr'" + with pytest.raises(AttributeError, match=match): cfg = Config.fromfile(str(error_attr)) cfg.error_attr error_module = tmp_path / 'error_module.py' - error_module.write_text("""import error_module""") - match = str(error_module).encode('unicode_escape').decode() + error_module.write_text("""import error_module;a=error_module""") + match = 'Failed to import error_module' with pytest.raises(ImportError, match=match): cfg = Config.fromfile(str(error_module)) - cfg.error_module + cfg.a # lazy-import and non-lazy-import should not be used mixed. # current text config, base lazy-import config @@ -1061,14 +1078,8 @@ def _compare_dict(a, b): osp.join(self.data_path, 'config/lazy_module_config/error_mix_using1.py')) - # Force to import in non-lazy-import mode - Config.fromfile( - osp.join(self.data_path, - 'config/lazy_module_config/error_mix_using1.py'), - lazy_import=False) - # current lazy-import config, base text config - with pytest.raises(RuntimeError, match='_base_ ='): + with pytest.raises(AttributeError, match='item2'): Config.fromfile( osp.join(self.data_path, 'config/lazy_module_config/error_mix_using2.py')) @@ -1089,7 +1100,7 @@ def _compare_dict(a, b): dumped_cfg = Config.fromfile(dumped_cfg_path) assert set(dumped_cfg.keys()) == { - 'path', 'name', 'suffix', 'chained', 'existed', 'cfgname' + 'path', 'name', 'suffix', 'chained', 'existed', 'cfgname', 'ex' } assert dumped_cfg.to_dict() == cfg.to_dict() diff --git a/tests/test_config/test_lazy.py b/tests/test_config/test_lazy.py index d69822814b..8e30cb9de9 100644 --- a/tests/test_config/test_lazy.py +++ b/tests/test_config/test_lazy.py @@ -1,122 +1,11 @@ # Copyright (c) OpenMMLab. All rights reserved. -import ast import copy -import os -import os.path as osp from importlib import import_module -from importlib.util import find_spec from unittest import TestCase -import numpy -import numpy.compat -import numpy.linalg as linalg -from rich.progress import Progress - import mmengine -from mmengine.config import Config -from mmengine.config.lazy import LazyAttr, LazyObject -from mmengine.config.utils import ImportTransformer, _gather_abs_import_lazyobj -from mmengine.fileio import LocalBackend, PetrelBackend - - -class TestImportTransformer(TestCase): - - @classmethod - def setUpClass(cls) -> None: - cls.data_dir = osp.join( # type: ignore - osp.dirname(__file__), '..', 'data', 'config', - 'lazy_module_config') - super().setUpClass() - - def test_lazy_module(self): - cfg_path = osp.join(self.data_dir, 'test_ast_transform.py') - with open(cfg_path) as f: - codestr = f.read() - codeobj = ast.parse(codestr) - global_dict = { - 'LazyObject': LazyObject, - } - base_dict = { - '._base_.default_runtime': { - 'default_scope': 'test_config' - }, - '._base_.scheduler': { - 'val_cfg': {} - }, - } - codeobj = ImportTransformer(global_dict, base_dict).visit(codeobj) - codeobj, _ = _gather_abs_import_lazyobj(codeobj) - codeobj = ast.fix_missing_locations(codeobj) - - exec(compile(codeobj, cfg_path, mode='exec'), global_dict, global_dict) - # 1. absolute import - # 1.1 import module as LazyObject - lazy_numpy = global_dict['numpy'] - self.assertIsInstance(lazy_numpy, LazyObject) - - # 1.2 getattr as LazyAttr - self.assertIsInstance(lazy_numpy.linalg, LazyAttr) - self.assertIsInstance(lazy_numpy.compat, LazyAttr) - - # 1.3 Build module from LazyObject. amp and functional can be accessed - imported_numpy = lazy_numpy.build() - self.assertIs(imported_numpy.linalg, linalg) - self.assertIs(imported_numpy.compat, numpy.compat) - - # 1.4.1 Build module from LazyAttr - imported_linalg = lazy_numpy.linalg.build() - imported_compat = lazy_numpy.compat.build() - self.assertIs(imported_compat, numpy.compat) - self.assertIs(imported_linalg, linalg) - - # 1.4.2 build class method from LazyAttr - start = global_dict['start'] - self.assertEqual(start.module, 'rich.progress.Progress') - self.assertEqual(str(start), 'start') - self.assertIs(start.build(), Progress.start) - - # 1.5 import ... as, and build module from LazyObject - lazy_linalg = global_dict['linalg'] - self.assertIsInstance(lazy_linalg, LazyObject) - self.assertIs(lazy_linalg.build(), linalg) - self.assertIsInstance(lazy_linalg.norm, LazyAttr) - self.assertIs(lazy_linalg.norm.build(), linalg.norm) - - # 1.6 import built in module - imported_os = global_dict['os'] - self.assertIs(imported_os, os) - - # 2. Relative import - # 2.1 from ... import ... - lazy_local_backend = global_dict['local'] - self.assertIsInstance(lazy_local_backend, LazyObject) - self.assertIs(lazy_local_backend.build(), LocalBackend) - - # 2.2 from ... import ... as ... - lazy_petrel_backend = global_dict['PetrelBackend'] - self.assertIsInstance(lazy_petrel_backend, LazyObject) - self.assertIs(lazy_petrel_backend.build(), PetrelBackend) - - # 2.3 from ... import builtin module or obj from `mmengine.Config` - self.assertIs(global_dict['find_module'], find_spec) - self.assertIs(global_dict['Config'], Config) - - # 3 test import base config - # 3.1 simple from ... import and from ... import ... as - self.assertEqual(global_dict['scope'], 'test_config') - self.assertDictEqual(global_dict['val_cfg'], {}) - - # 4. Error catching - cfg_path = osp.join(self.data_dir, - 'test_ast_transform_error_catching1.py') - with open(cfg_path) as f: - codestr = f.read() - codeobj = ast.parse(codestr) - global_dict = {'LazyObject': LazyObject} - with self.assertRaisesRegex( - RuntimeError, - r'Illegal syntax in config! `from xxx import \*`'): - codeobj = ImportTransformer(global_dict).visit(codeobj) +from mmengine.config.lazy import LazyObject +from mmengine.fileio import LocalBackend class TestLazyObject(TestCase): @@ -126,14 +15,6 @@ def test_init(self): LazyObject('mmengine.fileio') LazyObject('mmengine.fileio', 'LocalBackend') - # module must be str - with self.assertRaises(TypeError): - LazyObject(1) - - # imported must be a sequence of string or None - with self.assertRaises(TypeError): - LazyObject('mmengine', ['error_type']) - def test_build(self): lazy_mmengine = LazyObject('mmengine') self.assertIs(lazy_mmengine.build(), mmengine) @@ -142,37 +23,19 @@ def test_build(self): self.assertIs(lazy_mmengine_fileio.build(), import_module('mmengine.fileio')) - lazy_local_backend = LazyObject('mmengine.fileio', 'LocalBackend') + lazy_local_backend = LazyObject('LocalBackend', + LazyObject('mmengine.fileio')) self.assertIs(lazy_local_backend.build(), LocalBackend) - # TODO: The commented test is required, we need to test the built - # LazyObject can access the `mmengine.dataset`. We need to clean the - # environment to make sure the `dataset` is not imported before, and - # it is triggered by lazy_mmengine.build(). However, if we simply - # pop the `mmengine.dataset` will lead to other tests failed, of which - # reason is still unknown. We need to figure out the reason and fix it - # in the latter - - # sys.modules.pop('mmengine.config') - # sys.modules.pop('mmengine.fileio') - # sys.modules.pop('mmengine') - # lazy_mmengine = LazyObject(['mmengine', 'mmengine.dataset']) - # self.assertIs(lazy_mmengine.build().dataset, - # import_module('mmengine.config')) copied = copy.deepcopy(lazy_local_backend) self.assertDictEqual(copied.__dict__, lazy_local_backend.__dict__) - with self.assertRaises(RuntimeError): + with self.assertRaises(TypeError): lazy_mmengine() with self.assertRaises(ImportError): LazyObject('unknown').build() - -class TestLazyAttr(TestCase): - # Since LazyAttr should only be built from LazyObect, we only test - # the build method here. - def test_build(self): lazy_mmengine = LazyObject('mmengine') local_backend = lazy_mmengine.fileio.LocalBackend self.assertIs(local_backend.build(), LocalBackend) @@ -180,10 +43,8 @@ def test_build(self): copied = copy.deepcopy(local_backend) self.assertDictEqual(copied.__dict__, local_backend.__dict__) - with self.assertRaises(RuntimeError): + with self.assertRaises(TypeError): local_backend() - with self.assertRaisesRegex( - ImportError, - 'Failed to import mmengine.fileio.LocalBackend.unknown'): + with self.assertRaisesRegex(ImportError, 'Failed to import'): local_backend.unknown.build() diff --git a/tests/test_registry/test_registry.py b/tests/test_registry/test_registry.py index eb99b3dc8e..a8913d8769 100644 --- a/tests/test_registry/test_registry.py +++ b/tests/test_registry/test_registry.py @@ -323,12 +323,12 @@ class LittlePedigreeSamoyed: assert LITTLE_HOUNDS.get('mid_hound.PedigreeSamoyedddddd') is None # Get mmengine.utils by string - utils = LITTLE_HOUNDS.get('mmengine.utils') + utils = LITTLE_HOUNDS.get('') import mmengine.utils assert utils is mmengine.utils - unknown = LITTLE_HOUNDS.get('mmengine.unknown') - assert unknown is None + with pytest.raises(RuntimeError, match='Failed to get'): + LITTLE_HOUNDS.get('') def test__search_child(self): # Hierarchical Registry