From 787a7ec042c33ca0ccaf4972a08019bd8b065096 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Wed, 23 Oct 2024 17:55:06 -0700 Subject: [PATCH] refactor type annotations using Python 3.9+ syntax --- src/everett/__init__.py | 4 +- src/everett/ext/inifile.py | 10 ++--- src/everett/ext/yamlfile.py | 12 +++--- src/everett/manager.py | 59 +++++++++++++--------------- src/everett/sphinxext.py | 76 +++++++++---------------------------- 5 files changed, 58 insertions(+), 103 deletions(-) diff --git a/src/everett/__init__.py b/src/everett/__init__.py index 54c4f89..7ba919e 100644 --- a/src/everett/__init__.py +++ b/src/everett/__init__.py @@ -9,7 +9,7 @@ version as importlib_version, PackageNotFoundError, ) -from typing import Callable, List, Union +from typing import Callable, Union try: @@ -60,7 +60,7 @@ class DetailedConfigurationError(ConfigurationError): """Base class for configuration errors that have a msg, namespace, key, and parser.""" def __init__( - self, msg: str, namespace: Union[List[str], None], key: str, parser: Callable + self, msg: str, namespace: Union[list[str], None], key: str, parser: Callable ): self.msg = msg self.namespace = namespace diff --git a/src/everett/ext/inifile.py b/src/everett/ext/inifile.py index d384733..df97f31 100644 --- a/src/everett/ext/inifile.py +++ b/src/everett/ext/inifile.py @@ -12,7 +12,7 @@ import logging import os -from typing import Dict, List, Optional, Union +from typing import Optional, Union from configobj import ConfigObj @@ -124,7 +124,7 @@ class ConfigIniEnv: """ - def __init__(self, possible_paths: Union[str, List[str]]) -> None: + def __init__(self, possible_paths: Union[str, list[str]]) -> None: """ :param possible_paths: either a single string with a file path (e.g. ``"/etc/project.ini"`` or a list of strings with file paths @@ -147,11 +147,11 @@ def __init__(self, possible_paths: Union[str, List[str]]) -> None: if not self.path: logger.debug("No INI file found: %s", possible_paths) - def parse_ini_file(self, path: str) -> Dict: + def parse_ini_file(self, path: str) -> dict: """Parse ini file at ``path`` and return dict.""" cfgobj = ConfigObj(path, list_values=False) - def extract_section(namespace: List[str], d: Dict) -> Dict: + def extract_section(namespace: list[str], d: dict) -> dict: cfg = {} for key, val in d.items(): if isinstance(d[key], dict): @@ -164,7 +164,7 @@ def extract_section(namespace: List[str], d: Dict) -> Dict: return extract_section([], cfgobj.dict()) def get( - self, key: str, namespace: Optional[List[str]] = None + self, key: str, namespace: Optional[list[str]] = None ) -> Union[str, NoValue]: """Retrieve value for key.""" if not self.path: diff --git a/src/everett/ext/yamlfile.py b/src/everett/ext/yamlfile.py index b346374..36eacc8 100644 --- a/src/everett/ext/yamlfile.py +++ b/src/everett/ext/yamlfile.py @@ -12,7 +12,7 @@ import logging import os -from typing import Dict, List, Optional, Union +from typing import Optional, Union import yaml @@ -109,7 +109,7 @@ class ConfigYamlEnv: """ - def __init__(self, possible_paths: Union[str, List[str]]) -> None: + def __init__(self, possible_paths: Union[str, list[str]]) -> None: """ :param possible_paths: either a single string with a file path (e.g. ``"/etc/project.yaml"`` or a list of strings with file paths @@ -132,15 +132,15 @@ def __init__(self, possible_paths: Union[str, List[str]]) -> None: if not self.path: logger.debug("No YAML file found: %s", possible_paths) - def parse_yaml_file(self, path: str) -> Dict: + def parse_yaml_file(self, path: str) -> dict: """Parse yaml file at ``path`` and return a dict.""" - with open(path, "r") as fp: + with open(path) as fp: data = yaml.safe_load(fp) if not data: return {} - def traverse(namespace: List[str], d: Dict) -> Dict: + def traverse(namespace: list[str], d: dict) -> dict: cfg = {} for key, val in d.items(): if isinstance(val, dict): @@ -161,7 +161,7 @@ def traverse(namespace: List[str], d: Dict) -> Dict: return traverse([], data) def get( - self, key: str, namespace: Optional[List[str]] = None + self, key: str, namespace: Optional[list[str]] = None ) -> Union[str, NoValue]: """Retrieve value for key.""" if not self.path: diff --git a/src/everett/manager.py b/src/everett/manager.py index 05c329e..381c3a1 100644 --- a/src/everett/manager.py +++ b/src/everett/manager.py @@ -20,15 +20,10 @@ from typing import ( Any, Callable, - Dict, - Iterable, - List, - Mapping, Optional, - Tuple, - Type, Union, ) +from collections.abc import Iterable, Mapping from everett import ( ConfigurationError, @@ -101,7 +96,7 @@ def qualname(thing: Any) -> str: def build_msg( - namespace: Optional[List[str]], + namespace: Optional[list[str]], key: Optional[str], parser: Optional[Callable], msg: str = "", @@ -156,7 +151,7 @@ class Config: def __init__( self, default: Union[str, NoValue] = NO_VALUE, - alternate_keys: Optional[List[str]] = None, + alternate_keys: Optional[list[str]] = None, doc: str = "", parser: Callable = str, meta: Any = None, @@ -201,7 +196,7 @@ def __eq__(self, obj: Any) -> bool: ) -def get_config_for_class(cls: Type) -> Dict[str, Tuple[Option, Type]]: +def get_config_for_class(cls: type) -> dict[str, tuple[Option, type]]: """Roll up configuration options for this class and parent classes. This handles subclasses overriding configuration options in parent classes. @@ -229,8 +224,8 @@ def get_config_for_class(cls: Type) -> Dict[str, Tuple[Option, Type]]: def traverse_tree( - instance: Any, namespace: Optional[List[str]] = None -) -> Iterable[Tuple[List[str], str, Option, Any]]: + instance: Any, namespace: Optional[list[str]] = None +) -> Iterable[tuple[list[str], str, Option, Any]]: """Traverses a tree of objects and computes the configuration for it Note: This expects the tree not to have any loops or repeated nodes. @@ -270,7 +265,7 @@ def traverse_tree( return options -def parse_env_file(envfile: Iterable[str]) -> Dict: +def parse_env_file(envfile: Iterable[str]) -> dict: """Parse the content of an iterable of lines as ``.env``. Return a dict of config variables. @@ -482,7 +477,7 @@ def get_parser(parser: Callable) -> Callable: return parser -def listify(thing: Any) -> List[Any]: +def listify(thing: Any) -> list[Any]: """Convert thing to a list. If thing is a string, then returns a list of thing. Otherwise @@ -500,7 +495,7 @@ def listify(thing: Any) -> List[Any]: return thing -def generate_uppercase_key(key: str, namespace: Optional[List[str]] = None) -> str: +def generate_uppercase_key(key: str, namespace: Optional[list[str]] = None) -> str: """Given a key and a namespace, generates a final uppercase key. >>> generate_uppercase_key("foo") @@ -568,7 +563,7 @@ def __init__(self, parser: Callable, delimiter: str = ","): self.sub_parser = parser self.delimiter = delimiter - def __call__(self, value: str) -> List[Any]: + def __call__(self, value: str) -> list[Any]: parser = get_parser(self.sub_parser) if value: return [parser(token.strip()) for token in value.split(self.delimiter)] @@ -583,7 +578,7 @@ class ConfigOverrideEnv: """Override configuration layer for testing.""" def get( - self, key: str, namespace: Optional[List[str]] = None + self, key: str, namespace: Optional[list[str]] = None ) -> Union[str, NoValue]: """Retrieve value for key.""" global _CONFIG_OVERRIDE @@ -644,7 +639,7 @@ def __init__(self, obj: Any, force_lower: bool = True): self.obj = obj def get( - self, key: str, namespace: Optional[List[str]] = None + self, key: str, namespace: Optional[list[str]] = None ) -> Union[str, NoValue]: """Retrieve value for key.""" full_key = generate_uppercase_key(key, namespace) @@ -727,11 +722,11 @@ class ConfigDictEnv: """ - def __init__(self, cfg: Dict): + def __init__(self, cfg: dict): self.cfg = {key.upper(): val for key, val in cfg.items()} def get( - self, key: str, namespace: Optional[List[str]] = None + self, key: str, namespace: Optional[list[str]] = None ) -> Union[str, NoValue]: """Retrieve value for key.""" full_key = generate_uppercase_key(key, namespace) @@ -795,7 +790,7 @@ class ConfigEnvFileEnv: """ - def __init__(self, possible_paths: Union[str, List[str]]): + def __init__(self, possible_paths: Union[str, list[str]]): self.data = {} self.path = None @@ -812,7 +807,7 @@ def __init__(self, possible_paths: Union[str, List[str]]): break def get( - self, key: str, namespace: Optional[List[str]] = None + self, key: str, namespace: Optional[list[str]] = None ) -> Union[str, NoValue]: """Retrieve value for key.""" full_key = generate_uppercase_key(key, namespace) @@ -867,7 +862,7 @@ class ConfigOSEnv: """ def get( - self, key: str, namespace: Optional[List[str]] = None + self, key: str, namespace: Optional[list[str]] = None ) -> Union[str, NoValue]: """Retrieve value for key.""" full_key = generate_uppercase_key(key, namespace) @@ -890,7 +885,7 @@ def get_runtime_config( config: "ConfigManager", component: Any, traverse: Callable = traverse_tree, -) -> List[Tuple[List[str], str, Any, Option]]: +) -> list[tuple[list[str], str, Any, Option]]: """Returns configuration specification and values for a component tree For example, if you had a tree of components instantiated, you could @@ -989,7 +984,7 @@ class ConfigManager: def __init__( self, - environments: List[Any], + environments: list[Any], doc: str = "", msg_builder: Callable = build_msg, with_override: bool = True, @@ -1037,10 +1032,10 @@ def build_msg(namespace, key, parser, msg="", option_doc="", config_doc=""): self.doc = doc self.msg_builder = msg_builder - self.namespace: List[str] = [] + self.namespace: list[str] = [] self.bound_component: Any = None - self.bound_component_prefix: List[str] = [] + self.bound_component_prefix: list[str] = [] self.bound_component_options: Mapping[str, Any] = {} self.original_manager = self @@ -1084,7 +1079,7 @@ def basic_config(cls, env_file: str = ".env", doc: str = "") -> "ConfigManager": return cls(environments=[ConfigOSEnv(), ConfigEnvFileEnv([env_file])], doc=doc) @classmethod - def from_dict(cls, dict_config: Dict) -> "ConfigManager": + def from_dict(cls, dict_config: dict) -> "ConfigManager": """Create a ConfigManager with specified configuration as a Python dict. This is shorthand for:: @@ -1112,7 +1107,7 @@ def get_bound_component(self) -> Any: """ return self.bound_component - def get_namespace(self) -> List[str]: + def get_namespace(self) -> list[str]: """Retrieve the complete namespace for this config object. :returns: namespace as a list of strings @@ -1139,7 +1134,7 @@ def clone(self) -> "ConfigManager": return my_clone - def with_namespace(self, namespace: Union[List[str], str]) -> "ConfigManager": + def with_namespace(self, namespace: Union[list[str], str]) -> "ConfigManager": """Apply a namespace to this configuration. Namespaces accumulate as you add them. @@ -1200,10 +1195,10 @@ def with_options(self, component: Any) -> "ConfigManager": def __call__( self, key: str, - namespace: Union[List[str], str, None] = None, + namespace: Union[list[str], str, None] = None, default: Union[str, NoValue] = NO_VALUE, default_if_empty: bool = True, - alternate_keys: Optional[List[str]] = None, + alternate_keys: Optional[list[str]] = None, doc: str = "", parser: Callable = str, raise_error: bool = True, @@ -1475,7 +1470,7 @@ def __enter__(self) -> None: def __exit__( self, - exc_type: Optional[Type[BaseException]], + exc_type: Optional[type[BaseException]], exc_value: Optional[BaseException], traceback: Optional[TracebackType], ) -> None: diff --git a/src/everett/sphinxext.py b/src/everett/sphinxext.py index cd965e3..21ce65b 100644 --- a/src/everett/sphinxext.py +++ b/src/everett/sphinxext.py @@ -17,13 +17,10 @@ from typing import ( TYPE_CHECKING, Any, - Dict, - Generator, - List, Optional, - Tuple, Union, ) +from collections.abc import Generator from docutils import nodes from docutils.parsers.rst import Directive, directives @@ -52,7 +49,7 @@ LOGGER = logging.getLogger(__name__) -def split_clspath(clspath: str) -> List[str]: +def split_clspath(clspath: str) -> list[str]: """Split clspath into module and class names. Note: This is a really simplistic implementation. @@ -299,13 +296,13 @@ class EverettDomain(Domain): "component": XRefRole(), "option": XRefRole(), } - initial_data: Dict[str, dict] = { + initial_data: dict[str, dict] = { # (typ, clspath) -> sphinx document name "objects": {} } @property - def objects(self) -> Dict[Tuple[str, str], Tuple[str, str]]: + def objects(self) -> dict[tuple[str, str], tuple[str, str]]: return self.data.setdefault("objects", {}) def clear_doc(self, docname: str) -> None: @@ -315,7 +312,7 @@ def clear_doc(self, docname: str) -> None: del self.objects[key] # FIXME(willkg): What's the value in otherdata dict? - def merge_domaindata(self, docnames: List[str], otherdata: Dict[str, Any]) -> None: + def merge_domaindata(self, docnames: list[str], otherdata: dict[str, Any]) -> None: for key, val in otherdata["objects"].items(): if val[0] in docnames: self.objects[key] = val @@ -364,7 +361,7 @@ def generate_docs( component_index: str, docstring: str, sourcename: str, - option_data: List[Dict], + option_data: list[dict], more_content: Any, ) -> None: indent = " " @@ -391,7 +388,7 @@ def generate_docs( self.add_line("", sourcename) # First build a table of metric items - table: List[List[str]] = [] + table: list[list[str]] = [] table.append(["Setting", "Parser", "Required?"]) for option_item in option_data: ref = f"{component_name}.{option_item['key']}" @@ -466,7 +463,7 @@ def extract_configuration( obj: Any, namespace: Optional[str] = None, case: Optional[str] = None, - ) -> List[Dict]: + ) -> list[dict]: """Extracts configuration values from list of Everett configuration options :param obj: object/class to extract configuration from @@ -477,7 +474,7 @@ def extract_configuration( """ config = get_config_for_class(obj) - options: List[Dict] = [] + options: list[dict] = [] # Go through options and figure out relevant information for key, (option, _) in config.items(): @@ -502,7 +499,7 @@ def extract_configuration( ) return options - def run(self) -> List[nodes.Node]: + def run(self) -> list[nodes.Node]: self.reporter = self.state.document.reporter self.result = ViewList() @@ -553,7 +550,7 @@ def run(self) -> List[nodes.Node]: SETTING_RE = re.compile(r"^[A-Z_]+$") -def build_table(table: List[List[str]]) -> List[str]: +def build_table(table: list[list[str]]) -> list[str]: """Generates reST for a table. :param table: a 2d array of rows and columns @@ -561,7 +558,7 @@ def build_table(table: List[List[str]]) -> List[str]: :returns: list of strings """ - output: List[str] = [] + output: list[str] = [] col_size = [0] * len(table[0]) for row in table: @@ -591,43 +588,6 @@ def build_table(table: List[List[str]]) -> List[str]: return output -def get_value_from_ast_node(source: str, val: ast.AST) -> str: - """Wrapper for ast.get_source_segment. - - NOTE(willkg): ``ast.get_source_segment()`` was implemented in Python 3.8 - and when we drop support for Python 3.7, we can drop this code, too. - - This is to get the source code for the AST node in question so that we can - display it as is in the docs. For a wildly contrived example, if you did - something bizarre like:: - - config = ConfigManager.basic_config() - DEBUG = config("debug, parser=bool, default="False") - WEIRD = config("weird", parser=(bool if config("debug") else int)) - - then the node for ``bool if config("debug") else int`` will be an - ``ast.IfEq`` (or something like that) and this function will return:: - - bool if config("debug") else int - - and show that as the parser in the docs. - - :param source: the complete source text for the file being parsed - :param val: the ast node in question - - :returns: a string representation of the ast node for display in docs - - """ - if hasattr(ast, "get_source_segment"): - return ast.get_source_segment(source, val) or "?" - - # If we have < Python 3.8, return a "?" because it's not clear what it is. - # Python 3.6 and 3.7 don't have end_lineno and end_col_offset either, so I - # couldn't figure out a good way to figure out the source segment to - # return. - return "?" - - class AutoModuleConfigDirective(ConfigDirective): """Directive for documenting configuration for a module.""" @@ -670,7 +630,7 @@ def extract_configuration( variable_name: str, namespace: Optional[str] = None, case: Optional[str] = None, - ) -> List[Dict]: + ) -> list[dict]: """Extracts configuration values from a module at filepath :param filepath: the filepath to parse configuration from @@ -726,7 +686,7 @@ def extract_configuration( "meta", ] - def extract_value(source: str, val: ast.AST) -> Tuple[str, str]: + def extract_value(source: str, val: ast.AST) -> tuple[str, str]: """Returns (category, value)""" if isinstance(val, ast.Constant): return "constant", val.value @@ -736,14 +696,14 @@ def extract_value(source: str, val: ast.AST) -> Tuple[str, str]: _, left = extract_value(source, val.left) _, right = extract_value(source, val.right) return "binop", left + right - return "unknown", get_value_from_ast_node(source, val) + return "unknown", ast.get_source_segment(source, val) or "?" # Using a dict here avoids the case where configuration options are # defined multiple times configuration = {} for name, node in config_nodes: - args: Dict[str, Any] = { + args: dict[str, Any] = { "key": name, "default": NO_VALUE, "parser": "str", @@ -789,7 +749,7 @@ def extract_value(source: str, val: ast.AST) -> Tuple[str, str]: return list(configuration.values()) - def run(self) -> List[nodes.Node]: + def run(self) -> list[nodes.Node]: self.reporter = self.state.document.reporter self.result = ViewList() @@ -849,7 +809,7 @@ def run(self) -> List[nodes.Node]: # FIXME(willkg): this takes a Sphinx app -def setup(app: Any) -> Dict[str, Any]: +def setup(app: Any) -> dict[str, Any]: """Register domain and directive in Sphinx.""" app.add_domain(EverettDomain) app.add_directive("autocomponentconfig", AutoComponentConfigDirective)