From 0aedfa26a1db572c3b8e33ef8d7d9d842e22eff7 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 18 Jan 2024 12:41:45 -0700 Subject: [PATCH 01/61] Initial commit. --- docs/index.md | 4 + pydantic_settings/__init__.py | 4 + pydantic_settings/main.py | 33 +++++++- pydantic_settings/sources.py | 141 +++++++++++++++++++++++++++++++++- tests/test_settings.py | 10 ++- 5 files changed, 188 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index d1627c9d..0e66ab25 100644 --- a/docs/index.md +++ b/docs/index.md @@ -363,6 +363,7 @@ class Settings(BaseSettings): cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -575,6 +576,7 @@ class Settings(BaseSettings): cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -657,6 +659,7 @@ class Settings(BaseSettings): cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -693,6 +696,7 @@ class Settings(BaseSettings): cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, diff --git a/pydantic_settings/__init__.py b/pydantic_settings/__init__.py index 7b99f885..0c9682a8 100644 --- a/pydantic_settings/__init__.py +++ b/pydantic_settings/__init__.py @@ -1,5 +1,7 @@ from .main import BaseSettings, SettingsConfigDict from .sources import ( + CliSettingsSource, + CliSubCommand, DotEnvSettingsSource, EnvSettingsSource, InitSettingsSource, @@ -12,6 +14,8 @@ 'BaseSettings', 'DotEnvSettingsSource', 'EnvSettingsSource', + 'CliSettingsSource', + 'CliSubCommand', 'InitSettingsSource', 'PydanticBaseSettingsSource', 'SecretsSettingsSource', diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index d3ea63f2..e7937018 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -10,6 +10,7 @@ from .sources import ( ENV_FILE_SENTINEL, + CliSettingsSource, DotEnvSettingsSource, DotenvType, EnvSettingsSource, @@ -27,6 +28,9 @@ class SettingsConfigDict(ConfigDict, total=False): env_ignore_empty: bool env_nested_delimiter: str | None env_parse_none_str: str | None + cli_parse_args: bool + cli_hide_none: bool + cli_hide_json: bool secrets_dir: str | Path | None @@ -71,6 +75,9 @@ def __init__( _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_parse_none_str: str | None = None, + _cli_parse_args: bool | None = None, + _cli_hide_none: bool | None = None, + _cli_hide_json: bool | None = None, _secrets_dir: str | Path | None = None, **values: Any, ) -> None: @@ -85,6 +92,9 @@ def __init__( _env_ignore_empty=_env_ignore_empty, _env_nested_delimiter=_env_nested_delimiter, _env_parse_none_str=_env_parse_none_str, + _cli_parse_args=_cli_parse_args, + _cli_hide_none=_cli_hide_none, + _cli_hide_json=_cli_hide_json, _secrets_dir=_secrets_dir, ) ) @@ -94,6 +104,7 @@ def settings_customise_sources( cls, settings_cls: type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -104,6 +115,7 @@ def settings_customise_sources( Args: settings_cls: The Settings class. init_settings: The `InitSettingsSource` instance. + cli_settings: The `CliSettingsSource` instance. env_settings: The `EnvSettingsSource` instance. dotenv_settings: The `DotEnvSettingsSource` instance. file_secret_settings: The `SecretsSettingsSource` instance. @@ -111,7 +123,7 @@ def settings_customise_sources( Returns: A tuple containing the sources and their order for loading the settings values. """ - return init_settings, env_settings, dotenv_settings, file_secret_settings + return init_settings, cli_settings, env_settings, dotenv_settings, file_secret_settings def _settings_build_values( self, @@ -123,6 +135,9 @@ def _settings_build_values( _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_parse_none_str: str | None = None, + _cli_parse_args: bool | None = None, + _cli_hide_none: bool | None = None, + _cli_hide_json: bool | None = None, _secrets_dir: str | Path | None = None, ) -> dict[str, Any]: # Determine settings config values @@ -143,10 +158,22 @@ def _settings_build_values( env_parse_none_str = ( _env_parse_none_str if _env_parse_none_str is not None else self.model_config.get('env_parse_none_str') ) + + cli_parse_args = _cli_parse_args if _cli_parse_args is not None else self.model_config.get('cli_parse_args') + cli_hide_none = _cli_hide_none if _cli_hide_none is not None else self.model_config.get('cli_hide_none') + cli_hide_json = _cli_hide_json if _cli_hide_json is not None else self.model_config.get('cli_hide_json') + secrets_dir = _secrets_dir if _secrets_dir is not None else self.model_config.get('secrets_dir') # Configure built-in sources init_settings = InitSettingsSource(self.__class__, init_kwargs=init_kwargs) + cli_settings = CliSettingsSource( + self.__class__, + env_parse_none_str=env_parse_none_str, + cli_parse_args=cli_parse_args, + cli_hide_none=cli_hide_none, + cli_hide_json=cli_hide_json, + ) env_settings = EnvSettingsSource( self.__class__, case_sensitive=case_sensitive, @@ -173,6 +200,7 @@ def _settings_build_values( sources = self.settings_customise_sources( self.__class__, init_settings=init_settings, + cli_settings=cli_settings, env_settings=env_settings, dotenv_settings=dotenv_settings, file_secret_settings=file_secret_settings, @@ -195,6 +223,9 @@ def _settings_build_values( env_ignore_empty=False, env_nested_delimiter=None, env_parse_none_str=None, + cli_parse_args=False, + cli_hide_none=False, + cli_hide_json=False, secrets_dir=None, protected_namespaces=('model_', 'settings_'), ) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 335ba9dd..eae3d680 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -6,8 +6,9 @@ from abc import ABC, abstractmethod from collections import deque from dataclasses import is_dataclass +from inspect import isclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Sequence, Tuple, Union, cast +from typing import TYPE_CHECKING, Annotated, Any, Dict, List, Mapping, Sequence, Tuple, TypeVar, Union, cast from dotenv import dotenv_values from pydantic import AliasChoices, AliasPath, BaseModel, Json, TypeAdapter @@ -21,6 +22,7 @@ if TYPE_CHECKING: from pydantic_settings.main import BaseSettings +from argparse import SUPPRESS, ArgumentParser, _ArgumentGroup, _SubParsersAction DotenvType = Union[Path, str, List[Union[Path, str]], Tuple[Union[Path, str], ...]] @@ -30,6 +32,14 @@ ENV_FILE_SENTINEL: DotenvType = Path('') +class _CliSubCommand: + pass + + +T = TypeVar('T') +CliSubCommand = Annotated[T, _CliSubCommand] + + class EnvNoneType(str): pass @@ -457,7 +467,9 @@ def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, val """ is_complex, allow_parse_failure = self._field_is_complex(field) if is_complex or value_is_complex: - if value is None: + if isinstance(value, EnvNoneType): + return value + elif value is None: # field is complex but no value found so far, try explode_env_vars env_val_built = self.explode_env_vars(field_name, field, self.env_vars) if env_val_built: @@ -664,6 +676,131 @@ def __repr__(self) -> str: ) +class CliSettingsSource(EnvSettingsSource): + """ + Source class for loading settings values from CLI. + """ + + def __init__( + self, + settings_cls: type[BaseSettings], + env_parse_none_str: str | None = None, + cli_parse_args: bool | None = None, + cli_hide_none: bool | None = None, + cli_hide_json: bool | None = None, + ) -> None: + self._cli_arg_names: list = [] + self.cli_parse_args = cli_parse_args if cli_parse_args is not None else self.config.get('cli_parse_args', False) + self.cli_hide_none = cli_hide_none if cli_hide_none is not None else self.config.get('cli_hide_none', False) + self.cli_hide_json = cli_hide_json if cli_hide_json is not None else self.config.get('cli_hide_json', False) + if env_parse_none_str is None: + env_parse_none_str = 'None' if cli_hide_json is True else 'null' + super().__init__(settings_cls, env_nested_delimiter='.', env_parse_none_str=env_parse_none_str) + + def _load_env_vars(self) -> Mapping[str, str | None]: + if not self.cli_parse_args: + return {} + + self._cli_arg_names = [] + parser: ArgumentParser = self._add_fields_to_parser(ArgumentParser(), self.settings_cls) + return parse_env_vars( + vars(parser.parse_args()), self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str + ) + + def _add_fields_to_parser( + self, + parser: ArgumentParser, + model: type[BaseModel], + _arg_prefix: str = '', + _dest_prefix: str = '', + _group: _ArgumentGroup | None = None, + ) -> ArgumentParser: + subparsers: _SubParsersAction | None = None + for field_name, field_info in model.model_fields.items(): + field_types: tuple[Any, ...] = ( + (field_info.annotation,) if not get_args(field_info.annotation) else get_args(field_info.annotation) + ) + + if self.cli_hide_none: + field_types = tuple([type_ for type_ in field_types if type_ is not type(None)]) + + arg_name = f'{_arg_prefix}{field_name}' + if _CliSubCommand in field_info.metadata: + if subparsers is not None: + raise SettingsError( + f'detected a second subcommand definition at {model.__name__}.{field_name}, ' + 'only one per model is allowed' + ) + dest_name = f'{_dest_prefix}{arg_name}' + subparsers = parser.add_subparsers(title='subcommands', description='available subcommands') + for type_ in field_types: + if get_origin(type_) is Annotated and _CliSubCommand in get_args(type_): + raise SettingsError(f'subcommand is not outermost annotation for {model.__name__}.{field_name}') + if not (isclass(type_) or issubclass(type_, BaseModel)): # type: ignore + raise SettingsError( + 'only BaseModel derived types can be subcommands, ' + f'found {type_.__name__} in {model.__name__}.{field_name}' + ) + self._add_fields_to_parser( + subparsers.add_parser( + f'{type_.__name__.lower()}', help=type_.__doc__, formatter_class=parser.formatter_class + ), + type_, + _dest_prefix=f'{dest_name}.', + ) + elif arg_name not in self._cli_arg_names: + dest_name = f'{_dest_prefix}{field_name}' + objects: list[type[BaseModel]] = [ + type_ for type_ in field_types if isclass(type_) and issubclass(type_, BaseModel) + ] + + metavar: str = ','.join( + (['JSON'] if objects else []) + + [ + type_.__name__ if type_ is not type(None) else self.env_parse_none_str + for type_ in field_types + if type_ not in objects + ] + ) + metavar = f'{{{metavar}}}' if len(field_types) > 1 else metavar + + if objects: + object_group = ( + parser.add_argument_group(f'{arg_name} options', field_info.description) + if _group is None + else _group + ) + if not self.cli_hide_json: + self._cli_arg_names.append(arg_name) + object_group.add_argument( + f'--{arg_name}', + help=f'set {arg_name} from JSON string', + metavar=metavar, + default=SUPPRESS, + dest=dest_name, + ) + for object_ in objects: + self._add_fields_to_parser( + parser, + object_, + _arg_prefix=f'{arg_name}.', + _group=object_group, + _dest_prefix=f'{dest_name}.', + ) + elif _group is not None: + self._cli_arg_names.append(arg_name) + _group.add_argument( + f'--{arg_name}', help=field_info.description, metavar=metavar, default=SUPPRESS, dest=dest_name + ) + else: + self._cli_arg_names.append(arg_name) + parser.add_argument( + f'--{arg_name}', help=field_info.description, metavar=metavar, default=SUPPRESS, dest=dest_name + ) + + return parser + + def _get_env_var_key(key: str, case_sensitive: bool = False) -> str: return key if case_sensitive else key.lower() diff --git a/tests/test_settings.py b/tests/test_settings.py index b2371953..7491b5f6 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -606,6 +606,7 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -635,7 +636,7 @@ class Settings(BaseSettings, env_nested_delimiter='__'): @classmethod def settings_customise_sources( - cls, settings_cls, init_settings, env_settings, dotenv_settings, file_secret_settings + cls, settings_cls, init_settings, cli_settings, env_settings, dotenv_settings, file_secret_settings ): return env_settings, dotenv_settings, init_settings, file_secret_settings @@ -674,6 +675,7 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1309,6 +1311,7 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1355,6 +1358,7 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1428,6 +1432,7 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1456,6 +1461,7 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1490,6 +1496,7 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1514,6 +1521,7 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, From 9d22a39a76cd05c7daaff07b0ccd7c17f428ca12 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 18 Jan 2024 17:25:56 -0700 Subject: [PATCH 02/61] Draft complete. Needs testing. --- pydantic_settings/__init__.py | 2 + pydantic_settings/main.py | 34 +++-- pydantic_settings/sources.py | 254 +++++++++++++++++++++++----------- 3 files changed, 196 insertions(+), 94 deletions(-) diff --git a/pydantic_settings/__init__.py b/pydantic_settings/__init__.py index 0c9682a8..3e641125 100644 --- a/pydantic_settings/__init__.py +++ b/pydantic_settings/__init__.py @@ -1,5 +1,6 @@ from .main import BaseSettings, SettingsConfigDict from .sources import ( + CliPositionalArg, CliSettingsSource, CliSubCommand, DotEnvSettingsSource, @@ -16,6 +17,7 @@ 'EnvSettingsSource', 'CliSettingsSource', 'CliSubCommand', + 'CliPositionalArg', 'InitSettingsSource', 'PydanticBaseSettingsSource', 'SecretsSettingsSource', diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index e7937018..1b41a039 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -1,5 +1,6 @@ from __future__ import annotations as _annotations +import sys from pathlib import Path from typing import Any, ClassVar @@ -29,8 +30,8 @@ class SettingsConfigDict(ConfigDict, total=False): env_nested_delimiter: str | None env_parse_none_str: str | None cli_parse_args: bool - cli_hide_none: bool - cli_hide_json: bool + cli_hide_none_type: bool + cli_avoid_json: bool secrets_dir: str | Path | None @@ -76,8 +77,8 @@ def __init__( _env_nested_delimiter: str | None = None, _env_parse_none_str: str | None = None, _cli_parse_args: bool | None = None, - _cli_hide_none: bool | None = None, - _cli_hide_json: bool | None = None, + _cli_hide_none_type: bool | None = None, + _cli_avoid_json: bool | None = None, _secrets_dir: str | Path | None = None, **values: Any, ) -> None: @@ -93,8 +94,8 @@ def __init__( _env_nested_delimiter=_env_nested_delimiter, _env_parse_none_str=_env_parse_none_str, _cli_parse_args=_cli_parse_args, - _cli_hide_none=_cli_hide_none, - _cli_hide_json=_cli_hide_json, + _cli_hide_none_type=_cli_hide_none_type, + _cli_avoid_json=_cli_avoid_json, _secrets_dir=_secrets_dir, ) ) @@ -136,8 +137,8 @@ def _settings_build_values( _env_nested_delimiter: str | None = None, _env_parse_none_str: str | None = None, _cli_parse_args: bool | None = None, - _cli_hide_none: bool | None = None, - _cli_hide_json: bool | None = None, + _cli_hide_none_type: bool | None = None, + _cli_avoid_json: bool | None = None, _secrets_dir: str | Path | None = None, ) -> dict[str, Any]: # Determine settings config values @@ -160,8 +161,10 @@ def _settings_build_values( ) cli_parse_args = _cli_parse_args if _cli_parse_args is not None else self.model_config.get('cli_parse_args') - cli_hide_none = _cli_hide_none if _cli_hide_none is not None else self.model_config.get('cli_hide_none') - cli_hide_json = _cli_hide_json if _cli_hide_json is not None else self.model_config.get('cli_hide_json') + cli_hide_none_type = ( + _cli_hide_none_type if _cli_hide_none_type is not None else self.model_config.get('cli_hide_none_type') + ) + cli_avoid_json = _cli_avoid_json if _cli_avoid_json is not None else self.model_config.get('cli_avoid_json') secrets_dir = _secrets_dir if _secrets_dir is not None else self.model_config.get('secrets_dir') @@ -169,10 +172,11 @@ def _settings_build_values( init_settings = InitSettingsSource(self.__class__, init_kwargs=init_kwargs) cli_settings = CliSettingsSource( self.__class__, - env_parse_none_str=env_parse_none_str, + sys.argv[1:], + cli_parse_none_str=env_parse_none_str, cli_parse_args=cli_parse_args, - cli_hide_none=cli_hide_none, - cli_hide_json=cli_hide_json, + cli_hide_none_type=cli_hide_none_type, + cli_avoid_json=cli_avoid_json, ) env_settings = EnvSettingsSource( self.__class__, @@ -224,8 +228,8 @@ def _settings_build_values( env_nested_delimiter=None, env_parse_none_str=None, cli_parse_args=False, - cli_hide_none=False, - cli_hide_json=False, + cli_hide_none_type=False, + cli_avoid_json=False, secrets_dir=None, protected_namespaces=('model_', 'settings_'), ) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index eae3d680..110b7fd5 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -8,14 +8,16 @@ from dataclasses import is_dataclass from inspect import isclass from pathlib import Path -from typing import TYPE_CHECKING, Annotated, Any, Dict, List, Mapping, Sequence, Tuple, TypeVar, Union, cast +from types import FunctionType +from typing import TYPE_CHECKING, Annotated, Any, Dict, List, Literal, Mapping, Sequence, Tuple, TypeVar, Union, cast from dotenv import dotenv_values from pydantic import AliasChoices, AliasPath, BaseModel, Json, TypeAdapter -from pydantic._internal._typing_extra import origin_is_union +from pydantic._internal._repr import Representation +from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base from pydantic._internal._utils import deep_update, lenient_issubclass from pydantic.fields import FieldInfo -from typing_extensions import get_args, get_origin +from typing_extensions import TypeAliasType, get_args, get_origin from pydantic_settings.utils import path_type_label @@ -36,8 +38,13 @@ class _CliSubCommand: pass +class _CliPositionalArg: + pass + + T = TypeVar('T') CliSubCommand = Annotated[T, _CliSubCommand] +CliPositionalArg = Annotated[T, _CliPositionalArg] class EnvNoneType(str): @@ -684,28 +691,111 @@ class CliSettingsSource(EnvSettingsSource): def __init__( self, settings_cls: type[BaseSettings], - env_parse_none_str: str | None = None, + args: list[str], + cli_parse_none_str: str | None = None, cli_parse_args: bool | None = None, - cli_hide_none: bool | None = None, - cli_hide_json: bool | None = None, + cli_hide_none_type: bool | None = None, + cli_avoid_json: bool | None = None, ) -> None: - self._cli_arg_names: list = [] + self.args = args self.cli_parse_args = cli_parse_args if cli_parse_args is not None else self.config.get('cli_parse_args', False) - self.cli_hide_none = cli_hide_none if cli_hide_none is not None else self.config.get('cli_hide_none', False) - self.cli_hide_json = cli_hide_json if cli_hide_json is not None else self.config.get('cli_hide_json', False) - if env_parse_none_str is None: - env_parse_none_str = 'None' if cli_hide_json is True else 'null' - super().__init__(settings_cls, env_nested_delimiter='.', env_parse_none_str=env_parse_none_str) + self.cli_hide_none_type = ( + cli_hide_none_type if cli_hide_none_type is not None else self.config.get('cli_hide_none_type', False) + ) + self.cli_avoid_json = cli_avoid_json if cli_avoid_json is not None else self.config.get('cli_avoid_json', False) + if cli_parse_none_str is None: + cli_parse_none_str = 'None' if self.cli_avoid_json is True else 'null' + super().__init__(settings_cls, env_nested_delimiter='.', env_parse_none_str=cli_parse_none_str) def _load_env_vars(self) -> Mapping[str, str | None]: if not self.cli_parse_args: return {} - self._cli_arg_names = [] + self._cli_arg_names: list[str] = [] + self._cli_dict_arg_names: list[str] = [] parser: ArgumentParser = self._add_fields_to_parser(ArgumentParser(), self.settings_cls) + parsed_args: dict[str, list[str] | str] = vars(parser.parse_args(self.args)) + for field, val in parsed_args.items(): + if isinstance(val, list): + merge_list = [] + for sub_val in val: + if sub_val.startswith('[') and sub_val.endswith(']'): + sub_val = sub_val[1:-1] + merge_list.append(sub_val) + parsed_args[field] = ( + f'[{",".join(merge_list)}]' + if field not in self._cli_dict_arg_names + else self._merge_json_key_val_list_str(f'[{",".join(merge_list)}]') + ) + return parse_env_vars( - vars(parser.parse_args()), self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str + parsed_args, self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str # type: ignore + ) + + def _merge_json_key_val_list_str(self, key_val_list_str: str) -> str: + orig_key_val_list_str, key_val_list_str = key_val_list_str, key_val_list_str[1:-1] + key_val_dict: dict[str, str] = {} + obj_count = 0 + while key_val_list_str: + if obj_count != 0: + raise SettingsError(f'Parsing error encountered on JSON object {orig_key_val_list_str}') + for i in range(len(key_val_list_str)): + if key_val_list_str[i] == '{': + obj_count += 1 + elif key_val_list_str[i] == '}': + obj_count -= 1 + if obj_count == 0: + key_val_dict |= json.loads(key_val_list_str[: i + 1]) + key_val_list_str = key_val_list_str[i + 1 :].lstrip(',') + break + elif obj_count == 0: + val, quote_count = '', 0 + key, key_val_list_str = key_val_list_str.split('=', 1) + for i in range(len(key_val_list_str)): + if key_val_list_str[i] in ('"', "'"): + quote_count += 1 + if key_val_list_str[i] == ',' and quote_count % 2 == 0: + val, key_val_list_str = key_val_list_str[:i], key_val_list_str[i:].lstrip(',') + break + if not val: + val, key_val_list_str = key_val_list_str, '' + key_val_dict |= {key.strip('\'"'): val.strip('\'"')} + break + return json.dumps(key_val_dict) + + def _get_sub_models( + self, model: type[BaseModel], field_name: str, field_info: FieldInfo, subparsers: _SubParsersAction[Any] | None + ) -> list[type[BaseModel]]: + field_types: tuple[Any, ...] = ( + (field_info.annotation,) if not get_args(field_info.annotation) else get_args(field_info.annotation) ) + if self.cli_hide_none_type: + field_types = tuple([type_ for type_ in field_types if type_ is not type(None)]) + + sub_models: list[type[BaseModel]] = [] + for type_ in field_types: + if get_origin(type_) is Annotated: + if _CliSubCommand in get_args(type_): + raise SettingsError(f'CliSubCommand is not outermost annotation for {model.__name__}.{field_name}') + elif _CliPositionalArg in get_args(type_): + raise SettingsError( + f'CliPositionalArg is not outermost annotation for {model.__name__}.{field_name}' + ) + if isclass(type_) and issubclass(type_, BaseModel): + sub_models.append(type_) + elif _CliSubCommand in field_info.metadata and subparsers is not None: + raise SettingsError( + f'detected a second subcommand definition at {model.__name__}.{field_name}, ' + 'only one per model is allowed' + ) + if _CliPositionalArg in field_info.metadata: + if not field_info.is_required(): + raise SettingsError(f'positional argument {model.__name__}.{field_name} has a default value') + elif subparsers is not None: + raise SettingsError( + f'positional argument {model.__name__}.{field_name} ' 'is speficied after a subcommand definition' + ) + return sub_models def _add_fields_to_parser( self, @@ -715,91 +805,97 @@ def _add_fields_to_parser( _dest_prefix: str = '', _group: _ArgumentGroup | None = None, ) -> ArgumentParser: - subparsers: _SubParsersAction | None = None + subparsers: _SubParsersAction[Any] | None = None for field_name, field_info in model.model_fields.items(): - field_types: tuple[Any, ...] = ( - (field_info.annotation,) if not get_args(field_info.annotation) else get_args(field_info.annotation) - ) - - if self.cli_hide_none: - field_types = tuple([type_ for type_ in field_types if type_ is not type(None)]) - arg_name = f'{_arg_prefix}{field_name}' + sub_models: list[type[BaseModel]] = self._get_sub_models(model, field_name, field_info, subparsers) if _CliSubCommand in field_info.metadata: - if subparsers is not None: - raise SettingsError( - f'detected a second subcommand definition at {model.__name__}.{field_name}, ' - 'only one per model is allowed' - ) - dest_name = f'{_dest_prefix}{arg_name}' subparsers = parser.add_subparsers(title='subcommands', description='available subcommands') - for type_ in field_types: - if get_origin(type_) is Annotated and _CliSubCommand in get_args(type_): - raise SettingsError(f'subcommand is not outermost annotation for {model.__name__}.{field_name}') - if not (isclass(type_) or issubclass(type_, BaseModel)): # type: ignore - raise SettingsError( - 'only BaseModel derived types can be subcommands, ' - f'found {type_.__name__} in {model.__name__}.{field_name}' - ) + for model in sub_models: self._add_fields_to_parser( subparsers.add_parser( - f'{type_.__name__.lower()}', help=type_.__doc__, formatter_class=parser.formatter_class + f'{model.__name__.lower()}', help=model.__doc__, formatter_class=parser.formatter_class ), - type_, - _dest_prefix=f'{dest_name}.', + model, + _dest_prefix=f'{_dest_prefix}{arg_name}', ) elif arg_name not in self._cli_arg_names: - dest_name = f'{_dest_prefix}{field_name}' - objects: list[type[BaseModel]] = [ - type_ for type_ in field_types if isclass(type_) and issubclass(type_, BaseModel) - ] - - metavar: str = ','.join( - (['JSON'] if objects else []) - + [ - type_.__name__ if type_ is not type(None) else self.env_parse_none_str - for type_ in field_types - if type_ not in objects - ] - ) - metavar = f'{{{metavar}}}' if len(field_types) > 1 else metavar - - if objects: - object_group = ( - parser.add_argument_group(f'{arg_name} options', field_info.description) - if _group is None - else _group - ) - if not self.cli_hide_json: + arg_flag: str = '--' + kwargs: dict[str, Any] = {} + kwargs['default'] = SUPPRESS + kwargs['help'] = field_info.description + kwargs['dest'] = f'{_dest_prefix}{field_name}' + kwargs['metavar'] = self._format_metavar(field_info.annotation) + if get_origin(field_info.annotation) in (list, set, dict, Sequence): + kwargs['action'] = 'append' + if get_origin(field_info.annotation) is dict: + self._cli_dict_arg_names.append(arg_name) + if _CliPositionalArg in field_info.metadata: + del kwargs['dest'] + arg_flag = '' + + if sub_models and kwargs.get('action') != 'append': + model_group = parser.add_argument_group(f'{arg_name} options', field_info.description) + if not self.cli_avoid_json: self._cli_arg_names.append(arg_name) - object_group.add_argument( - f'--{arg_name}', - help=f'set {arg_name} from JSON string', - metavar=metavar, - default=SUPPRESS, - dest=dest_name, - ) - for object_ in objects: + kwargs['help'] = f'set {arg_name} from JSON string' + model_group.add_argument(f'{arg_flag}{arg_name}', **kwargs) + for model in sub_models: self._add_fields_to_parser( parser, - object_, + model, _arg_prefix=f'{arg_name}.', - _group=object_group, - _dest_prefix=f'{dest_name}.', + _group=model_group, + _dest_prefix=f"{kwargs['dest']}.", ) elif _group is not None: self._cli_arg_names.append(arg_name) - _group.add_argument( - f'--{arg_name}', help=field_info.description, metavar=metavar, default=SUPPRESS, dest=dest_name - ) + _group.add_argument(f'{arg_flag}{arg_name}', **kwargs) else: self._cli_arg_names.append(arg_name) - parser.add_argument( - f'--{arg_name}', help=field_info.description, metavar=metavar, default=SUPPRESS, dest=dest_name - ) - + parser.add_argument(f'{arg_flag}{arg_name}', **kwargs) return parser + def _get_modified_args(self, obj: Any) -> tuple[str, ...]: + if not self.cli_hide_none_type: + return get_args(obj) + else: + return tuple([type_ for type_ in get_args(obj) if type_ is not type(None)]) + + def _format_metavar(self, obj: Any) -> str: + """Pretty metavar representation of a type. Adapts logic from `pydantic._repr.display_as_type`.""" + if isinstance(obj, FunctionType): + return obj.__name__ + elif obj is ...: + return '...' + elif isinstance(obj, Representation): + return repr(obj) + elif isinstance(obj, TypeAliasType): + return str(obj) + + if not isinstance(obj, (typing_base, WithArgsTypes, type)): + obj = obj.__class__ + + if origin_is_union(get_origin(obj)): + args = ','.join(map(self._format_metavar, self._get_modified_args(obj))) + return f'{{{args}}}' if ',' in args else args + elif isinstance(obj, WithArgsTypes): + if get_origin(obj) == Literal: + args = ','.join(map(repr, self._get_modified_args(obj))) + return f'{{{args}}}' if ',' in args else args + else: + args = ','.join(map(self._format_metavar, self._get_modified_args(obj))) + try: + return f'{obj.__qualname__}[{args}]' + except AttributeError: + return str(obj) # handles TypeAliasType in 3.12 + elif obj is type(None): + return self.env_parse_none_str + elif isinstance(obj, type): + return obj.__qualname__ + else: + return repr(obj).replace('typing.', '').replace('typing_extensions.', '') + def _get_env_var_key(key: str, case_sensitive: bool = False) -> str: return key if case_sensitive else key.lower() From 2ef847323d3638881d8ac5a0cdfad9fa063536e0 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sat, 20 Jan 2024 16:36:54 -0700 Subject: [PATCH 03/61] Subcommand union discrimination not strong enough. Will update to be individualized. This will also help improve generated help text, which was a concern when using class doc strings. --- pydantic_settings/main.py | 25 +++++--- pydantic_settings/sources.py | 62 ++++++++++++------- tests/test_settings.py | 113 ++++++++++++++++++++++++++++++++++- 3 files changed, 171 insertions(+), 29 deletions(-) diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 1b41a039..29179f96 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -1,6 +1,5 @@ from __future__ import annotations as _annotations -import sys from pathlib import Path from typing import Any, ClassVar @@ -29,7 +28,8 @@ class SettingsConfigDict(ConfigDict, total=False): env_ignore_empty: bool env_nested_delimiter: str | None env_parse_none_str: str | None - cli_parse_args: bool + cli_prog_name: str | None + cli_parse_args: bool | list[str] | None cli_hide_none_type: bool cli_avoid_json: bool secrets_dir: str | Path | None @@ -64,6 +64,12 @@ class BaseSettings(BaseModel): _env_nested_delimiter: The nested env values delimiter. Defaults to `None`. _env_parse_none_str: The env string value that should be parsed (e.g. "null", "void", "None", etc.) into `None` type(None). Defaults to `None` type(None), which means no parsing should occur. + _cli_prog_name: The CLI program name to display in help text. Defaults to `None` if _cli_parse_args is `None`. + Otherwse, defaults to sys.argv[0]. + _cli_parse_args: The list of CLI arguments to parse. Defaults to None. + If set to `True`, defaults to sys.argv[1:]. + _cli_hide_none_type: Hide NoneType values in CLI help text. Defaults to `False`. + _cli_avoid_json: Avoid complex JSON objects in CLI help text. Defaults to `False`. _secrets_dir: The secret files directory. Defaults to `None`. """ @@ -76,7 +82,8 @@ def __init__( _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_parse_none_str: str | None = None, - _cli_parse_args: bool | None = None, + _cli_prog_name: str | None = None, + _cli_parse_args: bool | list[str] | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _secrets_dir: str | Path | None = None, @@ -93,6 +100,7 @@ def __init__( _env_ignore_empty=_env_ignore_empty, _env_nested_delimiter=_env_nested_delimiter, _env_parse_none_str=_env_parse_none_str, + _cli_prog_name=_cli_prog_name, _cli_parse_args=_cli_parse_args, _cli_hide_none_type=_cli_hide_none_type, _cli_avoid_json=_cli_avoid_json, @@ -136,7 +144,8 @@ def _settings_build_values( _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_parse_none_str: str | None = None, - _cli_parse_args: bool | None = None, + _cli_prog_name: str | None = None, + _cli_parse_args: bool | list[str] | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _secrets_dir: str | Path | None = None, @@ -160,6 +169,7 @@ def _settings_build_values( _env_parse_none_str if _env_parse_none_str is not None else self.model_config.get('env_parse_none_str') ) + cli_prog_name = _cli_prog_name if _cli_prog_name is not None else self.model_config.get('cli_prog_name') cli_parse_args = _cli_parse_args if _cli_parse_args is not None else self.model_config.get('cli_parse_args') cli_hide_none_type = ( _cli_hide_none_type if _cli_hide_none_type is not None else self.model_config.get('cli_hide_none_type') @@ -172,9 +182,9 @@ def _settings_build_values( init_settings = InitSettingsSource(self.__class__, init_kwargs=init_kwargs) cli_settings = CliSettingsSource( self.__class__, - sys.argv[1:], - cli_parse_none_str=env_parse_none_str, + cli_prog_name=cli_prog_name, cli_parse_args=cli_parse_args, + cli_parse_none_str=env_parse_none_str, cli_hide_none_type=cli_hide_none_type, cli_avoid_json=cli_avoid_json, ) @@ -227,7 +237,8 @@ def _settings_build_values( env_ignore_empty=False, env_nested_delimiter=None, env_parse_none_str=None, - cli_parse_args=False, + cli_prog_name=None, + cli_parse_args=None, cli_hide_none_type=False, cli_avoid_json=False, secrets_dir=None, diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 110b7fd5..54d430ad 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -2,6 +2,7 @@ import json import os +import sys import warnings from abc import ABC, abstractmethod from collections import deque @@ -691,14 +692,19 @@ class CliSettingsSource(EnvSettingsSource): def __init__( self, settings_cls: type[BaseSettings], - args: list[str], + cli_prog_name: str | None = None, + cli_parse_args: bool | list[str] | None = None, cli_parse_none_str: str | None = None, - cli_parse_args: bool | None = None, cli_hide_none_type: bool | None = None, cli_avoid_json: bool | None = None, ) -> None: - self.args = args - self.cli_parse_args = cli_parse_args if cli_parse_args is not None else self.config.get('cli_parse_args', False) + self.cli_prog_name = sys.argv[0] if cli_prog_name is None else cli_prog_name + self.cli_parse_args = cli_parse_args + if self.cli_parse_args not in (None, False): + if self.cli_parse_args is True: + self.cli_parse_args = sys.argv[1:] + elif not isinstance(self.cli_parse_args, list): + raise SettingsError(f'cli_parse_args must be List[str], recieved {type(self.cli_parse_args)}') self.cli_hide_none_type = ( cli_hide_none_type if cli_hide_none_type is not None else self.config.get('cli_hide_none_type', False) ) @@ -708,13 +714,14 @@ def __init__( super().__init__(settings_cls, env_nested_delimiter='.', env_parse_none_str=cli_parse_none_str) def _load_env_vars(self) -> Mapping[str, str | None]: - if not self.cli_parse_args: + if self.cli_parse_args in (None, False): return {} - self._cli_arg_names: list[str] = [] self._cli_dict_arg_names: list[str] = [] - parser: ArgumentParser = self._add_fields_to_parser(ArgumentParser(), self.settings_cls) - parsed_args: dict[str, list[str] | str] = vars(parser.parse_args(self.args)) + parser: ArgumentParser = self._add_fields_to_parser( + ArgumentParser(prog=self.cli_prog_name, description=self.settings_cls.__doc__), self.settings_cls + ) + parsed_args: dict[str, list[str] | str] = vars(parser.parse_args(self.cli_parse_args)) # type: ignore for field, val in parsed_args.items(): if isinstance(val, list): merge_list = [] @@ -801,58 +808,71 @@ def _add_fields_to_parser( self, parser: ArgumentParser, model: type[BaseModel], + _added_args: list[str] = [], _arg_prefix: str = '', - _dest_prefix: str = '', + _subcommand_prefix: str = '', _group: _ArgumentGroup | None = None, ) -> ArgumentParser: subparsers: _SubParsersAction[Any] | None = None for field_name, field_info in model.model_fields.items(): - arg_name = f'{_arg_prefix}{field_name}' sub_models: list[type[BaseModel]] = self._get_sub_models(model, field_name, field_info, subparsers) if _CliSubCommand in field_info.metadata: - subparsers = parser.add_subparsers(title='subcommands', description='available subcommands') + subparsers = parser.add_subparsers(title='subcommands', description=field_info.description) for model in sub_models: self._add_fields_to_parser( subparsers.add_parser( - f'{model.__name__.lower()}', help=model.__doc__, formatter_class=parser.formatter_class + f'{model.__name__.lower()}', + help=model.__doc__, + formatter_class=parser.formatter_class, + description=model.__doc__, ), model, - _dest_prefix=f'{_dest_prefix}{arg_name}', + _added_args=[], + _arg_prefix=f'{_arg_prefix}{field_name}.', + _subcommand_prefix=f'{_subcommand_prefix}{field_name}.', ) - elif arg_name not in self._cli_arg_names: + else: arg_flag: str = '--' kwargs: dict[str, Any] = {} + kwargs['dest'] = f'{_arg_prefix}{field_name}' + if kwargs['dest'] in _added_args: + continue + kwargs['default'] = SUPPRESS kwargs['help'] = field_info.description - kwargs['dest'] = f'{_dest_prefix}{field_name}' kwargs['metavar'] = self._format_metavar(field_info.annotation) if get_origin(field_info.annotation) in (list, set, dict, Sequence): kwargs['action'] = 'append' if get_origin(field_info.annotation) is dict: - self._cli_dict_arg_names.append(arg_name) + self._cli_dict_arg_names.append(kwargs['dest']) + arg_name = f'{_arg_prefix.replace(_subcommand_prefix, "", 1)}{field_name}' + print((arg_name, kwargs['dest'])) if _CliPositionalArg in field_info.metadata: + kwargs['metavar'] = field_name.upper() + arg_name = kwargs['dest'] del kwargs['dest'] arg_flag = '' if sub_models and kwargs.get('action') != 'append': model_group = parser.add_argument_group(f'{arg_name} options', field_info.description) if not self.cli_avoid_json: - self._cli_arg_names.append(arg_name) + _added_args.append(arg_name) kwargs['help'] = f'set {arg_name} from JSON string' model_group.add_argument(f'{arg_flag}{arg_name}', **kwargs) for model in sub_models: self._add_fields_to_parser( parser, model, - _arg_prefix=f'{arg_name}.', + _added_args=_added_args, + _arg_prefix=f'{_arg_prefix}{field_name}.', + _subcommand_prefix=_subcommand_prefix, _group=model_group, - _dest_prefix=f"{kwargs['dest']}.", ) elif _group is not None: - self._cli_arg_names.append(arg_name) + _added_args.append(arg_name) _group.add_argument(f'{arg_flag}{arg_name}', **kwargs) else: - self._cli_arg_names.append(arg_name) + _added_args.append(arg_name) parser.add_argument(f'{arg_flag}{arg_name}', **kwargs) return parser diff --git a/tests/test_settings.py b/tests/test_settings.py index 7491b5f6..53cb6d62 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -34,7 +34,7 @@ SecretsSettingsSource, SettingsConfigDict, ) -from pydantic_settings.sources import SettingsError, read_env_file +from pydantic_settings.sources import CliPositionalArg, CliSubCommand, SettingsError, read_env_file try: import dotenv @@ -1923,3 +1923,114 @@ class Settings(BaseSettings): s = Settings() assert s.data == {'foo': 'bar'} + + +def test_cli_nested_arg(): + class SubSubValue(BaseModel): + v6: str + + class SubValue(BaseModel): + v4: str + v5: int + sub_sub: SubSubValue + + class TopValue(BaseModel): + v1: str + v2: str + v3: str + sub: SubValue + + class Cfg(BaseSettings): + v0: str + v0_union: Union[SubValue, int] + top: TopValue + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return cli_settings, init_settings + + argv: list[str] = [] + argv += ['--top', '{"v1": "json-1", "v2": "json-2", "sub": {"v5": "xx"}}'] + argv += ['--top.sub.v5', '5'] + argv += ['--v0', '0'] + argv += ['--top.v2', '2'] + argv += ['--top.v3', '3'] + argv += ['--v0_union', '0'] + argv += ['--top.sub.sub_sub.v6', '6'] + argv += ['--top.sub.v4', '4'] + cfg = Cfg(_cli_parse_args=argv) + assert cfg.model_dump() == { + 'v0': '0', + 'v0_union': 0, + 'top': { + 'v1': 'json-1', + 'v2': '2', + 'v3': '3', + 'sub': {'v4': '4', 'v5': 5, 'sub_sub': {'v6': '6'}}, + }, + } + + +def test_cli_list_arg(): + pass + + +def test_cli_dict_arg(): + pass + + +def test_cli_literal(): + pass + + +def test_cli_positional_arg(): + pass + + +def test_cli_subcommand_with_positionals(): + class Clone(BaseModel, use_attribute_docstrings=True): + local: bool = False + shared: bool = False + repository: CliPositionalArg[str] + directory: CliPositionalArg[str] + + class Init(BaseModel, use_attribute_docstrings=True): + quiet: bool = False + bare: bool = False + directory: CliPositionalArg[str] + + class Git(BaseSettings, use_attribute_docstrings=True): + subcommand: CliSubCommand[Clone | Init] + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return cli_settings, init_settings + + git = Git(_cli_parse_args=['init', '--quiet', 'true', 'dir/path']) + assert git.model_dump() == {'subcommand': {'quiet': True, 'bare': False, 'directory': 'dir/path'}} + + git = Git(_cli_parse_args=['clone', 'repo', '.', '--shared', 'true']) + assert git.model_dump() == {'subcommand': {'local': False, 'shared': True, 'repository': 'repo', 'directory': '.'}} + +def test_cli_avoid_json(): + pass + + +def test_cli_hide_none_type(): + pass From 0ce72aca7134488768d75a6ba4b3fa9e9fc1a87d Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sat, 20 Jan 2024 21:28:35 -0700 Subject: [PATCH 04/61] Updated subcommands. --- pydantic_settings/sources.py | 116 ++++++++++++++++++++--------------- tests/test_settings.py | 33 ++++++++-- 2 files changed, 95 insertions(+), 54 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 54d430ad..4511b982 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -44,7 +44,7 @@ class _CliPositionalArg: T = TypeVar('T') -CliSubCommand = Annotated[T, _CliSubCommand] +CliSubCommand = Annotated[T | None, _CliSubCommand] CliPositionalArg = Annotated[T, _CliPositionalArg] @@ -718,22 +718,30 @@ def _load_env_vars(self) -> Mapping[str, str | None]: return {} self._cli_dict_arg_names: list[str] = [] + self._cli_subcommands: dict[str, list[str]] = {} parser: ArgumentParser = self._add_fields_to_parser( ArgumentParser(prog=self.cli_prog_name, description=self.settings_cls.__doc__), self.settings_cls ) parsed_args: dict[str, list[str] | str] = vars(parser.parse_args(self.cli_parse_args)) # type: ignore - for field, val in parsed_args.items(): - if isinstance(val, list): - merge_list = [] - for sub_val in val: - if sub_val.startswith('[') and sub_val.endswith(']'): - sub_val = sub_val[1:-1] - merge_list.append(sub_val) - parsed_args[field] = ( - f'[{",".join(merge_list)}]' - if field not in self._cli_dict_arg_names - else self._merge_json_key_val_list_str(f'[{",".join(merge_list)}]') - ) + if any(key for key in parsed_args.keys() if not key.endswith(':subcommand')): + for field, val in parsed_args.items(): + if isinstance(val, list): + merge_list = [] + for sub_val in val: + if sub_val.startswith('[') and sub_val.endswith(']'): + sub_val = sub_val[1:-1] + merge_list.append(sub_val) + parsed_args[field] = ( + f'[{",".join(merge_list)}]' + if field not in self._cli_dict_arg_names + else self._merge_json_key_val_list_str(f'[{",".join(merge_list)}]') + ) + elif field.endswith(':subcommand'): + self._cli_subcommands[field].remove(field.split(':')[0] + val) + + for subcommands in self._cli_subcommands.values(): + for subcommand in subcommands: + parsed_args[subcommand] = self.env_parse_none_str # type: ignore return parse_env_vars( parsed_args, self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str # type: ignore @@ -770,9 +778,7 @@ def _merge_json_key_val_list_str(self, key_val_list_str: str) -> str: break return json.dumps(key_val_dict) - def _get_sub_models( - self, model: type[BaseModel], field_name: str, field_info: FieldInfo, subparsers: _SubParsersAction[Any] | None - ) -> list[type[BaseModel]]: + def _get_sub_models(self, model: type[BaseModel], field_name: str, field_info: FieldInfo) -> list[type[BaseModel]]: field_types: tuple[Any, ...] = ( (field_info.annotation,) if not get_args(field_info.annotation) else get_args(field_info.annotation) ) @@ -790,20 +796,31 @@ def _get_sub_models( ) if isclass(type_) and issubclass(type_, BaseModel): sub_models.append(type_) - elif _CliSubCommand in field_info.metadata and subparsers is not None: - raise SettingsError( - f'detected a second subcommand definition at {model.__name__}.{field_name}, ' - 'only one per model is allowed' - ) - if _CliPositionalArg in field_info.metadata: - if not field_info.is_required(): - raise SettingsError(f'positional argument {model.__name__}.{field_name} has a default value') - elif subparsers is not None: - raise SettingsError( - f'positional argument {model.__name__}.{field_name} ' 'is speficied after a subcommand definition' - ) return sub_models + def _sort_arg_fields(self, model: type[BaseModel]) -> list[tuple[str, FieldInfo]]: + positional_args, subcommand_args, optional_args = [], [], [] + for field_name, field_info in model.model_fields.items(): + if _CliSubCommand in field_info.metadata: + if not field_info.is_required(): + raise SettingsError(f'subcommand argument {model.__name__}.{field_name} has a default value') + else: + field_types = [type_ for type_ in get_args(field_info.annotation) if type_ is not type(None)] + if len(field_types) != 1: + raise SettingsError(f'subcommand argument {model.__name__}.{field_name} has multiple types') + elif not (isclass(field_types[0]) and issubclass(field_types[0], BaseModel)): + raise SettingsError( + f'subcommand argument {model.__name__}.{field_name} is not derived from BaseModel' + ) + subcommand_args.append((field_name, field_info)) + elif _CliPositionalArg in field_info.metadata: + if not field_info.is_required(): + raise SettingsError(f'positional argument {model.__name__}.{field_name} has a default value') + positional_args.append((field_name, field_info)) + else: + optional_args.append((field_name, field_info)) + return positional_args + subcommand_args + optional_args + def _add_fields_to_parser( self, parser: ArgumentParser, @@ -814,39 +831,42 @@ def _add_fields_to_parser( _group: _ArgumentGroup | None = None, ) -> ArgumentParser: subparsers: _SubParsersAction[Any] | None = None - for field_name, field_info in model.model_fields.items(): - sub_models: list[type[BaseModel]] = self._get_sub_models(model, field_name, field_info, subparsers) + for field_name, field_info in self._sort_arg_fields(model): + sub_models: list[type[BaseModel]] = self._get_sub_models(model, field_name, field_info) if _CliSubCommand in field_info.metadata: - subparsers = parser.add_subparsers(title='subcommands', description=field_info.description) - for model in sub_models: - self._add_fields_to_parser( - subparsers.add_parser( - f'{model.__name__.lower()}', - help=model.__doc__, - formatter_class=parser.formatter_class, - description=model.__doc__, - ), - model, - _added_args=[], - _arg_prefix=f'{_arg_prefix}{field_name}.', - _subcommand_prefix=f'{_subcommand_prefix}{field_name}.', - ) + if subparsers is None: + subparsers = parser.add_subparsers(title='subcommands', dest=f'{_arg_prefix}:subcommand') + self._cli_subcommands[f'{_arg_prefix}:subcommand'] = [f'{_arg_prefix}{field_name}'] + else: + self._cli_subcommands[f'{_arg_prefix}:subcommand'].append(f'{_arg_prefix}{field_name}') + + model = sub_models[0] + self._add_fields_to_parser( + subparsers.add_parser( + field_name, + help=field_info.description, + formatter_class=parser.formatter_class, + description=model.__doc__, + ), + model, + _added_args=[], + _arg_prefix=f'{_arg_prefix}{field_name}.', + _subcommand_prefix=f'{_subcommand_prefix}{field_name}.', + ) else: arg_flag: str = '--' kwargs: dict[str, Any] = {} kwargs['dest'] = f'{_arg_prefix}{field_name}' if kwargs['dest'] in _added_args: continue - kwargs['default'] = SUPPRESS kwargs['help'] = field_info.description kwargs['metavar'] = self._format_metavar(field_info.annotation) - if get_origin(field_info.annotation) in (list, set, dict, Sequence): + if get_origin(field_info.annotation) in (list, set, dict, Sequence, Mapping): kwargs['action'] = 'append' - if get_origin(field_info.annotation) is dict: + if get_origin(field_info.annotation) in (dict, Mapping): self._cli_dict_arg_names.append(kwargs['dest']) arg_name = f'{_arg_prefix.replace(_subcommand_prefix, "", 1)}{field_name}' - print((arg_name, kwargs['dest'])) if _CliPositionalArg in field_info.metadata: kwargs['metavar'] = field_name.upper() arg_name = kwargs['dest'] diff --git a/tests/test_settings.py b/tests/test_settings.py index 53cb6d62..b238e22d 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1996,19 +1996,31 @@ def test_cli_positional_arg(): def test_cli_subcommand_with_positionals(): + class FooPlugin(BaseModel, use_attribute_docstrings=True): + my_feature: bool = False + + class BarPlugin(BaseModel, use_attribute_docstrings=True): + my_feature: bool = False + + class Plugins(BaseModel, use_attribute_docstrings=True): + foo: CliSubCommand[FooPlugin] + bar: CliSubCommand[BarPlugin] + class Clone(BaseModel, use_attribute_docstrings=True): - local: bool = False - shared: bool = False repository: CliPositionalArg[str] directory: CliPositionalArg[str] + local: bool = False + shared: bool = False class Init(BaseModel, use_attribute_docstrings=True): + directory: CliPositionalArg[str] quiet: bool = False bare: bool = False - directory: CliPositionalArg[str] class Git(BaseSettings, use_attribute_docstrings=True): - subcommand: CliSubCommand[Clone | Init] + clone: CliSubCommand[Clone] + init: CliSubCommand[Init] + plugins: CliSubCommand[Plugins] @classmethod def settings_customise_sources( @@ -2023,10 +2035,19 @@ def settings_customise_sources( return cli_settings, init_settings git = Git(_cli_parse_args=['init', '--quiet', 'true', 'dir/path']) - assert git.model_dump() == {'subcommand': {'quiet': True, 'bare': False, 'directory': 'dir/path'}} + assert git.model_dump() == { + 'clone': None, + 'init': {'directory': 'dir/path', 'quiet': True, 'bare': False}, + 'plugins': None, + } git = Git(_cli_parse_args=['clone', 'repo', '.', '--shared', 'true']) - assert git.model_dump() == {'subcommand': {'local': False, 'shared': True, 'repository': 'repo', 'directory': '.'}} + assert git.model_dump() == { + 'clone': {'repository': 'repo', 'directory': '.', 'local': False, 'shared': True}, + 'init': None, + 'plugins': None, + } + def test_cli_avoid_json(): pass From ce2c425cf5811ce55428127442b4dd7d43d474bb Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sun, 21 Jan 2024 14:25:49 -0700 Subject: [PATCH 05/61] Initial tests. --- pydantic_settings/sources.py | 70 ++++++---- tests/test_settings.py | 249 ++++++++++++++++++++++++++++++++--- 2 files changed, 276 insertions(+), 43 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 4511b982..b04b57ea 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -720,8 +720,14 @@ def _load_env_vars(self) -> Mapping[str, str | None]: self._cli_dict_arg_names: list[str] = [] self._cli_subcommands: dict[str, list[str]] = {} parser: ArgumentParser = self._add_fields_to_parser( - ArgumentParser(prog=self.cli_prog_name, description=self.settings_cls.__doc__), self.settings_cls + ArgumentParser(prog=self.cli_prog_name, description=self.settings_cls.__doc__), + model=self.settings_cls, + added_args=[], + arg_prefix='', + subcommand_prefix='', + group=None, ) + parsed_args: dict[str, list[str] | str] = vars(parser.parse_args(self.cli_parse_args)) # type: ignore if any(key for key in parsed_args.keys() if not key.endswith(':subcommand')): for field, val in parsed_args.items(): @@ -825,20 +831,20 @@ def _add_fields_to_parser( self, parser: ArgumentParser, model: type[BaseModel], - _added_args: list[str] = [], - _arg_prefix: str = '', - _subcommand_prefix: str = '', - _group: _ArgumentGroup | None = None, + added_args: list[str], + arg_prefix: str, + subcommand_prefix: str, + group: _ArgumentGroup | None, ) -> ArgumentParser: subparsers: _SubParsersAction[Any] | None = None for field_name, field_info in self._sort_arg_fields(model): sub_models: list[type[BaseModel]] = self._get_sub_models(model, field_name, field_info) if _CliSubCommand in field_info.metadata: if subparsers is None: - subparsers = parser.add_subparsers(title='subcommands', dest=f'{_arg_prefix}:subcommand') - self._cli_subcommands[f'{_arg_prefix}:subcommand'] = [f'{_arg_prefix}{field_name}'] + subparsers = parser.add_subparsers(title='subcommands', dest=f'{arg_prefix}:subcommand') + self._cli_subcommands[f'{arg_prefix}:subcommand'] = [f'{arg_prefix}{field_name}'] else: - self._cli_subcommands[f'{_arg_prefix}:subcommand'].append(f'{_arg_prefix}{field_name}') + self._cli_subcommands[f'{arg_prefix}:subcommand'].append(f'{arg_prefix}{field_name}') model = sub_models[0] self._add_fields_to_parser( @@ -849,24 +855,26 @@ def _add_fields_to_parser( description=model.__doc__, ), model, - _added_args=[], - _arg_prefix=f'{_arg_prefix}{field_name}.', - _subcommand_prefix=f'{_subcommand_prefix}{field_name}.', + added_args=[], + arg_prefix=f'{arg_prefix}{field_name}.', + subcommand_prefix=f'{subcommand_prefix}{field_name}.', + group=None, ) else: arg_flag: str = '--' kwargs: dict[str, Any] = {} - kwargs['dest'] = f'{_arg_prefix}{field_name}' - if kwargs['dest'] in _added_args: - continue kwargs['default'] = SUPPRESS kwargs['help'] = field_info.description + kwargs['dest'] = f'{arg_prefix}{field_name}' kwargs['metavar'] = self._format_metavar(field_info.annotation) - if get_origin(field_info.annotation) in (list, set, dict, Sequence, Mapping): + if kwargs['dest'] in added_args: + continue + if _annotation_contains_types(field_info.annotation, (list, set, dict, Sequence, Mapping)): kwargs['action'] = 'append' - if get_origin(field_info.annotation) in (dict, Mapping): + if _annotation_contains_types(field_info.annotation, (dict, Mapping)): self._cli_dict_arg_names.append(kwargs['dest']) - arg_name = f'{_arg_prefix.replace(_subcommand_prefix, "", 1)}{field_name}' + + arg_name = f'{arg_prefix.replace(subcommand_prefix, "", 1)}{field_name}' if _CliPositionalArg in field_info.metadata: kwargs['metavar'] = field_name.upper() arg_name = kwargs['dest'] @@ -876,23 +884,23 @@ def _add_fields_to_parser( if sub_models and kwargs.get('action') != 'append': model_group = parser.add_argument_group(f'{arg_name} options', field_info.description) if not self.cli_avoid_json: - _added_args.append(arg_name) + added_args.append(arg_name) kwargs['help'] = f'set {arg_name} from JSON string' model_group.add_argument(f'{arg_flag}{arg_name}', **kwargs) for model in sub_models: self._add_fields_to_parser( parser, model, - _added_args=_added_args, - _arg_prefix=f'{_arg_prefix}{field_name}.', - _subcommand_prefix=_subcommand_prefix, - _group=model_group, + added_args=added_args, + arg_prefix=f'{arg_prefix}{field_name}.', + subcommand_prefix=subcommand_prefix, + group=model_group, ) - elif _group is not None: - _added_args.append(arg_name) - _group.add_argument(f'{arg_flag}{arg_name}', **kwargs) + elif group is not None: + added_args.append(arg_name) + group.add_argument(f'{arg_flag}{arg_name}', **kwargs) else: - _added_args.append(arg_name) + added_args.append(arg_name) parser.add_argument(f'{arg_flag}{arg_name}', **kwargs) return parser @@ -989,3 +997,13 @@ def _annotation_is_complex_inner(annotation: type[Any] | None) -> bool: return lenient_issubclass(annotation, (BaseModel, Mapping, Sequence, tuple, set, frozenset, deque)) or is_dataclass( annotation ) + + +def _annotation_contains_types(annotation: type[Any] | None, types: tuple[Any, ...]) -> bool: + if origin_is_union(get_origin(annotation)) or isinstance(annotation, WithArgsTypes): + if get_origin(annotation) in types: + return True + for type_ in get_args(annotation): + if _annotation_contains_types(type_, types): + return True + return annotation in types diff --git a/tests/test_settings.py b/tests/test_settings.py index b238e22d..d15c451f 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1957,16 +1957,16 @@ def settings_customise_sources( ) -> Tuple[PydanticBaseSettingsSource, ...]: return cli_settings, init_settings - argv: list[str] = [] - argv += ['--top', '{"v1": "json-1", "v2": "json-2", "sub": {"v5": "xx"}}'] - argv += ['--top.sub.v5', '5'] - argv += ['--v0', '0'] - argv += ['--top.v2', '2'] - argv += ['--top.v3', '3'] - argv += ['--v0_union', '0'] - argv += ['--top.sub.sub_sub.v6', '6'] - argv += ['--top.sub.v4', '4'] - cfg = Cfg(_cli_parse_args=argv) + args: list[str] = [] + args += ['--top', '{"v1": "json-1", "v2": "json-2", "sub": {"v5": "xx"}}'] + args += ['--top.sub.v5', '5'] + args += ['--v0', '0'] + args += ['--top.v2', '2'] + args += ['--top.v3', '3'] + args += ['--v0_union', '0'] + args += ['--top.sub.sub_sub.v6', '6'] + args += ['--top.sub.v4', '4'] + cfg = Cfg(_cli_parse_args=args) assert cfg.model_dump() == { 'v0': '0', 'v0_union': 0, @@ -1980,19 +1980,156 @@ def settings_customise_sources( def test_cli_list_arg(): - pass + class Obj(BaseModel): + val: int + class Child(BaseModel): + num_list: Optional[List[int]] = None + obj_list: Optional[List[Obj]] = None + str_list: Optional[List[str]] = None + union_list: Optional[List[Obj | int]] = None -def test_cli_dict_arg(): - pass + class Cfg(BaseSettings): + num_list: Optional[List[int]] = None + obj_list: Optional[List[Obj]] = None + union_list: Optional[List[Obj | int]] = None + str_list: Optional[List[str]] = None + child: Optional[Child] = None + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return cli_settings, init_settings -def test_cli_literal(): - pass + args: list[str] = [] + args = ['--num_list', '[1,2]'] + args += ['--num_list', '3,4'] + args += ['--num_list', '5', '--num_list', '6'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == { + 'num_list': [1, 2, 3, 4, 5, 6], + 'obj_list': None, + 'union_list': None, + 'str_list': None, + 'child': None, + } + args = ['--obj_list', '[{"val":1},{"val":2}]'] + args += ['--obj_list', '{"val":3},{"val":4}'] + args += ['--obj_list', '{"val":5}', '--obj_list', '{"val":6}'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == { + 'num_list': None, + 'obj_list': [{'val': 1}, {'val': 2}, {'val': 3}, {'val': 4}, {'val': 5}, {'val': 6}], + 'union_list': None, + 'str_list': None, + 'child': None, + } -def test_cli_positional_arg(): - pass + args = ['--union_list', '[{"val":1},2]', '--union_list', '[3,{"val":4}]'] + args += ['--union_list', '{"val":5},6', '--union_list', '7,{"val":8}'] + args += ['--union_list', '{"val":9}', '--union_list', '10'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == { + 'num_list': None, + 'obj_list': None, + 'union_list': [{'val': 1}, 2, 3, {'val': 4}, {'val': 5}, 6, 7, {'val': 8}, {'val': 9}, 10], + 'str_list': None, + 'child': None, + } + + args = ['--str_list', '["0,0","1,1"]'] + args += ['--str_list', '"2,2","3,3"'] + args += ['--str_list', '"4,4"', '--str_list', '"5,5"'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == { + 'num_list': None, + 'obj_list': None, + 'union_list': None, + 'str_list': ['0,0', '1,1', '2,2', '3,3', '4,4', '5,5'], + 'child': None, + } + + +def test_cli_dict_arg(): + class Child(BaseModel): + check_dict: Dict[str, str] + + class Cfg(BaseSettings): + check_dict: Optional[Dict[str, str]] = None + child: Optional[Child] = None + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return cli_settings, init_settings + + args: list[str] = [] + args = ['--check_dict', '{"k1":"a","k2":"b"}'] + args += ['--check_dict', '{"k3":"c"},{"k4":"d"}'] + args += ['--check_dict', '{"k5":"e"}', '--check_dict', '{"k6":"f"}'] + args += ['--check_dict', '[k7=g,k8=h]'] + args += ['--check_dict', 'k9=i,k10=j'] + args += ['--check_dict', 'k11=k', '--check_dict', 'k12=l'] + args += ['--check_dict', '[{"k13":"m"},k14=n]', '--check_dict', '[k15=o,{"k16":"p"}]'] + args += ['--check_dict', '{"k17":"q"},k18=r', '--check_dict', 'k19=s,{"k20":"t"}'] + args += ['--check_dict', '{"k21":"u"},k22=v,{"k23":"w"}'] + args += ['--check_dict', 'k24=x,{"k25":"y"},k26=z'] + args += ['--check_dict', '[k27="x,y",k28="x,y"]'] + args += ['--check_dict', 'k29="x,y",k30="x,y"'] + args += ['--check_dict', 'k31="x,y"', '--check_dict', 'k32="x,y"'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == { + 'check_dict': { + 'k1': 'a', + 'k2': 'b', + 'k3': 'c', + 'k4': 'd', + 'k5': 'e', + 'k6': 'f', + 'k7': 'g', + 'k8': 'h', + 'k9': 'i', + 'k10': 'j', + 'k11': 'k', + 'k12': 'l', + 'k13': 'm', + 'k14': 'n', + 'k15': 'o', + 'k16': 'p', + 'k17': 'q', + 'k18': 'r', + 'k19': 's', + 'k20': 't', + 'k21': 'u', + 'k22': 'v', + 'k23': 'w', + 'k24': 'x', + 'k25': 'y', + 'k26': 'z', + 'k27': 'x,y', + 'k28': 'x,y', + 'k29': 'x,y', + 'k30': 'x,y', + 'k31': 'x,y', + 'k32': 'x,y', + }, + 'child': None, + } def test_cli_subcommand_with_positionals(): @@ -2049,6 +2186,84 @@ def settings_customise_sources( } +def test_cli_union_similar_sub_models(): + class ChildA(BaseModel): + name: str = 'child a' + diff_a: str = 'child a difference' + + class ChildB(BaseModel): + name: str = 'child b' + diff_b: str = 'child b difference' + + class Cfg(BaseSettings): + child: Union[ChildA, ChildB] + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + cli_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return cli_settings, init_settings + + cfg = Cfg(_cli_parse_args=['--child.name', 'new name a', '--child.diff_a', 'new diff a']) + assert cfg.model_dump() == {'child': {'name': 'new name a', 'diff_a': 'new diff a'}} + + +def test_cli_annotation_exceptions(): + class SubCmdAlt(BaseModel): + pass + + class SubCmd(BaseModel): + pass + + with pytest.raises(SettingsError): + + class SubCommandNotOutermost(BaseSettings): + subcmd: Union[int, CliSubCommand[SubCmd]] + + SubCommandNotOutermost(_cli_parse_args=['--help']) + + with pytest.raises(SettingsError): + + class SubCommandHasDefault(BaseSettings): + subcmd: CliSubCommand[SubCmd] = SubCmd() + + SubCommandHasDefault(_cli_parse_args=['--help']) + + with pytest.raises(SettingsError): + + class SubCommandMultipleTypes(BaseSettings): + subcmd: CliSubCommand[SubCmd | SubCmdAlt] + + SubCommandMultipleTypes(_cli_parse_args=['--help']) + + with pytest.raises(SettingsError): + + class SubCommandNotModel(BaseSettings): + subcmd: CliSubCommand[str] + + SubCommandNotModel(_cli_parse_args=['--help']) + + with pytest.raises(SettingsError): + + class PositionalArgNotOutermost(BaseSettings): + pos_arg: Union[int, CliPositionalArg[str]] + + PositionalArgNotOutermost(_cli_parse_args=['--help']) + + with pytest.raises(SettingsError): + + class PositionalArgHasDefault(BaseSettings): + pos_arg: CliPositionalArg[str] = 'bad' + + PositionalArgHasDefault(_cli_parse_args=['--help']) + + def test_cli_avoid_json(): pass From 5795c7cf6973406fdf70d7983a73d8486236c364 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sun, 21 Jan 2024 23:17:59 -0700 Subject: [PATCH 06/61] Remove use_attribute_docstrings. --- pydantic_settings/sources.py | 4 ++-- tests/test_settings.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index b04b57ea..d7450cc1 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -10,7 +10,7 @@ from inspect import isclass from pathlib import Path from types import FunctionType -from typing import TYPE_CHECKING, Annotated, Any, Dict, List, Literal, Mapping, Sequence, Tuple, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Sequence, Tuple, TypeVar, Union, cast from dotenv import dotenv_values from pydantic import AliasChoices, AliasPath, BaseModel, Json, TypeAdapter @@ -18,7 +18,7 @@ from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base from pydantic._internal._utils import deep_update, lenient_issubclass from pydantic.fields import FieldInfo -from typing_extensions import TypeAliasType, get_args, get_origin +from typing_extensions import Annotated, TypeAliasType, get_args, get_origin from pydantic_settings.utils import path_type_label diff --git a/tests/test_settings.py b/tests/test_settings.py index d15c451f..ffd4b8e2 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2133,28 +2133,28 @@ def settings_customise_sources( def test_cli_subcommand_with_positionals(): - class FooPlugin(BaseModel, use_attribute_docstrings=True): + class FooPlugin(BaseModel): my_feature: bool = False - class BarPlugin(BaseModel, use_attribute_docstrings=True): + class BarPlugin(BaseModel): my_feature: bool = False - class Plugins(BaseModel, use_attribute_docstrings=True): + class Plugins(BaseModel): foo: CliSubCommand[FooPlugin] bar: CliSubCommand[BarPlugin] - class Clone(BaseModel, use_attribute_docstrings=True): + class Clone(BaseModel): repository: CliPositionalArg[str] directory: CliPositionalArg[str] local: bool = False shared: bool = False - class Init(BaseModel, use_attribute_docstrings=True): + class Init(BaseModel): directory: CliPositionalArg[str] quiet: bool = False bare: bool = False - class Git(BaseSettings, use_attribute_docstrings=True): + class Git(BaseSettings): clone: CliSubCommand[Clone] init: CliSubCommand[Init] plugins: CliSubCommand[Plugins] From 3215797ad2e3ec341753382682392eb22689fbd6 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 23 Jan 2024 15:53:29 -0700 Subject: [PATCH 07/61] Various updates. --- docs/index.md | 4 - pydantic_settings/main.py | 19 ++- pydantic_settings/sources.py | 76 ++++++---- tests/test_settings.py | 283 +++++++++++++++-------------------- 4 files changed, 183 insertions(+), 199 deletions(-) diff --git a/docs/index.md b/docs/index.md index 0e66ab25..d1627c9d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -363,7 +363,6 @@ class Settings(BaseSettings): cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -576,7 +575,6 @@ class Settings(BaseSettings): cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -659,7 +657,6 @@ class Settings(BaseSettings): cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -696,7 +693,6 @@ class Settings(BaseSettings): cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 29179f96..26d2f3c7 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -32,6 +32,7 @@ class SettingsConfigDict(ConfigDict, total=False): cli_parse_args: bool | list[str] | None cli_hide_none_type: bool cli_avoid_json: bool + cli_enforce_required: bool secrets_dir: str | Path | None @@ -70,6 +71,7 @@ class BaseSettings(BaseModel): If set to `True`, defaults to sys.argv[1:]. _cli_hide_none_type: Hide NoneType values in CLI help text. Defaults to `False`. _cli_avoid_json: Avoid complex JSON objects in CLI help text. Defaults to `False`. + _cli_enforce_required: Enforce required fields at the CLI. Defaults to `False`. _secrets_dir: The secret files directory. Defaults to `None`. """ @@ -86,6 +88,7 @@ def __init__( _cli_parse_args: bool | list[str] | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, + _cli_enforce_required: bool | None = None, _secrets_dir: str | Path | None = None, **values: Any, ) -> None: @@ -104,6 +107,7 @@ def __init__( _cli_parse_args=_cli_parse_args, _cli_hide_none_type=_cli_hide_none_type, _cli_avoid_json=_cli_avoid_json, + _cli_enforce_required=_cli_enforce_required, _secrets_dir=_secrets_dir, ) ) @@ -113,7 +117,6 @@ def settings_customise_sources( cls, settings_cls: type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -124,7 +127,6 @@ def settings_customise_sources( Args: settings_cls: The Settings class. init_settings: The `InitSettingsSource` instance. - cli_settings: The `CliSettingsSource` instance. env_settings: The `EnvSettingsSource` instance. dotenv_settings: The `DotEnvSettingsSource` instance. file_secret_settings: The `SecretsSettingsSource` instance. @@ -132,7 +134,7 @@ def settings_customise_sources( Returns: A tuple containing the sources and their order for loading the settings values. """ - return init_settings, cli_settings, env_settings, dotenv_settings, file_secret_settings + return init_settings, env_settings, dotenv_settings, file_secret_settings def _settings_build_values( self, @@ -148,6 +150,7 @@ def _settings_build_values( _cli_parse_args: bool | list[str] | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, + _cli_enforce_required: bool | None = None, _secrets_dir: str | Path | None = None, ) -> dict[str, Any]: # Determine settings config values @@ -175,6 +178,11 @@ def _settings_build_values( _cli_hide_none_type if _cli_hide_none_type is not None else self.model_config.get('cli_hide_none_type') ) cli_avoid_json = _cli_avoid_json if _cli_avoid_json is not None else self.model_config.get('cli_avoid_json') + cli_enforce_required = ( + _cli_enforce_required + if _cli_enforce_required is not None + else self.model_config.get('cli_enforce_required') + ) secrets_dir = _secrets_dir if _secrets_dir is not None else self.model_config.get('secrets_dir') @@ -187,6 +195,7 @@ def _settings_build_values( cli_parse_none_str=env_parse_none_str, cli_hide_none_type=cli_hide_none_type, cli_avoid_json=cli_avoid_json, + cli_enforce_required=cli_enforce_required, ) env_settings = EnvSettingsSource( self.__class__, @@ -214,11 +223,12 @@ def _settings_build_values( sources = self.settings_customise_sources( self.__class__, init_settings=init_settings, - cli_settings=cli_settings, env_settings=env_settings, dotenv_settings=dotenv_settings, file_secret_settings=file_secret_settings, ) + if _cli_parse_args: + sources = (cli_settings,) + sources if sources: return deep_update(*reversed([source() for source in sources])) else: @@ -241,6 +251,7 @@ def _settings_build_values( cli_parse_args=None, cli_hide_none_type=False, cli_avoid_json=False, + cli_enforce_required=False, secrets_dir=None, protected_namespaces=('model_', 'settings_'), ) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index d7450cc1..5adfb53a 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -7,7 +7,6 @@ from abc import ABC, abstractmethod from collections import deque from dataclasses import is_dataclass -from inspect import isclass from pathlib import Path from types import FunctionType from typing import TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Sequence, Tuple, TypeVar, Union, cast @@ -16,7 +15,7 @@ from pydantic import AliasChoices, AliasPath, BaseModel, Json, TypeAdapter from pydantic._internal._repr import Representation from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base -from pydantic._internal._utils import deep_update, lenient_issubclass +from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass from pydantic.fields import FieldInfo from typing_extensions import Annotated, TypeAliasType, get_args, get_origin @@ -515,7 +514,7 @@ def _field_is_complex(self, field: FieldInfo) -> tuple[bool, bool]: return True, allow_parse_failure @staticmethod - def next_field(field: FieldInfo | None, key: str) -> FieldInfo | None: + def next_field(field: FieldInfo | Any | None, key: str) -> FieldInfo | None: """ Find the field in a sub model by key(env name) @@ -544,11 +543,25 @@ class Cfg(BaseSettings): Returns: Field if it finds the next field otherwise `None`. """ - if not field or origin_is_union(get_origin(field.annotation)): - # no support for Unions of complex BaseSettings fields + if not field: return None - elif field.annotation and hasattr(field.annotation, 'model_fields') and field.annotation.model_fields.get(key): - return field.annotation.model_fields[key] + if isinstance(field, FieldInfo): + if not hasattr(field, 'annotation'): + return None + annotation = field.annotation + else: + annotation = field + + if origin_is_union(get_origin(annotation)) or isinstance(annotation, WithArgsTypes): + type_ = get_origin(annotation) + if is_model_class(type_) and type_.model_fields.get(key): + return type_.model_fields.get(key) + for type_ in get_args(annotation): + type_has_key = EnvSettingsSource.next_field(type_, key) + if type_has_key: + return type_has_key + elif is_model_class(annotation) and annotation.model_fields.get(key): + return annotation.model_fields[key] return None @@ -697,6 +710,7 @@ def __init__( cli_parse_none_str: str | None = None, cli_hide_none_type: bool | None = None, cli_avoid_json: bool | None = None, + cli_enforce_required: bool | None = None, ) -> None: self.cli_prog_name = sys.argv[0] if cli_prog_name is None else cli_prog_name self.cli_parse_args = cli_parse_args @@ -711,6 +725,9 @@ def __init__( self.cli_avoid_json = cli_avoid_json if cli_avoid_json is not None else self.config.get('cli_avoid_json', False) if cli_parse_none_str is None: cli_parse_none_str = 'None' if self.cli_avoid_json is True else 'null' + self.cli_enforce_required = ( + cli_enforce_required if cli_enforce_required is not None else self.config.get('cli_enforce_required', False) + ) super().__init__(settings_cls, env_nested_delimiter='.', env_parse_none_str=cli_parse_none_str) def _load_env_vars(self) -> Mapping[str, str | None]: @@ -729,25 +746,32 @@ def _load_env_vars(self) -> Mapping[str, str | None]: ) parsed_args: dict[str, list[str] | str] = vars(parser.parse_args(self.cli_parse_args)) # type: ignore - if any(key for key in parsed_args.keys() if not key.endswith(':subcommand')): - for field, val in parsed_args.items(): - if isinstance(val, list): - merge_list = [] - for sub_val in val: - if sub_val.startswith('[') and sub_val.endswith(']'): - sub_val = sub_val[1:-1] - merge_list.append(sub_val) - parsed_args[field] = ( - f'[{",".join(merge_list)}]' - if field not in self._cli_dict_arg_names - else self._merge_json_key_val_list_str(f'[{",".join(merge_list)}]') - ) - elif field.endswith(':subcommand'): - self._cli_subcommands[field].remove(field.split(':')[0] + val) + selected_subcommands: list[str] = [] + for field_name, val in parsed_args.items(): + if isinstance(val, list): + merge_list = [] + for sub_val in val: + if sub_val.startswith('[') and sub_val.endswith(']'): + sub_val = sub_val[1:-1] + merge_list.append(sub_val) + parsed_args[field_name] = ( + f'[{",".join(merge_list)}]' + if field_name not in self._cli_dict_arg_names + else self._merge_json_key_val_list_str(f'[{",".join(merge_list)}]') + ) + elif field_name.endswith(':subcommand'): + selected_subcommands.append(field_name.split(':')[0] + val) for subcommands in self._cli_subcommands.values(): for subcommand in subcommands: - parsed_args[subcommand] = self.env_parse_none_str # type: ignore + if subcommand not in selected_subcommands: + parsed_args[subcommand] = self.env_parse_none_str # type: ignore + + parsed_args = {key: val for key, val in parsed_args.items() if not key.endswith(':subcommand')} + if selected_subcommands: + last_selected_subcommand = max(selected_subcommands, key=len) + if not any(field_name for field_name in parsed_args.keys() if f'{last_selected_subcommand}.' in field_name): + parsed_args[last_selected_subcommand] = '{}' return parse_env_vars( parsed_args, self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str # type: ignore @@ -800,7 +824,7 @@ def _get_sub_models(self, model: type[BaseModel], field_name: str, field_info: F raise SettingsError( f'CliPositionalArg is not outermost annotation for {model.__name__}.{field_name}' ) - if isclass(type_) and issubclass(type_, BaseModel): + if is_model_class(type_): sub_models.append(type_) return sub_models @@ -814,7 +838,7 @@ def _sort_arg_fields(self, model: type[BaseModel]) -> list[tuple[str, FieldInfo] field_types = [type_ for type_ in get_args(field_info.annotation) if type_ is not type(None)] if len(field_types) != 1: raise SettingsError(f'subcommand argument {model.__name__}.{field_name} has multiple types') - elif not (isclass(field_types[0]) and issubclass(field_types[0], BaseModel)): + elif not is_model_class(field_types[0]): raise SettingsError( f'subcommand argument {model.__name__}.{field_name} is not derived from BaseModel' ) @@ -867,6 +891,7 @@ def _add_fields_to_parser( kwargs['help'] = field_info.description kwargs['dest'] = f'{arg_prefix}{field_name}' kwargs['metavar'] = self._format_metavar(field_info.annotation) + kwargs['required'] = self.cli_enforce_required and field_info.is_required() if kwargs['dest'] in added_args: continue if _annotation_contains_types(field_info.annotation, (list, set, dict, Sequence, Mapping)): @@ -879,6 +904,7 @@ def _add_fields_to_parser( kwargs['metavar'] = field_name.upper() arg_name = kwargs['dest'] del kwargs['dest'] + del kwargs['required'] arg_flag = '' if sub_models and kwargs.get('action') != 'append': diff --git a/tests/test_settings.py b/tests/test_settings.py index ffd4b8e2..022e6818 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -606,7 +606,6 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -636,7 +635,7 @@ class Settings(BaseSettings, env_nested_delimiter='__'): @classmethod def settings_customise_sources( - cls, settings_cls, init_settings, cli_settings, env_settings, dotenv_settings, file_secret_settings + cls, settings_cls, init_settings, env_settings, dotenv_settings, file_secret_settings ): return env_settings, dotenv_settings, init_settings, file_secret_settings @@ -675,7 +674,6 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1311,7 +1309,6 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1358,7 +1355,6 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1432,7 +1428,6 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1461,7 +1456,6 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1496,7 +1490,6 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1521,7 +1514,6 @@ def settings_customise_sources( cls, settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, @@ -1945,18 +1937,6 @@ class Cfg(BaseSettings): v0_union: Union[SubValue, int] top: TopValue - @classmethod - def settings_customise_sources( - cls, - settings_cls: Type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> Tuple[PydanticBaseSettingsSource, ...]: - return cli_settings, init_settings - args: list[str] = [] args += ['--top', '{"v1": "json-1", "v2": "json-2", "sub": {"v5": "xx"}}'] args += ['--top.sub.v5', '5'] @@ -1996,66 +1976,68 @@ class Cfg(BaseSettings): str_list: Optional[List[str]] = None child: Optional[Child] = None - @classmethod - def settings_customise_sources( - cls, - settings_cls: Type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> Tuple[PydanticBaseSettingsSource, ...]: - return cli_settings, init_settings + def check_answer(cfg, prefix, expected): + if prefix: + assert cfg.model_dump() == { + 'num_list': None, + 'obj_list': None, + 'union_list': None, + 'str_list': None, + 'child': expected, + } + else: + expected['child'] = None + assert cfg.model_dump() == expected args: list[str] = [] - args = ['--num_list', '[1,2]'] - args += ['--num_list', '3,4'] - args += ['--num_list', '5', '--num_list', '6'] - cfg = Cfg(_cli_parse_args=args) - assert cfg.model_dump() == { - 'num_list': [1, 2, 3, 4, 5, 6], - 'obj_list': None, - 'union_list': None, - 'str_list': None, - 'child': None, - } - - args = ['--obj_list', '[{"val":1},{"val":2}]'] - args += ['--obj_list', '{"val":3},{"val":4}'] - args += ['--obj_list', '{"val":5}', '--obj_list', '{"val":6}'] - cfg = Cfg(_cli_parse_args=args) - assert cfg.model_dump() == { - 'num_list': None, - 'obj_list': [{'val': 1}, {'val': 2}, {'val': 3}, {'val': 4}, {'val': 5}, {'val': 6}], - 'union_list': None, - 'str_list': None, - 'child': None, - } - - args = ['--union_list', '[{"val":1},2]', '--union_list', '[3,{"val":4}]'] - args += ['--union_list', '{"val":5},6', '--union_list', '7,{"val":8}'] - args += ['--union_list', '{"val":9}', '--union_list', '10'] - cfg = Cfg(_cli_parse_args=args) - assert cfg.model_dump() == { - 'num_list': None, - 'obj_list': None, - 'union_list': [{'val': 1}, 2, 3, {'val': 4}, {'val': 5}, 6, 7, {'val': 8}, {'val': 9}, 10], - 'str_list': None, - 'child': None, - } - - args = ['--str_list', '["0,0","1,1"]'] - args += ['--str_list', '"2,2","3,3"'] - args += ['--str_list', '"4,4"', '--str_list', '"5,5"'] - cfg = Cfg(_cli_parse_args=args) - assert cfg.model_dump() == { - 'num_list': None, - 'obj_list': None, - 'union_list': None, - 'str_list': ['0,0', '1,1', '2,2', '3,3', '4,4', '5,5'], - 'child': None, - } + for prefix in ('', 'child.'): + args = [f'--{prefix}num_list', '[1,2]'] + args += [f'--{prefix}num_list', '3,4'] + args += [f'--{prefix}num_list', '5', f'--{prefix}num_list', '6'] + cfg = Cfg(_cli_parse_args=args) + expected= { + 'num_list': [1, 2, 3, 4, 5, 6], + 'obj_list': None, + 'union_list': None, + 'str_list': None, + } + check_answer(cfg, prefix, expected) + + args = [f'--{prefix}obj_list', '[{"val":1},{"val":2}]'] + args += [f'--{prefix}obj_list', '{"val":3},{"val":4}'] + args += [f'--{prefix}obj_list', '{"val":5}', f'--{prefix}obj_list', '{"val":6}'] + cfg = Cfg(_cli_parse_args=args) + expected= { + 'num_list': None, + 'obj_list': [{'val': 1}, {'val': 2}, {'val': 3}, {'val': 4}, {'val': 5}, {'val': 6}], + 'union_list': None, + 'str_list': None, + } + check_answer(cfg, prefix, expected) + + args = [f'--{prefix}union_list', '[{"val":1},2]', f'--{prefix}union_list', '[3,{"val":4}]'] + args += [f'--{prefix}union_list', '{"val":5},6', f'--{prefix}union_list', '7,{"val":8}'] + args += [f'--{prefix}union_list', '{"val":9}', f'--{prefix}union_list', '10'] + cfg = Cfg(_cli_parse_args=args) + expected= { + 'num_list': None, + 'obj_list': None, + 'union_list': [{'val': 1}, 2, 3, {'val': 4}, {'val': 5}, 6, 7, {'val': 8}, {'val': 9}, 10], + 'str_list': None, + } + check_answer(cfg, prefix, expected) + + args = [f'--{prefix}str_list', '["0,0","1,1"]'] + args += [f'--{prefix}str_list', '"2,2","3,3"'] + args += [f'--{prefix}str_list', '"4,4"', f'--{prefix}str_list', '"5,5"'] + cfg = Cfg(_cli_parse_args=args) + expected= { + 'num_list': None, + 'obj_list': None, + 'union_list': None, + 'str_list': ['0,0', '1,1', '2,2', '3,3', '4,4', '5,5'], + } + check_answer(cfg, prefix, expected) def test_cli_dict_arg(): @@ -2066,70 +2048,63 @@ class Cfg(BaseSettings): check_dict: Optional[Dict[str, str]] = None child: Optional[Child] = None - @classmethod - def settings_customise_sources( - cls, - settings_cls: Type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> Tuple[PydanticBaseSettingsSource, ...]: - return cli_settings, init_settings - args: list[str] = [] - args = ['--check_dict', '{"k1":"a","k2":"b"}'] - args += ['--check_dict', '{"k3":"c"},{"k4":"d"}'] - args += ['--check_dict', '{"k5":"e"}', '--check_dict', '{"k6":"f"}'] - args += ['--check_dict', '[k7=g,k8=h]'] - args += ['--check_dict', 'k9=i,k10=j'] - args += ['--check_dict', 'k11=k', '--check_dict', 'k12=l'] - args += ['--check_dict', '[{"k13":"m"},k14=n]', '--check_dict', '[k15=o,{"k16":"p"}]'] - args += ['--check_dict', '{"k17":"q"},k18=r', '--check_dict', 'k19=s,{"k20":"t"}'] - args += ['--check_dict', '{"k21":"u"},k22=v,{"k23":"w"}'] - args += ['--check_dict', 'k24=x,{"k25":"y"},k26=z'] - args += ['--check_dict', '[k27="x,y",k28="x,y"]'] - args += ['--check_dict', 'k29="x,y",k30="x,y"'] - args += ['--check_dict', 'k31="x,y"', '--check_dict', 'k32="x,y"'] - cfg = Cfg(_cli_parse_args=args) - assert cfg.model_dump() == { - 'check_dict': { - 'k1': 'a', - 'k2': 'b', - 'k3': 'c', - 'k4': 'd', - 'k5': 'e', - 'k6': 'f', - 'k7': 'g', - 'k8': 'h', - 'k9': 'i', - 'k10': 'j', - 'k11': 'k', - 'k12': 'l', - 'k13': 'm', - 'k14': 'n', - 'k15': 'o', - 'k16': 'p', - 'k17': 'q', - 'k18': 'r', - 'k19': 's', - 'k20': 't', - 'k21': 'u', - 'k22': 'v', - 'k23': 'w', - 'k24': 'x', - 'k25': 'y', - 'k26': 'z', - 'k27': 'x,y', - 'k28': 'x,y', - 'k29': 'x,y', - 'k30': 'x,y', - 'k31': 'x,y', - 'k32': 'x,y', - }, - 'child': None, - } + for prefix in ('', 'child.'): + args = [f'--{prefix}check_dict', '{"k1":"a","k2":"b"}'] + args += [f'--{prefix}check_dict', '{"k3":"c"},{"k4":"d"}'] + args += [f'--{prefix}check_dict', '{"k5":"e"}', f'--{prefix}check_dict', '{"k6":"f"}'] + args += [f'--{prefix}check_dict', '[k7=g,k8=h]'] + args += [f'--{prefix}check_dict', 'k9=i,k10=j'] + args += [f'--{prefix}check_dict', 'k11=k', f'--{prefix}check_dict', 'k12=l'] + args += [f'--{prefix}check_dict', '[{"k13":"m"},k14=n]', f'--{prefix}check_dict', '[k15=o,{"k16":"p"}]'] + args += [f'--{prefix}check_dict', '{"k17":"q"},k18=r', f'--{prefix}check_dict', 'k19=s,{"k20":"t"}'] + args += [f'--{prefix}check_dict', '{"k21":"u"},k22=v,{"k23":"w"}'] + args += [f'--{prefix}check_dict', 'k24=x,{"k25":"y"},k26=z'] + args += [f'--{prefix}check_dict', '[k27="x,y",k28="x,y"]'] + args += [f'--{prefix}check_dict', 'k29="x,y",k30="x,y"'] + args += [f'--{prefix}check_dict', 'k31="x,y"', f'--{prefix}check_dict', 'k32="x,y"'] + cfg = Cfg(_cli_parse_args=args) + expected: dict[str, Any] = { + 'check_dict': { + 'k1': 'a', + 'k2': 'b', + 'k3': 'c', + 'k4': 'd', + 'k5': 'e', + 'k6': 'f', + 'k7': 'g', + 'k8': 'h', + 'k9': 'i', + 'k10': 'j', + 'k11': 'k', + 'k12': 'l', + 'k13': 'm', + 'k14': 'n', + 'k15': 'o', + 'k16': 'p', + 'k17': 'q', + 'k18': 'r', + 'k19': 's', + 'k20': 't', + 'k21': 'u', + 'k22': 'v', + 'k23': 'w', + 'k24': 'x', + 'k25': 'y', + 'k26': 'z', + 'k27': 'x,y', + 'k28': 'x,y', + 'k29': 'x,y', + 'k30': 'x,y', + 'k31': 'x,y', + 'k32': 'x,y', + } + } + if prefix: + expected = {'check_dict': None, 'child': expected} + else: + expected['child'] = None + assert cfg.model_dump() == expected def test_cli_subcommand_with_positionals(): @@ -2159,18 +2134,6 @@ class Git(BaseSettings): init: CliSubCommand[Init] plugins: CliSubCommand[Plugins] - @classmethod - def settings_customise_sources( - cls, - settings_cls: Type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> Tuple[PydanticBaseSettingsSource, ...]: - return cli_settings, init_settings - git = Git(_cli_parse_args=['init', '--quiet', 'true', 'dir/path']) assert git.model_dump() == { 'clone': None, @@ -2198,18 +2161,6 @@ class ChildB(BaseModel): class Cfg(BaseSettings): child: Union[ChildA, ChildB] - @classmethod - def settings_customise_sources( - cls, - settings_cls: Type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - cli_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> Tuple[PydanticBaseSettingsSource, ...]: - return cli_settings, init_settings - cfg = Cfg(_cli_parse_args=['--child.name', 'new name a', '--child.diff_a', 'new diff a']) assert cfg.model_dump() == {'child': {'name': 'new name a', 'diff_a': 'new diff a'}} From dd5bf6ed230b6c4fefe28d631b9f02b084b9441c Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sun, 28 Jan 2024 13:20:18 -0700 Subject: [PATCH 08/61] Docs with various fixes and updates. --- docs/index.md | 530 ++++++++++++++++++++++++++++++++++- pydantic_settings/main.py | 13 + pydantic_settings/sources.py | 39 ++- tests/test_settings.py | 8 +- 4 files changed, 572 insertions(+), 18 deletions(-) diff --git a/docs/index.md b/docs/index.md index a05a2bce..1a0bd183 100644 --- a/docs/index.md +++ b/docs/index.md @@ -459,6 +459,525 @@ class Settings(BaseSettings): model_config = SettingsConfigDict(env_file='.env', extra='ignore') ``` +## Command Line Support + +Pydantic settings provides integrated CLI support, making it easy to quickly define CLI applications using Pydantic +models. There are two primary use cases for Pydantic settings CLI: + +1. When using a CLI to override fields in Pydantic models. +2. When using Pydantic models to define CLIs. + +By default, the experience is tailored towards use case #1 and builds on the foundations established in [parsing +environment variables](#parsing-environment-variables). If your use case primarily falls into #2, you will likely want +to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli). + +### The Basics + +To get started, let's look at a basic example for defining a Pydantic settings CLI: + +```py +from pydantic import BaseModel + +from pydantic_settings import BaseSettings + + +class DeepSubModel(BaseModel, use_attribute_docstrings=True): + """DeepSubModel class documentation.""" + + v4: list[int] + """the deeply nested sub model v4 option""" + + +class SubModel(BaseModel, use_attribute_docstrings=True): + """SubModel class documentation.""" + + v1: int + """the sub model v1 option""" + + deep: DeepSubModel + """The help summary for DeepSubModel and related options. This will be placed at top of group.""" + + +class Settings(BaseSettings, use_attribute_docstrings=True): + """The Settings class documentation will show in top level help text.""" + + v0: str + """the top level v0 option""" + + sub_model: SubModel + """The help summary for SubModel related options. This will be placed at top of group.""" + + +print(Settings(_cli_prog_name='app', _cli_parse_args=['--help'])) # (1)! +""" +usage: app [-h] [--v0 str] [--sub_model JSON] [--sub_model.v1 int] [--sub_model.deep JSON] + [--sub_model.deep.v4 list[int]] + +The Settings class documentation will show in top level help text. # (2)! + +options: + -h, --help show this help message and exit + --v0 str the top level v0 option # (3)! + +sub_model options: # (4)! + The help summary for SubModel related options. This will be placed at top of group. + + --sub_model JSON set sub_model from JSON string + --sub_model.v1 int the sub model v1 option # (5)! + +sub_model.deep options: + The help summary for DeepSubModel and related options. This will be placed at top of + group. # (6)! + + --sub_model.deep JSON # (7)! + set sub_model.deep from JSON string + --sub_model.deep.v4 list[int] + the deeply nested sub model v4 option +""" +``` + +1. Does `_cli_prog_name` and `_cli_parse_args` look familiar? They retain the same meanings as in argparse. + +2. Help text for application main or subcommands is populated from class docstrings. + +3. Help text for fields is populated from field descriptions. + +4. Nested models (e.g. `SubModel`, `DeepSubModel`) and their associated fields will always be grouped together. + +5. Note that nested fields look and act just like their environment variable counterparts. The CLI uses `.` as its + nested delimiter. + +6. Group help text is populated from field descriptions by default, but can be configured to pull from class docstrings + as well. + +7. Just like when parsing environment variables, top level models allow for JSON strings and nested fields taking + precedence. + +To enable CLI parsing, we simply set the `cli_parse_args` flag to a valid value, which retains similar conotations as +defined in argparse. In the above example, we parsed our args from the `['--help']` list that was passed into +`_cli_parse_args`. Alternatively, we could have set `_cli_parse_args=True` to parse args from the command line (i.e., +`sys.argv[1:]`). + +Lastly, a CLI settings source is always [**the topmost source**](#field-value-priority), and does not support [changing +its priority](#changing-priority). + +#### Enable CLI Argument Parsing + +`cli_parse_args: Optional[list[str] | bool] = None` + +* Default = `None` +* If `True`, parse from `sys.argv[1:]` +* If `list[str]`, parse from `list[str]` +* If `False` or `None`, do not parse CLI arguments + +#### Lists + +CLI argument parsing of lists supports intermixing of any of the below three styles: + + * JSON style `--field='[1,2]'` + * Argparse style `--field 1 --field 2` + * Lazy style `--field=1,2` + +```python +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + my_list: list[int] + + +print(Settings(_cli_parse_args=['--my_list', '[1,2]']).model_dump()) +#> {'my_list': [1, 2]} + +print(Settings(_cli_parse_args=['--my_list', '1', '--my_list', '2']).model_dump()) +#> {'my_list': [1, 2]} + +print(Settings(_cli_parse_args=['--my_list', '1,2']).model_dump()) +#> {'my_list': [1, 2]} +``` + +#### Dictionaries + +CLI argument parsing of dictionaries supports intermixing of any of the below two styles: + + * JSON style `--field='{"k1": 1, "k2": 2}'` + * Environment variable style `--field k1=1 --field k2=2` + +These can be used in conjunction with list forms as well, e.g: + + * `--field k1=1,k2=2 --field k3=3 --field '{"k4: 4}'` etc. + +```python +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + my_dict: dict[str, int] + + +print(Settings(_cli_parse_args=['--my_dict', '{"k1":1,"k2":2}']).model_dump()) +#> {'my_dict': {'k1': 1, 'k2': 2}} + +print(Settings(_cli_parse_args=['--my_dict', 'k1=1', '--my_dict', 'k2=2']).model_dump()) +#> {'my_dict': {'k1': 1, 'k2': 2}} +``` + +### Subcommands and Positional Arguments + +Subcommands and positional arguments are expressed using the `CliSubCommand` and `CliPositionalArg` annotations. These +annotations can only be applied to required fields (i.e. fields that do not have a default value). Furthermore, +subcommands must be a valid type derived from the pydantic `BaseModel` class. + +!!! note + CLI settings subcommands are limited to a single subparser per model. In other words, all subcommands for a model + are grouped under a single subparser; it does not allow for multiple subparsers with each subparser having its own + set of subcommands. For more information on subparsers, see [argparse + subcommands](https://docs.python.org/3/library/argparse.html#sub-commands). + +```py +from pydantic import BaseModel + +from pydantic_settings import ( + BaseSettings, + CliPositionalArg, + CliSubCommand, +) + + +class FooPlugin(BaseModel, use_attribute_docstrings=True): + """git-plugins-foo - Extra deep foo plugin command""" + + my_feature: bool = False + """Enable my feature on foo plugin""" + + +class BarPlugin(BaseModel, use_attribute_docstrings=True): + """git-plugins-bar - Extra deep bar plugin command""" + + my_feature: bool = False + """Enable my feature on bar plugin""" + + +class Plugins(BaseModel, use_attribute_docstrings=True): + """git-plugins - Fake plugins for GIT""" + + foo: CliSubCommand[FooPlugin] + """Foo is fake plugin""" + + bar: CliSubCommand[BarPlugin] + """Bar is also a fake plugin""" + + +class Clone(BaseModel, use_attribute_docstrings=True): + """git-clone - Clone a repository into a new directory""" + + repository: CliPositionalArg[str] + """The repository to clone""" + + directory: CliPositionalArg[str] + """The directory to clone into""" + + local: bool = False + """When the resposity to clone from is on a local machine, bypass ...""" + + +class Git(BaseSettings, use_attribute_docstrings=True): + """git - The stupid content tracker""" + + clone: CliSubCommand[Clone] + """Clone a repository into a new directory""" + + plugins: CliSubCommand[Plugins] + """Fake GIT plugion commands""" + + +print(Git(_cli_prog_name='git', _cli_parse_args=['--help'])) +""" +usage: git [-h] {clone,plugins} ... + +git - The stupid content tracker + +options: + -h, --help show this help message and exit + +subcommands: + {clone,plugins} + clone Clone a repository into a new directory + plugins Fake GIT plugion commands +""" + + +print(Git(_cli_prog_name='git', _cli_parse_args=['clone', '--help'])) +""" +usage: git clone [-h] [--local bool] [--shared bool] REPOSITORY DIRECTORY + +git-clone - Clone a repository into a new directory + +positional arguments: + REPOSITORY The repository to clone + DIRECTORY The directory to clone into + +options: + -h, --help show this help message and exit + --shared bool Force the clone process from a reposity on a local filesystem ... +""" + + +print(Git(_cli_prog_name='git', _cli_parse_args=['plugins', 'bar', '--help'])) +""" +usage: git plugins bar [-h] [--my_feature bool] + +git-plugins-bar - Extra deep bar plugin command + +options: + -h, --help show this help message and exit + --my_feature bool Enable my feature on bar plugin +""" +``` + +### Customizing the CLI Experience + +The below flags can be used to customise the CLI experience to your needs. + +#### Enforce Required Arguments at CLI + +Pydantic settings is designed to pull values in from various sources when instantating a model. This means a field that +is required is not strictly required from any single source (e.g. the CLI). Instead, all that matters is that one of the +sources provides the required value. + +However, if your use case [aligns more with #2](#command-line-support), using Pydantic models to define CLIs, you will +likely want required fields to be _strictly required at the CLI_. We can enable this behavior by using the +`cli_enforce_required` flag as shown below. + +```py +import os + +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings, use_attribute_docstrings=True): + my_required_field: str + """a top level required field""" + + +os.environ['MY_REQUIRED_FIELD'] = 'hello from environment' + +print(Settings(_cli_parse_args=[], _cli_enforce_required=False).model_dump()) +""" +{'my_required_field': 'hello from environment'} +""" + +print(Settings(_cli_parse_args=[], _cli_enforce_required=True).model_dump()) +""" +usage: example.py [-h] --my_required_field str +example.py: error: the following arguments are required: --my_required_field +""" +``` + +`cli_enforce_required: Optional[bool] = None` + +* Default = `None` +* If `True`, strictly enforce required fields at the CLI +* If `False` or `None`, do not enforce required fields at the CLI + +#### Hide None Type Values + +Hide `None` values from the CLI help text. + +```py +from typing import Optional + +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + v0: Optional[str] + """the top level v0 option""" + + +print(Settings(_cli_parse_args=['--help'], _cli_hide_none_type=False)) +""" +usage: example.py [-h] [--v0 {str,null}] + +options: + -h, --help show this help message and exit + --v0 {str,null} the top level v0 option +""" + +print(Settings(_cli_parse_args=['--help'], _cli_hide_none_type=True)) +""" +usage: example.py [-h] [--v0 str] + +options: + -h, --help show this help message and exit + --v0 str the top level v0 option +""" +``` + +`cli_hide_none_type: Optional[bool] = None` + +* Default = `None` +* If `True`, hide `None` type values from CLI help text +* If `False` or `None`, show `None` type values in CLI help text + +#### Avoid Adding JSON CLI Options + +Avoid adding complex fields that result in JSON strings at the CLI. + +```py +from pydantic import BaseModel + +from pydantic_settings import BaseSettings + + +class SubModel(BaseModel, use_attribute_docstrings=True): + v1: int + """the sub model v1 option""" + + +class Settings(BaseSettings, use_attribute_docstrings=True): + sub_model: SubModel + """The help summary for SubModel related options""" + + +print(Settings(_cli_parse_args=['--help'], _cli_avoid_json=False)) +""" +usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + +options: + -h, --help show this help message and exit + +sub_model options: + The help summary for SubModel related options + + --sub_model JSON set sub_model from JSON string + --sub_model.v1 int the sub model v1 option +""" + +print(Settings(_cli_parse_args=['--help'], _cli_avoid_json=True)) +""" +usage: example.py [-h] [--sub_model.v1 int] + +options: + -h, --help show this help message and exit + +sub_model options: + The help summary for SubModel related options + + --sub_model.v1 int the sub model v1 option +""" +``` + +`cli_avoid_json: Optional[bool] = None` + +* Default = `None` +* If `True`, avoid adding complex JSON fields to CLI +* If `False` or `None`, add complex JSON fields to CLI + +#### Use Class Docstring for Group Help Text + +By default, when populating the group help text for nested models it will pull from the field descriptions. +Alternatively, we can also configure CLI settings to pull from the class docstring instead. + +!!! note + If the field is a union of nested models the group help text will always be pulled from the field description; + even if `cli_use_class_docs_for_groups` is set to `True`. + +```py +from pydantic import BaseModel + +from pydantic_settings import BaseSettings + + +class SubModel(BaseModel, use_attribute_docstrings=True): + """The help text from the class docstring""" + + v1: int + """the sub model v1 option""" + + +class Settings(BaseSettings, use_attribute_docstrings=True): + """My application help text.""" + + sub_model: SubModel + """The help text from the field description""" + + +print(Settings(_cli_parse_args=['--help'], _cli_use_class_docs_for_groups=False)) +""" +usage: counter_example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + +My application help text. + +options: + -h, --help show this help message and exit + +sub_model options: + The help text from the field description + + --sub_model JSON set sub_model from JSON string + --sub_model.v1 int the sub model v1 option +""" + + +print(Settings(_cli_parse_args=['--help'], _cli_use_class_docs_for_groups=True)) +""" +usage: counter_example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + +My application help text. + +options: + -h, --help show this help message and exit + +sub_model options: + The help text from the class docstring + + --sub_model JSON set sub_model from JSON string + --sub_model.v1 int the sub model v1 option +""" +``` + +`cli_use_class_docs_for_groups: Optional[bool] = None` + +* Default = `None` +* If `True`, use class docstrings for CLI group help text +* If `False` or `None`, use field description for CLI group help text + +#### Change the Displayed Program Name + +Change the default program name displayed in the help text usage. By default, it will derive the name of the currently +executing program from `sys.argv[0]`, just like argparse. + +```py +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + pass + + +print(Settings(_cli_parse_args=['--help'])) +""" +usage: example.py [-h] + +options: + -h, --help show this help message and exit +""" + +print(Settings(_cli_parse_args=['--help'], _cli_prog_name='appdantic?')) +""" +usage: appdantic? [-h] + +options: + -h, --help show this help message and exit +""" +``` + +`cli_prog_name: Optional[str] = None` + +* Default = `None` +* If `str`, use `str` as program name +* If `None`, use `sys.argv[0]` as program name ## Secrets @@ -536,11 +1055,12 @@ docker service create --name pydantic-with-secrets --secret my_secret_data pydan In the case where a value is specified for the same `Settings` field in multiple ways, the selected value is determined as follows (in descending order of priority): -1. Arguments passed to the `Settings` class initialiser. -2. Environment variables, e.g. `my_prefix_special_function` as described above. -3. Variables loaded from a dotenv (`.env`) file. -4. Variables loaded from the secrets directory. -5. The default field values for the `Settings` model. +1. If `cli_parse_args` is enabled, arguments passed in at the CLI. +2. Arguments passed to the `Settings` class initialiser. +3. Environment variables, e.g. `my_prefix_special_function` as described above. +4. Variables loaded from a dotenv (`.env`) file. +5. Variables loaded from the secrets directory. +6. The default field values for the `Settings` model. ## Customise settings sources diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 26d2f3c7..673ffdb1 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -33,6 +33,7 @@ class SettingsConfigDict(ConfigDict, total=False): cli_hide_none_type: bool cli_avoid_json: bool cli_enforce_required: bool + cli_use_class_docs_for_groups: bool secrets_dir: str | Path | None @@ -72,6 +73,8 @@ class BaseSettings(BaseModel): _cli_hide_none_type: Hide NoneType values in CLI help text. Defaults to `False`. _cli_avoid_json: Avoid complex JSON objects in CLI help text. Defaults to `False`. _cli_enforce_required: Enforce required fields at the CLI. Defaults to `False`. + _cli_use_class_docs_for_groups: Use class docstrings in CLI group help text instead of field descriptions. + Defaults to `False`. _secrets_dir: The secret files directory. Defaults to `None`. """ @@ -89,6 +92,7 @@ def __init__( _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, + _cli_use_class_docs_for_groups: bool | None = None, _secrets_dir: str | Path | None = None, **values: Any, ) -> None: @@ -108,6 +112,7 @@ def __init__( _cli_hide_none_type=_cli_hide_none_type, _cli_avoid_json=_cli_avoid_json, _cli_enforce_required=_cli_enforce_required, + _cli_use_class_docs_for_groups=_cli_use_class_docs_for_groups, _secrets_dir=_secrets_dir, ) ) @@ -151,6 +156,7 @@ def _settings_build_values( _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, + _cli_use_class_docs_for_groups: bool | None = None, _secrets_dir: str | Path | None = None, ) -> dict[str, Any]: # Determine settings config values @@ -183,6 +189,11 @@ def _settings_build_values( if _cli_enforce_required is not None else self.model_config.get('cli_enforce_required') ) + cli_use_class_docs_for_groups = ( + _cli_use_class_docs_for_groups + if _cli_use_class_docs_for_groups is not None + else self.model_config.get('cli_use_class_docs_for_groups') + ) secrets_dir = _secrets_dir if _secrets_dir is not None else self.model_config.get('secrets_dir') @@ -196,6 +207,7 @@ def _settings_build_values( cli_hide_none_type=cli_hide_none_type, cli_avoid_json=cli_avoid_json, cli_enforce_required=cli_enforce_required, + cli_use_class_docs_for_groups=cli_use_class_docs_for_groups, ) env_settings = EnvSettingsSource( self.__class__, @@ -252,6 +264,7 @@ def _settings_build_values( cli_hide_none_type=False, cli_avoid_json=False, cli_enforce_required=False, + cli_use_class_docs_for_groups=False, secrets_dir=None, protected_namespaces=('model_', 'settings_'), ) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 68d454ee..514b5bca 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -43,7 +43,7 @@ class _CliPositionalArg: T = TypeVar('T') -CliSubCommand = Annotated[T | None, _CliSubCommand] +CliSubCommand = Annotated[Union[T, None], _CliSubCommand] CliPositionalArg = Annotated[T, _CliPositionalArg] @@ -707,6 +707,7 @@ def __init__( cli_hide_none_type: bool | None = None, cli_avoid_json: bool | None = None, cli_enforce_required: bool | None = None, + cli_use_class_docs_for_groups: bool | None = None, ) -> None: self.cli_prog_name = sys.argv[0] if cli_prog_name is None else cli_prog_name self.cli_parse_args = cli_parse_args @@ -724,6 +725,11 @@ def __init__( self.cli_enforce_required = ( cli_enforce_required if cli_enforce_required is not None else self.config.get('cli_enforce_required', False) ) + self.cli_use_class_docs_for_groups = ( + cli_use_class_docs_for_groups + if cli_use_class_docs_for_groups is not None + else self.config.get('cli_use_class_docs_for_groups', False) + ) super().__init__(settings_cls, env_nested_delimiter='.', env_parse_none_str=cli_parse_none_str) def _load_env_vars(self) -> Mapping[str, str | None]: @@ -755,7 +761,7 @@ def _load_env_vars(self) -> Mapping[str, str | None]: if field_name not in self._cli_dict_arg_names else self._merge_json_key_val_list_str(f'[{",".join(merge_list)}]') ) - elif field_name.endswith(':subcommand'): + elif field_name.endswith(':subcommand') and val is not None: selected_subcommands.append(field_name.split(':')[0] + val) for subcommands in self._cli_subcommands.values(): @@ -861,10 +867,14 @@ def _add_fields_to_parser( sub_models: list[type[BaseModel]] = self._get_sub_models(model, field_name, field_info) if _CliSubCommand in field_info.metadata: if subparsers is None: - subparsers = parser.add_subparsers(title='subcommands', dest=f'{arg_prefix}:subcommand') + subparsers = parser.add_subparsers( + title='subcommands', dest=f'{arg_prefix}:subcommand', required=self.cli_enforce_required + ) self._cli_subcommands[f'{arg_prefix}:subcommand'] = [f'{arg_prefix}{field_name}'] else: self._cli_subcommands[f'{arg_prefix}:subcommand'].append(f'{arg_prefix}{field_name}') + metavar = ','.join(self._cli_subcommands[f'{arg_prefix}:subcommand']) + subparsers.metavar = f'{{{metavar}}}' model = sub_models[0] self._add_fields_to_parser( @@ -886,7 +896,7 @@ def _add_fields_to_parser( kwargs['default'] = SUPPRESS kwargs['help'] = field_info.description kwargs['dest'] = f'{arg_prefix}{field_name}' - kwargs['metavar'] = self._format_metavar(field_info.annotation) + kwargs['metavar'] = self._metavar_format(field_info.annotation) kwargs['required'] = self.cli_enforce_required and field_info.is_required() if kwargs['dest'] in added_args: continue @@ -904,7 +914,12 @@ def _add_fields_to_parser( arg_flag = '' if sub_models and kwargs.get('action') != 'append': - model_group = parser.add_argument_group(f'{arg_name} options', field_info.description) + group_help_text = ( + sub_models[0].__doc__ + if self.cli_use_class_docs_for_groups and len(sub_models) == 1 + else field_info.description + ) + model_group = parser.add_argument_group(f'{arg_name} options', group_help_text) if not self.cli_avoid_json: added_args.append(arg_name) kwargs['help'] = f'set {arg_name} from JSON string' @@ -932,7 +947,11 @@ def _get_modified_args(self, obj: Any) -> tuple[str, ...]: else: return tuple([type_ for type_ in get_args(obj) if type_ is not type(None)]) - def _format_metavar(self, obj: Any) -> str: + def _metavar_format_list(self, args: list[str]) -> str: + args = args if 'JSON' not in args else [arg for arg in args[args.index('JSON') + 1 :] if arg != 'JSON'] + return ','.join(args) + + def _metavar_format(self, obj: Any) -> str: """Pretty metavar representation of a type. Adapts logic from `pydantic._repr.display_as_type`.""" if isinstance(obj, FunctionType): return obj.__name__ @@ -947,20 +966,22 @@ def _format_metavar(self, obj: Any) -> str: obj = obj.__class__ if origin_is_union(get_origin(obj)): - args = ','.join(map(self._format_metavar, self._get_modified_args(obj))) + args = self._metavar_format_list(list(map(self._metavar_format, self._get_modified_args(obj)))) return f'{{{args}}}' if ',' in args else args elif isinstance(obj, WithArgsTypes): if get_origin(obj) == Literal: - args = ','.join(map(repr, self._get_modified_args(obj))) + args = self._metavar_format_list(list(map(repr, self._get_modified_args(obj)))) return f'{{{args}}}' if ',' in args else args else: - args = ','.join(map(self._format_metavar, self._get_modified_args(obj))) + args = self._metavar_format_list(list(map(self._metavar_format, self._get_modified_args(obj)))) try: return f'{obj.__qualname__}[{args}]' except AttributeError: return str(obj) # handles TypeAliasType in 3.12 elif obj is type(None): return self.env_parse_none_str + elif is_model_class(obj): + return 'JSON' elif isinstance(obj, type): return obj.__qualname__ else: diff --git a/tests/test_settings.py b/tests/test_settings.py index 974b0974..7f9c5795 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2073,7 +2073,7 @@ def check_answer(cfg, prefix, expected): args += [f'--{prefix}num_list', '3,4'] args += [f'--{prefix}num_list', '5', f'--{prefix}num_list', '6'] cfg = Cfg(_cli_parse_args=args) - expected= { + expected = { 'num_list': [1, 2, 3, 4, 5, 6], 'obj_list': None, 'union_list': None, @@ -2085,7 +2085,7 @@ def check_answer(cfg, prefix, expected): args += [f'--{prefix}obj_list', '{"val":3},{"val":4}'] args += [f'--{prefix}obj_list', '{"val":5}', f'--{prefix}obj_list', '{"val":6}'] cfg = Cfg(_cli_parse_args=args) - expected= { + expected = { 'num_list': None, 'obj_list': [{'val': 1}, {'val': 2}, {'val': 3}, {'val': 4}, {'val': 5}, {'val': 6}], 'union_list': None, @@ -2097,7 +2097,7 @@ def check_answer(cfg, prefix, expected): args += [f'--{prefix}union_list', '{"val":5},6', f'--{prefix}union_list', '7,{"val":8}'] args += [f'--{prefix}union_list', '{"val":9}', f'--{prefix}union_list', '10'] cfg = Cfg(_cli_parse_args=args) - expected= { + expected = { 'num_list': None, 'obj_list': None, 'union_list': [{'val': 1}, 2, 3, {'val': 4}, {'val': 5}, 6, 7, {'val': 8}, {'val': 9}, 10], @@ -2109,7 +2109,7 @@ def check_answer(cfg, prefix, expected): args += [f'--{prefix}str_list', '"2,2","3,3"'] args += [f'--{prefix}str_list', '"4,4"', f'--{prefix}str_list', '"5,5"'] cfg = Cfg(_cli_parse_args=args) - expected= { + expected = { 'num_list': None, 'obj_list': None, 'union_list': None, From abc109522487a2a3b93e72ae0e2539f9a7d13319 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sun, 28 Jan 2024 13:41:36 -0700 Subject: [PATCH 09/61] Use Union. --- docs/index.md | 2 +- tests/test_settings.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index 1a0bd183..c01199de 100644 --- a/docs/index.md +++ b/docs/index.md @@ -563,7 +563,7 @@ its priority](#changing-priority). #### Enable CLI Argument Parsing -`cli_parse_args: Optional[list[str] | bool] = None` +`cli_parse_args: Optional[Union[list[str], bool]] = None` * Default = `None` * If `True`, parse from `sys.argv[1:]` diff --git a/tests/test_settings.py b/tests/test_settings.py index 7f9c5795..916cd38e 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2045,12 +2045,12 @@ class Child(BaseModel): num_list: Optional[List[int]] = None obj_list: Optional[List[Obj]] = None str_list: Optional[List[str]] = None - union_list: Optional[List[Obj | int]] = None + union_list: Optional[List[Union[Obj, int]]] = None class Cfg(BaseSettings): num_list: Optional[List[int]] = None obj_list: Optional[List[Obj]] = None - union_list: Optional[List[Obj | int]] = None + union_list: Optional[List[Union[Obj, int]]] = None str_list: Optional[List[str]] = None child: Optional[Child] = None @@ -2267,7 +2267,7 @@ class SubCommandHasDefault(BaseSettings): with pytest.raises(SettingsError): class SubCommandMultipleTypes(BaseSettings): - subcmd: CliSubCommand[SubCmd | SubCmdAlt] + subcmd: CliSubCommand[Union[SubCmd, SubCmdAlt]] SubCommandMultipleTypes(_cli_parse_args=['--help']) From fcc4d2d6f89cfeda729323ffd5f3ce29048f95f9 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 29 Jan 2024 11:48:22 -0700 Subject: [PATCH 10/61] Test and doc updates. --- docs/index.md | 44 ++++++------- tests/test_settings.py | 136 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 154 insertions(+), 26 deletions(-) diff --git a/docs/index.md b/docs/index.md index c01199de..f9243c2c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -475,7 +475,7 @@ to enable [enforcing required arguments at the CLI](#enforce-required-arguments- To get started, let's look at a basic example for defining a Pydantic settings CLI: -```py +```py test="skip" from pydantic import BaseModel from pydantic_settings import BaseSettings @@ -508,7 +508,7 @@ class Settings(BaseSettings, use_attribute_docstrings=True): """The help summary for SubModel related options. This will be placed at top of group.""" -print(Settings(_cli_prog_name='app', _cli_parse_args=['--help'])) # (1)! +Settings(_cli_prog_name='app', _cli_parse_args=['--help']) # (1)! """ usage: app [-h] [--v0 str] [--sub_model JSON] [--sub_model.v1 int] [--sub_model.deep JSON] [--sub_model.deep.v4 list[int]] @@ -634,7 +634,7 @@ subcommands must be a valid type derived from the pydantic `BaseModel` class. set of subcommands. For more information on subparsers, see [argparse subcommands](https://docs.python.org/3/library/argparse.html#sub-commands). -```py +```py test="skip" from pydantic import BaseModel from pydantic_settings import ( @@ -691,7 +691,7 @@ class Git(BaseSettings, use_attribute_docstrings=True): """Fake GIT plugion commands""" -print(Git(_cli_prog_name='git', _cli_parse_args=['--help'])) +Git(_cli_prog_name='git', _cli_parse_args=['--help']) """ usage: git [-h] {clone,plugins} ... @@ -707,7 +707,7 @@ subcommands: """ -print(Git(_cli_prog_name='git', _cli_parse_args=['clone', '--help'])) +Git(_cli_prog_name='git', _cli_parse_args=['clone', '--help']) """ usage: git clone [-h] [--local bool] [--shared bool] REPOSITORY DIRECTORY @@ -719,11 +719,11 @@ positional arguments: options: -h, --help show this help message and exit - --shared bool Force the clone process from a reposity on a local filesystem ... + --local bool When the resposity to clone from is on a local machine, bypass ... """ -print(Git(_cli_prog_name='git', _cli_parse_args=['plugins', 'bar', '--help'])) +Git(_cli_prog_name='git', _cli_parse_args=['plugins', 'bar', '--help']) """ usage: git plugins bar [-h] [--my_feature bool] @@ -749,7 +749,7 @@ However, if your use case [aligns more with #2](#command-line-support), using Py likely want required fields to be _strictly required at the CLI_. We can enable this behavior by using the `cli_enforce_required` flag as shown below. -```py +```py test="skip" import os from pydantic_settings import BaseSettings @@ -784,7 +784,7 @@ example.py: error: the following arguments are required: --my_required_field Hide `None` values from the CLI help text. -```py +```py test="skip" from typing import Optional from pydantic_settings import BaseSettings @@ -795,7 +795,7 @@ class Settings(BaseSettings): """the top level v0 option""" -print(Settings(_cli_parse_args=['--help'], _cli_hide_none_type=False)) +Settings(_cli_parse_args=['--help'], _cli_hide_none_type=False) """ usage: example.py [-h] [--v0 {str,null}] @@ -804,7 +804,7 @@ options: --v0 {str,null} the top level v0 option """ -print(Settings(_cli_parse_args=['--help'], _cli_hide_none_type=True)) +Settings(_cli_parse_args=['--help'], _cli_hide_none_type=True) """ usage: example.py [-h] [--v0 str] @@ -824,7 +824,7 @@ options: Avoid adding complex fields that result in JSON strings at the CLI. -```py +```py test="skip" from pydantic import BaseModel from pydantic_settings import BaseSettings @@ -840,7 +840,7 @@ class Settings(BaseSettings, use_attribute_docstrings=True): """The help summary for SubModel related options""" -print(Settings(_cli_parse_args=['--help'], _cli_avoid_json=False)) +Settings(_cli_parse_args=['--help'], _cli_avoid_json=False) """ usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] @@ -854,7 +854,7 @@ sub_model options: --sub_model.v1 int the sub model v1 option """ -print(Settings(_cli_parse_args=['--help'], _cli_avoid_json=True)) +Settings(_cli_parse_args=['--help'], _cli_avoid_json=True) """ usage: example.py [-h] [--sub_model.v1 int] @@ -883,7 +883,7 @@ Alternatively, we can also configure CLI settings to pull from the class docstri If the field is a union of nested models the group help text will always be pulled from the field description; even if `cli_use_class_docs_for_groups` is set to `True`. -```py +```py test="skip" from pydantic import BaseModel from pydantic_settings import BaseSettings @@ -903,9 +903,9 @@ class Settings(BaseSettings, use_attribute_docstrings=True): """The help text from the field description""" -print(Settings(_cli_parse_args=['--help'], _cli_use_class_docs_for_groups=False)) +Settings(_cli_parse_args=['--help'], _cli_use_class_docs_for_groups=False) """ -usage: counter_example.py [-h] [--sub_model JSON] [--sub_model.v1 int] +usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] My application help text. @@ -920,9 +920,9 @@ sub_model options: """ -print(Settings(_cli_parse_args=['--help'], _cli_use_class_docs_for_groups=True)) +Settings(_cli_parse_args=['--help'], _cli_use_class_docs_for_groups=True) """ -usage: counter_example.py [-h] [--sub_model JSON] [--sub_model.v1 int] +usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] My application help text. @@ -948,7 +948,7 @@ sub_model options: Change the default program name displayed in the help text usage. By default, it will derive the name of the currently executing program from `sys.argv[0]`, just like argparse. -```py +```py test="skip" from pydantic_settings import BaseSettings @@ -956,7 +956,7 @@ class Settings(BaseSettings): pass -print(Settings(_cli_parse_args=['--help'])) +Settings(_cli_parse_args=['--help']) """ usage: example.py [-h] @@ -964,7 +964,7 @@ options: -h, --help show this help message and exit """ -print(Settings(_cli_parse_args=['--help'], _cli_prog_name='appdantic?')) +Settings(_cli_parse_args=['--help'], _cli_prog_name='appdantic?') """ usage: appdantic? [-h] diff --git a/tests/test_settings.py b/tests/test_settings.py index 916cd38e..a9a7b2ab 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2293,9 +2293,137 @@ class PositionalArgHasDefault(BaseSettings): PositionalArgHasDefault(_cli_parse_args=['--help']) -def test_cli_avoid_json(): - pass +def test_cli_avoid_json(capsys): + class SubModel(BaseModel): + v1: int + + class Settings(BaseSettings): + sub_model: SubModel + + with pytest.raises(SystemExit): + Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_avoid_json=False) + + assert ( + capsys.readouterr().out + == """usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + +options: + -h, --help show this help message and exit + +sub_model options: + --sub_model JSON set sub_model from JSON string + --sub_model.v1 int +""" + ) + + with pytest.raises(SystemExit): + Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_avoid_json=True) + + assert ( + capsys.readouterr().out + == """usage: example.py [-h] [--sub_model.v1 int] + +options: + -h, --help show this help message and exit + +sub_model options: + --sub_model.v1 int +""" + ) + + +def test_cli_hide_none_type(capsys): + class Settings(BaseSettings): + v0: Optional[str] + + with pytest.raises(SystemExit): + Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_hide_none_type=False) + + assert ( + capsys.readouterr().out + == """usage: example.py [-h] [--v0 {str,null}] + +options: + -h, --help show this help message and exit + --v0 {str,null} +""" + ) + + with pytest.raises(SystemExit): + Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_hide_none_type=True) + + assert ( + capsys.readouterr().out + == """usage: example.py [-h] [--v0 str] + +options: + -h, --help show this help message and exit + --v0 str +""" + ) + + +def test_cli_use_class_docs_for_groups(capsys): + class SubModel(BaseModel): + """The help text from the class docstring""" + + v1: int + + class Settings(BaseSettings): + """My application help text.""" + sub_model: SubModel = Field(description='The help text from the field description') + + with pytest.raises(SystemExit): + Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_use_class_docs_for_groups=False) + + assert ( + capsys.readouterr().out + == """usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + +My application help text. + +options: + -h, --help show this help message and exit + +sub_model options: + The help text from the field description + + --sub_model JSON set sub_model from JSON string + --sub_model.v1 int +""" + ) + + with pytest.raises(SystemExit): + Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_use_class_docs_for_groups=True) + + assert ( + capsys.readouterr().out + == """usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + +My application help text. + +options: + -h, --help show this help message and exit + +sub_model options: + The help text from the class docstring + + --sub_model JSON set sub_model from JSON string + --sub_model.v1 int +""" + ) + + +def test_cli_enforce_required(env): + class Settings(BaseSettings, use_attribute_docstrings=True): + my_required_field: str + + env.set('MY_REQUIRED_FIELD', 'hello from environment') + + assert Settings(_cli_parse_args=[], _cli_enforce_required=False).model_dump() == { + 'my_required_field': 'hello from environment' + } -def test_cli_hide_none_type(): - pass + with pytest.raises(SystemExit): + Settings(_cli_parse_args=[], _cli_enforce_required=True).model_dump() From 783d1c99f81f174ca96d19a03d49f463e55d85f2 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 29 Jan 2024 11:59:55 -0700 Subject: [PATCH 11/61] Python 3.8 and 3.9 argparse help text fixes. --- tests/test_settings.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index a9a7b2ab..202ba064 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2300,14 +2300,16 @@ class SubModel(BaseModel): class Settings(BaseSettings): sub_model: SubModel + argparse_options_text = 'options' if sys.version_info > (3, 9) else 'optional arguments' + with pytest.raises(SystemExit): Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_avoid_json=False) assert ( capsys.readouterr().out - == """usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + == f"""usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] -options: +{argparse_options_text}: -h, --help show this help message and exit sub_model options: @@ -2321,9 +2323,9 @@ class Settings(BaseSettings): assert ( capsys.readouterr().out - == """usage: example.py [-h] [--sub_model.v1 int] + == f"""usage: example.py [-h] [--sub_model.v1 int] -options: +{argparse_options_text}: -h, --help show this help message and exit sub_model options: @@ -2336,16 +2338,18 @@ def test_cli_hide_none_type(capsys): class Settings(BaseSettings): v0: Optional[str] + argparse_options_text = 'options' if sys.version_info > (3, 9) else 'optional arguments' + with pytest.raises(SystemExit): Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_hide_none_type=False) assert ( capsys.readouterr().out - == """usage: example.py [-h] [--v0 {str,null}] + == f"""usage: example.py [-h] [--v0 {{str,null}}] -options: +{argparse_options_text}: -h, --help show this help message and exit - --v0 {str,null} + --v0 {{str,null}} """ ) @@ -2354,9 +2358,9 @@ class Settings(BaseSettings): assert ( capsys.readouterr().out - == """usage: example.py [-h] [--v0 str] + == f"""usage: example.py [-h] [--v0 str] -options: +{argparse_options_text}: -h, --help show this help message and exit --v0 str """ @@ -2374,16 +2378,18 @@ class Settings(BaseSettings): sub_model: SubModel = Field(description='The help text from the field description') + argparse_options_text = 'options' if sys.version_info > (3, 9) else 'optional arguments' + with pytest.raises(SystemExit): Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_use_class_docs_for_groups=False) assert ( capsys.readouterr().out - == """usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + == f"""usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] My application help text. -options: +{argparse_options_text}: -h, --help show this help message and exit sub_model options: @@ -2399,11 +2405,11 @@ class Settings(BaseSettings): assert ( capsys.readouterr().out - == """usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + == f"""usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] My application help text. -options: +{argparse_options_text}: -h, --help show this help message and exit sub_model options: @@ -2416,7 +2422,7 @@ class Settings(BaseSettings): def test_cli_enforce_required(env): - class Settings(BaseSettings, use_attribute_docstrings=True): + class Settings(BaseSettings): my_required_field: str env.set('MY_REQUIRED_FIELD', 'hello from environment') From ff5b7edb2c7bc3cc8ad920e5250d946df4f631eb Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 29 Jan 2024 12:16:50 -0700 Subject: [PATCH 12/61] More Python 3.8 and 3.9 test fixes. --- docs/index.md | 24 +++++++++++++++--------- tests/test_settings.py | 13 ++++++++++--- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/docs/index.md b/docs/index.md index f9243c2c..b8c809bf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -476,6 +476,8 @@ to enable [enforcing required arguments at the CLI](#enforce-required-arguments- To get started, let's look at a basic example for defining a Pydantic settings CLI: ```py test="skip" +from typing import List + from pydantic import BaseModel from pydantic_settings import BaseSettings @@ -484,7 +486,7 @@ from pydantic_settings import BaseSettings class DeepSubModel(BaseModel, use_attribute_docstrings=True): """DeepSubModel class documentation.""" - v4: list[int] + v4: List[int] """the deeply nested sub model v4 option""" @@ -511,7 +513,7 @@ class Settings(BaseSettings, use_attribute_docstrings=True): Settings(_cli_prog_name='app', _cli_parse_args=['--help']) # (1)! """ usage: app [-h] [--v0 str] [--sub_model JSON] [--sub_model.v1 int] [--sub_model.deep JSON] - [--sub_model.deep.v4 list[int]] + [--sub_model.deep.v4 List[int]] The Settings class documentation will show in top level help text. # (2)! @@ -531,7 +533,7 @@ sub_model.deep options: --sub_model.deep JSON # (7)! set sub_model.deep from JSON string - --sub_model.deep.v4 list[int] + --sub_model.deep.v4 List[int] the deeply nested sub model v4 option """ ``` @@ -563,11 +565,11 @@ its priority](#changing-priority). #### Enable CLI Argument Parsing -`cli_parse_args: Optional[Union[list[str], bool]] = None` +`cli_parse_args: Optional[Union[List[str], bool]] = None` * Default = `None` * If `True`, parse from `sys.argv[1:]` -* If `list[str]`, parse from `list[str]` +* If `List[str]`, parse from `List[str]` * If `False` or `None`, do not parse CLI arguments #### Lists @@ -578,12 +580,14 @@ CLI argument parsing of lists supports intermixing of any of the below three sty * Argparse style `--field 1 --field 2` * Lazy style `--field=1,2` -```python +```py +from typing import List + from pydantic_settings import BaseSettings class Settings(BaseSettings): - my_list: list[int] + my_list: List[int] print(Settings(_cli_parse_args=['--my_list', '[1,2]']).model_dump()) @@ -607,12 +611,14 @@ These can be used in conjunction with list forms as well, e.g: * `--field k1=1,k2=2 --field k3=3 --field '{"k4: 4}'` etc. -```python +```py +from typing import Dict + from pydantic_settings import BaseSettings class Settings(BaseSettings): - my_dict: dict[str, int] + my_dict: Dict[str, int] print(Settings(_cli_parse_args=['--my_dict', '{"k1":1,"k2":2}']).model_dump()) diff --git a/tests/test_settings.py b/tests/test_settings.py index 202ba064..6ae8d366 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2292,6 +2292,13 @@ class PositionalArgHasDefault(BaseSettings): PositionalArgHasDefault(_cli_parse_args=['--help']) + with pytest.raises(SettingsError): + + class InvalidCliParseArgsType(BaseSettings): + val: int + + PositionalArgHasDefault(_cli_parse_args='invalid type') + def test_cli_avoid_json(capsys): class SubModel(BaseModel): @@ -2300,7 +2307,7 @@ class SubModel(BaseModel): class Settings(BaseSettings): sub_model: SubModel - argparse_options_text = 'options' if sys.version_info > (3, 9) else 'optional arguments' + argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' with pytest.raises(SystemExit): Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_avoid_json=False) @@ -2338,7 +2345,7 @@ def test_cli_hide_none_type(capsys): class Settings(BaseSettings): v0: Optional[str] - argparse_options_text = 'options' if sys.version_info > (3, 9) else 'optional arguments' + argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' with pytest.raises(SystemExit): Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_hide_none_type=False) @@ -2378,7 +2385,7 @@ class Settings(BaseSettings): sub_model: SubModel = Field(description='The help text from the field description') - argparse_options_text = 'options' if sys.version_info > (3, 9) else 'optional arguments' + argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' with pytest.raises(SystemExit): Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_use_class_docs_for_groups=False) From 7ea4c97ee2065acd45dac8441bec81eee9caac7c Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 29 Jan 2024 15:23:33 -0700 Subject: [PATCH 13/61] More Python 3.8 and 3.9 fixes. --- pydantic_settings/sources.py | 9 ++++----- tests/test_settings.py | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 514b5bca..d41e4226 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -1047,10 +1047,9 @@ def _union_is_complex(annotation: type[Any] | None, metadata: list[Any]) -> bool def _annotation_contains_types(annotation: type[Any] | None, types: tuple[Any, ...]) -> bool: - if origin_is_union(get_origin(annotation)) or isinstance(annotation, WithArgsTypes): - if get_origin(annotation) in types: + if get_origin(annotation) in types: + return True + for type_ in get_args(annotation): + if _annotation_contains_types(type_, types): return True - for type_ in get_args(annotation): - if _annotation_contains_types(type_, types): - return True return annotation in types diff --git a/tests/test_settings.py b/tests/test_settings.py index 6ae8d366..62ca66e5 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2015,7 +2015,7 @@ class Cfg(BaseSettings): v0_union: Union[SubValue, int] top: TopValue - args: list[str] = [] + args: List[str] = [] args += ['--top', '{"v1": "json-1", "v2": "json-2", "sub": {"v5": "xx"}}'] args += ['--top.sub.v5', '5'] args += ['--v0', '0'] @@ -2067,7 +2067,7 @@ def check_answer(cfg, prefix, expected): expected['child'] = None assert cfg.model_dump() == expected - args: list[str] = [] + args: List[str] = [] for prefix in ('', 'child.'): args = [f'--{prefix}num_list', '[1,2]'] args += [f'--{prefix}num_list', '3,4'] @@ -2126,7 +2126,7 @@ class Cfg(BaseSettings): check_dict: Optional[Dict[str, str]] = None child: Optional[Child] = None - args: list[str] = [] + args: List[str] = [] for prefix in ('', 'child.'): args = [f'--{prefix}check_dict', '{"k1":"a","k2":"b"}'] args += [f'--{prefix}check_dict', '{"k3":"c"},{"k4":"d"}'] @@ -2142,7 +2142,7 @@ class Cfg(BaseSettings): args += [f'--{prefix}check_dict', 'k29="x,y",k30="x,y"'] args += [f'--{prefix}check_dict', 'k31="x,y"', f'--{prefix}check_dict', 'k32="x,y"'] cfg = Cfg(_cli_parse_args=args) - expected: dict[str, Any] = { + expected: Dict[str, Any] = { 'check_dict': { 'k1': 'a', 'k2': 'b', From d55d69941e389510d685a01933e298a0a139b6a8 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 29 Jan 2024 15:28:48 -0700 Subject: [PATCH 14/61] Python 3.8 dict union fix. --- pydantic_settings/sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index d41e4226..e8680eb6 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -792,7 +792,7 @@ def _merge_json_key_val_list_str(self, key_val_list_str: str) -> str: elif key_val_list_str[i] == '}': obj_count -= 1 if obj_count == 0: - key_val_dict |= json.loads(key_val_list_str[: i + 1]) + key_val_dict.update(json.loads(key_val_list_str[: i + 1])) key_val_list_str = key_val_list_str[i + 1 :].lstrip(',') break elif obj_count == 0: @@ -806,7 +806,7 @@ def _merge_json_key_val_list_str(self, key_val_list_str: str) -> str: break if not val: val, key_val_list_str = key_val_list_str, '' - key_val_dict |= {key.strip('\'"'): val.strip('\'"')} + key_val_dict.update({key.strip('\'"'): val.strip('\'"')}) break return json.dumps(key_val_dict) From cb3b2508489a71e45744016989c8cda814cd5d1d Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 29 Jan 2024 15:36:03 -0700 Subject: [PATCH 15/61] Mypy lint fix? --- pydantic_settings/sources.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index e8680eb6..9a8169c7 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -11,13 +11,14 @@ from types import FunctionType from typing import TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Sequence, Tuple, TypeVar, Union, cast +import typing_extensions from dotenv import dotenv_values from pydantic import AliasChoices, AliasPath, BaseModel, Json, TypeAdapter from pydantic._internal._repr import Representation from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass from pydantic.fields import FieldInfo -from typing_extensions import Annotated, TypeAliasType, get_args, get_origin +from typing_extensions import Annotated, get_args, get_origin from pydantic_settings.utils import path_type_label @@ -959,7 +960,7 @@ def _metavar_format(self, obj: Any) -> str: return '...' elif isinstance(obj, Representation): return repr(obj) - elif isinstance(obj, TypeAliasType): + elif isinstance(obj, typing_extensions.TypeAliasType): return str(obj) if not isinstance(obj, (typing_base, WithArgsTypes, type)): From d1692a32ee650cf6fd54bd4e0ae28ecb3a417083 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 29 Jan 2024 18:27:39 -0700 Subject: [PATCH 16/61] Add test case for nested dictionaries. --- pydantic_settings/sources.py | 74 ++++++++++++++++++------------------ tests/test_settings.py | 24 ++++++++++++ 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 9a8169c7..9030e39c 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -784,31 +784,34 @@ def _merge_json_key_val_list_str(self, key_val_list_str: str) -> str: orig_key_val_list_str, key_val_list_str = key_val_list_str, key_val_list_str[1:-1] key_val_dict: dict[str, str] = {} obj_count = 0 - while key_val_list_str: - if obj_count != 0: - raise SettingsError(f'Parsing error encountered on JSON object {orig_key_val_list_str}') - for i in range(len(key_val_list_str)): - if key_val_list_str[i] == '{': - obj_count += 1 - elif key_val_list_str[i] == '}': - obj_count -= 1 - if obj_count == 0: - key_val_dict.update(json.loads(key_val_list_str[: i + 1])) - key_val_list_str = key_val_list_str[i + 1 :].lstrip(',') - break - elif obj_count == 0: - val, quote_count = '', 0 - key, key_val_list_str = key_val_list_str.split('=', 1) - for i in range(len(key_val_list_str)): - if key_val_list_str[i] in ('"', "'"): - quote_count += 1 - if key_val_list_str[i] == ',' and quote_count % 2 == 0: - val, key_val_list_str = key_val_list_str[:i], key_val_list_str[i:].lstrip(',') + try: + while key_val_list_str: + assert obj_count == 0 + for i in range(len(key_val_list_str)): + if key_val_list_str[i] == '{': + obj_count += 1 + elif key_val_list_str[i] == '}': + obj_count -= 1 + if obj_count == 0: + key_val_dict.update(json.loads(key_val_list_str[: i + 1])) + key_val_list_str = key_val_list_str[i + 1 :].lstrip(',') break - if not val: - val, key_val_list_str = key_val_list_str, '' - key_val_dict.update({key.strip('\'"'): val.strip('\'"')}) - break + elif obj_count == 0: + val, quote_count = '', 0 + key, key_val_list_str = key_val_list_str.split('=', 1) + for i in range(len(key_val_list_str)): + if key_val_list_str[i] in ('"', "'"): + quote_count += 1 + if key_val_list_str[i] == ',' and quote_count % 2 == 0: + val, key_val_list_str = key_val_list_str[:i], key_val_list_str[i:].lstrip(',') + break + if not val: + val, key_val_list_str = key_val_list_str, '' + key_val_dict.update({key.strip('\'"'): val.strip('\'"')}) + break + except Exception: + raise SettingsError(f'Parsing error encountered on JSON object {orig_key_val_list_str}') + return json.dumps(key_val_dict) def _get_sub_models(self, model: type[BaseModel], field_name: str, field_info: FieldInfo) -> list[type[BaseModel]]: @@ -820,13 +823,10 @@ def _get_sub_models(self, model: type[BaseModel], field_name: str, field_info: F sub_models: list[type[BaseModel]] = [] for type_ in field_types: - if get_origin(type_) is Annotated: - if _CliSubCommand in get_args(type_): - raise SettingsError(f'CliSubCommand is not outermost annotation for {model.__name__}.{field_name}') - elif _CliPositionalArg in get_args(type_): - raise SettingsError( - f'CliPositionalArg is not outermost annotation for {model.__name__}.{field_name}' - ) + if _annotation_contains_types(type_, (_CliSubCommand,), is_include_origin=False): + raise SettingsError(f'CliSubCommand is not outermost annotation for {model.__name__}.{field_name}') + elif _annotation_contains_types(type_, (_CliPositionalArg,), is_include_origin=False): + raise SettingsError(f'CliPositionalArg is not outermost annotation for {model.__name__}.{field_name}') if is_model_class(type_): sub_models.append(type_) return sub_models @@ -901,9 +901,11 @@ def _add_fields_to_parser( kwargs['required'] = self.cli_enforce_required and field_info.is_required() if kwargs['dest'] in added_args: continue - if _annotation_contains_types(field_info.annotation, (list, set, dict, Sequence, Mapping)): + if _annotation_contains_types( + field_info.annotation, (list, set, dict, Sequence, Mapping), is_include_origin=True + ): kwargs['action'] = 'append' - if _annotation_contains_types(field_info.annotation, (dict, Mapping)): + if _annotation_contains_types(field_info.annotation, (dict, Mapping), is_include_origin=True): self._cli_dict_arg_names.append(kwargs['dest']) arg_name = f'{arg_prefix.replace(subcommand_prefix, "", 1)}{field_name}' @@ -1047,10 +1049,10 @@ def _union_is_complex(annotation: type[Any] | None, metadata: list[Any]) -> bool return any(_annotation_is_complex(arg, metadata) for arg in get_args(annotation)) -def _annotation_contains_types(annotation: type[Any] | None, types: tuple[Any, ...]) -> bool: - if get_origin(annotation) in types: +def _annotation_contains_types(annotation: type[Any] | None, types: tuple[Any, ...], is_include_origin: bool) -> bool: + if is_include_origin is True and get_origin(annotation) in types: return True for type_ in get_args(annotation): - if _annotation_contains_types(type_, types): + if _annotation_contains_types(type_, types, is_include_origin=True): return True return annotation in types diff --git a/tests/test_settings.py b/tests/test_settings.py index 62ca66e5..b5349c1f 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2185,6 +2185,23 @@ class Cfg(BaseSettings): assert cfg.model_dump() == expected +def test_cli_nested_dict_arg(): + class Cfg(BaseSettings): + check_dict: Dict[str, Any] + + args = ['--check_dict', '{"k1":{"a": 1}},{"k2":{"b": 2}}'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == {'check_dict': {'k1': {'a': 1}, 'k2': {'b': 2}}} + + with pytest.raises(SettingsError): + args = ['--check_dict', '{"k1":{"a": 1}},"k2":{"b": 2}}'] + cfg = Cfg(_cli_parse_args=args) + + with pytest.raises(SettingsError): + args = ['--check_dict', '{"k1":{"a": 1}},{"k2":{"b": 2}'] + cfg = Cfg(_cli_parse_args=args) + + def test_cli_subcommand_with_positionals(): class FooPlugin(BaseModel): my_feature: bool = False @@ -2226,6 +2243,13 @@ class Git(BaseSettings): 'plugins': None, } + git = Git(_cli_parse_args=['plugins', 'bar']) + assert git.model_dump() == { + 'clone': None, + 'init': None, + 'plugins': {'foo': None, 'bar': {'my_feature': False}}, + } + def test_cli_union_similar_sub_models(): class ChildA(BaseModel): From b107d91bfd4fa1122aee832f8fdfc34f4955932a Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 30 Jan 2024 12:25:47 -0700 Subject: [PATCH 17/61] Additional test cases. --- pydantic_settings/sources.py | 22 +- tests/test_settings.py | 425 +++++++++++++++++++++-------------- 2 files changed, 270 insertions(+), 177 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 9030e39c..5c62fa3f 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -14,10 +14,10 @@ import typing_extensions from dotenv import dotenv_values from pydantic import AliasChoices, AliasPath, BaseModel, Json, TypeAdapter -from pydantic._internal._repr import Representation from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass from pydantic.fields import FieldInfo +from pydantic.v1.utils import Representation from typing_extensions import Annotated, get_args, get_origin from pydantic_settings.utils import path_type_label @@ -718,18 +718,24 @@ def __init__( elif not isinstance(self.cli_parse_args, list): raise SettingsError(f'cli_parse_args must be List[str], recieved {type(self.cli_parse_args)}') self.cli_hide_none_type = ( - cli_hide_none_type if cli_hide_none_type is not None else self.config.get('cli_hide_none_type', False) + cli_hide_none_type + if cli_hide_none_type is not None + else settings_cls.model_config.get('cli_hide_none_type', False) + ) + self.cli_avoid_json = ( + cli_avoid_json if cli_avoid_json is not None else settings_cls.model_config.get('cli_avoid_json', False) ) - self.cli_avoid_json = cli_avoid_json if cli_avoid_json is not None else self.config.get('cli_avoid_json', False) if cli_parse_none_str is None: cli_parse_none_str = 'None' if self.cli_avoid_json is True else 'null' self.cli_enforce_required = ( - cli_enforce_required if cli_enforce_required is not None else self.config.get('cli_enforce_required', False) + cli_enforce_required + if cli_enforce_required is not None + else settings_cls.model_config.get('cli_enforce_required', False) ) self.cli_use_class_docs_for_groups = ( cli_use_class_docs_for_groups if cli_use_class_docs_for_groups is not None - else self.config.get('cli_use_class_docs_for_groups', False) + else settings_cls.model_config.get('cli_use_class_docs_for_groups', False) ) super().__init__(settings_cls, env_nested_delimiter='.', env_parse_none_str=cli_parse_none_str) @@ -951,7 +957,8 @@ def _get_modified_args(self, obj: Any) -> tuple[str, ...]: return tuple([type_ for type_ in get_args(obj) if type_ is not type(None)]) def _metavar_format_list(self, args: list[str]) -> str: - args = args if 'JSON' not in args else [arg for arg in args[args.index('JSON') + 1 :] if arg != 'JSON'] + if 'JSON' in args: + args = args[: args.index('JSON') + 1] + [arg for arg in args[args.index('JSON') + 1 :] if arg != 'JSON'] return ','.join(args) def _metavar_format(self, obj: Any) -> str: @@ -977,10 +984,7 @@ def _metavar_format(self, obj: Any) -> str: return f'{{{args}}}' if ',' in args else args else: args = self._metavar_format_list(list(map(self._metavar_format, self._get_modified_args(obj)))) - try: return f'{obj.__qualname__}[{args}]' - except AttributeError: - return str(obj) # handles TypeAliasType in 3.12 elif obj is type(None): return self.env_parse_none_str elif is_model_class(obj): diff --git a/tests/test_settings.py b/tests/test_settings.py index b5349c1f..21037cc2 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -4,7 +4,7 @@ import uuid from datetime import datetime, timezone from pathlib import Path -from typing import Any, Callable, Dict, Generic, List, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Any, Callable, Dict, Generic, List, Literal, Optional, Set, Tuple, Type, TypeVar, Union import pytest from annotated_types import MinLen @@ -23,6 +23,7 @@ dataclasses as pydantic_dataclasses, ) from pydantic.fields import FieldInfo +from pydantic.v1.utils import Representation from typing_extensions import Annotated from pydantic_settings import ( @@ -34,7 +35,7 @@ SecretsSettingsSource, SettingsConfigDict, ) -from pydantic_settings.sources import CliPositionalArg, CliSubCommand, SettingsError, read_env_file +from pydantic_settings.sources import CliPositionalArg, CliSettingsSource, CliSubCommand, SettingsError, read_env_file try: import dotenv @@ -42,6 +43,18 @@ dotenv = None +def foobar(a, b, c=4): + pass + + +T = TypeVar('T') + + +class LoggedVar(Generic[T]): + def get(self) -> T: + ... + + class SimpleSettings(BaseSettings): apple: str @@ -2037,7 +2050,8 @@ class Cfg(BaseSettings): } -def test_cli_list_arg(): +@pytest.mark.parametrize('prefix', ['', 'child.']) +def test_cli_list_arg(prefix): class Obj(BaseModel): val: int @@ -2068,57 +2082,57 @@ def check_answer(cfg, prefix, expected): assert cfg.model_dump() == expected args: List[str] = [] - for prefix in ('', 'child.'): - args = [f'--{prefix}num_list', '[1,2]'] - args += [f'--{prefix}num_list', '3,4'] - args += [f'--{prefix}num_list', '5', f'--{prefix}num_list', '6'] - cfg = Cfg(_cli_parse_args=args) - expected = { - 'num_list': [1, 2, 3, 4, 5, 6], - 'obj_list': None, - 'union_list': None, - 'str_list': None, - } - check_answer(cfg, prefix, expected) + args = [f'--{prefix}num_list', '[1,2]'] + args += [f'--{prefix}num_list', '3,4'] + args += [f'--{prefix}num_list', '5', f'--{prefix}num_list', '6'] + cfg = Cfg(_cli_parse_args=args) + expected = { + 'num_list': [1, 2, 3, 4, 5, 6], + 'obj_list': None, + 'union_list': None, + 'str_list': None, + } + check_answer(cfg, prefix, expected) - args = [f'--{prefix}obj_list', '[{"val":1},{"val":2}]'] - args += [f'--{prefix}obj_list', '{"val":3},{"val":4}'] - args += [f'--{prefix}obj_list', '{"val":5}', f'--{prefix}obj_list', '{"val":6}'] - cfg = Cfg(_cli_parse_args=args) - expected = { - 'num_list': None, - 'obj_list': [{'val': 1}, {'val': 2}, {'val': 3}, {'val': 4}, {'val': 5}, {'val': 6}], - 'union_list': None, - 'str_list': None, - } - check_answer(cfg, prefix, expected) + args = [f'--{prefix}obj_list', '[{"val":1},{"val":2}]'] + args += [f'--{prefix}obj_list', '{"val":3},{"val":4}'] + args += [f'--{prefix}obj_list', '{"val":5}', f'--{prefix}obj_list', '{"val":6}'] + cfg = Cfg(_cli_parse_args=args) + expected = { + 'num_list': None, + 'obj_list': [{'val': 1}, {'val': 2}, {'val': 3}, {'val': 4}, {'val': 5}, {'val': 6}], + 'union_list': None, + 'str_list': None, + } + check_answer(cfg, prefix, expected) - args = [f'--{prefix}union_list', '[{"val":1},2]', f'--{prefix}union_list', '[3,{"val":4}]'] - args += [f'--{prefix}union_list', '{"val":5},6', f'--{prefix}union_list', '7,{"val":8}'] - args += [f'--{prefix}union_list', '{"val":9}', f'--{prefix}union_list', '10'] - cfg = Cfg(_cli_parse_args=args) - expected = { - 'num_list': None, - 'obj_list': None, - 'union_list': [{'val': 1}, 2, 3, {'val': 4}, {'val': 5}, 6, 7, {'val': 8}, {'val': 9}, 10], - 'str_list': None, - } - check_answer(cfg, prefix, expected) + args = [f'--{prefix}union_list', '[{"val":1},2]', f'--{prefix}union_list', '[3,{"val":4}]'] + args += [f'--{prefix}union_list', '{"val":5},6', f'--{prefix}union_list', '7,{"val":8}'] + args += [f'--{prefix}union_list', '{"val":9}', f'--{prefix}union_list', '10'] + cfg = Cfg(_cli_parse_args=args) + expected = { + 'num_list': None, + 'obj_list': None, + 'union_list': [{'val': 1}, 2, 3, {'val': 4}, {'val': 5}, 6, 7, {'val': 8}, {'val': 9}, 10], + 'str_list': None, + } + check_answer(cfg, prefix, expected) - args = [f'--{prefix}str_list', '["0,0","1,1"]'] - args += [f'--{prefix}str_list', '"2,2","3,3"'] - args += [f'--{prefix}str_list', '"4,4"', f'--{prefix}str_list', '"5,5"'] - cfg = Cfg(_cli_parse_args=args) - expected = { - 'num_list': None, - 'obj_list': None, - 'union_list': None, - 'str_list': ['0,0', '1,1', '2,2', '3,3', '4,4', '5,5'], - } - check_answer(cfg, prefix, expected) + args = [f'--{prefix}str_list', '["0,0","1,1"]'] + args += [f'--{prefix}str_list', '"2,2","3,3"'] + args += [f'--{prefix}str_list', '"4,4"', f'--{prefix}str_list', '"5,5"'] + cfg = Cfg(_cli_parse_args=args) + expected = { + 'num_list': None, + 'obj_list': None, + 'union_list': None, + 'str_list': ['0,0', '1,1', '2,2', '3,3', '4,4', '5,5'], + } + check_answer(cfg, prefix, expected) -def test_cli_dict_arg(): +@pytest.mark.parametrize('prefix', ['', 'child.']) +def test_cli_dict_arg(prefix): class Child(BaseModel): check_dict: Dict[str, str] @@ -2127,62 +2141,61 @@ class Cfg(BaseSettings): child: Optional[Child] = None args: List[str] = [] - for prefix in ('', 'child.'): - args = [f'--{prefix}check_dict', '{"k1":"a","k2":"b"}'] - args += [f'--{prefix}check_dict', '{"k3":"c"},{"k4":"d"}'] - args += [f'--{prefix}check_dict', '{"k5":"e"}', f'--{prefix}check_dict', '{"k6":"f"}'] - args += [f'--{prefix}check_dict', '[k7=g,k8=h]'] - args += [f'--{prefix}check_dict', 'k9=i,k10=j'] - args += [f'--{prefix}check_dict', 'k11=k', f'--{prefix}check_dict', 'k12=l'] - args += [f'--{prefix}check_dict', '[{"k13":"m"},k14=n]', f'--{prefix}check_dict', '[k15=o,{"k16":"p"}]'] - args += [f'--{prefix}check_dict', '{"k17":"q"},k18=r', f'--{prefix}check_dict', 'k19=s,{"k20":"t"}'] - args += [f'--{prefix}check_dict', '{"k21":"u"},k22=v,{"k23":"w"}'] - args += [f'--{prefix}check_dict', 'k24=x,{"k25":"y"},k26=z'] - args += [f'--{prefix}check_dict', '[k27="x,y",k28="x,y"]'] - args += [f'--{prefix}check_dict', 'k29="x,y",k30="x,y"'] - args += [f'--{prefix}check_dict', 'k31="x,y"', f'--{prefix}check_dict', 'k32="x,y"'] - cfg = Cfg(_cli_parse_args=args) - expected: Dict[str, Any] = { - 'check_dict': { - 'k1': 'a', - 'k2': 'b', - 'k3': 'c', - 'k4': 'd', - 'k5': 'e', - 'k6': 'f', - 'k7': 'g', - 'k8': 'h', - 'k9': 'i', - 'k10': 'j', - 'k11': 'k', - 'k12': 'l', - 'k13': 'm', - 'k14': 'n', - 'k15': 'o', - 'k16': 'p', - 'k17': 'q', - 'k18': 'r', - 'k19': 's', - 'k20': 't', - 'k21': 'u', - 'k22': 'v', - 'k23': 'w', - 'k24': 'x', - 'k25': 'y', - 'k26': 'z', - 'k27': 'x,y', - 'k28': 'x,y', - 'k29': 'x,y', - 'k30': 'x,y', - 'k31': 'x,y', - 'k32': 'x,y', - } + args = [f'--{prefix}check_dict', '{"k1":"a","k2":"b"}'] + args += [f'--{prefix}check_dict', '{"k3":"c"},{"k4":"d"}'] + args += [f'--{prefix}check_dict', '{"k5":"e"}', f'--{prefix}check_dict', '{"k6":"f"}'] + args += [f'--{prefix}check_dict', '[k7=g,k8=h]'] + args += [f'--{prefix}check_dict', 'k9=i,k10=j'] + args += [f'--{prefix}check_dict', 'k11=k', f'--{prefix}check_dict', 'k12=l'] + args += [f'--{prefix}check_dict', '[{"k13":"m"},k14=n]', f'--{prefix}check_dict', '[k15=o,{"k16":"p"}]'] + args += [f'--{prefix}check_dict', '{"k17":"q"},k18=r', f'--{prefix}check_dict', 'k19=s,{"k20":"t"}'] + args += [f'--{prefix}check_dict', '{"k21":"u"},k22=v,{"k23":"w"}'] + args += [f'--{prefix}check_dict', 'k24=x,{"k25":"y"},k26=z'] + args += [f'--{prefix}check_dict', '[k27="x,y",k28="x,y"]'] + args += [f'--{prefix}check_dict', 'k29="x,y",k30="x,y"'] + args += [f'--{prefix}check_dict', 'k31="x,y"', f'--{prefix}check_dict', 'k32="x,y"'] + cfg = Cfg(_cli_parse_args=args) + expected: Dict[str, Any] = { + 'check_dict': { + 'k1': 'a', + 'k2': 'b', + 'k3': 'c', + 'k4': 'd', + 'k5': 'e', + 'k6': 'f', + 'k7': 'g', + 'k8': 'h', + 'k9': 'i', + 'k10': 'j', + 'k11': 'k', + 'k12': 'l', + 'k13': 'm', + 'k14': 'n', + 'k15': 'o', + 'k16': 'p', + 'k17': 'q', + 'k18': 'r', + 'k19': 's', + 'k20': 't', + 'k21': 'u', + 'k22': 'v', + 'k23': 'w', + 'k24': 'x', + 'k25': 'y', + 'k26': 'z', + 'k27': 'x,y', + 'k28': 'x,y', + 'k29': 'x,y', + 'k30': 'x,y', + 'k31': 'x,y', + 'k32': 'x,y', } - if prefix: - expected = {'check_dict': None, 'child': expected} - else: - expected['child'] = None - assert cfg.model_dump() == expected + } + if prefix: + expected = {'check_dict': None, 'child': expected} + else: + expected['child'] = None + assert cfg.model_dump() == expected def test_cli_nested_dict_arg(): @@ -2267,54 +2280,57 @@ class Cfg(BaseSettings): assert cfg.model_dump() == {'child': {'name': 'new name a', 'diff_a': 'new diff a'}} -def test_cli_annotation_exceptions(): +def test_cli_annotation_exceptions(monkeypatch): class SubCmdAlt(BaseModel): pass class SubCmd(BaseModel): pass - with pytest.raises(SettingsError): + with monkeypatch.context() as m: + m.setattr(sys, 'argv', ['example.py', '--help']) - class SubCommandNotOutermost(BaseSettings): - subcmd: Union[int, CliSubCommand[SubCmd]] + with pytest.raises(SettingsError): - SubCommandNotOutermost(_cli_parse_args=['--help']) + class SubCommandNotOutermost(BaseSettings): + subcmd: Union[int, CliSubCommand[SubCmd]] - with pytest.raises(SettingsError): + SubCommandNotOutermost(_cli_parse_args=True) - class SubCommandHasDefault(BaseSettings): - subcmd: CliSubCommand[SubCmd] = SubCmd() + with pytest.raises(SettingsError): - SubCommandHasDefault(_cli_parse_args=['--help']) + class SubCommandHasDefault(BaseSettings): + subcmd: CliSubCommand[SubCmd] = SubCmd() - with pytest.raises(SettingsError): + SubCommandHasDefault(_cli_parse_args=True) - class SubCommandMultipleTypes(BaseSettings): - subcmd: CliSubCommand[Union[SubCmd, SubCmdAlt]] + with pytest.raises(SettingsError): - SubCommandMultipleTypes(_cli_parse_args=['--help']) + class SubCommandMultipleTypes(BaseSettings): + subcmd: CliSubCommand[Union[SubCmd, SubCmdAlt]] - with pytest.raises(SettingsError): + SubCommandMultipleTypes(_cli_parse_args=True) - class SubCommandNotModel(BaseSettings): - subcmd: CliSubCommand[str] + with pytest.raises(SettingsError): - SubCommandNotModel(_cli_parse_args=['--help']) + class SubCommandNotModel(BaseSettings): + subcmd: CliSubCommand[str] - with pytest.raises(SettingsError): + SubCommandNotModel(_cli_parse_args=True) - class PositionalArgNotOutermost(BaseSettings): - pos_arg: Union[int, CliPositionalArg[str]] + with pytest.raises(SettingsError): - PositionalArgNotOutermost(_cli_parse_args=['--help']) + class PositionalArgNotOutermost(BaseSettings): + pos_arg: Union[int, CliPositionalArg[str]] - with pytest.raises(SettingsError): + PositionalArgNotOutermost(_cli_parse_args=True) - class PositionalArgHasDefault(BaseSettings): - pos_arg: CliPositionalArg[str] = 'bad' + with pytest.raises(SettingsError): - PositionalArgHasDefault(_cli_parse_args=['--help']) + class PositionalArgHasDefault(BaseSettings): + pos_arg: CliPositionalArg[str] = 'bad' + + PositionalArgHasDefault(_cli_parse_args=True) with pytest.raises(SettingsError): @@ -2324,7 +2340,7 @@ class InvalidCliParseArgsType(BaseSettings): PositionalArgHasDefault(_cli_parse_args='invalid type') -def test_cli_avoid_json(capsys): +def test_cli_avoid_json(capsys, monkeypatch): class SubModel(BaseModel): v1: int @@ -2333,12 +2349,15 @@ class Settings(BaseSettings): argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' - with pytest.raises(SystemExit): - Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_avoid_json=False) + with monkeypatch.context() as m: + m.setattr(sys, 'argv', ['example.py', '--help']) - assert ( - capsys.readouterr().out - == f"""usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + with pytest.raises(SystemExit): + Settings(_cli_parse_args=True, _cli_avoid_json=False) + + assert ( + capsys.readouterr().out + == f"""usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] {argparse_options_text}: -h, --help show this help message and exit @@ -2347,14 +2366,14 @@ class Settings(BaseSettings): --sub_model JSON set sub_model from JSON string --sub_model.v1 int """ - ) + ) - with pytest.raises(SystemExit): - Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_avoid_json=True) + with pytest.raises(SystemExit): + Settings(_cli_parse_args=True, _cli_avoid_json=True) - assert ( - capsys.readouterr().out - == f"""usage: example.py [-h] [--sub_model.v1 int] + assert ( + capsys.readouterr().out + == f"""usage: example.py [-h] [--sub_model.v1 int] {argparse_options_text}: -h, --help show this help message and exit @@ -2362,43 +2381,46 @@ class Settings(BaseSettings): sub_model options: --sub_model.v1 int """ - ) + ) -def test_cli_hide_none_type(capsys): +def test_cli_hide_none_type(capsys, monkeypatch): class Settings(BaseSettings): v0: Optional[str] argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' - with pytest.raises(SystemExit): - Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_hide_none_type=False) + with monkeypatch.context() as m: + m.setattr(sys, 'argv', ['example.py', '--help']) - assert ( - capsys.readouterr().out - == f"""usage: example.py [-h] [--v0 {{str,null}}] + with pytest.raises(SystemExit): + Settings(_cli_parse_args=True, _cli_hide_none_type=False) + + assert ( + capsys.readouterr().out + == f"""usage: example.py [-h] [--v0 {{str,null}}] {argparse_options_text}: -h, --help show this help message and exit --v0 {{str,null}} """ - ) + ) - with pytest.raises(SystemExit): - Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_hide_none_type=True) + with pytest.raises(SystemExit): + Settings(_cli_parse_args=True, _cli_hide_none_type=True) - assert ( - capsys.readouterr().out - == f"""usage: example.py [-h] [--v0 str] + assert ( + capsys.readouterr().out + == f"""usage: example.py [-h] [--v0 str] {argparse_options_text}: -h, --help show this help message and exit --v0 str """ - ) + ) -def test_cli_use_class_docs_for_groups(capsys): +def test_cli_use_class_docs_for_groups(capsys, monkeypatch): class SubModel(BaseModel): """The help text from the class docstring""" @@ -2411,12 +2433,15 @@ class Settings(BaseSettings): argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' - with pytest.raises(SystemExit): - Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_use_class_docs_for_groups=False) + with monkeypatch.context() as m: + m.setattr(sys, 'argv', ['example.py', '--help']) - assert ( - capsys.readouterr().out - == f"""usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + with pytest.raises(SystemExit): + Settings(_cli_parse_args=True, _cli_use_class_docs_for_groups=False) + + assert ( + capsys.readouterr().out + == f"""usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] My application help text. @@ -2429,14 +2454,14 @@ class Settings(BaseSettings): --sub_model JSON set sub_model from JSON string --sub_model.v1 int """ - ) + ) - with pytest.raises(SystemExit): - Settings(_cli_prog_name='example.py', _cli_parse_args=['--help'], _cli_use_class_docs_for_groups=True) + with pytest.raises(SystemExit): + Settings(_cli_parse_args=True, _cli_use_class_docs_for_groups=True) - assert ( - capsys.readouterr().out - == f"""usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] + assert ( + capsys.readouterr().out + == f"""usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] My application help text. @@ -2449,7 +2474,7 @@ class Settings(BaseSettings): --sub_model JSON set sub_model from JSON string --sub_model.v1 int """ - ) + ) def test_cli_enforce_required(env): @@ -2464,3 +2489,67 @@ class Settings(BaseSettings): with pytest.raises(SystemExit): Settings(_cli_parse_args=[], _cli_enforce_required=True).model_dump() + + +@pytest.mark.parametrize( + 'value,expected', + [ + (str, 'str'), + ('foobar', 'str'), + ('SomeForwardRefString', 'str'), # included to document current behavior; could be changed + (List['SomeForwardRef'], "List[ForwardRef('SomeForwardRef')]"), # noqa: F821 + (Union[str, int], '{str,int}'), + (list, 'list'), + (List, 'List'), + ([1, 2, 3], 'list'), + (List[Dict[str, int]], 'List[Dict[str,int]]'), + (Tuple[str, int, float], 'Tuple[str,int,float]'), + (Tuple[str, ...], 'Tuple[str,...]'), + (Union[int, List[str], Tuple[str, int]], '{int,List[str],Tuple[str,int]}'), + (foobar, 'foobar'), + (LoggedVar, 'LoggedVar'), + (LoggedVar(), 'LoggedVar'), + (Representation(), 'Representation()'), + (Literal[1, 2, 3], '{1,2,3}'), + (SimpleSettings, 'JSON'), + (Union[SimpleSettings, SettingWithIgnoreEmpty], 'JSON'), + (Union[SimpleSettings, str, SettingWithIgnoreEmpty], '{JSON,str}'), + (Union[str, SimpleSettings, SettingWithIgnoreEmpty], '{str,JSON}'), + ], +) +def test_cli_metavar_format(value, expected): + assert CliSettingsSource(SimpleSettings)._metavar_format(value) == expected + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason='requires python 3.10 or higher') +@pytest.mark.parametrize( + 'value_gen,expected', + [ + (lambda: str, 'str'), + (lambda: 'SomeForwardRefString', 'str'), # included to document current behavior; could be changed + (lambda: List['SomeForwardRef'], "List[ForwardRef('SomeForwardRef')]"), # noqa: F821 + (lambda: str | int, '{str,int}'), + (lambda: list, 'list'), + (lambda: List, 'List'), + (lambda: list[int], 'list[int]'), + (lambda: List[int], 'List[int]'), + (lambda: list[dict[str, int]], 'list[dict[str,int]]'), + (lambda: list[Union[str, int]], 'list[{str,int}]'), + (lambda: list[str | int], 'list[{str,int}]'), + (lambda: LoggedVar[int], 'LoggedVar[int]'), + (lambda: LoggedVar[Dict[int, str]], 'LoggedVar[Dict[int,str]]'), + ], +) +def test_cli_metavar_format_310(value_gen, expected): + value = value_gen() + assert CliSettingsSource(SimpleSettings)._metavar_format(value) == expected + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason='requires python 3.12 or higher') +def test_cli_metavar_format_type_alias_312(): + exec( + """ +type TypeAliasInt = int +assert CliSettingsSource(SimpleSettings)._metavar_format(TypeAliasInt) == 'TypeAliasInt' +""" + ) From 7e7713eda12bfe67fa8845539f6b1a4769be2d85 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 30 Jan 2024 15:32:11 -0700 Subject: [PATCH 18/61] Python 3.8 and 3.9 format fixes. --- pydantic_settings/sources.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 5c62fa3f..902dd498 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -9,7 +9,7 @@ from dataclasses import is_dataclass from pathlib import Path from types import FunctionType -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Sequence, Tuple, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Sequence, Tuple, TypeVar, Union, cast import typing_extensions from dotenv import dotenv_values @@ -961,7 +961,7 @@ def _metavar_format_list(self, args: list[str]) -> str: args = args[: args.index('JSON') + 1] + [arg for arg in args[args.index('JSON') + 1 :] if arg != 'JSON'] return ','.join(args) - def _metavar_format(self, obj: Any) -> str: + def _metavar_format_recurse(self, obj: Any) -> str: """Pretty metavar representation of a type. Adapts logic from `pydantic._repr.display_as_type`.""" if isinstance(obj, FunctionType): return obj.__name__ @@ -976,15 +976,14 @@ def _metavar_format(self, obj: Any) -> str: obj = obj.__class__ if origin_is_union(get_origin(obj)): - args = self._metavar_format_list(list(map(self._metavar_format, self._get_modified_args(obj)))) + args = self._metavar_format_list(list(map(self._metavar_format_recurse, self._get_modified_args(obj)))) + return f'{{{args}}}' if ',' in args else args + elif get_origin(obj) == typing_extensions.Literal: + args = self._metavar_format_list(list(map(repr, self._get_modified_args(obj)))) return f'{{{args}}}' if ',' in args else args elif isinstance(obj, WithArgsTypes): - if get_origin(obj) == Literal: - args = self._metavar_format_list(list(map(repr, self._get_modified_args(obj)))) - return f'{{{args}}}' if ',' in args else args - else: - args = self._metavar_format_list(list(map(self._metavar_format, self._get_modified_args(obj)))) - return f'{obj.__qualname__}[{args}]' + args = self._metavar_format_list(list(map(self._metavar_format_recurse, self._get_modified_args(obj)))) + return f'{obj.__qualname__}[{args}]' elif obj is type(None): return self.env_parse_none_str elif is_model_class(obj): @@ -994,6 +993,9 @@ def _metavar_format(self, obj: Any) -> str: else: return repr(obj).replace('typing.', '').replace('typing_extensions.', '') + def _metavar_format(self, args: list[str]) -> str: + return self._metavar_format_recurse(args).replace(', ', ',') + def _get_env_var_key(key: str, case_sensitive: bool = False) -> str: return key if case_sensitive else key.lower() From ca39690d0e25528885976759960c256561524a1e Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 30 Jan 2024 16:24:46 -0700 Subject: [PATCH 19/61] Fix for typing vs typing_extensions Literal. --- pydantic_settings/sources.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 902dd498..c9b7e0ca 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -3,6 +3,7 @@ import json import os import sys +import typing import warnings from abc import ABC, abstractmethod from collections import deque @@ -978,7 +979,7 @@ def _metavar_format_recurse(self, obj: Any) -> str: if origin_is_union(get_origin(obj)): args = self._metavar_format_list(list(map(self._metavar_format_recurse, self._get_modified_args(obj)))) return f'{{{args}}}' if ',' in args else args - elif get_origin(obj) == typing_extensions.Literal: + elif get_origin(obj) in (typing_extensions.Literal, typing.Literal): args = self._metavar_format_list(list(map(repr, self._get_modified_args(obj)))) return f'{{{args}}}' if ',' in args else args elif isinstance(obj, WithArgsTypes): From 09bdce28461007a4f01d197b89cc3c4c8c6a333d Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 30 Jan 2024 16:38:31 -0700 Subject: [PATCH 20/61] Add test case for typing vs typing_extensions Literal. --- tests/test_settings.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index 21037cc2..4a2c29ac 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,12 +1,14 @@ import dataclasses import os import sys +import typing import uuid from datetime import datetime, timezone from pathlib import Path -from typing import Any, Callable, Dict, Generic, List, Literal, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Any, Callable, Dict, Generic, List, Optional, Set, Tuple, Type, TypeVar, Union import pytest +import typing_extensions from annotated_types import MinLen from pydantic import ( AliasChoices, @@ -2510,7 +2512,8 @@ class Settings(BaseSettings): (LoggedVar, 'LoggedVar'), (LoggedVar(), 'LoggedVar'), (Representation(), 'Representation()'), - (Literal[1, 2, 3], '{1,2,3}'), + (typing.Literal[1, 2, 3], '{1,2,3}'), + (typing_extensions.Literal[1, 2, 3], '{1,2,3}'), (SimpleSettings, 'JSON'), (Union[SimpleSettings, SettingWithIgnoreEmpty], 'JSON'), (Union[SimpleSettings, str, SettingWithIgnoreEmpty], '{JSON,str}'), From 20a83f1aaa3b0aa17573ca791a387f5c772ad137 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 30 Jan 2024 20:54:43 -0700 Subject: [PATCH 21/61] Mypy fix for _metavar_format function update. --- pydantic_settings/sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index c9b7e0ca..c645cc16 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -994,8 +994,8 @@ def _metavar_format_recurse(self, obj: Any) -> str: else: return repr(obj).replace('typing.', '').replace('typing_extensions.', '') - def _metavar_format(self, args: list[str]) -> str: - return self._metavar_format_recurse(args).replace(', ', ',') + def _metavar_format(self, obj: Any) -> str: + return self._metavar_format_recurse(obj).replace(', ', ',') def _get_env_var_key(key: str, case_sensitive: bool = False) -> str: From f4bf3ee946e052da087f3bb0b6620986dfd4b739 Mon Sep 17 00:00:00 2001 From: Hasan Ramezani Date: Wed, 31 Jan 2024 09:46:59 +0100 Subject: [PATCH 22/61] Update pydantic_settings/sources.py --- pydantic_settings/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index c645cc16..22afda6c 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -970,7 +970,7 @@ def _metavar_format_recurse(self, obj: Any) -> str: return '...' elif isinstance(obj, Representation): return repr(obj) - elif isinstance(obj, typing_extensions.TypeAliasType): + elif isinstance(obj, typing_extensions.TypeAliasType): # type: ignore return str(obj) if not isinstance(obj, (typing_base, WithArgsTypes, type)): From 3a4949cb1e429e5eaa689f82776ea8be59c9b00d Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Wed, 31 Jan 2024 08:53:45 -0700 Subject: [PATCH 23/61] Handle Representation from pydantic._internal and pydantic.v1. --- pydantic_settings/sources.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 22afda6c..681218a6 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -12,13 +12,14 @@ from types import FunctionType from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Sequence, Tuple, TypeVar, Union, cast +import pydantic._internal._repr +import pydantic.v1.utils import typing_extensions from dotenv import dotenv_values from pydantic import AliasChoices, AliasPath, BaseModel, Json, TypeAdapter from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass from pydantic.fields import FieldInfo -from pydantic.v1.utils import Representation from typing_extensions import Annotated, get_args, get_origin from pydantic_settings.utils import path_type_label @@ -968,7 +969,7 @@ def _metavar_format_recurse(self, obj: Any) -> str: return obj.__name__ elif obj is ...: return '...' - elif isinstance(obj, Representation): + elif isinstance(obj, (pydantic._internal._repr.Representation, pydantic.v1.utils.Representation)): return repr(obj) elif isinstance(obj, typing_extensions.TypeAliasType): # type: ignore return str(obj) From 048338040bc6303bf7945f0427498ecdbf679064 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Wed, 31 Jan 2024 09:19:19 -0700 Subject: [PATCH 24/61] Fix for _cli_parse_args to cli_parse_args. --- pydantic_settings/main.py | 2 +- tests/test_settings.py | 46 ++++++++++++++++++++++----------------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 673ffdb1..76fe0227 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -239,7 +239,7 @@ def _settings_build_values( dotenv_settings=dotenv_settings, file_secret_settings=file_secret_settings, ) - if _cli_parse_args: + if cli_parse_args: sources = (cli_settings,) + sources if sources: return deep_update(*reversed([source() for source in sources])) diff --git a/tests/test_settings.py b/tests/test_settings.py index 4a2c29ac..b1173232 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2294,52 +2294,52 @@ class SubCmd(BaseModel): with pytest.raises(SettingsError): - class SubCommandNotOutermost(BaseSettings): + class SubCommandNotOutermost(BaseSettings, cli_parse_args=True): subcmd: Union[int, CliSubCommand[SubCmd]] - SubCommandNotOutermost(_cli_parse_args=True) + SubCommandNotOutermost() with pytest.raises(SettingsError): - class SubCommandHasDefault(BaseSettings): + class SubCommandHasDefault(BaseSettings, cli_parse_args=True): subcmd: CliSubCommand[SubCmd] = SubCmd() - SubCommandHasDefault(_cli_parse_args=True) + SubCommandHasDefault() with pytest.raises(SettingsError): - class SubCommandMultipleTypes(BaseSettings): + class SubCommandMultipleTypes(BaseSettings, cli_parse_args=True): subcmd: CliSubCommand[Union[SubCmd, SubCmdAlt]] - SubCommandMultipleTypes(_cli_parse_args=True) + SubCommandMultipleTypes() with pytest.raises(SettingsError): - class SubCommandNotModel(BaseSettings): + class SubCommandNotModel(BaseSettings, cli_parse_args=True): subcmd: CliSubCommand[str] - SubCommandNotModel(_cli_parse_args=True) + SubCommandNotModel() with pytest.raises(SettingsError): - class PositionalArgNotOutermost(BaseSettings): + class PositionalArgNotOutermost(BaseSettings, cli_parse_args=True): pos_arg: Union[int, CliPositionalArg[str]] - PositionalArgNotOutermost(_cli_parse_args=True) + PositionalArgNotOutermost() with pytest.raises(SettingsError): - class PositionalArgHasDefault(BaseSettings): + class PositionalArgHasDefault(BaseSettings, cli_parse_args=True): pos_arg: CliPositionalArg[str] = 'bad' - PositionalArgHasDefault(_cli_parse_args=True) + PositionalArgHasDefault() with pytest.raises(SettingsError): - class InvalidCliParseArgsType(BaseSettings): + class InvalidCliParseArgsType(BaseSettings, cli_parse_args='invalid type'): val: int - PositionalArgHasDefault(_cli_parse_args='invalid type') + InvalidCliParseArgsType() def test_cli_avoid_json(capsys, monkeypatch): @@ -2349,13 +2349,15 @@ class SubModel(BaseModel): class Settings(BaseSettings): sub_model: SubModel + model_config = SettingsConfigDict(cli_parse_args=True) + argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' with monkeypatch.context() as m: m.setattr(sys, 'argv', ['example.py', '--help']) with pytest.raises(SystemExit): - Settings(_cli_parse_args=True, _cli_avoid_json=False) + Settings(_cli_avoid_json=False) assert ( capsys.readouterr().out @@ -2371,7 +2373,7 @@ class Settings(BaseSettings): ) with pytest.raises(SystemExit): - Settings(_cli_parse_args=True, _cli_avoid_json=True) + Settings(_cli_avoid_json=True) assert ( capsys.readouterr().out @@ -2390,13 +2392,15 @@ def test_cli_hide_none_type(capsys, monkeypatch): class Settings(BaseSettings): v0: Optional[str] + model_config = SettingsConfigDict(cli_parse_args=True) + argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' with monkeypatch.context() as m: m.setattr(sys, 'argv', ['example.py', '--help']) with pytest.raises(SystemExit): - Settings(_cli_parse_args=True, _cli_hide_none_type=False) + Settings(_cli_hide_none_type=False) assert ( capsys.readouterr().out @@ -2409,7 +2413,7 @@ class Settings(BaseSettings): ) with pytest.raises(SystemExit): - Settings(_cli_parse_args=True, _cli_hide_none_type=True) + Settings(_cli_hide_none_type=True) assert ( capsys.readouterr().out @@ -2433,13 +2437,15 @@ class Settings(BaseSettings): sub_model: SubModel = Field(description='The help text from the field description') + model_config = SettingsConfigDict(cli_parse_args=True) + argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' with monkeypatch.context() as m: m.setattr(sys, 'argv', ['example.py', '--help']) with pytest.raises(SystemExit): - Settings(_cli_parse_args=True, _cli_use_class_docs_for_groups=False) + Settings(_cli_use_class_docs_for_groups=False) assert ( capsys.readouterr().out @@ -2459,7 +2465,7 @@ class Settings(BaseSettings): ) with pytest.raises(SystemExit): - Settings(_cli_parse_args=True, _cli_use_class_docs_for_groups=True) + Settings(_cli_use_class_docs_for_groups=True) assert ( capsys.readouterr().out From ff018cee5cf36e74780a958cacdeca0a54cdf1a6 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Wed, 31 Jan 2024 14:22:20 -0700 Subject: [PATCH 25/61] Complex test cases and fixes for env parse none str. --- pydantic_settings/sources.py | 4 +++- tests/test_settings.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 681218a6..564a5506 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -601,7 +601,9 @@ def explode_env_vars(self, field_name: str, field: FieldInfo, env_vars: Mapping[ except ValueError as e: if not allow_json_failure: raise e - env_var[last_key] = env_val + + if last_key not in env_var or not isinstance(env_val, EnvNoneType) or env_var[last_key] is {}: + env_var[last_key] = env_val return result diff --git a/tests/test_settings.py b/tests/test_settings.py index b1173232..4b08a2f0 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1917,6 +1917,26 @@ class NestedSettings(BaseSettings, env_nested_delimiter='__'): assert s.nested.deep['z'] is None assert s.nested.keep['z'] == 'None' + env.set('nested__deep', 'None') + + with pytest.raises(ValidationError): + s = NestedSettings() + s = NestedSettings(_env_parse_none_str='None') + assert s.nested.x is None + assert s.nested.y == 'y_override' + assert s.nested.deep['z'] is None + assert s.nested.keep['z'] == 'None' + + env.pop('nested__deep__z') + + with pytest.raises(ValidationError): + s = NestedSettings() + s = NestedSettings(_env_parse_none_str='None') + assert s.nested.x is None + assert s.nested.y == 'y_override' + assert s.nested.deep is None + assert s.nested.keep['z'] == 'None' + def test_env_json_field_dict(env): class Settings(BaseSettings): From a8d15b1491a63fe6588bee93490144c33dd346d3 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 6 Feb 2024 11:47:57 -0700 Subject: [PATCH 26/61] Remove empty groups from parsing and help text. --- pydantic_settings/sources.py | 7 ++++-- tests/test_settings.py | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 564a5506..954701ee 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -871,7 +871,7 @@ def _add_fields_to_parser( added_args: list[str], arg_prefix: str, subcommand_prefix: str, - group: _ArgumentGroup | None, + group: _ArgumentGroup | Tuple[str, str] | None, ) -> ArgumentParser: subparsers: _SubParsersAction[Any] | None = None for field_name, field_info in self._sort_arg_fields(model): @@ -932,10 +932,11 @@ def _add_fields_to_parser( if self.cli_use_class_docs_for_groups and len(sub_models) == 1 else field_info.description ) - model_group = parser.add_argument_group(f'{arg_name} options', group_help_text) + model_group = (f'{arg_name} options', group_help_text) if not self.cli_avoid_json: added_args.append(arg_name) kwargs['help'] = f'set {arg_name} from JSON string' + model_group = parser.add_argument_group(*model_group) model_group.add_argument(f'{arg_flag}{arg_name}', **kwargs) for model in sub_models: self._add_fields_to_parser( @@ -947,6 +948,8 @@ def _add_fields_to_parser( group=model_group, ) elif group is not None: + if isinstance(group, tuple): + group = parser.add_argument_group(*group) added_args.append(arg_name) group.add_argument(f'{arg_flag}{arg_name}', **kwargs) else: diff --git a/tests/test_settings.py b/tests/test_settings.py index 4b08a2f0..f6a80caa 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2408,6 +2408,49 @@ class Settings(BaseSettings): ) + +def test_cli_remove_empty_groups(capsys, monkeypatch): + class SubModel(BaseModel): + pass + + class Settings(BaseSettings): + sub_model: SubModel + + model_config = SettingsConfigDict(cli_parse_args=True) + + argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' + + with monkeypatch.context() as m: + m.setattr(sys, 'argv', ['example.py', '--help']) + + with pytest.raises(SystemExit): + Settings(_cli_avoid_json=False) + + assert ( + capsys.readouterr().out + == f"""usage: example.py [-h] [--sub_model JSON] + +{argparse_options_text}: + -h, --help show this help message and exit + +sub_model options: + --sub_model JSON set sub_model from JSON string +""" + ) + + with pytest.raises(SystemExit): + Settings(_cli_avoid_json=True) + + assert ( + capsys.readouterr().out + == f"""usage: example.py [-h] + +{argparse_options_text}: + -h, --help show this help message and exit +""" + ) + + def test_cli_hide_none_type(capsys, monkeypatch): class Settings(BaseSettings): v0: Optional[str] From 90403eebd9afb5e86103bf6918593613ddab6a18 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 6 Feb 2024 11:52:59 -0700 Subject: [PATCH 27/61] Lint fix. --- pydantic_settings/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 954701ee..c9452d98 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -871,7 +871,7 @@ def _add_fields_to_parser( added_args: list[str], arg_prefix: str, subcommand_prefix: str, - group: _ArgumentGroup | Tuple[str, str] | None, + group: _ArgumentGroup | tuple[str, str] | None, ) -> ArgumentParser: subparsers: _SubParsersAction[Any] | None = None for field_name, field_info in self._sort_arg_fields(model): From 61a4745c4211aaf0a426b2fcefabcbfa342be837 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 6 Feb 2024 12:20:32 -0700 Subject: [PATCH 28/61] Lint and formatting. --- pydantic_settings/sources.py | 16 +++++++++------- tests/test_settings.py | 1 - 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index c9452d98..af6dc7ab 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -871,7 +871,7 @@ def _add_fields_to_parser( added_args: list[str], arg_prefix: str, subcommand_prefix: str, - group: _ArgumentGroup | tuple[str, str] | None, + group: _ArgumentGroup | Dict[str, Any] | None, ) -> ArgumentParser: subparsers: _SubParsersAction[Any] | None = None for field_name, field_info in self._sort_arg_fields(model): @@ -927,16 +927,18 @@ def _add_fields_to_parser( arg_flag = '' if sub_models and kwargs.get('action') != 'append': - group_help_text = ( + model_group: _ArgumentGroup | None = None + model_group_kwargs: dict[str, Any] = {} + model_group_kwargs['title'] = f'{arg_name} options' + model_group_kwargs['description'] = ( sub_models[0].__doc__ if self.cli_use_class_docs_for_groups and len(sub_models) == 1 else field_info.description ) - model_group = (f'{arg_name} options', group_help_text) if not self.cli_avoid_json: added_args.append(arg_name) kwargs['help'] = f'set {arg_name} from JSON string' - model_group = parser.add_argument_group(*model_group) + model_group = parser.add_argument_group(**model_group_kwargs) model_group.add_argument(f'{arg_flag}{arg_name}', **kwargs) for model in sub_models: self._add_fields_to_parser( @@ -945,11 +947,11 @@ def _add_fields_to_parser( added_args=added_args, arg_prefix=f'{arg_prefix}{field_name}.', subcommand_prefix=subcommand_prefix, - group=model_group, + group=model_group if model_group else model_group_kwargs, ) elif group is not None: - if isinstance(group, tuple): - group = parser.add_argument_group(*group) + if not isinstance(group, _ArgumentGroup): + group = parser.add_argument_group(**group) added_args.append(arg_name) group.add_argument(f'{arg_flag}{arg_name}', **kwargs) else: diff --git a/tests/test_settings.py b/tests/test_settings.py index f6a80caa..b7919a33 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2408,7 +2408,6 @@ class Settings(BaseSettings): ) - def test_cli_remove_empty_groups(capsys, monkeypatch): class SubModel(BaseModel): pass From c38ed6a98307d85ba06c74fe3a0b02d43a2bca46 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 6 Feb 2024 12:22:51 -0700 Subject: [PATCH 29/61] Lint again. --- pydantic_settings/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index af6dc7ab..22009501 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -871,7 +871,7 @@ def _add_fields_to_parser( added_args: list[str], arg_prefix: str, subcommand_prefix: str, - group: _ArgumentGroup | Dict[str, Any] | None, + group: _ArgumentGroup | dict[str, Any] | None, ) -> ArgumentParser: subparsers: _SubParsersAction[Any] | None = None for field_name, field_info in self._sort_arg_fields(model): From cb9c1c36ff9973954ef04a880483eb796c3d0a7c Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 6 Feb 2024 17:56:14 -0700 Subject: [PATCH 30/61] Enum support and strip annotations. --- pydantic_settings/sources.py | 25 ++++++++++++++++++++++++- tests/test_settings.py | 20 ++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 22009501..ccb6a97c 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -8,6 +8,7 @@ from abc import ABC, abstractmethod from collections import deque from dataclasses import is_dataclass +from enum import Enum from pathlib import Path from types import FunctionType from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Sequence, Tuple, TypeVar, Union, cast @@ -27,7 +28,7 @@ if TYPE_CHECKING: from pydantic_settings.main import BaseSettings -from argparse import SUPPRESS, ArgumentParser, _ArgumentGroup, _SubParsersAction +from argparse import SUPPRESS, Action, ArgumentParser, Namespace, _ArgumentGroup, _SubParsersAction DotenvType = Union[Path, str, List[Union[Path, str]], Tuple[Union[Path, str], ...]] @@ -50,6 +51,22 @@ class _CliPositionalArg: CliPositionalArg = Annotated[T, _CliPositionalArg] +class _CliEnumAction(Action): + """ + CLI argparse action handler for enum types + """ + + def __init__(self, **kwargs: Any): + self._enum = kwargs.pop('type') + kwargs['choices'] = tuple(val.name for val in self._enum) + super().__init__(**kwargs) + + def __call__( + self, parser: ArgumentParser, namespace: Namespace, value: Any, option_string: str | None = None + ) -> None: + setattr(namespace, self.dest, self._enum[value]) + + class EnvNoneType(str): pass @@ -917,6 +934,10 @@ def _add_fields_to_parser( kwargs['action'] = 'append' if _annotation_contains_types(field_info.annotation, (dict, Mapping), is_include_origin=True): self._cli_dict_arg_names.append(kwargs['dest']) + elif lenient_issubclass(field_info.annotation, Enum): + kwargs['type'] = field_info.annotation + kwargs['action'] = _CliEnumAction + del kwargs['metavar'] arg_name = f'{arg_prefix.replace(subcommand_prefix, "", 1)}{field_name}' if _CliPositionalArg in field_info.metadata: @@ -972,6 +993,8 @@ def _metavar_format_list(self, args: list[str]) -> str: def _metavar_format_recurse(self, obj: Any) -> str: """Pretty metavar representation of a type. Adapts logic from `pydantic._repr.display_as_type`.""" + while get_origin(obj) == Annotated: + obj = get_args(obj)[0] if isinstance(obj, FunctionType): return obj.__name__ elif obj is ...: diff --git a/tests/test_settings.py b/tests/test_settings.py index b7919a33..0fcd62fc 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -4,6 +4,7 @@ import typing import uuid from datetime import datetime, timezone +from enum import IntEnum from pathlib import Path from typing import Any, Callable, Dict, Generic, List, Optional, Set, Tuple, Type, TypeVar, Union @@ -14,6 +15,7 @@ AliasChoices, AliasPath, BaseModel, + DirectoryPath, Field, HttpUrl, Json, @@ -2302,6 +2304,22 @@ class Cfg(BaseSettings): assert cfg.model_dump() == {'child': {'name': 'new name a', 'diff_a': 'new diff a'}} +def test_cli_enum(): + class Fruit(IntEnum): + apple = 0 + banna = 1 + orange = 2 + + class Cfg(BaseSettings): + fruit: Fruit + + cfg = Cfg(_cli_parse_args=['--fruit', 'orange']) + assert cfg.model_dump() == {'fruit': Fruit.orange} + + with pytest.raises(SystemExit): + Cfg(_cli_parse_args=['--fruit', 'lettuce']) + + def test_cli_annotation_exceptions(monkeypatch): class SubCmdAlt(BaseModel): pass @@ -2586,6 +2604,8 @@ class Settings(BaseSettings): (Union[SimpleSettings, SettingWithIgnoreEmpty], 'JSON'), (Union[SimpleSettings, str, SettingWithIgnoreEmpty], '{JSON,str}'), (Union[str, SimpleSettings, SettingWithIgnoreEmpty], '{str,JSON}'), + (Annotated[SimpleSettings, 'annotation'], 'JSON'), + (DirectoryPath, 'Path'), ], ) def test_cli_metavar_format(value, expected): From a984f32c963e44be6c77b5bd422af8ab33268a3e Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sat, 17 Feb 2024 12:24:15 -0700 Subject: [PATCH 31/61] Update pydantic_settings/main.py Co-authored-by: Samuel Colvin --- pydantic_settings/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 76fe0227..8add6c04 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -70,7 +70,7 @@ class BaseSettings(BaseModel): Otherwse, defaults to sys.argv[0]. _cli_parse_args: The list of CLI arguments to parse. Defaults to None. If set to `True`, defaults to sys.argv[1:]. - _cli_hide_none_type: Hide NoneType values in CLI help text. Defaults to `False`. + _cli_hide_none_type: Hide `None` values in CLI help text. Defaults to `False`. _cli_avoid_json: Avoid complex JSON objects in CLI help text. Defaults to `False`. _cli_enforce_required: Enforce required fields at the CLI. Defaults to `False`. _cli_use_class_docs_for_groups: Use class docstrings in CLI group help text instead of field descriptions. From 84709307042b0ec6719f5a047b653a69e15b4301 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 4 Mar 2024 00:36:24 -0700 Subject: [PATCH 32/61] Initial updates for external parser support. --- docs/index.md | 235 ++++++-------------- pydantic_settings/main.py | 41 +++- pydantic_settings/sources.py | 420 +++++++++++++++++++++++++---------- tests/test_settings.py | 239 +++++++++++++++++++- 4 files changed, 637 insertions(+), 298 deletions(-) diff --git a/docs/index.md b/docs/index.md index b8c809bf..699f605b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -563,15 +563,6 @@ defined in argparse. In the above example, we parsed our args from the `['--help Lastly, a CLI settings source is always [**the topmost source**](#field-value-priority), and does not support [changing its priority](#changing-priority). -#### Enable CLI Argument Parsing - -`cli_parse_args: Optional[Union[List[str], bool]] = None` - -* Default = `None` -* If `True`, parse from `sys.argv[1:]` -* If `List[str]`, parse from `List[str]` -* If `False` or `None`, do not parse CLI arguments - #### Lists CLI argument parsing of lists supports intermixing of any of the below three styles: @@ -641,7 +632,7 @@ subcommands must be a valid type derived from the pydantic `BaseModel` class. subcommands](https://docs.python.org/3/library/argparse.html#sub-commands). ```py test="skip" -from pydantic import BaseModel +from pydantic import BaseModel, Field from pydantic_settings import ( BaseSettings, @@ -650,54 +641,54 @@ from pydantic_settings import ( ) -class FooPlugin(BaseModel, use_attribute_docstrings=True): +class FooPlugin(BaseModel): """git-plugins-foo - Extra deep foo plugin command""" - my_feature: bool = False - """Enable my feature on foo plugin""" + my_feature: bool = Field( + default=False, description='Enable my feature on foo plugin' + ) -class BarPlugin(BaseModel, use_attribute_docstrings=True): +class BarPlugin(BaseModel): """git-plugins-bar - Extra deep bar plugin command""" - my_feature: bool = False - """Enable my feature on bar plugin""" + my_feature: bool = Field( + default=False, description='Enable my feature on bar plugin' + ) -class Plugins(BaseModel, use_attribute_docstrings=True): +class Plugins(BaseModel): """git-plugins - Fake plugins for GIT""" - foo: CliSubCommand[FooPlugin] - """Foo is fake plugin""" + foo: CliSubCommand[FooPlugin] = Field(description='Foo is fake plugin') - bar: CliSubCommand[BarPlugin] - """Bar is also a fake plugin""" + bar: CliSubCommand[BarPlugin] = Field(description='Bar is also a fake plugin') -class Clone(BaseModel, use_attribute_docstrings=True): +class Clone(BaseModel): """git-clone - Clone a repository into a new directory""" - repository: CliPositionalArg[str] - """The repository to clone""" + repository: CliPositionalArg[str] = Field(description='The repository to clone') - directory: CliPositionalArg[str] - """The directory to clone into""" + directory: CliPositionalArg[str] = Field(description='The directory to clone into') - local: bool = False - """When the resposity to clone from is on a local machine, bypass ...""" + local: bool = Field( + default=False, + description='When the resposity to clone from is on a local machine, bypass ...', + ) -class Git(BaseSettings, use_attribute_docstrings=True): +class Git(BaseSettings, cli_prog_name='git'): """git - The stupid content tracker""" - clone: CliSubCommand[Clone] - """Clone a repository into a new directory""" + clone: CliSubCommand[Clone] = Field( + description='Clone a repository into a new directory' + ) - plugins: CliSubCommand[Plugins] - """Fake GIT plugion commands""" + plugins: CliSubCommand[Plugins] = Field(description='Fake GIT plugin commands') -Git(_cli_prog_name='git', _cli_parse_args=['--help']) +Git(_cli_parse_args=['--help']) """ usage: git [-h] {clone,plugins} ... @@ -709,11 +700,11 @@ options: subcommands: {clone,plugins} clone Clone a repository into a new directory - plugins Fake GIT plugion commands + plugins Fake GIT plugin commands """ -Git(_cli_prog_name='git', _cli_parse_args=['clone', '--help']) +Git(_cli_parse_args=['clone', '--help']) """ usage: git clone [-h] [--local bool] [--shared bool] REPOSITORY DIRECTORY @@ -729,7 +720,7 @@ options: """ -Git(_cli_prog_name='git', _cli_parse_args=['plugins', 'bar', '--help']) +Git(_cli_parse_args=['plugins', 'bar', '--help']) """ usage: git plugins bar [-h] [--my_feature bool] @@ -753,64 +744,49 @@ sources provides the required value. However, if your use case [aligns more with #2](#command-line-support), using Pydantic models to define CLIs, you will likely want required fields to be _strictly required at the CLI_. We can enable this behavior by using the -`cli_enforce_required` flag as shown below. +`cli_enforce_required`. -```py test="skip" +```py import os +from pydantic import Field + from pydantic_settings import BaseSettings -class Settings(BaseSettings, use_attribute_docstrings=True): - my_required_field: str - """a top level required field""" +class Settings(BaseSettings, cli_enforce_required=True): + my_required_field: str = Field(description='a top level required field') os.environ['MY_REQUIRED_FIELD'] = 'hello from environment' -print(Settings(_cli_parse_args=[], _cli_enforce_required=False).model_dump()) -""" -{'my_required_field': 'hello from environment'} -""" - -print(Settings(_cli_parse_args=[], _cli_enforce_required=True).model_dump()) -""" +try: + print(Settings(_cli_parse_args=[]).model_dump()) + """ usage: example.py [-h] --my_required_field str example.py: error: the following arguments are required: --my_required_field """ +except SystemExit: + pass ``` -`cli_enforce_required: Optional[bool] = None` - -* Default = `None` -* If `True`, strictly enforce required fields at the CLI -* If `False` or `None`, do not enforce required fields at the CLI - #### Hide None Type Values -Hide `None` values from the CLI help text. +Hide `None` values from the CLI help text by enabling `cli_hide_none_type`. -```py test="skip" +```py from typing import Optional -from pydantic_settings import BaseSettings - +from pydantic import Field -class Settings(BaseSettings): - v0: Optional[str] - """the top level v0 option""" +from pydantic_settings import BaseSettings, CliSettingsSource -Settings(_cli_parse_args=['--help'], _cli_hide_none_type=False) -""" -usage: example.py [-h] [--v0 {str,null}] +class Settings(BaseSettings, cli_hide_none_type=True): + v0: Optional[str] = Field(description='the top level v0 option') -options: - -h, --help show this help message and exit - --v0 {str,null} the top level v0 option -""" -Settings(_cli_parse_args=['--help'], _cli_hide_none_type=True) +print(CliSettingsSource(Settings, cli_prog_name='example.py').root_parser.format_help()) """ usage: example.py [-h] [--v0 str] @@ -820,47 +796,27 @@ options: """ ``` -`cli_hide_none_type: Optional[bool] = None` - -* Default = `None` -* If `True`, hide `None` type values from CLI help text -* If `False` or `None`, show `None` type values in CLI help text - #### Avoid Adding JSON CLI Options -Avoid adding complex fields that result in JSON strings at the CLI. - -```py test="skip" -from pydantic import BaseModel +Avoid adding complex fields that result in JSON strings at the CLI by enabling `cli_avoid_json`. -from pydantic_settings import BaseSettings - - -class SubModel(BaseModel, use_attribute_docstrings=True): - v1: int - """the sub model v1 option""" +```py +from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings, CliSettingsSource -class Settings(BaseSettings, use_attribute_docstrings=True): - sub_model: SubModel - """The help summary for SubModel related options""" +class SubModel(BaseModel): + v1: int = Field(description='the sub model v1 option') -Settings(_cli_parse_args=['--help'], _cli_avoid_json=False) -""" -usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] -options: - -h, --help show this help message and exit +class Settings(BaseSettings, cli_avoid_json=True): + sub_model: SubModel = Field( + description='The help summary for SubModel related options' + ) -sub_model options: - The help summary for SubModel related options - --sub_model JSON set sub_model from JSON string - --sub_model.v1 int the sub model v1 option -""" - -Settings(_cli_parse_args=['--help'], _cli_avoid_json=True) +print(CliSettingsSource(Settings, cli_prog_name='example.py').root_parser.format_help()) """ usage: example.py [-h] [--sub_model.v1 int] @@ -874,12 +830,6 @@ sub_model options: """ ``` -`cli_avoid_json: Optional[bool] = None` - -* Default = `None` -* If `True`, avoid adding complex JSON fields to CLI -* If `False` or `None`, add complex JSON fields to CLI - #### Use Class Docstring for Group Help Text By default, when populating the group help text for nested models it will pull from the field descriptions. @@ -889,27 +839,25 @@ Alternatively, we can also configure CLI settings to pull from the class docstri If the field is a union of nested models the group help text will always be pulled from the field description; even if `cli_use_class_docs_for_groups` is set to `True`. -```py test="skip" -from pydantic import BaseModel +```py +from pydantic import BaseModel, Field -from pydantic_settings import BaseSettings +from pydantic_settings import BaseSettings, CliSettingsSource -class SubModel(BaseModel, use_attribute_docstrings=True): - """The help text from the class docstring""" +class SubModel(BaseModel): + """The help text from the class docstring.""" - v1: int - """the sub model v1 option""" + v1: int = Field(description='the sub model v1 option') -class Settings(BaseSettings, use_attribute_docstrings=True): +class Settings(BaseSettings, cli_use_class_docs_for_groups=True): """My application help text.""" - sub_model: SubModel - """The help text from the field description""" + sub_model: SubModel = Field(description='The help text from the field description') -Settings(_cli_parse_args=['--help'], _cli_use_class_docs_for_groups=False) +print(CliSettingsSource(Settings, cli_prog_name='example.py').root_parser.format_help()) """ usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] @@ -919,72 +867,35 @@ options: -h, --help show this help message and exit sub_model options: - The help text from the field description - - --sub_model JSON set sub_model from JSON string - --sub_model.v1 int the sub model v1 option -""" - - -Settings(_cli_parse_args=['--help'], _cli_use_class_docs_for_groups=True) -""" -usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] - -My application help text. - -options: - -h, --help show this help message and exit - -sub_model options: - The help text from the class docstring + The help text from the class docstring. --sub_model JSON set sub_model from JSON string --sub_model.v1 int the sub model v1 option """ ``` -`cli_use_class_docs_for_groups: Optional[bool] = None` - -* Default = `None` -* If `True`, use class docstrings for CLI group help text -* If `False` or `None`, use field description for CLI group help text - #### Change the Displayed Program Name -Change the default program name displayed in the help text usage. By default, it will derive the name of the currently +Change the default program name displayed in the help text usage by setting `cli_prog_name`. By default, it will derive the name of the currently executing program from `sys.argv[0]`, just like argparse. -```py test="skip" -from pydantic_settings import BaseSettings +```py +from pydantic_settings import BaseSettings, CliSettingsSource -class Settings(BaseSettings): +class Settings(BaseSettings, cli_prog_name='appdantic'): pass -Settings(_cli_parse_args=['--help']) -""" -usage: example.py [-h] - -options: - -h, --help show this help message and exit -""" - -Settings(_cli_parse_args=['--help'], _cli_prog_name='appdantic?') +print(CliSettingsSource(Settings).root_parser.format_help()) """ -usage: appdantic? [-h] +usage: appdantic [-h] options: -h, --help show this help message and exit """ ``` -`cli_prog_name: Optional[str] = None` - -* Default = `None` -* If `str`, use `str` as program name -* If `None`, use `sys.argv[0]` as program name - ## Secrets Placing secret values in files is a common pattern to provide sensitive configuration to an application. diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 8add6c04..01e5c159 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -30,10 +30,12 @@ class SettingsConfigDict(ConfigDict, total=False): env_parse_none_str: str | None cli_prog_name: str | None cli_parse_args: bool | list[str] | None + cli_settings_source: CliSettingsSource[Any] | None cli_hide_none_type: bool cli_avoid_json: bool cli_enforce_required: bool cli_use_class_docs_for_groups: bool + cli_prefix: str secrets_dir: str | Path | None @@ -70,11 +72,13 @@ class BaseSettings(BaseModel): Otherwse, defaults to sys.argv[0]. _cli_parse_args: The list of CLI arguments to parse. Defaults to None. If set to `True`, defaults to sys.argv[1:]. + _cli_settings_source: Override the default CLI settings source with a user defined instance. Defaults to None. _cli_hide_none_type: Hide `None` values in CLI help text. Defaults to `False`. _cli_avoid_json: Avoid complex JSON objects in CLI help text. Defaults to `False`. _cli_enforce_required: Enforce required fields at the CLI. Defaults to `False`. _cli_use_class_docs_for_groups: Use class docstrings in CLI group help text instead of field descriptions. Defaults to `False`. + _cli_prefix: The root parser command line arguments prefix. Defaults to "". _secrets_dir: The secret files directory. Defaults to `None`. """ @@ -89,10 +93,12 @@ def __init__( _env_parse_none_str: str | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | None = None, + _cli_settings_source: CliSettingsSource[Any] | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, + _cli_prefix: str | None = None, _secrets_dir: str | Path | None = None, **values: Any, ) -> None: @@ -109,10 +115,12 @@ def __init__( _env_parse_none_str=_env_parse_none_str, _cli_prog_name=_cli_prog_name, _cli_parse_args=_cli_parse_args, + _cli_settings_source=_cli_settings_source, _cli_hide_none_type=_cli_hide_none_type, _cli_avoid_json=_cli_avoid_json, _cli_enforce_required=_cli_enforce_required, _cli_use_class_docs_for_groups=_cli_use_class_docs_for_groups, + _cli_prefix=_cli_prefix, _secrets_dir=_secrets_dir, ) ) @@ -153,10 +161,12 @@ def _settings_build_values( _env_parse_none_str: str | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | None = None, + _cli_settings_source: CliSettingsSource[Any] | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, + _cli_prefix: str | None = None, _secrets_dir: str | Path | None = None, ) -> dict[str, Any]: # Determine settings config values @@ -180,6 +190,9 @@ def _settings_build_values( cli_prog_name = _cli_prog_name if _cli_prog_name is not None else self.model_config.get('cli_prog_name') cli_parse_args = _cli_parse_args if _cli_parse_args is not None else self.model_config.get('cli_parse_args') + cli_settings_source = ( + _cli_settings_source if _cli_settings_source is not None else self.model_config.get('cli_settings_source') + ) cli_hide_none_type = ( _cli_hide_none_type if _cli_hide_none_type is not None else self.model_config.get('cli_hide_none_type') ) @@ -194,20 +207,26 @@ def _settings_build_values( if _cli_use_class_docs_for_groups is not None else self.model_config.get('cli_use_class_docs_for_groups') ) + cli_prefix = _cli_prefix if _cli_prefix is not None else self.model_config.get('cli_prefix') secrets_dir = _secrets_dir if _secrets_dir is not None else self.model_config.get('secrets_dir') # Configure built-in sources init_settings = InitSettingsSource(self.__class__, init_kwargs=init_kwargs) - cli_settings = CliSettingsSource( - self.__class__, - cli_prog_name=cli_prog_name, - cli_parse_args=cli_parse_args, - cli_parse_none_str=env_parse_none_str, - cli_hide_none_type=cli_hide_none_type, - cli_avoid_json=cli_avoid_json, - cli_enforce_required=cli_enforce_required, - cli_use_class_docs_for_groups=cli_use_class_docs_for_groups, + cli_settings = ( + CliSettingsSource( + self.__class__, + cli_prog_name=cli_prog_name, + cli_parse_args=cli_parse_args, + cli_parse_none_str=env_parse_none_str, + cli_hide_none_type=cli_hide_none_type, + cli_avoid_json=cli_avoid_json, + cli_enforce_required=cli_enforce_required, + cli_use_class_docs_for_groups=cli_use_class_docs_for_groups, + cli_prefix=cli_prefix, + ) + if cli_settings_source is None + else cli_settings_source ) env_settings = EnvSettingsSource( self.__class__, @@ -239,7 +258,7 @@ def _settings_build_values( dotenv_settings=dotenv_settings, file_secret_settings=file_secret_settings, ) - if cli_parse_args: + if cli_parse_args or cli_settings_source: sources = (cli_settings,) + sources if sources: return deep_update(*reversed([source() for source in sources])) @@ -261,10 +280,12 @@ def _settings_build_values( env_parse_none_str=None, cli_prog_name=None, cli_parse_args=None, + cli_settings_source=None, cli_hide_none_type=False, cli_avoid_json=False, cli_enforce_required=False, cli_use_class_docs_for_groups=False, + cli_prefix='', secrets_dir=None, protected_namespaces=('model_', 'settings_'), ) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index ccb6a97c..f4af0f14 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -11,7 +11,21 @@ from enum import Enum from pathlib import Path from types import FunctionType -from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Sequence, Tuple, TypeVar, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generic, + List, + Mapping, + Sequence, + Tuple, + TypeVar, + Union, + cast, + overload, +) import pydantic._internal._repr import pydantic.v1.utils @@ -28,7 +42,7 @@ if TYPE_CHECKING: from pydantic_settings.main import BaseSettings -from argparse import SUPPRESS, Action, ArgumentParser, Namespace, _ArgumentGroup, _SubParsersAction +from argparse import SUPPRESS, ArgumentParser, HelpFormatter, Namespace, _SubParsersAction DotenvType = Union[Path, str, List[Union[Path, str]], Tuple[Union[Path, str], ...]] @@ -51,22 +65,6 @@ class _CliPositionalArg: CliPositionalArg = Annotated[T, _CliPositionalArg] -class _CliEnumAction(Action): - """ - CLI argparse action handler for enum types - """ - - def __init__(self, **kwargs: Any): - self._enum = kwargs.pop('type') - kwargs['choices'] = tuple(val.name for val in self._enum) - super().__init__(**kwargs) - - def __call__( - self, parser: ArgumentParser, namespace: Namespace, value: Any, option_string: str | None = None - ) -> None: - setattr(namespace, self.dest, self._enum[value]) - - class EnvNoneType(str): pass @@ -715,9 +713,35 @@ def __repr__(self) -> str: ) -class CliSettingsSource(EnvSettingsSource): +class CliSettingsSource(EnvSettingsSource, Generic[T]): """ Source class for loading settings values from CLI. + + The root parser to connect the CLI settings source to. This will add fields from the `settings_cls` to the root parser as + arguments and associate the internal CLI settings source parsing logic with the root parser. + + Note: + The parser methods must support the same attributes as their `argparse` library counterparts. + + Args: + cli_prog_name: The CLI program name to display in help text. Defaults to `None` if cli_parse_args is `None`. + Otherwse, defaults to sys.argv[0]. + cli_parse_args: The list of CLI arguments to parse. Defaults to None. + If set to `True`, defaults to sys.argv[1:]. + cli_settings_source: Override the default CLI settings source with a user defined instance. Defaults to None. + cli_hide_none_type: Hide `None` values in CLI help text. Defaults to `False`. + cli_avoid_json: Avoid complex JSON objects in CLI help text. Defaults to `False`. + cli_enforce_required: Enforce required fields at the CLI. Defaults to `False`. + cli_use_class_docs_for_groups: Use class docstrings in CLI group help text instead of field descriptions. + Defaults to `False`. + cli_prefix: Prefix for command line arguments added under the root parser. Defaults to "". + root_parser: The root parser object. + parse_args_method: The root parser parse args method. Defaults to `argparse.ArgumentParser.parse_args`. + add_argument_method: The root parser add argument method. Defaults to `argparse.ArgumentParser.add_argument`. + add_argument_group_method: The root parser add argument group method. Defaults to `argparse.ArgumentParser.add_argument_group`. + add_parser_method: The root parser add new parser (sub-command) method. Defaults to `argparse._SubParsersAction.add_parser`. + add_subparsers_method: The root parser add subparsers (sub-commands) method. Defaults to `argparse.ArgumentParser.add_subparsers`. + formatter_class: A class for customizing the root parser help text. Defaults to `argparse.HelpFormatter`. """ def __init__( @@ -730,14 +754,18 @@ def __init__( cli_avoid_json: bool | None = None, cli_enforce_required: bool | None = None, cli_use_class_docs_for_groups: bool | None = None, + cli_prefix: str | None = None, + root_parser: Any = None, + parse_args_method: Callable[..., Any] | None = ArgumentParser.parse_args, + add_argument_method: Callable[..., Any] | None = ArgumentParser.add_argument, + add_argument_group_method: Callable[..., Any] | None = ArgumentParser.add_argument_group, + add_parser_method: Callable[..., Any] | None = _SubParsersAction.add_parser, + add_subparsers_method: Callable[..., Any] | None = ArgumentParser.add_subparsers, + formatter_class: Any = HelpFormatter, ) -> None: - self.cli_prog_name = sys.argv[0] if cli_prog_name is None else cli_prog_name - self.cli_parse_args = cli_parse_args - if self.cli_parse_args not in (None, False): - if self.cli_parse_args is True: - self.cli_parse_args = sys.argv[1:] - elif not isinstance(self.cli_parse_args, list): - raise SettingsError(f'cli_parse_args must be List[str], recieved {type(self.cli_parse_args)}') + self.cli_prog_name = ( + cli_prog_name if cli_prog_name is not None else settings_cls.model_config.get('cli_prog_name', sys.argv[0]) + ) self.cli_hide_none_type = ( cli_hide_none_type if cli_hide_none_type is not None @@ -758,37 +786,98 @@ def __init__( if cli_use_class_docs_for_groups is not None else settings_cls.model_config.get('cli_use_class_docs_for_groups', False) ) - super().__init__(settings_cls, env_nested_delimiter='.', env_parse_none_str=cli_parse_none_str) + self.cli_prefix = cli_prefix if cli_prefix is not None else settings_cls.model_config.get('cli_prefix', '') + if self.cli_prefix: + if cli_prefix.startswith('.') or cli_prefix.endswith('.') or not cli_prefix.replace('.', '').isidentifier(): # type: ignore + raise SettingsError(f'CLI settings source prefix is invalid: {cli_prefix}') + self.cli_prefix += '.' + + super().__init__( + settings_cls, env_nested_delimiter='.', env_parse_none_str=cli_parse_none_str, env_prefix=self.cli_prefix + ) + + root_parser = ( + ArgumentParser(prog=self.cli_prog_name, description=settings_cls.__doc__) + if root_parser is None + else root_parser + ) + self._connect_root_parser( + root_parser=root_parser, + parse_args_method=parse_args_method, + add_argument_method=add_argument_method, + add_argument_group_method=add_argument_group_method, + add_parser_method=add_parser_method, + add_subparsers_method=add_subparsers_method, + formatter_class=formatter_class, + ) + if cli_parse_args not in (None, False): + if cli_parse_args is True: + cli_parse_args = sys.argv[1:] + elif not isinstance(cli_parse_args, list): + raise SettingsError(f'cli_parse_args must be List[str], recieved {type(cli_parse_args)}') + self._load_env_vars(parsed_args=self._parse_args(self.root_parser, cli_parse_args)) + + @overload + def __call__(self) -> dict[str, Any]: + ... + + @overload + def __call__(self, *, args: list[str]) -> dict[str, Any]: + ... + + @overload + def __call__(self, *, parsed_args: Namespace | dict[str, list[str] | str]) -> dict[str, Any]: + ... + + def __call__( + self, *, args: list[str] | None = None, parsed_args: Namespace | dict[str, list[str] | str] | None = None + ) -> dict[str, Any] | CliSettingsSource[T]: + """ + Loads parsed command line arguments into the CLI settings source. If parsed args are `None` + (the default) will return the CLI settings source vars dicitionary. + + Note: + The parsed args must be in `argparse.Namespace` or vars dictionary (e.g., vars(argparse.Namespace)) + format. + + Args: + args: + parsed_args: The parsed args to load. + + Returns: + CliSettingsSource: The object instance itself. + """ + if args is not None and parsed_args is not None: + raise SettingsError('args and parsed_args are mutually exclusive') + elif args is not None: + return self._load_env_vars(parsed_args=self._parse_args(self.root_parser, args)) + elif parsed_args is not None: + return self._load_env_vars(parsed_args=parsed_args) + else: + return super().__call__() + + @overload def _load_env_vars(self) -> Mapping[str, str | None]: - if self.cli_parse_args in (None, False): + ... + + @overload + def _load_env_vars(self, *, parsed_args: Namespace | dict[str, list[str] | str]) -> CliSettingsSource[T]: + ... + + def _load_env_vars( + self, *, parsed_args: Namespace | dict[str, list[str] | str] | None = None + ) -> Mapping[str, str | None] | CliSettingsSource[T]: + if parsed_args is None: return {} - self._cli_dict_arg_names: list[str] = [] - self._cli_subcommands: dict[str, list[str]] = {} - parser: ArgumentParser = self._add_fields_to_parser( - ArgumentParser(prog=self.cli_prog_name, description=self.settings_cls.__doc__), - model=self.settings_cls, - added_args=[], - arg_prefix='', - subcommand_prefix='', - group=None, - ) + if isinstance(parsed_args, Namespace): + parsed_args = vars(parsed_args) - parsed_args: dict[str, list[str] | str] = vars(parser.parse_args(self.cli_parse_args)) # type: ignore selected_subcommands: list[str] = [] for field_name, val in parsed_args.items(): if isinstance(val, list): - merge_list = [] - for sub_val in val: - if sub_val.startswith('[') and sub_val.endswith(']'): - sub_val = sub_val[1:-1] - merge_list.append(sub_val) - parsed_args[field_name] = ( - f'[{",".join(merge_list)}]' - if field_name not in self._cli_dict_arg_names - else self._merge_json_key_val_list_str(f'[{",".join(merge_list)}]') - ) + parsed_args[field_name] = self._merge_parsed_list(val, field_name) elif field_name.endswith(':subcommand') and val is not None: selected_subcommands.append(field_name.split(':')[0] + val) @@ -803,43 +892,86 @@ def _load_env_vars(self) -> Mapping[str, str | None]: if not any(field_name for field_name in parsed_args.keys() if f'{last_selected_subcommand}.' in field_name): parsed_args[last_selected_subcommand] = '{}' - return parse_env_vars( + self.env_vars = parse_env_vars( parsed_args, self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str # type: ignore ) - def _merge_json_key_val_list_str(self, key_val_list_str: str) -> str: - orig_key_val_list_str, key_val_list_str = key_val_list_str, key_val_list_str[1:-1] - key_val_dict: dict[str, str] = {} - obj_count = 0 + return self + + def _merge_parsed_list(self, parsed_list: list[str], field_name: str) -> str: try: - while key_val_list_str: - assert obj_count == 0 - for i in range(len(key_val_list_str)): - if key_val_list_str[i] == '{': - obj_count += 1 - elif key_val_list_str[i] == '}': - obj_count -= 1 - if obj_count == 0: - key_val_dict.update(json.loads(key_val_list_str[: i + 1])) - key_val_list_str = key_val_list_str[i + 1 :].lstrip(',') - break - elif obj_count == 0: - val, quote_count = '', 0 - key, key_val_list_str = key_val_list_str.split('=', 1) - for i in range(len(key_val_list_str)): - if key_val_list_str[i] in ('"', "'"): - quote_count += 1 - if key_val_list_str[i] == ',' and quote_count % 2 == 0: - val, key_val_list_str = key_val_list_str[:i], key_val_list_str[i:].lstrip(',') - break - if not val: - val, key_val_list_str = key_val_list_str, '' - key_val_dict.update({key.strip('\'"'): val.strip('\'"')}) - break - except Exception: - raise SettingsError(f'Parsing error encountered on JSON object {orig_key_val_list_str}') - - return json.dumps(key_val_dict) + merged_list: list[str] = [] + is_last_consumed_a_value = False + is_dict_list = field_name in self._cli_dict_arg_names + for val in parsed_list: + if val.startswith('[') and val.endswith(']'): + val = val[1:-1] + while val: + if val.startswith(','): + val = self._consume_comma(val, merged_list, is_last_consumed_a_value) + is_last_consumed_a_value = False + else: + if val.startswith('{') or val.startswith('['): + val = self._consume_object_or_array(val, merged_list) + else: + val = self._consume_string_or_number(val, merged_list, is_dict_list) + is_last_consumed_a_value = True + if not is_last_consumed_a_value: + val = self._consume_comma(val, merged_list, is_last_consumed_a_value) + + if not is_dict_list: + return f'[{",".join(merged_list)}]' + else: + merged_dict: dict[str, str] = {} + for item in merged_list: + merged_dict.update(json.loads(item)) + return json.dumps(merged_dict) + except Exception as e: + raise SettingsError(f'Parsing error encountered for {field_name}: {e}') + + def _consume_comma(self, item: str, merged_list: list[str], is_last_consumed_a_value: bool) -> str: + if not is_last_consumed_a_value: + merged_list.append('""') + return item[1:] + + def _consume_object_or_array(self, item: str, merged_list: list[str]) -> str: + count = 1 + close_delim = '}' if item.startswith('{') else ']' + for consumed in range(1, len(item)): + if item[consumed] in ('{', '['): + count += 1 + elif item[consumed] in ('}', ']'): + count -= 1 + if item[consumed] == close_delim and count == 0: + merged_list.append(item[: consumed + 1]) + return item[consumed + 1 :] + raise SettingsError(f'Missing end delimiter "{close_delim}"') + + def _consume_string_or_number(self, item: str, merged_list: list[str], is_dict_list: bool) -> str: + consumed = 0 + is_find_end_quote = False + while consumed < len(item): + if item[consumed] == '"' and (consumed == 0 or item[consumed - 1] != '\\'): + is_find_end_quote = not is_find_end_quote + if not is_find_end_quote and item[consumed] == ',': + break + consumed += 1 + if is_find_end_quote: + raise SettingsError('Mismatched quotes') + val_string = item[:consumed].strip() + if not is_dict_list: + try: + float(val_string) + except ValueError: + if val_string == self.env_parse_none_str: + val_string = 'null' + if val_string not in ('true', 'false', 'null') and not val_string.startswith('"'): + val_string = f'"{val_string}"' + merged_list.append(val_string) + else: + key, val = (kv.strip('"') for kv in val_string.split('=', 1)) + merged_list.append(json.dumps({key: val})) + return item[consumed:] def _get_sub_models(self, model: type[BaseModel], field_name: str, field_info: FieldInfo) -> list[type[BaseModel]]: field_types: tuple[Any, ...] = ( @@ -881,38 +1013,86 @@ def _sort_arg_fields(self, model: type[BaseModel]) -> list[tuple[str, FieldInfo] optional_args.append((field_name, field_info)) return positional_args + subcommand_args + optional_args - def _add_fields_to_parser( + @property + def root_parser(self) -> T: + """The connected root parser instance.""" + return self._root_parser + + def _connect_parser_method( + self, parser_method: Callable[..., Any] | None, method_name: str, *args: Any, **kwargs: Any + ) -> Callable[..., Any]: + if parser_method: + return parser_method + + def none_parser_method(*args: Any, **kwargs: Any) -> Any: + raise SettingsError( + f'cannot connect CLI settings source root parser: {method_name} is set to `None` but is needed for connecting' + ) + + return none_parser_method + + def _connect_root_parser( self, - parser: ArgumentParser, + root_parser: T, + parse_args_method: Callable[..., Any] | None = ArgumentParser.parse_args, + add_argument_method: Callable[..., Any] | None = ArgumentParser.add_argument, + add_argument_group_method: Callable[..., Any] | None = ArgumentParser.add_argument_group, + add_parser_method: Callable[..., Any] | None = _SubParsersAction.add_parser, + add_subparsers_method: Callable[..., Any] | None = ArgumentParser.add_subparsers, + formatter_class: Any = HelpFormatter, + ) -> None: + self._root_parser = root_parser + self._parse_args = self._connect_parser_method(parse_args_method, 'parsed_args_method') + self._add_argument = self._connect_parser_method(add_argument_method, 'add_argument_method') + self._add_argument_group = self._connect_parser_method(add_argument_group_method, 'add_argument_group_method') + self._add_parser = self._connect_parser_method(add_parser_method, 'add_parser_method') + self._add_subparsers = self._connect_parser_method(add_subparsers_method, 'add_subparsers_method') + self._formatter_class = formatter_class + self._cli_dict_arg_names: list[str] = [] + self._cli_subcommands: dict[str, list[str]] = {} + self._add_parser_args( + parser=self.root_parser, + model=self.settings_cls, + added_args=[], + arg_prefix=self.env_prefix, + subcommand_prefix=self.env_prefix, + group=None, + ) + + def _add_parser_args( + self, + parser: Any, model: type[BaseModel], added_args: list[str], arg_prefix: str, subcommand_prefix: str, - group: _ArgumentGroup | dict[str, Any] | None, + group: Any, ) -> ArgumentParser: - subparsers: _SubParsersAction[Any] | None = None + subparsers: Any = None for field_name, field_info in self._sort_arg_fields(model): sub_models: list[type[BaseModel]] = self._get_sub_models(model, field_name, field_info) if _CliSubCommand in field_info.metadata: if subparsers is None: - subparsers = parser.add_subparsers( - title='subcommands', dest=f'{arg_prefix}:subcommand', required=self.cli_enforce_required + subparsers = self._add_subparsers( + parser, title='subcommands', dest=f'{arg_prefix}:subcommand', required=self.cli_enforce_required ) self._cli_subcommands[f'{arg_prefix}:subcommand'] = [f'{arg_prefix}{field_name}'] else: self._cli_subcommands[f'{arg_prefix}:subcommand'].append(f'{arg_prefix}{field_name}') - metavar = ','.join(self._cli_subcommands[f'{arg_prefix}:subcommand']) - subparsers.metavar = f'{{{metavar}}}' + if hasattr(subparsers, 'metavar'): + metavar = ','.join(self._cli_subcommands[f'{arg_prefix}:subcommand']) + subparsers.metavar = f'{{{metavar}}}' model = sub_models[0] - self._add_fields_to_parser( - subparsers.add_parser( + self._add_parser_args( + parser=self._add_parser( + subparsers, field_name, help=field_info.description, - formatter_class=parser.formatter_class, + formatter_class=self._formatter_class, description=model.__doc__, ), - model, + model=model, added_args=[], arg_prefix=f'{arg_prefix}{field_name}.', subcommand_prefix=f'{subcommand_prefix}{field_name}.', @@ -929,17 +1109,21 @@ def _add_fields_to_parser( if kwargs['dest'] in added_args: continue if _annotation_contains_types( - field_info.annotation, (list, set, dict, Sequence, Mapping), is_include_origin=True + _strip_annotated(field_info.annotation), + (list, set, dict, Sequence, Mapping), + is_include_origin=True, ): kwargs['action'] = 'append' - if _annotation_contains_types(field_info.annotation, (dict, Mapping), is_include_origin=True): + if _annotation_contains_types( + _strip_annotated(field_info.annotation), (dict, Mapping), is_include_origin=True + ): self._cli_dict_arg_names.append(kwargs['dest']) - elif lenient_issubclass(field_info.annotation, Enum): - kwargs['type'] = field_info.annotation - kwargs['action'] = _CliEnumAction - del kwargs['metavar'] - arg_name = f'{arg_prefix.replace(subcommand_prefix, "", 1)}{field_name}' + arg_name = ( + f'{arg_prefix}{field_name}' + if subcommand_prefix == self.env_prefix + else f'{arg_prefix.replace(subcommand_prefix, "", 1)}{field_name}' + ) if _CliPositionalArg in field_info.metadata: kwargs['metavar'] = field_name.upper() arg_name = kwargs['dest'] @@ -948,7 +1132,7 @@ def _add_fields_to_parser( arg_flag = '' if sub_models and kwargs.get('action') != 'append': - model_group: _ArgumentGroup | None = None + model_group: Any = None model_group_kwargs: dict[str, Any] = {} model_group_kwargs['title'] = f'{arg_name} options' model_group_kwargs['description'] = ( @@ -959,25 +1143,25 @@ def _add_fields_to_parser( if not self.cli_avoid_json: added_args.append(arg_name) kwargs['help'] = f'set {arg_name} from JSON string' - model_group = parser.add_argument_group(**model_group_kwargs) - model_group.add_argument(f'{arg_flag}{arg_name}', **kwargs) + model_group = self._add_argument_group(parser, **model_group_kwargs) + self._add_argument(model_group, f'{arg_flag}{arg_name}', **kwargs) for model in sub_models: - self._add_fields_to_parser( - parser, - model, + self._add_parser_args( + parser=parser, + model=model, added_args=added_args, arg_prefix=f'{arg_prefix}{field_name}.', subcommand_prefix=subcommand_prefix, group=model_group if model_group else model_group_kwargs, ) elif group is not None: - if not isinstance(group, _ArgumentGroup): - group = parser.add_argument_group(**group) + if isinstance(group, dict): + group = self._add_argument_group(parser, **group) added_args.append(arg_name) - group.add_argument(f'{arg_flag}{arg_name}', **kwargs) + self._add_argument(group, f'{arg_flag}{arg_name}', **kwargs) else: added_args.append(arg_name) - parser.add_argument(f'{arg_flag}{arg_name}', **kwargs) + self._add_argument(parser, f'{arg_flag}{arg_name}', **kwargs) return parser def _get_modified_args(self, obj: Any) -> tuple[str, ...]: @@ -993,8 +1177,7 @@ def _metavar_format_list(self, args: list[str]) -> str: def _metavar_format_recurse(self, obj: Any) -> str: """Pretty metavar representation of a type. Adapts logic from `pydantic._repr.display_as_type`.""" - while get_origin(obj) == Annotated: - obj = get_args(obj)[0] + obj = _strip_annotated(obj) if isinstance(obj, FunctionType): return obj.__name__ elif obj is ...: @@ -1011,7 +1194,10 @@ def _metavar_format_recurse(self, obj: Any) -> str: args = self._metavar_format_list(list(map(self._metavar_format_recurse, self._get_modified_args(obj)))) return f'{{{args}}}' if ',' in args else args elif get_origin(obj) in (typing_extensions.Literal, typing.Literal): - args = self._metavar_format_list(list(map(repr, self._get_modified_args(obj)))) + args = self._metavar_format_list(list(map(str, self._get_modified_args(obj)))) + return f'{{{args}}}' if ',' in args else args + elif lenient_issubclass(obj, Enum): + args = self._metavar_format_list([val.name for val in obj]) return f'{{{args}}}' if ',' in args else args elif isinstance(obj, WithArgsTypes): args = self._metavar_format_list(list(map(self._metavar_format_recurse, self._get_modified_args(obj)))) @@ -1094,3 +1280,9 @@ def _annotation_contains_types(annotation: type[Any] | None, types: tuple[Any, . if _annotation_contains_types(type_, types, is_include_origin=True): return True return annotation in types + + +def _strip_annotated(annotation: Any) -> Any: + while get_origin(annotation) == Annotated: + annotation = get_args(annotation)[0] + return annotation diff --git a/tests/test_settings.py b/tests/test_settings.py index 0fcd62fc..22d0bf86 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,3 +1,4 @@ +import argparse import dataclasses import os import sys @@ -6,7 +7,7 @@ from datetime import datetime, timezone from enum import IntEnum from pathlib import Path -from typing import Any, Callable, Dict, Generic, List, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Any, Callable, Dict, Generic, List, Literal, Optional, Set, Tuple, Type, TypeVar, Union import pytest import typing_extensions @@ -54,6 +55,42 @@ def foobar(a, b, c=4): T = TypeVar('T') +class FruitsEnum(IntEnum): + pear = 0 + kiwi = 1 + lime = 2 + + +class CliDummyArgGroup(BaseModel, arbitrary_types_allowed=True): + group: argparse._ArgumentGroup + + def add_argument(self, *args, **kwargs) -> None: + self.group.add_argument(*args, **kwargs) + + +class CliDummySubParsers(BaseModel, arbitrary_types_allowed=True): + sub_parser: argparse._SubParsersAction + + def add_parser(self, *args, **kwargs) -> 'CliDummyParser': + return CliDummyParser(parser=self.sub_parser.add_parser(*args, **kwargs)) + + +class CliDummyParser(BaseModel, arbitrary_types_allowed=True): + parser: argparse.ArgumentParser = Field(default_factory=lambda: argparse.ArgumentParser()) + + def add_argument(self, *args, **kwargs) -> None: + self.parser.add_argument(*args, **kwargs) + + def add_argument_group(self, *args, **kwargs) -> CliDummyArgGroup: + return CliDummyArgGroup(group=self.parser.add_argument_group(*args, **kwargs)) + + def add_subparsers(self, *args, **kwargs) -> CliDummySubParsers: + return CliDummySubParsers(sub_parser=self.parser.add_subparsers(*args, **kwargs)) + + def parse_args(self, *args, **kwargs) -> argparse.Namespace: + return self.parser.parse_args(*args, **kwargs) + + class LoggedVar(Generic[T]): def get(self) -> T: ... @@ -2155,6 +2192,27 @@ def check_answer(cfg, prefix, expected): check_answer(cfg, prefix, expected) +def test_cli_list_json_value_parsing(): + class Cfg(BaseSettings): + json_list: list[str | bool | None] + + assert Cfg( + _cli_parse_args=[ + '--json_list', + 'true,"true"', + '--json_list', + 'false,"false"', + '--json_list', + 'null,"null"', + '--json_list', + 'hi,"bye"', + ] + ).model_dump() == {'json_list': [True, 'true', False, 'false', None, 'null', 'hi', 'bye']} + + assert Cfg(_cli_parse_args=['--json_list', '"","","",""']).model_dump() == {'json_list': ['', '', '', '']} + assert Cfg(_cli_parse_args=['--json_list', ',,,']).model_dump() == {'json_list': ['', '', '', '']} + + @pytest.mark.parametrize('prefix', ['', 'child.']) def test_cli_dict_arg(prefix): class Child(BaseModel): @@ -2221,6 +2279,12 @@ class Cfg(BaseSettings): expected['child'] = None assert cfg.model_dump() == expected + with pytest.raises(SettingsError): + cfg = Cfg(_cli_parse_args=[f'--{prefix}check_dict', 'k9="i']) + + with pytest.raises(SettingsError): + cfg = Cfg(_cli_parse_args=[f'--{prefix}check_dict', 'k9=i"']) + def test_cli_nested_dict_arg(): class Cfg(BaseSettings): @@ -2304,20 +2368,15 @@ class Cfg(BaseSettings): assert cfg.model_dump() == {'child': {'name': 'new name a', 'diff_a': 'new diff a'}} -def test_cli_enum(): - class Fruit(IntEnum): - apple = 0 - banna = 1 - orange = 2 - +def test_cli_literal(): class Cfg(BaseSettings): - fruit: Fruit + pet: Literal['dog', 'cat', 'bird'] - cfg = Cfg(_cli_parse_args=['--fruit', 'orange']) - assert cfg.model_dump() == {'fruit': Fruit.orange} + cfg = Cfg(_cli_parse_args=['--pet', 'cat']) + assert cfg.model_dump() == {'pet': 'cat'} - with pytest.raises(SystemExit): - Cfg(_cli_parse_args=['--fruit', 'lettuce']) + with pytest.raises(ValidationError): + Cfg(_cli_parse_args=['--pet', 'rock']) def test_cli_annotation_exceptions(monkeypatch): @@ -2579,6 +2638,159 @@ class Settings(BaseSettings): Settings(_cli_parse_args=[], _cli_enforce_required=True).model_dump() +@pytest.mark.parametrize('parser_type', [pytest.Parser, argparse.ArgumentParser, CliDummyParser]) +@pytest.mark.parametrize('prefix', ['', 'cfg']) +def test_cli_user_settings_source(parser_type, prefix): + class Cfg(BaseSettings): + pet: Literal['dog', 'cat', 'bird'] = 'bird' + + if parser_type is pytest.Parser: + parser = pytest.Parser(_ispytest=True) + parse_args = parser.parse + add_arg = parser.addoption + cli_cfg_settings = CliSettingsSource( + Cfg, + cli_prefix=prefix, + root_parser=parser, + parse_args_method=pytest.Parser.parse, + add_argument_method=pytest.Parser.addoption, + add_argument_group_method=pytest.Parser.getgroup, + add_parser_method=None, + add_subparsers_method=None, + formatter_class=None, + ) + elif parser_type is CliDummyParser: + parser = CliDummyParser() + parse_args = parser.parse_args + add_arg = parser.add_argument + cli_cfg_settings = CliSettingsSource( + Cfg, + cli_prefix=prefix, + root_parser=parser, + parse_args_method=CliDummyParser.parse_args, + add_argument_method=CliDummyParser.add_argument, + add_argument_group_method=CliDummyParser.add_argument_group, + add_parser_method=CliDummySubParsers.add_parser, + add_subparsers_method=CliDummyParser.add_subparsers, + ) + else: + parser = argparse.ArgumentParser() + parse_args = parser.parse_args + add_arg = parser.add_argument + cli_cfg_settings = CliSettingsSource(Cfg, cli_prefix=prefix, root_parser=parser) + + add_arg('--fruit', choices=['pear', 'kiwi', 'lime']) + + args = ['--fruit', 'pear'] + parsed_args = parse_args(args) + assert Cfg(_cli_settings_source=cli_cfg_settings(parsed_args=parsed_args)).model_dump() == {'pet': 'bird'} + assert Cfg(_cli_settings_source=cli_cfg_settings(args=args)).model_dump() == {'pet': 'bird'} + + arg_prefix = f'{prefix}.' if prefix else '' + args = ['--fruit', 'kiwi', f'--{arg_prefix}pet', 'dog'] + parsed_args = parse_args(args) + assert Cfg(_cli_settings_source=cli_cfg_settings(parsed_args=parsed_args)).model_dump() == {'pet': 'dog'} + assert Cfg(_cli_settings_source=cli_cfg_settings(args=args)).model_dump() == {'pet': 'dog'} + + parsed_args = parse_args(['--fruit', 'kiwi', f'--{arg_prefix}pet', 'cat']) + assert Cfg(_cli_settings_source=cli_cfg_settings(parsed_args=vars(parsed_args))).model_dump() == {'pet': 'cat'} + + +@pytest.mark.parametrize('prefix', ['', 'cfg']) +def test_cli_dummy_user_settings_with_subcommand(prefix): + class DogCommands(BaseModel): + name: str = 'Bob' + command: Literal['roll', 'bark', 'sit'] = 'sit' + + class Cfg(BaseSettings): + pet: Literal['dog', 'cat', 'bird'] = 'bird' + command: CliSubCommand[DogCommands] + + parser = CliDummyParser() + cli_cfg_settings = CliSettingsSource( + Cfg, + root_parser=parser, + cli_prefix=prefix, + parse_args_method=CliDummyParser.parse_args, + add_argument_method=CliDummyParser.add_argument, + add_argument_group_method=CliDummyParser.add_argument_group, + add_parser_method=CliDummySubParsers.add_parser, + add_subparsers_method=CliDummyParser.add_subparsers, + ) + + parser.add_argument('--fruit', choices=['pear', 'kiwi', 'lime']) + + args = ['--fruit', 'pear'] + parsed_args = parser.parse_args(args) + assert Cfg(_cli_settings_source=cli_cfg_settings(parsed_args=parsed_args)).model_dump() == { + 'pet': 'bird', + 'command': None, + } + assert Cfg(_cli_settings_source=cli_cfg_settings(args=args)).model_dump() == { + 'pet': 'bird', + 'command': None, + } + + arg_prefix = f'{prefix}.' if prefix else '' + args = ['--fruit', 'kiwi', f'--{arg_prefix}pet', 'dog'] + parsed_args = parser.parse_args(args) + assert Cfg(_cli_settings_source=cli_cfg_settings(parsed_args=parsed_args)).model_dump() == { + 'pet': 'dog', + 'command': None, + } + assert Cfg(_cli_settings_source=cli_cfg_settings(args=args)).model_dump() == { + 'pet': 'dog', + 'command': None, + } + + parsed_args = parser.parse_args(['--fruit', 'kiwi', f'--{arg_prefix}pet', 'cat']) + assert Cfg(_cli_settings_source=cli_cfg_settings(parsed_args=vars(parsed_args))).model_dump() == { + 'pet': 'cat', + 'command': None, + } + + args = ['--fruit', 'kiwi', f'--{arg_prefix}pet', 'dog', 'command', '--name', 'ralph', '--command', 'roll'] + parsed_args = parser.parse_args(args) + assert Cfg(_cli_settings_source=cli_cfg_settings(parsed_args=vars(parsed_args))).model_dump() == { + 'pet': 'dog', + 'command': {'name': 'ralph', 'command': 'roll'}, + } + assert Cfg(_cli_settings_source=cli_cfg_settings(args=args)).model_dump() == { + 'pet': 'dog', + 'command': {'name': 'ralph', 'command': 'roll'}, + } + + +def test_cli_user_settings_source_exceptions(): + class Cfg(BaseSettings): + pet: Literal['dog', 'cat', 'bird'] = 'bird' + + with pytest.raises(SettingsError): + args = ['--pet', 'dog'] + parsed_args = {'pet': 'dog'} + cli_cfg_settings = CliSettingsSource(Cfg) + Cfg(_cli_settings_source=cli_cfg_settings(args=args, parsed_args=parsed_args)) + + with pytest.raises(SettingsError): + CliSettingsSource(Cfg, cli_prefix='.cfg') + + with pytest.raises(SettingsError): + CliSettingsSource(Cfg, cli_prefix='cfg.') + + with pytest.raises(SettingsError): + CliSettingsSource(Cfg, cli_prefix='123') + + class Food(BaseModel): + fruit: FruitsEnum = FruitsEnum.kiwi + + class CfgWithSubCommand(BaseSettings): + pet: Literal['dog', 'cat', 'bird'] = 'bird' + food: CliSubCommand[Food] + + with pytest.raises(SettingsError): + CliSettingsSource(CfgWithSubCommand, add_subparsers_method=None) + + @pytest.mark.parametrize( 'value,expected', [ @@ -2600,12 +2812,15 @@ class Settings(BaseSettings): (Representation(), 'Representation()'), (typing.Literal[1, 2, 3], '{1,2,3}'), (typing_extensions.Literal[1, 2, 3], '{1,2,3}'), + (typing.Literal['a', 'b', 'c'], '{a,b,c}'), + (typing_extensions.Literal['a', 'b', 'c'], '{a,b,c}'), (SimpleSettings, 'JSON'), (Union[SimpleSettings, SettingWithIgnoreEmpty], 'JSON'), (Union[SimpleSettings, str, SettingWithIgnoreEmpty], '{JSON,str}'), (Union[str, SimpleSettings, SettingWithIgnoreEmpty], '{str,JSON}'), (Annotated[SimpleSettings, 'annotation'], 'JSON'), (DirectoryPath, 'Path'), + (FruitsEnum, '{pear,kiwi,lime}'), ], ) def test_cli_metavar_format(value, expected): From 0eaba11a053dac576ab99a41fa4b2027a7f8c8f6 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 4 Mar 2024 15:49:24 -0700 Subject: [PATCH 33/61] Doc updates. --- docs/index.md | 242 +++++++++++++++++++++-------------- pydantic_settings/sources.py | 73 ++++++++--- 2 files changed, 198 insertions(+), 117 deletions(-) diff --git a/docs/index.md b/docs/index.md index 699f605b..f0d3561d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -473,95 +473,118 @@ to enable [enforcing required arguments at the CLI](#enforce-required-arguments- ### The Basics -To get started, let's look at a basic example for defining a Pydantic settings CLI: +To get started, let's revisit the example presented in [parsing environment variables](#parsing-environment-variables) but using a Pydantic settings CLI: -```py test="skip" -from typing import List +```py +import sys from pydantic import BaseModel -from pydantic_settings import BaseSettings - - -class DeepSubModel(BaseModel, use_attribute_docstrings=True): - """DeepSubModel class documentation.""" - - v4: List[int] - """the deeply nested sub model v4 option""" +from pydantic_settings import BaseSettings, SettingsConfigDict -class SubModel(BaseModel, use_attribute_docstrings=True): - """SubModel class documentation.""" +class DeepSubModel(BaseModel): + v4: str - v1: int - """the sub model v1 option""" +class SubModel(BaseModel): + v1: str + v2: bytes + v3: int deep: DeepSubModel - """The help summary for DeepSubModel and related options. This will be placed at top of group.""" -class Settings(BaseSettings, use_attribute_docstrings=True): - """The Settings class documentation will show in top level help text.""" +class Settings(BaseSettings): + model_config = SettingsConfigDict(cli_parse_args=True) v0: str - """the top level v0 option""" - sub_model: SubModel - """The help summary for SubModel related options. This will be placed at top of group.""" -Settings(_cli_prog_name='app', _cli_parse_args=['--help']) # (1)! +sys.argv = [ + 'example.py', + '--v0=0', + '--sub_model={"v1": "json-1", "v2": "json-2"}', + '--sub_model.v2=nested-2', + '--sub_model.v3=3', + '--sub_model.deep.v4=v4', +] + +print(Settings().model_dump()) """ -usage: app [-h] [--v0 str] [--sub_model JSON] [--sub_model.v1 int] [--sub_model.deep JSON] - [--sub_model.deep.v4 List[int]] +{ + 'v0': '0', + 'sub_model': {'v1': 'json-1', 'v2': b'nested-2', 'v3': 3, 'deep': {'v4': 'v4'}}, +} +""" +``` -The Settings class documentation will show in top level help text. # (2)! +To enable CLI parsing, we simply set the `cli_parse_args` flag to a valid value, which retains similar conotations as +defined in `argparse`. Alternatively, we can also directly provided the args to parse at time of instantiation: + +```py test="skip" lint="skip" +Settings( + _cli_parse_args=[ + '--v0=0', + '--sub_model={"v1": "json-1", "v2": "json-2"}', + '--sub_model.v2=nested-2', + '--sub_model.v3=3', + '--sub_model.deep.v4=v4', + ] +) +``` -options: - -h, --help show this help message and exit - --v0 str the top level v0 option # (3)! +Note that a CLI settings source is always [**the topmost source**](#field-value-priority) and does not support [changing +its priority](#changing-priority). -sub_model options: # (4)! - The help summary for SubModel related options. This will be placed at top of group. +#### Integrating with Existing Parsers - --sub_model JSON set sub_model from JSON string - --sub_model.v1 int the sub model v1 option # (5)! +A CLI settings source can be integrated with existing parsers by overriding the default CLI settings source with a user +defined one that specifies the `root_parser` object. -sub_model.deep options: - The help summary for DeepSubModel and related options. This will be placed at top of - group. # (6)! +```py +import sys +from argparse import ArgumentParser - --sub_model.deep JSON # (7)! - set sub_model.deep from JSON string - --sub_model.deep.v4 List[int] - the deeply nested sub model v4 option -""" -``` +from pydantic_settings import BaseSettings, CliSettingsSource -1. Does `_cli_prog_name` and `_cli_parse_args` look familiar? They retain the same meanings as in argparse. +parser = ArgumentParser() +parser.add_argument('--food', choices=['pear', 'kiwi', 'lime']) -2. Help text for application main or subcommands is populated from class docstrings. -3. Help text for fields is populated from field descriptions. +class Settings(BaseSettings): + name: str = 'Bob' -4. Nested models (e.g. `SubModel`, `DeepSubModel`) and their associated fields will always be grouped together. -5. Note that nested fields look and act just like their environment variable counterparts. The CLI uses `.` as its - nested delimiter. +# Set existing `parser` as the `root_parser` object for the user defined settings source +cli_settings = CliSettingsSource(Settings, root_parser=parser) -6. Group help text is populated from field descriptions by default, but can be configured to pull from class docstrings - as well. +# Parse and load CLI settings from the command line into the settings source. +sys.argv = ['example.py', '--food', 'kiwi', '--name', 'waldo'] +print(Settings(_cli_settings_source=cli_settings(args=True)).model_dump()) +#> {'name': 'waldo'} -7. Just like when parsing environment variables, top level models allow for JSON strings and nested fields taking - precedence. +# Load CLI settings from pre-parsed arguments. i.e., the parsing occurs elsewhere and we +# just need to load the pre-parsed args into the settings source. +parsed_args = parser.parse_args(['--food', 'kiwi', '--name', 'ralph']) +print(Settings(_cli_settings_source=cli_settings(parsed_args=parsed_args)).model_dump()) +#> {'name': 'ralph'} +``` -To enable CLI parsing, we simply set the `cli_parse_args` flag to a valid value, which retains similar conotations as -defined in argparse. In the above example, we parsed our args from the `['--help']` list that was passed into -`_cli_parse_args`. Alternatively, we could have set `_cli_parse_args=True` to parse args from the command line (i.e., -`sys.argv[1:]`). +A `CliSettingsSource` connects with a `root_parser` object by using parser methods to add `settings_cls` fields as +command line arguments. The `CliSettingsSource` internal parser representation is based on the `argparse` library, and +therefore, requires parser methods that support the same attributes as their `argparse` counterparts. The available +parser methods that can be customised, along with their argparse counterparts (the defaults), are listed below: -Lastly, a CLI settings source is always [**the topmost source**](#field-value-priority), and does not support [changing -its priority](#changing-priority). +* `parse_args_method` - argparse.ArgumentParser.parse_args +* `add_argument_method` - argparse.ArgumentParser.add_argument +* `add_argument_group_method` - argparse.ArgumentParser.add\_argument_group +* `add_parser_method` - argparse.\_SubParsersAction.add_parser +* `add_subparsers_method` - argparse.ArgumentParser.add_subparsers +* `formatter_class` - argparse.HelpFormatter + +For a non-argparse parser the parser methods can be set to `None` if not supported. The CLI settings will only raise an +error when connecting to the root parser if a parser method is necessary but set to `None`. #### Lists @@ -572,22 +595,26 @@ CLI argument parsing of lists supports intermixing of any of the below three sty * Lazy style `--field=1,2` ```py +import sys from typing import List from pydantic_settings import BaseSettings -class Settings(BaseSettings): +class Settings(BaseSettings, cli_parse_args=True): my_list: List[int] -print(Settings(_cli_parse_args=['--my_list', '[1,2]']).model_dump()) +sys.argv = ['example.py', '--my_list', '[1,2]'] +print(Settings().model_dump()) #> {'my_list': [1, 2]} -print(Settings(_cli_parse_args=['--my_list', '1', '--my_list', '2']).model_dump()) +sys.argv = ['example.py', '--my_list', '1', '--my_list', '2'] +print(Settings().model_dump()) #> {'my_list': [1, 2]} -print(Settings(_cli_parse_args=['--my_list', '1,2']).model_dump()) +sys.argv = ['example.py', '--my_list', '1,2'] +print(Settings().model_dump()) #> {'my_list': [1, 2]} ``` @@ -603,19 +630,22 @@ These can be used in conjunction with list forms as well, e.g: * `--field k1=1,k2=2 --field k3=3 --field '{"k4: 4}'` etc. ```py +import sys from typing import Dict from pydantic_settings import BaseSettings -class Settings(BaseSettings): +class Settings(BaseSettings, cli_parse_args=True): my_dict: Dict[str, int] -print(Settings(_cli_parse_args=['--my_dict', '{"k1":1,"k2":2}']).model_dump()) +sys.argv = ['example.py', '--my_dict', '{"k1":1,"k2":2}'] +print(Settings().model_dump()) #> {'my_dict': {'k1': 1, 'k2': 2}} -print(Settings(_cli_parse_args=['--my_dict', 'k1=1', '--my_dict', 'k2=2']).model_dump()) +sys.argv = ['example.py', '--my_dict', 'k1=1', '--my_dict', 'k2=2'] +print(Settings().model_dump()) #> {'my_dict': {'k1': 1, 'k2': 2}} ``` @@ -631,7 +661,9 @@ subcommands must be a valid type derived from the pydantic `BaseModel` class. set of subcommands. For more information on subparsers, see [argparse subcommands](https://docs.python.org/3/library/argparse.html#sub-commands). -```py test="skip" +```py +import sys + from pydantic import BaseModel, Field from pydantic_settings import ( @@ -678,7 +710,7 @@ class Clone(BaseModel): ) -class Git(BaseSettings, cli_prog_name='git'): +class Git(BaseSettings, cli_parse_args=True, cli_prog_name='git'): """git - The stupid content tracker""" clone: CliSubCommand[Clone] = Field( @@ -688,7 +720,12 @@ class Git(BaseSettings, cli_prog_name='git'): plugins: CliSubCommand[Plugins] = Field(description='Fake GIT plugin commands') -Git(_cli_parse_args=['--help']) +try: + sys.argv = ['example.py', '--help'] + Git() +except SystemExit as e: + print(e) + #> 0 """ usage: git [-h] {clone,plugins} ... @@ -704,7 +741,12 @@ subcommands: """ -Git(_cli_parse_args=['clone', '--help']) +try: + sys.argv = ['example.py', 'clone', '--help'] + Git() +except SystemExit as e: + print(e) + #> 0 """ usage: git clone [-h] [--local bool] [--shared bool] REPOSITORY DIRECTORY @@ -720,7 +762,12 @@ options: """ -Git(_cli_parse_args=['plugins', 'bar', '--help']) +try: + sys.argv = ['example.py', 'plugins', 'bar', '--help'] + Git() +except SystemExit as e: + print(e) + #> 0 """ usage: git plugins bar [-h] [--my_feature bool] @@ -736,6 +783,28 @@ options: The below flags can be used to customise the CLI experience to your needs. +#### Change the Displayed Program Name + +Change the default program name displayed in the help text usage by setting `cli_prog_name`. By default, it will derive the name of the currently +executing program from `sys.argv[0]`, just like argparse. + +```py +from pydantic_settings import BaseSettings, CliSettingsSource + + +class Settings(BaseSettings, cli_prog_name='appdantic'): + pass + + +print(CliSettingsSource(Settings).root_parser.format_help()) +""" +usage: appdantic [-h] + +options: + -h, --help show this help message and exit +""" +``` + #### Enforce Required Arguments at CLI Pydantic settings is designed to pull values in from various sources when instantating a model. This means a field that @@ -748,26 +817,29 @@ likely want required fields to be _strictly required at the CLI_. We can enable ```py import os +import sys from pydantic import Field from pydantic_settings import BaseSettings -class Settings(BaseSettings, cli_enforce_required=True): +class Settings(BaseSettings, cli_parse_args=True, cli_enforce_required=True): my_required_field: str = Field(description='a top level required field') os.environ['MY_REQUIRED_FIELD'] = 'hello from environment' try: - print(Settings(_cli_parse_args=[]).model_dump()) - """ + sys.argv = ['example.py'] + Settings() +except SystemExit as e: + print(e) + #> 2 +""" usage: example.py [-h] --my_required_field str example.py: error: the following arguments are required: --my_required_field """ -except SystemExit: - pass ``` #### Hide None Type Values @@ -874,28 +946,6 @@ sub_model options: """ ``` -#### Change the Displayed Program Name - -Change the default program name displayed in the help text usage by setting `cli_prog_name`. By default, it will derive the name of the currently -executing program from `sys.argv[0]`, just like argparse. - -```py -from pydantic_settings import BaseSettings, CliSettingsSource - - -class Settings(BaseSettings, cli_prog_name='appdantic'): - pass - - -print(CliSettingsSource(Settings).root_parser.format_help()) -""" -usage: appdantic [-h] - -options: - -h, --help show this help message and exit -""" -``` - ## Secrets Placing secret values in files is a common pattern to provide sensitive configuration to an application. diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index f4af0f14..93862b3b 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -717,18 +717,18 @@ class CliSettingsSource(EnvSettingsSource, Generic[T]): """ Source class for loading settings values from CLI. - The root parser to connect the CLI settings source to. This will add fields from the `settings_cls` to the root parser as - arguments and associate the internal CLI settings source parsing logic with the root parser. - Note: - The parser methods must support the same attributes as their `argparse` library counterparts. + A `CliSettingsSource` connects with a `root_parser` object by using the parser methods to add + `settings_cls` fields as command line arguments. The `CliSettingsSource` internal parser representation + is based upon the `argparse` parsing library, and therefore, requires the parser methods to support + the same attributes as their `argparse` library counterparts. Args: cli_prog_name: The CLI program name to display in help text. Defaults to `None` if cli_parse_args is `None`. Otherwse, defaults to sys.argv[0]. cli_parse_args: The list of CLI arguments to parse. Defaults to None. If set to `True`, defaults to sys.argv[1:]. - cli_settings_source: Override the default CLI settings source with a user defined instance. Defaults to None. + cli_settings_source: Override the default CLI settings source with a user defined instance. Defaults to `None`. cli_hide_none_type: Hide `None` values in CLI help text. Defaults to `False`. cli_avoid_json: Avoid complex JSON objects in CLI help text. Defaults to `False`. cli_enforce_required: Enforce required fields at the CLI. Defaults to `False`. @@ -738,9 +738,12 @@ class CliSettingsSource(EnvSettingsSource, Generic[T]): root_parser: The root parser object. parse_args_method: The root parser parse args method. Defaults to `argparse.ArgumentParser.parse_args`. add_argument_method: The root parser add argument method. Defaults to `argparse.ArgumentParser.add_argument`. - add_argument_group_method: The root parser add argument group method. Defaults to `argparse.ArgumentParser.add_argument_group`. - add_parser_method: The root parser add new parser (sub-command) method. Defaults to `argparse._SubParsersAction.add_parser`. - add_subparsers_method: The root parser add subparsers (sub-commands) method. Defaults to `argparse.ArgumentParser.add_subparsers`. + add_argument_group_method: The root parser add argument group method. + Defaults to `argparse.ArgumentParser.add_argument_group`. + add_parser_method: The root parser add new parser (sub-command) method. + Defaults to `argparse._SubParsersAction.add_parser`. + add_subparsers_method: The root parser add subparsers (sub-commands) method. + Defaults to `argparse.ArgumentParser.add_subparsers`. formatter_class: A class for customizing the root parser help text. Defaults to `argparse.HelpFormatter`. """ @@ -748,7 +751,7 @@ def __init__( self, settings_cls: type[BaseSettings], cli_prog_name: str | None = None, - cli_parse_args: bool | list[str] | None = None, + cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, cli_parse_none_str: str | None = None, cli_hide_none_type: bool | None = None, cli_avoid_json: bool | None = None, @@ -814,8 +817,10 @@ def __init__( if cli_parse_args not in (None, False): if cli_parse_args is True: cli_parse_args = sys.argv[1:] - elif not isinstance(cli_parse_args, list): - raise SettingsError(f'cli_parse_args must be List[str], recieved {type(cli_parse_args)}') + elif not isinstance(cli_parse_args, (list, tuple)): + raise SettingsError( + f'cli_parse_args must be List[str] or Tuple[str, ...], recieved {type(cli_parse_args)}' + ) self._load_env_vars(parsed_args=self._parse_args(self.root_parser, cli_parse_args)) @overload @@ -823,34 +828,47 @@ def __call__(self) -> dict[str, Any]: ... @overload - def __call__(self, *, args: list[str]) -> dict[str, Any]: + def __call__(self, *, args: list[str] | tuple[str, ...] | bool) -> dict[str, Any]: + """ + Parse and load the command line arguments list into the CLI settings source. + + Args: + args: The command line arguments to parse and load. Defaults to `None`. If set to `True`, defaults + to sys.argv[1:]. If set to `False`, defaults to []. + + Returns: + CliSettingsSource: The object instance itself. + """ ... @overload def __call__(self, *, parsed_args: Namespace | dict[str, list[str] | str]) -> dict[str, Any]: - ... - - def __call__( - self, *, args: list[str] | None = None, parsed_args: Namespace | dict[str, list[str] | str] | None = None - ) -> dict[str, Any] | CliSettingsSource[T]: """ - Loads parsed command line arguments into the CLI settings source. If parsed args are `None` - (the default) will return the CLI settings source vars dicitionary. + Loads parsed command line arguments into the CLI settings source. Note: The parsed args must be in `argparse.Namespace` or vars dictionary (e.g., vars(argparse.Namespace)) format. Args: - args: parsed_args: The parsed args to load. Returns: CliSettingsSource: The object instance itself. """ + ... + + def __call__( + self, + *, + args: list[str] | tuple[str, ...] | bool | None = None, + parsed_args: Namespace | dict[str, list[str] | str] | None = None, + ) -> dict[str, Any] | CliSettingsSource[T]: if args is not None and parsed_args is not None: - raise SettingsError('args and parsed_args are mutually exclusive') + raise SettingsError('`args` and `parsed_args` are mutually exclusive') elif args is not None: + if args is True: + args = sys.argv[1:] if args else [] return self._load_env_vars(parsed_args=self._parse_args(self.root_parser, args)) elif parsed_args is not None: return self._load_env_vars(parsed_args=parsed_args) @@ -863,6 +881,19 @@ def _load_env_vars(self) -> Mapping[str, str | None]: @overload def _load_env_vars(self, *, parsed_args: Namespace | dict[str, list[str] | str]) -> CliSettingsSource[T]: + """ + Loads the parsed command line arguments into the CLI environment settings variables. + + Note: + The parsed args must be in `argparse.Namespace` or vars dictionary (e.g., vars(argparse.Namespace)) + format. + + Args: + parsed_args: The parsed args to load. + + Returns: + CliSettingsSource: The object instance itself. + """ ... def _load_env_vars( From 336108fcb508e751dcf32ee64be97bdd6a8991b2 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 4 Mar 2024 15:54:49 -0700 Subject: [PATCH 34/61] Add tuple type. --- pydantic_settings/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 01e5c159..ce8bcf6e 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -29,7 +29,7 @@ class SettingsConfigDict(ConfigDict, total=False): env_nested_delimiter: str | None env_parse_none_str: str | None cli_prog_name: str | None - cli_parse_args: bool | list[str] | None + cli_parse_args: bool | list[str] | tuple[str, ...] | None cli_settings_source: CliSettingsSource[Any] | None cli_hide_none_type: bool cli_avoid_json: bool @@ -92,7 +92,7 @@ def __init__( _env_nested_delimiter: str | None = None, _env_parse_none_str: str | None = None, _cli_prog_name: str | None = None, - _cli_parse_args: bool | list[str] | None = None, + _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: CliSettingsSource[Any] | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, @@ -160,7 +160,7 @@ def _settings_build_values( _env_nested_delimiter: str | None = None, _env_parse_none_str: str | None = None, _cli_prog_name: str | None = None, - _cli_parse_args: bool | list[str] | None = None, + _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: CliSettingsSource[Any] | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, From ae6fa739702a72ce05ad01ab606d4686ce4f50fc Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 5 Mar 2024 13:26:11 -0700 Subject: [PATCH 35/61] Doc and test prep for literals and enums. --- docs/index.md | 29 +++++++++++++++++++++++++++++ pydantic_settings/sources.py | 14 +++++++++----- tests/test_settings.py | 20 +++++++++++++++++++- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/docs/index.md b/docs/index.md index f0d3561d..c52af9f0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -649,6 +649,35 @@ print(Settings().model_dump()) #> {'my_dict': {'k1': 1, 'k2': 2}} ``` +#### Literals and Enums + +CLI argument parsing of literals and enums are converted into CLI choices. + + +```py test="skip" +import sys +from enum import IntEnum +from typing import Literal + +from pydantic_settings import BaseSettings + + +class Fruit(IntEnum): + pear = 0 + kiwi = 1 + lime = 2 + + +class Settings(BaseSettings, cli_parse_args=True): + fruit: Fruit + pet: Literal['dog', 'cat', 'bird'] + + +sys.argv = ['example.py', '--fruit', 'lime', '--pet', 'cat'] +print(Settings().model_dump()) +#> {'fruit': , 'pet': 'cat'} +``` + ### Subcommands and Positional Arguments Subcommands and positional arguments are expressed using the `CliSubCommand` and `CliPositionalArg` annotations. These diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 93862b3b..deda78df 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -828,13 +828,15 @@ def __call__(self) -> dict[str, Any]: ... @overload - def __call__(self, *, args: list[str] | tuple[str, ...] | bool) -> dict[str, Any]: + def __call__(self, *, args: list[str] | tuple[str, ...] | bool) -> CliSettingsSource[T]: """ Parse and load the command line arguments list into the CLI settings source. Args: - args: The command line arguments to parse and load. Defaults to `None`. If set to `True`, defaults - to sys.argv[1:]. If set to `False`, defaults to []. + args: + The command line arguments to parse and load. Defaults to `None`, which means do not parse + command line arguments. If set to `True`, defaults to sys.argv[1:]. If set to `False`, does + not parse command line arguments. Returns: CliSettingsSource: The object instance itself. @@ -842,7 +844,7 @@ def __call__(self, *, args: list[str] | tuple[str, ...] | bool) -> dict[str, Any ... @overload - def __call__(self, *, parsed_args: Namespace | dict[str, list[str] | str]) -> dict[str, Any]: + def __call__(self, *, parsed_args: Namespace | dict[str, list[str] | str]) -> CliSettingsSource[T]: """ Loads parsed command line arguments into the CLI settings source. @@ -867,8 +869,10 @@ def __call__( if args is not None and parsed_args is not None: raise SettingsError('`args` and `parsed_args` are mutually exclusive') elif args is not None: + if args is False: + return self if args is True: - args = sys.argv[1:] if args else [] + args = sys.argv[1:] return self._load_env_vars(parsed_args=self._parse_args(self.root_parser, args)) elif parsed_args is not None: return self._load_env_vars(parsed_args=parsed_args) diff --git a/tests/test_settings.py b/tests/test_settings.py index 22d0bf86..968b3bd3 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2368,7 +2368,25 @@ class Cfg(BaseSettings): assert cfg.model_dump() == {'child': {'name': 'new name a', 'diff_a': 'new diff a'}} -def test_cli_literal(): +# TODO Remove skip once environment parsing support for enums is added +@pytest.mark.skip +def test_cli_enums(): + class Pet(IntEnum): + dog = 0 + cat = 1 + bird = 2 + + class Cfg(BaseSettings): + pet: Pet + + cfg = Cfg(_cli_parse_args=['--pet', 'cat']) + assert cfg.model_dump() == {'pet': Pet.cat} + + with pytest.raises(ValidationError): + Cfg(_cli_parse_args=['--pet', 'rock']) + + +def test_cli_literals(): class Cfg(BaseSettings): pet: Literal['dog', 'cat', 'bird'] From 9679c107daf910ffca6ea769b4d1ed1eb47e1024 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 11 Mar 2024 20:47:17 -0600 Subject: [PATCH 36/61] Enable CLI enum support. --- docs/index.md | 3 +-- pydantic_settings/sources.py | 6 +++++- tests/test_settings.py | 7 ------- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/index.md b/docs/index.md index bcc4a227..4b6934ef 100644 --- a/docs/index.md +++ b/docs/index.md @@ -652,8 +652,7 @@ print(Settings().model_dump()) CLI argument parsing of literals and enums are converted into CLI choices. - -```py test="skip" +```py import sys from enum import IntEnum from typing import Literal diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 3696c0ad..6797f498 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -851,7 +851,11 @@ def __init__( self.cli_prefix += '.' super().__init__( - settings_cls, env_nested_delimiter='.', env_parse_none_str=cli_parse_none_str, env_prefix=self.cli_prefix + settings_cls, + env_nested_delimiter='.', + env_parse_none_str=cli_parse_none_str, + env_parse_enums=True, + env_prefix=self.cli_prefix, ) root_parser = ( diff --git a/tests/test_settings.py b/tests/test_settings.py index fa6d65f6..df2dfa93 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1908,11 +1908,6 @@ class Settings(BaseSettings): def test_env_parse_enums(env): - class FruitsEnum(IntEnum): - pear = 0 - kiwi = 1 - lime = 2 - class Settings(BaseSettings): fruit: FruitsEnum @@ -2392,8 +2387,6 @@ class Cfg(BaseSettings): assert cfg.model_dump() == {'child': {'name': 'new name a', 'diff_a': 'new diff a'}} -# TODO Remove skip once environment parsing support for enums is added -@pytest.mark.skip def test_cli_enums(): class Pet(IntEnum): dog = 0 From 197114d4ed80132e0081474b03d8e3774212376c Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 12 Mar 2024 10:38:58 -0600 Subject: [PATCH 37/61] Exception validation and skip doc tests using --help. --- docs/index.md | 43 ++++++++++++++--------- tests/test_settings.py | 77 ++++++++++++++++++++++++++++++++---------- 2 files changed, 86 insertions(+), 34 deletions(-) diff --git a/docs/index.md b/docs/index.md index 4b6934ef..e123c885 100644 --- a/docs/index.md +++ b/docs/index.md @@ -815,15 +815,18 @@ The below flags can be used to customise the CLI experience to your needs. Change the default program name displayed in the help text usage by setting `cli_prog_name`. By default, it will derive the name of the currently executing program from `sys.argv[0]`, just like argparse. -```py -from pydantic_settings import BaseSettings, CliSettingsSource +```py test="skip" +import sys + +from pydantic_settings import BaseSettings -class Settings(BaseSettings, cli_prog_name='appdantic'): +class Settings(BaseSettings, cli_parse_args=True, cli_prog_name='appdantic'): pass -print(CliSettingsSource(Settings).root_parser.format_help()) +sys.argv = ['example.py', '--help'] +Settings() """ usage: appdantic [-h] @@ -873,19 +876,21 @@ example.py: error: the following arguments are required: --my_required_field Hide `None` values from the CLI help text by enabling `cli_hide_none_type`. -```py +```py test="skip" +import sys from typing import Optional from pydantic import Field -from pydantic_settings import BaseSettings, CliSettingsSource +from pydantic_settings import BaseSettings -class Settings(BaseSettings, cli_hide_none_type=True): +class Settings(BaseSettings, cli_parse_args=True, cli_hide_none_type=True): v0: Optional[str] = Field(description='the top level v0 option') -print(CliSettingsSource(Settings, cli_prog_name='example.py').root_parser.format_help()) +sys.argv = ['example.py', '--help'] +Settings() """ usage: example.py [-h] [--v0 str] @@ -899,23 +904,26 @@ options: Avoid adding complex fields that result in JSON strings at the CLI by enabling `cli_avoid_json`. -```py +```py test="skip" +import sys + from pydantic import BaseModel, Field -from pydantic_settings import BaseSettings, CliSettingsSource +from pydantic_settings import BaseSettings class SubModel(BaseModel): v1: int = Field(description='the sub model v1 option') -class Settings(BaseSettings, cli_avoid_json=True): +class Settings(BaseSettings, cli_parse_args=True, cli_avoid_json=True): sub_model: SubModel = Field( description='The help summary for SubModel related options' ) -print(CliSettingsSource(Settings, cli_prog_name='example.py').root_parser.format_help()) +sys.argv = ['example.py', '--help'] +Settings() """ usage: example.py [-h] [--sub_model.v1 int] @@ -938,10 +946,12 @@ Alternatively, we can also configure CLI settings to pull from the class docstri If the field is a union of nested models the group help text will always be pulled from the field description; even if `cli_use_class_docs_for_groups` is set to `True`. -```py +```py test="skip" +import sys + from pydantic import BaseModel, Field -from pydantic_settings import BaseSettings, CliSettingsSource +from pydantic_settings import BaseSettings class SubModel(BaseModel): @@ -950,13 +960,14 @@ class SubModel(BaseModel): v1: int = Field(description='the sub model v1 option') -class Settings(BaseSettings, cli_use_class_docs_for_groups=True): +class Settings(BaseSettings, cli_parse_args=True, cli_use_class_docs_for_groups=True): """My application help text.""" sub_model: SubModel = Field(description='The help text from the field description') -print(CliSettingsSource(Settings, cli_prog_name='example.py').root_parser.format_help()) +sys.argv = ['example.py', '--help'] +Settings() """ usage: example.py [-h] [--sub_model JSON] [--sub_model.v1 int] diff --git a/tests/test_settings.py b/tests/test_settings.py index df2dfa93..73b61b91 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2213,7 +2213,7 @@ def check_answer(cfg, prefix, expected): def test_cli_list_json_value_parsing(): class Cfg(BaseSettings): - json_list: list[str | bool | None] + json_list: list[Union[str, bool, None]] assert Cfg( _cli_parse_args=[ @@ -2298,11 +2298,13 @@ class Cfg(BaseSettings): expected['child'] = None assert cfg.model_dump() == expected - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: cfg = Cfg(_cli_parse_args=[f'--{prefix}check_dict', 'k9="i']) + assert str(exc_info.value) == f'Parsing error encountered for {prefix}check_dict: Mismatched quotes' with pytest.raises(SettingsError): cfg = Cfg(_cli_parse_args=[f'--{prefix}check_dict', 'k9=i"']) + assert str(exc_info.value) == f'Parsing error encountered for {prefix}check_dict: Mismatched quotes' def test_cli_nested_dict_arg(): @@ -2313,13 +2315,18 @@ class Cfg(BaseSettings): cfg = Cfg(_cli_parse_args=args) assert cfg.model_dump() == {'check_dict': {'k1': {'a': 1}, 'k2': {'b': 2}}} - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: args = ['--check_dict', '{"k1":{"a": 1}},"k2":{"b": 2}}'] cfg = Cfg(_cli_parse_args=args) + assert ( + str(exc_info.value) + == 'Parsing error encountered for check_dict: not enough values to unpack (expected 2, got 1)' + ) - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: args = ['--check_dict', '{"k1":{"a": 1}},{"k2":{"b": 2}'] cfg = Cfg(_cli_parse_args=args) + assert str(exc_info.value) == 'Parsing error encountered for check_dict: Missing end delimiter "}"' def test_cli_subcommand_with_positionals(): @@ -2399,8 +2406,16 @@ class Cfg(BaseSettings): cfg = Cfg(_cli_parse_args=['--pet', 'cat']) assert cfg.model_dump() == {'pet': Pet.cat} - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc_info: Cfg(_cli_parse_args=['--pet', 'rock']) + assert exc_info.value.errors(include_url=False) == [ + { + 'type': 'int_parsing', + 'loc': ('pet',), + 'msg': 'Input should be a valid integer, unable to parse string as an integer', + 'input': 'rock', + } + ] def test_cli_literals(): @@ -2410,8 +2425,17 @@ class Cfg(BaseSettings): cfg = Cfg(_cli_parse_args=['--pet', 'cat']) assert cfg.model_dump() == {'pet': 'cat'} - with pytest.raises(ValidationError): + with pytest.raises(ValidationError) as exc_info: Cfg(_cli_parse_args=['--pet', 'rock']) + assert exc_info.value.errors(include_url=False) == [ + { + 'ctx': {'expected': "'dog', 'cat' or 'bird'"}, + 'type': 'literal_error', + 'loc': ('pet',), + 'msg': "Input should be 'dog', 'cat' or 'bird'", + 'input': 'rock', + } + ] def test_cli_annotation_exceptions(monkeypatch): @@ -2424,54 +2448,63 @@ class SubCmd(BaseModel): with monkeypatch.context() as m: m.setattr(sys, 'argv', ['example.py', '--help']) - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: class SubCommandNotOutermost(BaseSettings, cli_parse_args=True): subcmd: Union[int, CliSubCommand[SubCmd]] SubCommandNotOutermost() + assert str(exc_info.value) == 'CliSubCommand is not outermost annotation for SubCommandNotOutermost.subcmd' - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: class SubCommandHasDefault(BaseSettings, cli_parse_args=True): subcmd: CliSubCommand[SubCmd] = SubCmd() SubCommandHasDefault() + assert str(exc_info.value) == 'subcommand argument SubCommandHasDefault.subcmd has a default value' - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: class SubCommandMultipleTypes(BaseSettings, cli_parse_args=True): subcmd: CliSubCommand[Union[SubCmd, SubCmdAlt]] SubCommandMultipleTypes() + assert str(exc_info.value) == 'subcommand argument SubCommandMultipleTypes.subcmd has multiple types' - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: class SubCommandNotModel(BaseSettings, cli_parse_args=True): subcmd: CliSubCommand[str] SubCommandNotModel() + assert str(exc_info.value) == 'subcommand argument SubCommandNotModel.subcmd is not derived from BaseModel' - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: class PositionalArgNotOutermost(BaseSettings, cli_parse_args=True): pos_arg: Union[int, CliPositionalArg[str]] PositionalArgNotOutermost() + assert ( + str(exc_info.value) == 'CliPositionalArg is not outermost annotation for PositionalArgNotOutermost.pos_arg' + ) - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: class PositionalArgHasDefault(BaseSettings, cli_parse_args=True): pos_arg: CliPositionalArg[str] = 'bad' PositionalArgHasDefault() + assert str(exc_info.value) == 'positional argument PositionalArgHasDefault.pos_arg has a default value' - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: class InvalidCliParseArgsType(BaseSettings, cli_parse_args='invalid type'): val: int InvalidCliParseArgsType() + assert str(exc_info.value) == "cli_parse_args must be List[str] or Tuple[str, ...], recieved " def test_cli_avoid_json(capsys, monkeypatch): @@ -2800,20 +2833,24 @@ def test_cli_user_settings_source_exceptions(): class Cfg(BaseSettings): pet: Literal['dog', 'cat', 'bird'] = 'bird' - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: args = ['--pet', 'dog'] parsed_args = {'pet': 'dog'} cli_cfg_settings = CliSettingsSource(Cfg) Cfg(_cli_settings_source=cli_cfg_settings(args=args, parsed_args=parsed_args)) + assert str(exc_info.value) == '`args` and `parsed_args` are mutually exclusive' - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: CliSettingsSource(Cfg, cli_prefix='.cfg') + assert str(exc_info.value) == 'CLI settings source prefix is invalid: .cfg' - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: CliSettingsSource(Cfg, cli_prefix='cfg.') + assert str(exc_info.value) == 'CLI settings source prefix is invalid: cfg.' - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: CliSettingsSource(Cfg, cli_prefix='123') + assert str(exc_info.value) == 'CLI settings source prefix is invalid: 123' class Food(BaseModel): fruit: FruitsEnum = FruitsEnum.kiwi @@ -2822,8 +2859,12 @@ class CfgWithSubCommand(BaseSettings): pet: Literal['dog', 'cat', 'bird'] = 'bird' food: CliSubCommand[Food] - with pytest.raises(SettingsError): + with pytest.raises(SettingsError) as exc_info: CliSettingsSource(CfgWithSubCommand, add_subparsers_method=None) + assert ( + str(exc_info.value) + == 'cannot connect CLI settings source root parser: add_subparsers_method is set to `None` but is needed for connecting' + ) @pytest.mark.parametrize( From 431bcb122cd86c227f6bece0eb711a1b4d29f6a6 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 12 Mar 2024 10:43:46 -0600 Subject: [PATCH 38/61] Python 3.8 fix. --- tests/test_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index 73b61b91..a11479ec 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2213,7 +2213,7 @@ def check_answer(cfg, prefix, expected): def test_cli_list_json_value_parsing(): class Cfg(BaseSettings): - json_list: list[Union[str, bool, None]] + json_list: List[Union[str, bool, None]] assert Cfg( _cli_parse_args=[ From 0eeee791acd3a39aad3db29623622e0457d71905 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 12 Mar 2024 10:48:06 -0600 Subject: [PATCH 39/61] Lint fixes. --- pydantic_settings/sources.py | 3 +-- tests/test_settings.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 6797f498..d56d39c4 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -883,8 +883,7 @@ def __init__( self._load_env_vars(parsed_args=self._parse_args(self.root_parser, cli_parse_args)) @overload - def __call__(self) -> dict[str, Any]: - ... + def __call__(self) -> dict[str, Any]: ... @overload def __call__(self, *, args: list[str] | tuple[str, ...] | bool) -> CliSettingsSource[T]: diff --git a/tests/test_settings.py b/tests/test_settings.py index a11479ec..6e275e94 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -104,8 +104,7 @@ def parse_args(self, *args, **kwargs) -> argparse.Namespace: class LoggedVar(Generic[T]): - def get(self) -> T: - ... + def get(self) -> T: ... class SimpleSettings(BaseSettings): From e04fa937e0452a0518902b60e61189ac6e03816e Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 12 Mar 2024 10:51:02 -0600 Subject: [PATCH 40/61] Lint fixes. --- pydantic_settings/sources.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index d56d39c4..6fcdafc7 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -938,8 +938,7 @@ def __call__( return super().__call__() @overload - def _load_env_vars(self) -> Mapping[str, str | None]: - ... + def _load_env_vars(self) -> Mapping[str, str | None]: ... @overload def _load_env_vars(self, *, parsed_args: Namespace | dict[str, list[str] | str]) -> CliSettingsSource[T]: From 7d735abebee822673ed2142c4d6d1eac81ebbeb9 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 12 Mar 2024 10:53:17 -0600 Subject: [PATCH 41/61] Mypy fix. --- pydantic_settings/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 6fcdafc7..16d29672 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -1276,7 +1276,7 @@ def _metavar_format_recurse(self, obj: Any) -> str: return '...' elif isinstance(obj, (pydantic._internal._repr.Representation, pydantic.v1.utils.Representation)): return repr(obj) - elif isinstance(obj, typing_extensions.TypeAliasType): # type: ignore + elif isinstance(obj, typing_extensions.TypeAliasType): return str(obj) if not isinstance(obj, (typing_base, WithArgsTypes, type)): From 1ce348aa4c20214a6a269e04a00938094a0f8a04 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 12 Mar 2024 11:28:41 -0600 Subject: [PATCH 42/61] Move integration doc section down. --- docs/index.md | 98 +++++++++++++++++++++++++-------------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/docs/index.md b/docs/index.md index e123c885..62515219 100644 --- a/docs/index.md +++ b/docs/index.md @@ -536,55 +536,6 @@ Settings( Note that a CLI settings source is always [**the topmost source**](#field-value-priority) and does not support [changing its priority](#changing-priority). -#### Integrating with Existing Parsers - -A CLI settings source can be integrated with existing parsers by overriding the default CLI settings source with a user -defined one that specifies the `root_parser` object. - -```py -import sys -from argparse import ArgumentParser - -from pydantic_settings import BaseSettings, CliSettingsSource - -parser = ArgumentParser() -parser.add_argument('--food', choices=['pear', 'kiwi', 'lime']) - - -class Settings(BaseSettings): - name: str = 'Bob' - - -# Set existing `parser` as the `root_parser` object for the user defined settings source -cli_settings = CliSettingsSource(Settings, root_parser=parser) - -# Parse and load CLI settings from the command line into the settings source. -sys.argv = ['example.py', '--food', 'kiwi', '--name', 'waldo'] -print(Settings(_cli_settings_source=cli_settings(args=True)).model_dump()) -#> {'name': 'waldo'} - -# Load CLI settings from pre-parsed arguments. i.e., the parsing occurs elsewhere and we -# just need to load the pre-parsed args into the settings source. -parsed_args = parser.parse_args(['--food', 'kiwi', '--name', 'ralph']) -print(Settings(_cli_settings_source=cli_settings(parsed_args=parsed_args)).model_dump()) -#> {'name': 'ralph'} -``` - -A `CliSettingsSource` connects with a `root_parser` object by using parser methods to add `settings_cls` fields as -command line arguments. The `CliSettingsSource` internal parser representation is based on the `argparse` library, and -therefore, requires parser methods that support the same attributes as their `argparse` counterparts. The available -parser methods that can be customised, along with their argparse counterparts (the defaults), are listed below: - -* `parse_args_method` - argparse.ArgumentParser.parse_args -* `add_argument_method` - argparse.ArgumentParser.add_argument -* `add_argument_group_method` - argparse.ArgumentParser.add\_argument_group -* `add_parser_method` - argparse.\_SubParsersAction.add_parser -* `add_subparsers_method` - argparse.ArgumentParser.add_subparsers -* `formatter_class` - argparse.HelpFormatter - -For a non-argparse parser the parser methods can be set to `None` if not supported. The CLI settings will only raise an -error when connecting to the root parser if a parser method is necessary but set to `None`. - #### Lists CLI argument parsing of lists supports intermixing of any of the below three styles: @@ -989,6 +940,55 @@ sub_model options: So if you provide extra values in a dotenv file, whether they start with `env_prefix` or not, a `ValidationError` will be raised. +### Integrating with Existing Parsers + +A CLI settings source can be integrated with existing parsers by overriding the default CLI settings source with a user +defined one that specifies the `root_parser` object. + +```py +import sys +from argparse import ArgumentParser + +from pydantic_settings import BaseSettings, CliSettingsSource + +parser = ArgumentParser() +parser.add_argument('--food', choices=['pear', 'kiwi', 'lime']) + + +class Settings(BaseSettings): + name: str = 'Bob' + + +# Set existing `parser` as the `root_parser` object for the user defined settings source +cli_settings = CliSettingsSource(Settings, root_parser=parser) + +# Parse and load CLI settings from the command line into the settings source. +sys.argv = ['example.py', '--food', 'kiwi', '--name', 'waldo'] +print(Settings(_cli_settings_source=cli_settings(args=True)).model_dump()) +#> {'name': 'waldo'} + +# Load CLI settings from pre-parsed arguments. i.e., the parsing occurs elsewhere and we +# just need to load the pre-parsed args into the settings source. +parsed_args = parser.parse_args(['--food', 'kiwi', '--name', 'ralph']) +print(Settings(_cli_settings_source=cli_settings(parsed_args=parsed_args)).model_dump()) +#> {'name': 'ralph'} +``` + +A `CliSettingsSource` connects with a `root_parser` object by using parser methods to add `settings_cls` fields as +command line arguments. The `CliSettingsSource` internal parser representation is based on the `argparse` library, and +therefore, requires parser methods that support the same attributes as their `argparse` counterparts. The available +parser methods that can be customised, along with their argparse counterparts (the defaults), are listed below: + +* `parse_args_method` - argparse.ArgumentParser.parse_args +* `add_argument_method` - argparse.ArgumentParser.add_argument +* `add_argument_group_method` - argparse.ArgumentParser.add\_argument_group +* `add_parser_method` - argparse.\_SubParsersAction.add_parser +* `add_subparsers_method` - argparse.ArgumentParser.add_subparsers +* `formatter_class` - argparse.HelpFormatter + +For a non-argparse parser the parser methods can be set to `None` if not supported. The CLI settings will only raise an +error when connecting to the root parser if a parser method is necessary but set to `None`. + ## Secrets Placing secret values in files is a common pattern to provide sensitive configuration to an application. From 16feca732ee028f5dd86863f6cfe0a01edb25300 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sun, 24 Mar 2024 13:01:26 -0600 Subject: [PATCH 43/61] Fix unioned dicts and hide_none_type metavar formatting. --- pydantic_settings/sources.py | 81 +++++++++++++++-------- tests/test_settings.py | 121 +++++++++++++++++++++++++++++++++-- 2 files changed, 171 insertions(+), 31 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 16d29672..75ffba87 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -928,7 +928,7 @@ def __call__( raise SettingsError('`args` and `parsed_args` are mutually exclusive') elif args is not None: if args is False: - return self + return self._load_env_vars(parsed_args={}) if args is True: args = sys.argv[1:] return self._load_env_vars(parsed_args=self._parse_args(self.root_parser, args)) @@ -994,7 +994,21 @@ def _merge_parsed_list(self, parsed_list: list[str], field_name: str) -> str: try: merged_list: list[str] = [] is_last_consumed_a_value = False - is_dict_list = field_name in self._cli_dict_arg_names + merge_type = self._cli_dict_args.get(field_name, list) + if ( + merge_type is list + or not origin_is_union(get_origin(merge_type)) + or not any( + type_ + for type_ in get_args(merge_type) + if type_ is not type(None) and get_origin(type_) not in (dict, Mapping) + ) + ): + inferred_type = merge_type + else: + inferred_type = ( + list if parsed_list and (len(parsed_list) > 1 or parsed_list[0].startswith('[')) else str + ) for val in parsed_list: if val.startswith('[') and val.endswith(']'): val = val[1:-1] @@ -1006,12 +1020,20 @@ def _merge_parsed_list(self, parsed_list: list[str], field_name: str) -> str: if val.startswith('{') or val.startswith('['): val = self._consume_object_or_array(val, merged_list) else: - val = self._consume_string_or_number(val, merged_list, is_dict_list) + try: + val = self._consume_string_or_number(val, merged_list, merge_type) + except ValueError as e: + if merge_type is inferred_type: + raise e + merge_type = inferred_type + val = self._consume_string_or_number(val, merged_list, merge_type) is_last_consumed_a_value = True if not is_last_consumed_a_value: val = self._consume_comma(val, merged_list, is_last_consumed_a_value) - if not is_dict_list: + if merge_type is str: + return merged_list[0] + elif merge_type is list: return f'[{",".join(merged_list)}]' else: merged_dict: dict[str, str] = {} @@ -1039,8 +1061,8 @@ def _consume_object_or_array(self, item: str, merged_list: list[str]) -> str: return item[consumed + 1 :] raise SettingsError(f'Missing end delimiter "{close_delim}"') - def _consume_string_or_number(self, item: str, merged_list: list[str], is_dict_list: bool) -> str: - consumed = 0 + def _consume_string_or_number(self, item: str, merged_list: list[str], merge_type: type[Any] | None) -> str: + consumed = 0 if merge_type is not str else len(item) is_find_end_quote = False while consumed < len(item): if item[consumed] == '"' and (consumed == 0 or item[consumed - 1] != '\\'): @@ -1051,7 +1073,7 @@ def _consume_string_or_number(self, item: str, merged_list: list[str], is_dict_l if is_find_end_quote: raise SettingsError('Mismatched quotes') val_string = item[:consumed].strip() - if not is_dict_list: + if merge_type in (list, str): try: float(val_string) except ValueError: @@ -1140,7 +1162,7 @@ def _connect_root_parser( self._add_parser = self._connect_parser_method(add_parser_method, 'add_parser_method') self._add_subparsers = self._connect_parser_method(add_subparsers_method, 'add_subparsers_method') self._formatter_class = formatter_class - self._cli_dict_arg_names: list[str] = [] + self._cli_dict_args: dict[str, type[Any] | None] = {} self._cli_subcommands: dict[str, list[str]] = {} self._add_parser_args( parser=self.root_parser, @@ -1201,15 +1223,11 @@ def _add_parser_args( if kwargs['dest'] in added_args: continue if _annotation_contains_types( - _strip_annotated(field_info.annotation), - (list, set, dict, Sequence, Mapping), - is_include_origin=True, + field_info.annotation, (list, set, dict, Sequence, Mapping), is_strip_annotated=True ): kwargs['action'] = 'append' - if _annotation_contains_types( - _strip_annotated(field_info.annotation), (dict, Mapping), is_include_origin=True - ): - self._cli_dict_arg_names.append(kwargs['dest']) + if _annotation_contains_types(field_info.annotation, (dict, Mapping), is_strip_annotated=True): + self._cli_dict_args[kwargs['dest']] = field_info.annotation arg_name = ( f'{arg_prefix}{field_name}' @@ -1262,10 +1280,14 @@ def _get_modified_args(self, obj: Any) -> tuple[str, ...]: else: return tuple([type_ for type_ in get_args(obj) if type_ is not type(None)]) - def _metavar_format_list(self, args: list[str]) -> str: + def _metavar_format_choices(self, args: list[str], obj_qualname: str | None = None) -> str: if 'JSON' in args: args = args[: args.index('JSON') + 1] + [arg for arg in args[args.index('JSON') + 1 :] if arg != 'JSON'] - return ','.join(args) + metavar = ','.join(args) + if obj_qualname: + return f'{obj_qualname}[{metavar}]' + else: + return metavar if len(args) == 1 else f'{{{metavar}}}' def _metavar_format_recurse(self, obj: Any) -> str: """Pretty metavar representation of a type. Adapts logic from `pydantic._repr.display_as_type`.""" @@ -1283,17 +1305,15 @@ def _metavar_format_recurse(self, obj: Any) -> str: obj = obj.__class__ if origin_is_union(get_origin(obj)): - args = self._metavar_format_list(list(map(self._metavar_format_recurse, self._get_modified_args(obj)))) - return f'{{{args}}}' if ',' in args else args + return self._metavar_format_choices(list(map(self._metavar_format_recurse, self._get_modified_args(obj)))) elif get_origin(obj) in (typing_extensions.Literal, typing.Literal): - args = self._metavar_format_list(list(map(str, self._get_modified_args(obj)))) - return f'{{{args}}}' if ',' in args else args + return self._metavar_format_choices(list(map(str, self._get_modified_args(obj)))) elif lenient_issubclass(obj, Enum): - args = self._metavar_format_list([val.name for val in obj]) - return f'{{{args}}}' if ',' in args else args + return self._metavar_format_choices([val.name for val in obj]) elif isinstance(obj, WithArgsTypes): - args = self._metavar_format_list(list(map(self._metavar_format_recurse, self._get_modified_args(obj)))) - return f'{obj.__qualname__}[{args}]' + return self._metavar_format_choices( + list(map(self._metavar_format_recurse, self._get_modified_args(obj))), obj_qualname=obj.__qualname__ + ) elif obj is type(None): return self.env_parse_none_str elif is_model_class(obj): @@ -1456,11 +1476,18 @@ def _union_is_complex(annotation: type[Any] | None, metadata: list[Any]) -> bool return any(_annotation_is_complex(arg, metadata) for arg in get_args(annotation)) -def _annotation_contains_types(annotation: type[Any] | None, types: tuple[Any, ...], is_include_origin: bool) -> bool: +def _annotation_contains_types( + annotation: type[Any] | None, + types: tuple[Any, ...], + is_include_origin: bool = True, + is_strip_annotated: bool = False, +) -> bool: + if is_strip_annotated: + annotation = _strip_annotated(annotation) if is_include_origin is True and get_origin(annotation) in types: return True for type_ in get_args(annotation): - if _annotation_contains_types(type_, types, is_include_origin=True): + if _annotation_contains_types(type_, types, is_include_origin=True, is_strip_annotated=is_strip_annotated): return True return annotation in types diff --git a/tests/test_settings.py b/tests/test_settings.py index 6e275e94..91071812 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2306,6 +2306,96 @@ class Cfg(BaseSettings): assert str(exc_info.value) == f'Parsing error encountered for {prefix}check_dict: Mismatched quotes' +def test_cli_union_dict_arg(): + class Cfg(BaseSettings): + union_str_dict: Union[str, Dict[str, Any]] + + with pytest.raises(ValidationError) as exc_info: + args = ['--union_str_dict', 'hello world', '--union_str_dict', 'hello world'] + cfg = Cfg(_cli_parse_args=args) + assert exc_info.value.errors(include_url=False) == [ + { + 'input': [ + 'hello world', + 'hello world', + ], + 'loc': ( + 'union_str_dict', + 'str', + ), + 'msg': 'Input should be a valid string', + 'type': 'string_type', + }, + { + 'input': [ + 'hello world', + 'hello world', + ], + 'loc': ( + 'union_str_dict', + 'dict[str,any]', + ), + 'msg': 'Input should be a valid dictionary', + 'type': 'dict_type', + }, + ] + + args = ['--union_str_dict', 'hello world'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == {'union_str_dict': 'hello world'} + + args = ['--union_str_dict', '{"hello": "world"}'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == {'union_str_dict': {'hello': 'world'}} + + args = ['--union_str_dict', 'hello=world'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == {'union_str_dict': {'hello': 'world'}} + + class Cfg(BaseSettings): + union_list_dict: Union[List[str], Dict[str, Any]] + + with pytest.raises(ValidationError) as exc_info: + args = ['--union_list_dict', 'hello,world'] + cfg = Cfg(_cli_parse_args=args) + assert exc_info.value.errors(include_url=False) == [ + { + 'input': 'hello,world', + 'loc': ( + 'union_list_dict', + 'list[str]', + ), + 'msg': 'Input should be a valid list', + 'type': 'list_type', + }, + { + 'input': 'hello,world', + 'loc': ( + 'union_list_dict', + 'dict[str,any]', + ), + 'msg': 'Input should be a valid dictionary', + 'type': 'dict_type', + }, + ] + + args = ['--union_list_dict', 'hello,world', '--union_list_dict', 'hello,world'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == {'union_list_dict': ['hello', 'world', 'hello', 'world']} + + args = ['--union_list_dict', '[hello,world]'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == {'union_list_dict': ['hello', 'world']} + + args = ['--union_list_dict', '{"hello": "world"}'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == {'union_list_dict': {'hello': 'world'}} + + args = ['--union_list_dict', 'hello=world'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == {'union_list_dict': {'hello': 'world'}} + + def test_cli_nested_dict_arg(): class Cfg(BaseSettings): check_dict: Dict[str, Any] @@ -2752,15 +2842,18 @@ class Cfg(BaseSettings): parsed_args = parse_args(args) assert Cfg(_cli_settings_source=cli_cfg_settings(parsed_args=parsed_args)).model_dump() == {'pet': 'bird'} assert Cfg(_cli_settings_source=cli_cfg_settings(args=args)).model_dump() == {'pet': 'bird'} + assert Cfg(_cli_settings_source=cli_cfg_settings(args=False)).model_dump() == {'pet': 'bird'} arg_prefix = f'{prefix}.' if prefix else '' args = ['--fruit', 'kiwi', f'--{arg_prefix}pet', 'dog'] parsed_args = parse_args(args) assert Cfg(_cli_settings_source=cli_cfg_settings(parsed_args=parsed_args)).model_dump() == {'pet': 'dog'} assert Cfg(_cli_settings_source=cli_cfg_settings(args=args)).model_dump() == {'pet': 'dog'} + assert Cfg(_cli_settings_source=cli_cfg_settings(args=False)).model_dump() == {'pet': 'bird'} parsed_args = parse_args(['--fruit', 'kiwi', f'--{arg_prefix}pet', 'cat']) assert Cfg(_cli_settings_source=cli_cfg_settings(parsed_args=vars(parsed_args))).model_dump() == {'pet': 'cat'} + assert Cfg(_cli_settings_source=cli_cfg_settings(args=False)).model_dump() == {'pet': 'bird'} @pytest.mark.parametrize('prefix', ['', 'cfg']) @@ -2898,8 +2991,18 @@ class CfgWithSubCommand(BaseSettings): (FruitsEnum, '{pear,kiwi,lime}'), ], ) -def test_cli_metavar_format(value, expected): - assert CliSettingsSource(SimpleSettings)._metavar_format(value) == expected +@pytest.mark.parametrize('hide_none_type', [True, False]) +def test_cli_metavar_format(hide_none_type, value, expected): + cli_settings = CliSettingsSource(SimpleSettings, cli_hide_none_type=hide_none_type) + if hide_none_type: + if value in ('foobar', 'SomeForwardRefString'): + expected = f"ForwardRef('{value}')" # forward ref implicit cast + if typing_extensions.get_origin(value) is Union: + args = typing_extensions.get_args(value) + value = Union[args + (None,) if args else (value, None)] + elif value != [1, 2, 3]: + value = Union[(value, None)] + assert cli_settings._metavar_format(value) == expected @pytest.mark.skipif(sys.version_info < (3, 10), reason='requires python 3.10 or higher') @@ -2921,9 +3024,19 @@ def test_cli_metavar_format(value, expected): (lambda: LoggedVar[Dict[int, str]], 'LoggedVar[Dict[int,str]]'), ], ) -def test_cli_metavar_format_310(value_gen, expected): +@pytest.mark.parametrize('hide_none_type', [True, False]) +def test_cli_metavar_format_310(hide_none_type, value_gen, expected): value = value_gen() - assert CliSettingsSource(SimpleSettings)._metavar_format(value) == expected + cli_settings = CliSettingsSource(SimpleSettings, cli_hide_none_type=hide_none_type) + if hide_none_type: + if value == 'SomeForwardRefString': + expected = f"ForwardRef('{value}')" # forward ref implicit cast + if typing_extensions.get_origin(value) is Union: + args = typing_extensions.get_args(value) + value = Union[args + (None,) if args else (value, None)] + else: + value = Union[(value, None)] + assert cli_settings._metavar_format(value) == expected @pytest.mark.skipif(sys.version_info < (3, 12), reason='requires python 3.12 or higher') From 309f1c605e7438b1ed95b4ba176cc3293587187c Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sun, 24 Mar 2024 13:36:19 -0600 Subject: [PATCH 44/61] Test case updates. --- tests/test_settings.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index 91071812..67d18828 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2995,12 +2995,14 @@ class CfgWithSubCommand(BaseSettings): def test_cli_metavar_format(hide_none_type, value, expected): cli_settings = CliSettingsSource(SimpleSettings, cli_hide_none_type=hide_none_type) if hide_none_type: + if value == [1, 2, 3] or isinstance(value, LoggedVar) or isinstance(value, Representation): + pytest.skip() if value in ('foobar', 'SomeForwardRefString'): expected = f"ForwardRef('{value}')" # forward ref implicit cast if typing_extensions.get_origin(value) is Union: args = typing_extensions.get_args(value) value = Union[args + (None,) if args else (value, None)] - elif value != [1, 2, 3]: + else: value = Union[(value, None)] assert cli_settings._metavar_format(value) == expected @@ -3009,12 +3011,7 @@ def test_cli_metavar_format(hide_none_type, value, expected): @pytest.mark.parametrize( 'value_gen,expected', [ - (lambda: str, 'str'), - (lambda: 'SomeForwardRefString', 'str'), # included to document current behavior; could be changed - (lambda: List['SomeForwardRef'], "List[ForwardRef('SomeForwardRef')]"), # noqa: F821 (lambda: str | int, '{str,int}'), - (lambda: list, 'list'), - (lambda: List, 'List'), (lambda: list[int], 'list[int]'), (lambda: List[int], 'List[int]'), (lambda: list[dict[str, int]], 'list[dict[str,int]]'), From 04c51ca39fcffd0a26db837163f5ede2f8c26b3e Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sun, 24 Mar 2024 14:17:23 -0600 Subject: [PATCH 45/61] Add string inference. --- pydantic_settings/sources.py | 5 ++++- tests/test_settings.py | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 75ffba87..fe8cc687 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -1083,7 +1083,10 @@ def _consume_string_or_number(self, item: str, merged_list: list[str], merge_typ val_string = f'"{val_string}"' merged_list.append(val_string) else: - key, val = (kv.strip('"') for kv in val_string.split('=', 1)) + key, val = (kv for kv in val_string.split('=', 1)) + if key.startswith('"') and not key.endswith('"') and not val.startswith('"') and val.endswith('"'): + raise ValueError(f'Dictionary key=val parameter is a quoted string: {val_string}') + key, val = key.strip('"'), val.strip('"') merged_list.append(json.dumps({key: val})) return item[consumed:] diff --git a/tests/test_settings.py b/tests/test_settings.py index 67d18828..afb06585 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2352,6 +2352,10 @@ class Cfg(BaseSettings): cfg = Cfg(_cli_parse_args=args) assert cfg.model_dump() == {'union_str_dict': {'hello': 'world'}} + args = ['--union_str_dict', '"hello=world"'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == {'union_str_dict': 'hello=world'} + class Cfg(BaseSettings): union_list_dict: Union[List[str], Dict[str, Any]] @@ -2395,6 +2399,34 @@ class Cfg(BaseSettings): cfg = Cfg(_cli_parse_args=args) assert cfg.model_dump() == {'union_list_dict': {'hello': 'world'}} + with pytest.raises(ValidationError) as exc_info: + args = ['--union_list_dict', '"hello=world"'] + cfg = Cfg(_cli_parse_args=args) + assert exc_info.value.errors(include_url=False) == [ + { + 'input': 'hello=world', + 'loc': ( + 'union_list_dict', + 'list[str]', + ), + 'msg': 'Input should be a valid list', + 'type': 'list_type', + }, + { + 'input': 'hello=world', + 'loc': ( + 'union_list_dict', + 'dict[str,any]', + ), + 'msg': 'Input should be a valid dictionary', + 'type': 'dict_type', + }, + ] + + args = ['--union_list_dict', '["hello=world"]'] + cfg = Cfg(_cli_parse_args=args) + assert cfg.model_dump() == {'union_list_dict': ['hello=world']} + def test_cli_nested_dict_arg(): class Cfg(BaseSettings): @@ -3026,8 +3058,6 @@ def test_cli_metavar_format_310(hide_none_type, value_gen, expected): value = value_gen() cli_settings = CliSettingsSource(SimpleSettings, cli_hide_none_type=hide_none_type) if hide_none_type: - if value == 'SomeForwardRefString': - expected = f"ForwardRef('{value}')" # forward ref implicit cast if typing_extensions.get_origin(value) is Union: args = typing_extensions.get_args(value) value = Union[args + (None,) if args else (value, None)] From 385b77009f9c149204cdf3103ce0dbf969c5af1e Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Fri, 29 Mar 2024 12:42:44 -0600 Subject: [PATCH 46/61] Remove v1 import. --- pydantic_settings/sources.py | 5 ++--- tests/test_settings.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 1239bcea..94ade017 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -27,11 +27,10 @@ overload, ) -import pydantic._internal._repr -import pydantic.v1.utils import typing_extensions from dotenv import dotenv_values from pydantic import AliasChoices, AliasPath, BaseModel, Json +from pydantic._internal._repr import Representation from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass from pydantic.fields import FieldInfo @@ -1305,7 +1304,7 @@ def _metavar_format_recurse(self, obj: Any) -> str: return obj.__name__ elif obj is ...: return '...' - elif isinstance(obj, (pydantic._internal._repr.Representation, pydantic.v1.utils.Representation)): + elif isinstance(obj, Representation): return repr(obj) elif isinstance(obj, typing_extensions.TypeAliasType): return str(obj) diff --git a/tests/test_settings.py b/tests/test_settings.py index 581eda71..6fa62cf3 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -28,8 +28,8 @@ from pydantic import ( dataclasses as pydantic_dataclasses, ) +from pydantic._internal._repr import Representation from pydantic.fields import FieldInfo -from pydantic.v1.utils import Representation from pytest_mock import MockerFixture from typing_extensions import Annotated From 4ee2bbdb084e0926394f0a561e97a1ce41e556af Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Fri, 29 Mar 2024 12:53:59 -0600 Subject: [PATCH 47/61] Docs fix. --- docs/index.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/index.md b/docs/index.md index 1081df42..21419f1b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -458,6 +458,12 @@ class Settings(BaseSettings): model_config = SettingsConfigDict(env_file='.env', extra='ignore') ``` + +!!! note + Pydantic settings loads all the values from dotenv file and passes it to the model, regardless of the model's `env_prefix`. + So if you provide extra values in a dotenv file, whether they start with `env_prefix` or not, + a `ValidationError` will be raised. + ## Command Line Support Pydantic settings provides integrated CLI support, making it easy to quickly define CLI applications using Pydantic @@ -935,11 +941,6 @@ sub_model options: """ ``` -!!! note - Pydantic settings loads all the values from dotenv file and passes it to the model, regardless of the model's `env_prefix`. - So if you provide extra values in a dotenv file, whether they start with `env_prefix` or not, - a `ValidationError` will be raised. - ### Integrating with Existing Parsers A CLI settings source can be integrated with existing parsers by overriding the default CLI settings source with a user From 58a3d8f39deafabe837a4d3feada69ad464c13c3 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sun, 19 May 2024 09:31:35 -0600 Subject: [PATCH 48/61] Add support for alias fields. --- pydantic_settings/sources.py | 49 ++++++++++++++++++++---------------- tests/test_settings.py | 13 ++++++++++ 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 94ade017..6d7d0305 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -1095,7 +1095,9 @@ def _consume_string_or_number(self, item: str, merged_list: list[str], merge_typ merged_list.append(json.dumps({key: val})) return item[consumed:] - def _get_sub_models(self, model: type[BaseModel], field_name: str, field_info: FieldInfo) -> list[type[BaseModel]]: + def _get_sub_models( + self, model: type[BaseModel], resolved_name: str, field_info: FieldInfo + ) -> list[type[BaseModel]]: field_types: tuple[Any, ...] = ( (field_info.annotation,) if not get_args(field_info.annotation) else get_args(field_info.annotation) ) @@ -1105,9 +1107,11 @@ def _get_sub_models(self, model: type[BaseModel], field_name: str, field_info: F sub_models: list[type[BaseModel]] = [] for type_ in field_types: if _annotation_contains_types(type_, (_CliSubCommand,), is_include_origin=False): - raise SettingsError(f'CliSubCommand is not outermost annotation for {model.__name__}.{field_name}') + raise SettingsError(f'CliSubCommand is not outermost annotation for {model.__name__}.{resolved_name}') elif _annotation_contains_types(type_, (_CliPositionalArg,), is_include_origin=False): - raise SettingsError(f'CliPositionalArg is not outermost annotation for {model.__name__}.{field_name}') + raise SettingsError( + f'CliPositionalArg is not outermost annotation for {model.__name__}.{resolved_name}' + ) if is_model_class(type_): sub_models.append(type_) return sub_models @@ -1115,24 +1119,25 @@ def _get_sub_models(self, model: type[BaseModel], field_name: str, field_info: F def _sort_arg_fields(self, model: type[BaseModel]) -> list[tuple[str, FieldInfo]]: positional_args, subcommand_args, optional_args = [], [], [] for field_name, field_info in model.model_fields.items(): + resolved_name = field_name if field_info.alias is None else field_info.alias if _CliSubCommand in field_info.metadata: if not field_info.is_required(): - raise SettingsError(f'subcommand argument {model.__name__}.{field_name} has a default value') + raise SettingsError(f'subcommand argument {model.__name__}.{resolved_name} has a default value') else: field_types = [type_ for type_ in get_args(field_info.annotation) if type_ is not type(None)] if len(field_types) != 1: - raise SettingsError(f'subcommand argument {model.__name__}.{field_name} has multiple types') + raise SettingsError(f'subcommand argument {model.__name__}.{resolved_name} has multiple types') elif not is_model_class(field_types[0]): raise SettingsError( - f'subcommand argument {model.__name__}.{field_name} is not derived from BaseModel' + f'subcommand argument {model.__name__}.{resolved_name} is not derived from BaseModel' ) - subcommand_args.append((field_name, field_info)) + subcommand_args.append((resolved_name, field_info)) elif _CliPositionalArg in field_info.metadata: if not field_info.is_required(): - raise SettingsError(f'positional argument {model.__name__}.{field_name} has a default value') - positional_args.append((field_name, field_info)) + raise SettingsError(f'positional argument {model.__name__}.{resolved_name} has a default value') + positional_args.append((resolved_name, field_info)) else: - optional_args.append((field_name, field_info)) + optional_args.append((resolved_name, field_info)) return positional_args + subcommand_args + optional_args @property @@ -1191,16 +1196,16 @@ def _add_parser_args( group: Any, ) -> ArgumentParser: subparsers: Any = None - for field_name, field_info in self._sort_arg_fields(model): - sub_models: list[type[BaseModel]] = self._get_sub_models(model, field_name, field_info) + for resolved_name, field_info in self._sort_arg_fields(model): + sub_models: list[type[BaseModel]] = self._get_sub_models(model, resolved_name, field_info) if _CliSubCommand in field_info.metadata: if subparsers is None: subparsers = self._add_subparsers( parser, title='subcommands', dest=f'{arg_prefix}:subcommand', required=self.cli_enforce_required ) - self._cli_subcommands[f'{arg_prefix}:subcommand'] = [f'{arg_prefix}{field_name}'] + self._cli_subcommands[f'{arg_prefix}:subcommand'] = [f'{arg_prefix}{resolved_name}'] else: - self._cli_subcommands[f'{arg_prefix}:subcommand'].append(f'{arg_prefix}{field_name}') + self._cli_subcommands[f'{arg_prefix}:subcommand'].append(f'{arg_prefix}{resolved_name}') if hasattr(subparsers, 'metavar'): metavar = ','.join(self._cli_subcommands[f'{arg_prefix}:subcommand']) subparsers.metavar = f'{{{metavar}}}' @@ -1209,15 +1214,15 @@ def _add_parser_args( self._add_parser_args( parser=self._add_parser( subparsers, - field_name, + resolved_name, help=field_info.description, formatter_class=self._formatter_class, description=model.__doc__, ), model=model, added_args=[], - arg_prefix=f'{arg_prefix}{field_name}.', - subcommand_prefix=f'{subcommand_prefix}{field_name}.', + arg_prefix=f'{arg_prefix}{resolved_name}.', + subcommand_prefix=f'{subcommand_prefix}{resolved_name}.', group=None, ) else: @@ -1225,7 +1230,7 @@ def _add_parser_args( kwargs: dict[str, Any] = {} kwargs['default'] = SUPPRESS kwargs['help'] = field_info.description - kwargs['dest'] = f'{arg_prefix}{field_name}' + kwargs['dest'] = f'{arg_prefix}{resolved_name}' kwargs['metavar'] = self._metavar_format(field_info.annotation) kwargs['required'] = self.cli_enforce_required and field_info.is_required() if kwargs['dest'] in added_args: @@ -1238,12 +1243,12 @@ def _add_parser_args( self._cli_dict_args[kwargs['dest']] = field_info.annotation arg_name = ( - f'{arg_prefix}{field_name}' + f'{arg_prefix}{resolved_name}' if subcommand_prefix == self.env_prefix - else f'{arg_prefix.replace(subcommand_prefix, "", 1)}{field_name}' + else f'{arg_prefix.replace(subcommand_prefix, "", 1)}{resolved_name}' ) if _CliPositionalArg in field_info.metadata: - kwargs['metavar'] = field_name.upper() + kwargs['metavar'] = resolved_name.upper() arg_name = kwargs['dest'] del kwargs['dest'] del kwargs['required'] @@ -1268,7 +1273,7 @@ def _add_parser_args( parser=parser, model=model, added_args=added_args, - arg_prefix=f'{arg_prefix}{field_name}.', + arg_prefix=f'{arg_prefix}{resolved_name}.', subcommand_prefix=subcommand_prefix, group=model_group if model_group else model_group_kwargs, ) diff --git a/tests/test_settings.py b/tests/test_settings.py index 6fa62cf3..f5aafb42 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2131,6 +2131,19 @@ class Cfg(BaseSettings): } +def test_cli_alias_arg(): + class Animal(BaseModel): + name: str + + class Cfg(BaseSettings): + apple: str = Field(alias='alias') + pet: Animal = Field(alias='critter') + + cfg = Cfg(_cli_parse_args=['--alias', 'foo', '--critter.name', 'harry']) + assert cfg.model_dump() == {'apple': 'foo', 'pet': {'name': 'harry'}} + assert cfg.model_dump(by_alias=True) == {'alias': 'foo', 'critter': {'name': 'harry'}} + + @pytest.mark.parametrize('prefix', ['', 'child.']) def test_cli_list_arg(prefix): class Obj(BaseModel): From 1b41f173b2ab9646c22926fbdb9d86d9b26bdaf0 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 23 May 2024 11:38:37 -0600 Subject: [PATCH 49/61] Add support for pydantic dataclasses. --- pydantic_settings/sources.py | 6 ++++-- tests/test_settings.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 6d7d0305..f0b76396 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -33,6 +33,7 @@ from pydantic._internal._repr import Representation from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass +from pydantic.dataclasses import is_pydantic_dataclass from pydantic.fields import FieldInfo from typing_extensions import Annotated, get_args, get_origin @@ -1112,13 +1113,14 @@ def _get_sub_models( raise SettingsError( f'CliPositionalArg is not outermost annotation for {model.__name__}.{resolved_name}' ) - if is_model_class(type_): + if is_model_class(type_) or is_pydantic_dataclass(type_): sub_models.append(type_) return sub_models def _sort_arg_fields(self, model: type[BaseModel]) -> list[tuple[str, FieldInfo]]: positional_args, subcommand_args, optional_args = [], [], [] - for field_name, field_info in model.model_fields.items(): + fields = model.__pydantic_fields__ if is_pydantic_dataclass(model) else model.model_fields + for field_name, field_info in fields.items(): resolved_name = field_name if field_info.alias is None else field_info.alias if _CliSubCommand in field_info.metadata: if not field_info.is_required(): diff --git a/tests/test_settings.py b/tests/test_settings.py index f5aafb42..014f4f3c 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2144,6 +2144,21 @@ class Cfg(BaseSettings): assert cfg.model_dump(by_alias=True) == {'alias': 'foo', 'critter': {'name': 'harry'}} +def test_cli_nested_dataclass_arg(): + @pydantic_dataclasses.dataclass + class MyDataclass: + foo: int + bar: str + + class Settings(BaseSettings): + n: MyDataclass + + s = Settings(_cli_parse_args=['--n.foo', '123', '--n.bar', 'bar value']) + assert isinstance(s.n, MyDataclass) + assert s.n.foo == 123 + assert s.n.bar == 'bar value' + + @pytest.mark.parametrize('prefix', ['', 'child.']) def test_cli_list_arg(prefix): class Obj(BaseModel): From 8606f78f60d57de89b6aa3e64eba7a3c2a731ef6 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 23 May 2024 15:40:18 -0600 Subject: [PATCH 50/61] Add support for CLISettingsSource prioritization. --- docs/index.md | 4 ++-- pydantic_settings/main.py | 5 +++-- tests/test_settings.py | 27 +++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index 21419f1b..924ccf3a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -539,8 +539,8 @@ Settings( ) ``` -Note that a CLI settings source is always [**the topmost source**](#field-value-priority) and does not support [changing -its priority](#changing-priority). +Note that a CLI settings source is [**the topmost source**](#field-value-priority) by default unless its [priority value +is changed](#changing-priority). #### Lists diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index e78e0d3d..270599ea 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -292,8 +292,9 @@ def _settings_build_values( dotenv_settings=dotenv_settings, file_secret_settings=file_secret_settings, ) - if cli_parse_args or cli_settings_source: - sources = (cli_settings,) + sources + if not any([source for source in sources if isinstance(source, CliSettingsSource)]): + if cli_parse_args or cli_settings_source: + sources = (cli_settings,) + sources if sources: return deep_update(*reversed([source() for source in sources])) else: diff --git a/tests/test_settings.py b/tests/test_settings.py index 014f4f3c..3b747911 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2131,6 +2131,33 @@ class Cfg(BaseSettings): } +def test_cli_source_prioritization(env): + class CfgDefault(BaseSettings): + foo: str + + class CfgPrioritized(BaseSettings): + foo: str + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return env_settings, CliSettingsSource(settings_cls, cli_parse_args=['--foo', 'FOO FROM CLI']) + + env.set('FOO', 'FOO FROM ENV') + + cfg = CfgDefault(_cli_parse_args=['--foo', 'FOO FROM CLI']) + assert cfg.model_dump() == {'foo': 'FOO FROM CLI'} + + cfg = CfgPrioritized() + assert cfg.model_dump() == {'foo': 'FOO FROM ENV'} + + def test_cli_alias_arg(): class Animal(BaseModel): name: str From 155ffe344989e7ba791d05adecb8f482d25b4637 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 23 May 2024 17:11:26 -0600 Subject: [PATCH 51/61] Fixes. --- pydantic_settings/sources.py | 5 +++-- tests/test_settings.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 264a9e79..73b0fcb6 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -674,7 +674,8 @@ def explode_env_vars(self, field_name: str, field: FieldInfo, env_vars: Mapping[ if not allow_json_failure: raise e if isinstance(env_var, dict): - env_var[last_key] = env_val + if last_key not in env_var or not isinstance(env_val, EnvNoneType) or env_var[last_key] is {}: + env_var[last_key] = env_val return result @@ -1114,7 +1115,7 @@ def _get_sub_models( f'CliPositionalArg is not outermost annotation for {model.__name__}.{resolved_name}' ) if is_model_class(type_) or is_pydantic_dataclass(type_): - sub_models.append(type_) + sub_models.append(type_) # type: ignore return sub_models def _sort_arg_fields(self, model: type[BaseModel]) -> list[tuple[str, FieldInfo]]: diff --git a/tests/test_settings.py b/tests/test_settings.py index cebaf816..49b6363b 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -8,7 +8,7 @@ from datetime import datetime, timezone from enum import IntEnum from pathlib import Path -from typing import Any, Callable, Dict, Generic, Hashable, List, Literal, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Any, Callable, Dict, Generic, Hashable, List, Optional, Set, Tuple, Type, TypeVar, Union import pytest import typing_extensions @@ -1955,13 +1955,13 @@ class Settings(BaseSettings): with pytest.raises(ValidationError) as exc_info: env.set('FRUIT', 'kiwi') s = Settings() + print(exc_info.value.errors(include_url=False)) assert exc_info.value.errors(include_url=False) == [ { - 'type': 'enum', + 'type': 'int_parsing', 'loc': ('fruit',), - 'msg': 'Input should be 0, 1 or 2', + 'msg': 'Input should be a valid integer, unable to parse string as an integer', 'input': 'kiwi', - 'ctx': {'expected': '0, 1 or 2'}, } ] From 5f430b067f4c1b20e63498ccbc2fbacb2115b239 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 23 May 2024 17:27:15 -0600 Subject: [PATCH 52/61] Fixes. --- tests/test_settings.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index 49b6363b..3c45e0aa 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1955,13 +1955,13 @@ class Settings(BaseSettings): with pytest.raises(ValidationError) as exc_info: env.set('FRUIT', 'kiwi') s = Settings() - print(exc_info.value.errors(include_url=False)) assert exc_info.value.errors(include_url=False) == [ { - 'type': 'int_parsing', + 'type': 'enum', 'loc': ('fruit',), - 'msg': 'Input should be a valid integer, unable to parse string as an integer', + 'msg': 'Input should be 0, 1 or 2', 'input': 'kiwi', + 'ctx': {'expected': '0, 1 or 2'}, } ] @@ -2629,10 +2629,11 @@ class Cfg(BaseSettings): Cfg(_cli_parse_args=['--pet', 'rock']) assert exc_info.value.errors(include_url=False) == [ { - 'type': 'int_parsing', + 'type': 'enum', 'loc': ('pet',), - 'msg': 'Input should be a valid integer, unable to parse string as an integer', + 'msg': 'Input should be 0, 1 or 2', 'input': 'rock', + 'ctx': {'expected': '0, 1 or 2'}, } ] From 53163a835f17d30228bee64309b9c51ead21363f Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 23 May 2024 17:30:05 -0600 Subject: [PATCH 53/61] Lint. --- pydantic_settings/sources.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 73b0fcb6..b9a5f910 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -992,7 +992,10 @@ def _load_env_vars( parsed_args[last_selected_subcommand] = '{}' self.env_vars = parse_env_vars( - parsed_args, self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str # type: ignore + parsed_args, + self.case_sensitive, + self.env_ignore_empty, + self.env_parse_none_str, # type: ignore ) return self @@ -1115,7 +1118,7 @@ def _get_sub_models( f'CliPositionalArg is not outermost annotation for {model.__name__}.{resolved_name}' ) if is_model_class(type_) or is_pydantic_dataclass(type_): - sub_models.append(type_) # type: ignore + sub_models.append(type_) # type: ignore return sub_models def _sort_arg_fields(self, model: type[BaseModel]) -> list[tuple[str, FieldInfo]]: From 1f62254a25ce4dce4eeded73c9448aaa9d902560 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 23 May 2024 17:40:08 -0600 Subject: [PATCH 54/61] Lint. --- pydantic_settings/sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index b9a5f910..d0fd37f1 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -992,10 +992,10 @@ def _load_env_vars( parsed_args[last_selected_subcommand] = '{}' self.env_vars = parse_env_vars( - parsed_args, + cast(Mapping[str, str | None], parsed_args), self.case_sensitive, self.env_ignore_empty, - self.env_parse_none_str, # type: ignore + self.env_parse_none_str, ) return self From 0016a5e4a289dca8941bc70f327cd5d72b0fcd7b Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 23 May 2024 17:46:08 -0600 Subject: [PATCH 55/61] Lint. --- pydantic_settings/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index d0fd37f1..16536265 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -992,7 +992,7 @@ def _load_env_vars( parsed_args[last_selected_subcommand] = '{}' self.env_vars = parse_env_vars( - cast(Mapping[str, str | None], parsed_args), + cast(Mapping[str, str], parsed_args), self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str, From b555ad11a94da4ca24bf60b928a992ae0b179c90 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Fri, 24 May 2024 00:21:11 -0600 Subject: [PATCH 56/61] Add support for case-insensitive matching. --- pydantic_settings/main.py | 1 + pydantic_settings/sources.py | 78 +++++++++++++++++++++++++++--------- tests/test_settings.py | 19 +++++++++ 3 files changed, 78 insertions(+), 20 deletions(-) diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 270599ea..73e50c4b 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -256,6 +256,7 @@ def _settings_build_values( cli_enforce_required=cli_enforce_required, cli_use_class_docs_for_groups=cli_use_class_docs_for_groups, cli_prefix=cli_prefix, + case_sensitive=case_sensitive, ) if cli_settings_source is None else cli_settings_source diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 16536265..391ec195 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -2,6 +2,8 @@ import json import os +import re +import shlex import sys import typing import warnings @@ -98,6 +100,10 @@ class _CliPositionalArg: pass +class _CliInternalArgParser(ArgumentParser): + pass + + T = TypeVar('T') CliSubCommand = Annotated[Union[T, None], _CliSubCommand] CliPositionalArg = Annotated[T, _CliPositionalArg] @@ -797,6 +803,9 @@ class CliSettingsSource(EnvSettingsSource, Generic[T]): cli_use_class_docs_for_groups: Use class docstrings in CLI group help text instead of field descriptions. Defaults to `False`. cli_prefix: Prefix for command line arguments added under the root parser. Defaults to "". + case_sensitive: Whether CLI "--arg" names should be read with case-sensitivity. Defaults to `True`. + Note: Case-insensitive matching is only supported on the internal root parser and does not apply to CLI + subcommands. root_parser: The root parser object. parse_args_method: The root parser parse args method. Defaults to `argparse.ArgumentParser.parse_args`. add_argument_method: The root parser add argument method. Defaults to `argparse.ArgumentParser.add_argument`. @@ -820,6 +829,7 @@ def __init__( cli_enforce_required: bool | None = None, cli_use_class_docs_for_groups: bool | None = None, cli_prefix: str | None = None, + case_sensitive: bool = True, root_parser: Any = None, parse_args_method: Callable[..., Any] | None = ArgumentParser.parse_args, add_argument_method: Callable[..., Any] | None = ArgumentParser.add_argument, @@ -857,16 +867,20 @@ def __init__( raise SettingsError(f'CLI settings source prefix is invalid: {cli_prefix}') self.cli_prefix += '.' + if not case_sensitive and root_parser is not None: + raise SettingsError('Case-insensitive matching is only supported on the internal root parser') + super().__init__( settings_cls, env_nested_delimiter='.', env_parse_none_str=cli_parse_none_str, env_parse_enums=True, env_prefix=self.cli_prefix, + case_sensitive=case_sensitive, ) root_parser = ( - ArgumentParser(prog=self.cli_prog_name, description=settings_cls.__doc__) + _CliInternalArgParser(prog=self.cli_prog_name, description=settings_cls.__doc__) if root_parser is None else root_parser ) @@ -1101,7 +1115,7 @@ def _consume_string_or_number(self, item: str, merged_list: list[str], merge_typ return item[consumed:] def _get_sub_models( - self, model: type[BaseModel], resolved_name: str, field_info: FieldInfo + self, model: type[BaseModel], field_name: str, field_info: FieldInfo ) -> list[type[BaseModel]]: field_types: tuple[Any, ...] = ( (field_info.annotation,) if not get_args(field_info.annotation) else get_args(field_info.annotation) @@ -1112,38 +1126,39 @@ def _get_sub_models( sub_models: list[type[BaseModel]] = [] for type_ in field_types: if _annotation_contains_types(type_, (_CliSubCommand,), is_include_origin=False): - raise SettingsError(f'CliSubCommand is not outermost annotation for {model.__name__}.{resolved_name}') + raise SettingsError(f'CliSubCommand is not outermost annotation for {model.__name__}.{field_name}') elif _annotation_contains_types(type_, (_CliPositionalArg,), is_include_origin=False): raise SettingsError( - f'CliPositionalArg is not outermost annotation for {model.__name__}.{resolved_name}' + f'CliPositionalArg is not outermost annotation for {model.__name__}.{field_name}' ) if is_model_class(type_) or is_pydantic_dataclass(type_): sub_models.append(type_) # type: ignore return sub_models - def _sort_arg_fields(self, model: type[BaseModel]) -> list[tuple[str, FieldInfo]]: + def _sort_arg_fields(self, model: type[BaseModel]) -> list[tuple[str, str, FieldInfo]]: positional_args, subcommand_args, optional_args = [], [], [] fields = model.__pydantic_fields__ if is_pydantic_dataclass(model) else model.model_fields for field_name, field_info in fields.items(): resolved_name = field_name if field_info.alias is None else field_info.alias + resolved_name = resolved_name.lower() if not self.case_sensitive else resolved_name if _CliSubCommand in field_info.metadata: if not field_info.is_required(): - raise SettingsError(f'subcommand argument {model.__name__}.{resolved_name} has a default value') + raise SettingsError(f'subcommand argument {model.__name__}.{field_name} has a default value') else: field_types = [type_ for type_ in get_args(field_info.annotation) if type_ is not type(None)] if len(field_types) != 1: - raise SettingsError(f'subcommand argument {model.__name__}.{resolved_name} has multiple types') + raise SettingsError(f'subcommand argument {model.__name__}.{field_name} has multiple types') elif not is_model_class(field_types[0]): raise SettingsError( f'subcommand argument {model.__name__}.{resolved_name} is not derived from BaseModel' ) - subcommand_args.append((resolved_name, field_info)) + subcommand_args.append((field_name, resolved_name, field_info)) elif _CliPositionalArg in field_info.metadata: if not field_info.is_required(): - raise SettingsError(f'positional argument {model.__name__}.{resolved_name} has a default value') - positional_args.append((resolved_name, field_info)) + raise SettingsError(f'positional argument {model.__name__}.{field_name} has a default value') + positional_args.append((field_name, resolved_name, field_info)) else: - optional_args.append((resolved_name, field_info)) + optional_args.append((field_name, resolved_name, field_info)) return positional_args + subcommand_args + optional_args @property @@ -1154,15 +1169,38 @@ def root_parser(self) -> T: def _connect_parser_method( self, parser_method: Callable[..., Any] | None, method_name: str, *args: Any, **kwargs: Any ) -> Callable[..., Any]: - if parser_method: - return parser_method + if not parser_method: - def none_parser_method(*args: Any, **kwargs: Any) -> Any: - raise SettingsError( - f'cannot connect CLI settings source root parser: {method_name} is set to `None` but is needed for connecting' - ) + def none_parser_method(*args: Any, **kwargs: Any) -> Any: + raise SettingsError( + f'cannot connect CLI settings source root parser: {method_name} is set to `None` but is needed for connecting' + ) + + return none_parser_method + + elif ( + self.case_sensitive is False + and method_name == 'parsed_args_method' + and isinstance(self._root_parser, _CliInternalArgParser) + ): + + def parse_args_insensitive_method( + root_parser: _CliInternalArgParser, + args: list[str] | tuple[str, ...] | None = None, + namespace: Namespace | None = None, + ) -> Any: + insensitive_args = [] + for arg in shlex.split(shlex.join(args)) if args else []: + matched = re.match(r'^(--[^\s=]+)(.*)', arg) + if matched: + arg = matched.group(1).lower() + matched.group(2) + insensitive_args.append(arg) + return parser_method(root_parser, insensitive_args, namespace) + + return parse_args_insensitive_method - return none_parser_method + else: + return parser_method def _connect_root_parser( self, @@ -1202,8 +1240,8 @@ def _add_parser_args( group: Any, ) -> ArgumentParser: subparsers: Any = None - for resolved_name, field_info in self._sort_arg_fields(model): - sub_models: list[type[BaseModel]] = self._get_sub_models(model, resolved_name, field_info) + for field_name, resolved_name, field_info in self._sort_arg_fields(model): + sub_models: list[type[BaseModel]] = self._get_sub_models(model, field_name, field_info) if _CliSubCommand in field_info.metadata: if subparsers is None: subparsers = self._add_subparsers( diff --git a/tests/test_settings.py b/tests/test_settings.py index 3c45e0aa..f658d015 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2212,6 +2212,25 @@ class Cfg(BaseSettings): assert cfg.model_dump(by_alias=True) == {'alias': 'foo', 'critter': {'name': 'harry'}} +def test_cli_case_insensitve_arg(): + class Cfg(BaseSettings): + Foo: str + Bar: str + + cfg = Cfg(_cli_parse_args=['--FOO=--VAL', '--BAR', '"--VAL"']) + assert cfg.model_dump() == {'Foo': '--VAL', 'Bar': '"--VAL"'} + + cfg = Cfg(_cli_parse_args=['--Foo=--VAL', '--Bar', '"--VAL"'], _case_sensitive=True) + assert cfg.model_dump() == {'Foo': '--VAL', 'Bar': '"--VAL"'} + + with pytest.raises(SystemExit): + Cfg(_cli_parse_args=['--FOO=--VAL', '--BAR', '"--VAL"'], _case_sensitive=True) + + with pytest.raises(SettingsError) as exc_info: + CliSettingsSource(Cfg, root_parser=CliDummyParser(), case_sensitive=False) + assert str(exc_info.value) == 'Case-insensitive matching is only supported on the internal root parser' + + def test_cli_nested_dataclass_arg(): @pydantic_dataclasses.dataclass class MyDataclass: From 3c27f27b48ec7c94590e2a0ef7bf129c9490dfe5 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Fri, 24 May 2024 00:25:11 -0600 Subject: [PATCH 57/61] Lint. --- pydantic_settings/sources.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 391ec195..1f409dc4 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -829,7 +829,7 @@ def __init__( cli_enforce_required: bool | None = None, cli_use_class_docs_for_groups: bool | None = None, cli_prefix: str | None = None, - case_sensitive: bool = True, + case_sensitive: bool | None = True, root_parser: Any = None, parse_args_method: Callable[..., Any] | None = ArgumentParser.parse_args, add_argument_method: Callable[..., Any] | None = ArgumentParser.add_argument, @@ -867,6 +867,7 @@ def __init__( raise SettingsError(f'CLI settings source prefix is invalid: {cli_prefix}') self.cli_prefix += '.' + case_sensitive = case_sensitive if case_sensitive is not None else True if not case_sensitive and root_parser is not None: raise SettingsError('Case-insensitive matching is only supported on the internal root parser') From a3fe71ce2a0ebcb3bf67ace8d642f51eaede785b Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Fri, 24 May 2024 00:34:54 -0600 Subject: [PATCH 58/61] Lint. --- pydantic_settings/sources.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 1f409dc4..38e18bb5 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -1115,9 +1115,7 @@ def _consume_string_or_number(self, item: str, merged_list: list[str], merge_typ merged_list.append(json.dumps({key: val})) return item[consumed:] - def _get_sub_models( - self, model: type[BaseModel], field_name: str, field_info: FieldInfo - ) -> list[type[BaseModel]]: + def _get_sub_models(self, model: type[BaseModel], field_name: str, field_info: FieldInfo) -> list[type[BaseModel]]: field_types: tuple[Any, ...] = ( (field_info.annotation,) if not get_args(field_info.annotation) else get_args(field_info.annotation) ) @@ -1129,9 +1127,7 @@ def _get_sub_models( if _annotation_contains_types(type_, (_CliSubCommand,), is_include_origin=False): raise SettingsError(f'CliSubCommand is not outermost annotation for {model.__name__}.{field_name}') elif _annotation_contains_types(type_, (_CliPositionalArg,), is_include_origin=False): - raise SettingsError( - f'CliPositionalArg is not outermost annotation for {model.__name__}.{field_name}' - ) + raise SettingsError(f'CliPositionalArg is not outermost annotation for {model.__name__}.{field_name}') if is_model_class(type_) or is_pydantic_dataclass(type_): sub_models.append(type_) # type: ignore return sub_models @@ -1170,17 +1166,9 @@ def root_parser(self) -> T: def _connect_parser_method( self, parser_method: Callable[..., Any] | None, method_name: str, *args: Any, **kwargs: Any ) -> Callable[..., Any]: - if not parser_method: - - def none_parser_method(*args: Any, **kwargs: Any) -> Any: - raise SettingsError( - f'cannot connect CLI settings source root parser: {method_name} is set to `None` but is needed for connecting' - ) - - return none_parser_method - - elif ( - self.case_sensitive is False + if ( + parser_method is not None + and self.case_sensitive is False and method_name == 'parsed_args_method' and isinstance(self._root_parser, _CliInternalArgParser) ): @@ -1196,10 +1184,19 @@ def parse_args_insensitive_method( if matched: arg = matched.group(1).lower() + matched.group(2) insensitive_args.append(arg) - return parser_method(root_parser, insensitive_args, namespace) + return parser_method(root_parser, insensitive_args, namespace) # type: ignore return parse_args_insensitive_method + elif parser_method is None: + + def none_parser_method(*args: Any, **kwargs: Any) -> Any: + raise SettingsError( + f'cannot connect CLI settings source root parser: {method_name} is set to `None` but is needed for connecting' + ) + + return none_parser_method + else: return parser_method From a1688c77a3c01d4de91645d6796ea5a2f1eddc4b Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Fri, 24 May 2024 09:55:01 -0600 Subject: [PATCH 59/61] Add CliSettingsSource prioritization doc. --- docs/index.md | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 455b8078..f144c095 100644 --- a/docs/index.md +++ b/docs/index.md @@ -540,7 +540,42 @@ Settings( ``` Note that a CLI settings source is [**the topmost source**](#field-value-priority) by default unless its [priority value -is changed](#changing-priority). +is customised](#customise-settings-sources): + +```py +import os +import sys +from typing import Tuple, Type + +from pydantic_settings import ( + BaseSettings, + CliSettingsSource, + PydanticBaseSettingsSource, +) + + +class Settings(BaseSettings): + my_foo: str + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return env_settings, CliSettingsSource(settings_cls, cli_parse_args=True) + + +os.environ['MY_FOO'] = 'from environment' + +sys.argv = ['example.py', '--my_foo=from cli'] + +print(Settings().model_dump()) +#> {'my_foo': 'from environment'} +``` #### Lists From 56146e95c3982b446597a5eb99ed6bcc828c85ec Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Sun, 2 Jun 2024 02:07:45 -0600 Subject: [PATCH 60/61] Update help text and formalize cli_parse_none_str. --- docs/index.md | 41 +++++++++++++++++++++++++++++------- pydantic_settings/main.py | 14 +++++++++++- pydantic_settings/sources.py | 29 +++++++++++++++++++------ tests/test_settings.py | 12 +++++------ 4 files changed, 75 insertions(+), 21 deletions(-) diff --git a/docs/index.md b/docs/index.md index f144c095..dc770d9e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -478,7 +478,8 @@ to enable [enforcing required arguments at the CLI](#enforce-required-arguments- ### The Basics -To get started, let's revisit the example presented in [parsing environment variables](#parsing-environment-variables) but using a Pydantic settings CLI: +To get started, let's revisit the example presented in [parsing environment variables](#parsing-environment-variables) +but using a Pydantic settings CLI: ```py import sys @@ -777,7 +778,7 @@ positional arguments: options: -h, --help show this help message and exit - --local bool When the resposity to clone from is on a local machine, bypass ... + --local bool When the resposity to clone from is on a local machine, bypass ... (default: False) """ @@ -794,7 +795,7 @@ git-plugins-bar - Extra deep bar plugin command options: -h, --help show this help message and exit - --my_feature bool Enable my feature on bar plugin + --my_feature bool Enable my feature on bar plugin (default: False) """ ``` @@ -804,8 +805,8 @@ The below flags can be used to customise the CLI experience to your needs. #### Change the Displayed Program Name -Change the default program name displayed in the help text usage by setting `cli_prog_name`. By default, it will derive the name of the currently -executing program from `sys.argv[0]`, just like argparse. +Change the default program name displayed in the help text usage by setting `cli_prog_name`. By default, it will derive +the name of the currently executing program from `sys.argv[0]`, just like argparse. ```py test="skip" import sys @@ -864,6 +865,30 @@ example.py: error: the following arguments are required: --my_required_field """ ``` +#### Change the None Type Parse String + +Change the CLI string value that will be parsed (e.g. "null", "void", "None", etc.) into `None` type(None) by setting +`cli_parse_none_str`. By default it will use the `env_parse_none_str` value if set. Otherwise, it will default to "null" +if `cli_avoid_json` is `False`, and "None" if `cli_avoid_json` is `True`. + +```py +import sys +from typing import Optional + +from pydantic import Field + +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings, cli_parse_args=True, cli_parse_none_str='void'): + v1: Optional[int] = Field(description='the top level v0 option') + + +sys.argv = ['example.py', '--v1', 'void'] +print(Settings().model_dump()) +#> {'v1': None} +``` + #### Hide None Type Values Hide `None` values from the CLI help text by enabling `cli_hide_none_type`. @@ -888,7 +913,7 @@ usage: example.py [-h] [--v0 str] options: -h, --help show this help message and exit - --v0 str the top level v0 option + --v0 str the top level v0 option (required) """ ``` @@ -925,7 +950,7 @@ options: sub_model options: The help summary for SubModel related options - --sub_model.v1 int the sub model v1 option + --sub_model.v1 int the sub model v1 option (required) """ ``` @@ -972,7 +997,7 @@ sub_model options: The help text from the class docstring. --sub_model JSON set sub_model from JSON string - --sub_model.v1 int the sub model v1 option + --sub_model.v1 int the sub model v1 option (required) """ ``` diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 73e50c4b..3b3ba434 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -33,6 +33,7 @@ class SettingsConfigDict(ConfigDict, total=False): cli_prog_name: str | None cli_parse_args: bool | list[str] | tuple[str, ...] | None cli_settings_source: CliSettingsSource[Any] | None + cli_parse_none_str: str | None cli_hide_none_type: bool cli_avoid_json: bool cli_enforce_required: bool @@ -101,6 +102,9 @@ class BaseSettings(BaseModel): _cli_parse_args: The list of CLI arguments to parse. Defaults to None. If set to `True`, defaults to sys.argv[1:]. _cli_settings_source: Override the default CLI settings source with a user defined instance. Defaults to None. + _cli_parse_none_str: The CLI string value that should be parsed (e.g. "null", "void", "None", etc.) into + `None` type(None). Defaults to _env_parse_none_str value if set. Otherwise, defaults to "null" if + _cli_avoid_json is `False`, and "None" if _cli_avoid_json is `True`. _cli_hide_none_type: Hide `None` values in CLI help text. Defaults to `False`. _cli_avoid_json: Avoid complex JSON objects in CLI help text. Defaults to `False`. _cli_enforce_required: Enforce required fields at the CLI. Defaults to `False`. @@ -123,6 +127,7 @@ def __init__( _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: CliSettingsSource[Any] | None = None, + _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, @@ -146,6 +151,7 @@ def __init__( _cli_prog_name=_cli_prog_name, _cli_parse_args=_cli_parse_args, _cli_settings_source=_cli_settings_source, + _cli_parse_none_str=_cli_parse_none_str, _cli_hide_none_type=_cli_hide_none_type, _cli_avoid_json=_cli_avoid_json, _cli_enforce_required=_cli_enforce_required, @@ -193,6 +199,7 @@ def _settings_build_values( _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: CliSettingsSource[Any] | None = None, + _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, @@ -225,6 +232,10 @@ def _settings_build_values( cli_settings_source = ( _cli_settings_source if _cli_settings_source is not None else self.model_config.get('cli_settings_source') ) + cli_parse_none_str = ( + _cli_parse_none_str if _cli_parse_none_str is not None else self.model_config.get('cli_parse_none_str') + ) + cli_parse_none_str = cli_parse_none_str if not env_parse_none_str else env_parse_none_str cli_hide_none_type = ( _cli_hide_none_type if _cli_hide_none_type is not None else self.model_config.get('cli_hide_none_type') ) @@ -250,7 +261,7 @@ def _settings_build_values( self.__class__, cli_prog_name=cli_prog_name, cli_parse_args=cli_parse_args, - cli_parse_none_str=env_parse_none_str, + cli_parse_none_str=cli_parse_none_str, cli_hide_none_type=cli_hide_none_type, cli_avoid_json=cli_avoid_json, cli_enforce_required=cli_enforce_required, @@ -318,6 +329,7 @@ def _settings_build_values( cli_prog_name=None, cli_parse_args=None, cli_settings_source=None, + cli_parse_none_str=None, cli_hide_none_type=False, cli_avoid_json=False, cli_enforce_required=False, diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 38e18bb5..fd4f2a13 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -37,6 +37,7 @@ from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass from pydantic.dataclasses import is_pydantic_dataclass from pydantic.fields import FieldInfo +from pydantic_core import PydanticUndefined from typing_extensions import Annotated, _AnnotatedAlias, get_args, get_origin from pydantic_settings.utils import path_type_label @@ -797,6 +798,8 @@ class CliSettingsSource(EnvSettingsSource, Generic[T]): cli_parse_args: The list of CLI arguments to parse. Defaults to None. If set to `True`, defaults to sys.argv[1:]. cli_settings_source: Override the default CLI settings source with a user defined instance. Defaults to `None`. + cli_parse_none_str: The CLI string value that should be parsed (e.g. "null", "void", "None", etc.) into `None` + type(None). Defaults to "null" if cli_avoid_json is `False`, and "None" if cli_avoid_json is `True`. cli_hide_none_type: Hide `None` values in CLI help text. Defaults to `False`. cli_avoid_json: Avoid complex JSON objects in CLI help text. Defaults to `False`. cli_enforce_required: Enforce required fields at the CLI. Defaults to `False`. @@ -851,6 +854,7 @@ def __init__( ) if cli_parse_none_str is None: cli_parse_none_str = 'None' if self.cli_avoid_json is True else 'null' + self.cli_parse_none_str = cli_parse_none_str self.cli_enforce_required = ( cli_enforce_required if cli_enforce_required is not None @@ -874,7 +878,7 @@ def __init__( super().__init__( settings_cls, env_nested_delimiter='.', - env_parse_none_str=cli_parse_none_str, + env_parse_none_str=self.cli_parse_none_str, env_parse_enums=True, env_prefix=self.cli_prefix, case_sensitive=case_sensitive, @@ -998,7 +1002,7 @@ def _load_env_vars( for subcommands in self._cli_subcommands.values(): for subcommand in subcommands: if subcommand not in selected_subcommands: - parsed_args[subcommand] = self.env_parse_none_str # type: ignore + parsed_args[subcommand] = self.cli_parse_none_str parsed_args = {key: val for key, val in parsed_args.items() if not key.endswith(':subcommand')} if selected_subcommands: @@ -1010,7 +1014,7 @@ def _load_env_vars( cast(Mapping[str, str], parsed_args), self.case_sensitive, self.env_ignore_empty, - self.env_parse_none_str, + self.cli_parse_none_str, ) return self @@ -1102,7 +1106,7 @@ def _consume_string_or_number(self, item: str, merged_list: list[str], merge_typ try: float(val_string) except ValueError: - if val_string == self.env_parse_none_str: + if val_string == self.cli_parse_none_str: val_string = 'null' if val_string not in ('true', 'false', 'null') and not val_string.startswith('"'): val_string = f'"{val_string}"' @@ -1271,7 +1275,7 @@ def _add_parser_args( arg_flag: str = '--' kwargs: dict[str, Any] = {} kwargs['default'] = SUPPRESS - kwargs['help'] = field_info.description + kwargs['help'] = self._help_format(field_info) kwargs['dest'] = f'{arg_prefix}{resolved_name}' kwargs['metavar'] = self._metavar_format(field_info.annotation) kwargs['required'] = self.cli_enforce_required and field_info.is_required() @@ -1370,7 +1374,7 @@ def _metavar_format_recurse(self, obj: Any) -> str: list(map(self._metavar_format_recurse, self._get_modified_args(obj))), obj_qualname=obj.__qualname__ ) elif obj is type(None): - return self.env_parse_none_str + return self.cli_parse_none_str elif is_model_class(obj): return 'JSON' elif isinstance(obj, type): @@ -1381,6 +1385,19 @@ def _metavar_format_recurse(self, obj: Any) -> str: def _metavar_format(self, obj: Any) -> str: return self._metavar_format_recurse(obj).replace(', ', ',') + def _help_format(self, field_info: FieldInfo) -> str: + _help = field_info.description if field_info.description else '' + if field_info.is_required() and _CliPositionalArg not in field_info.metadata: + _help += ' (required)' if _help else '(required)' + elif field_info.default is not PydanticUndefined: + default = ( + f'(default: {field_info.default})' + if field_info.default is not None + else f'(default: {self.cli_parse_none_str})' + ) + _help += f' {default}' if _help else default + return _help + class ConfigFileSourceMixin(ABC): def _read_files(self, files: PathType | None) -> dict[str, Any]: diff --git a/tests/test_settings.py b/tests/test_settings.py index f658d015..d1b8514d 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2772,7 +2772,7 @@ class Settings(BaseSettings): sub_model options: --sub_model JSON set sub_model from JSON string - --sub_model.v1 int + --sub_model.v1 int (required) """ ) @@ -2787,7 +2787,7 @@ class Settings(BaseSettings): -h, --help show this help message and exit sub_model options: - --sub_model.v1 int + --sub_model.v1 int (required) """ ) @@ -2854,7 +2854,7 @@ class Settings(BaseSettings): {argparse_options_text}: -h, --help show this help message and exit - --v0 {{str,null}} + --v0 {{str,null}} (required) """ ) @@ -2867,7 +2867,7 @@ class Settings(BaseSettings): {argparse_options_text}: -h, --help show this help message and exit - --v0 str + --v0 str (required) """ ) @@ -2906,7 +2906,7 @@ class Settings(BaseSettings): The help text from the field description --sub_model JSON set sub_model from JSON string - --sub_model.v1 int + --sub_model.v1 int (required) """ ) @@ -2926,7 +2926,7 @@ class Settings(BaseSettings): The help text from the class docstring --sub_model JSON set sub_model from JSON string - --sub_model.v1 int + --sub_model.v1 int (required) """ ) From d3f2de2ceff65e77b06ac66c2e4ed6d0e17f1348 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 3 Jun 2024 02:28:25 -0600 Subject: [PATCH 61/61] Add help text for default factory. --- pydantic_settings/sources.py | 19 ++++++++++--------- tests/test_settings.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index fd4f2a13..069e50a6 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -852,7 +852,7 @@ def __init__( self.cli_avoid_json = ( cli_avoid_json if cli_avoid_json is not None else settings_cls.model_config.get('cli_avoid_json', False) ) - if cli_parse_none_str is None: + if not cli_parse_none_str: cli_parse_none_str = 'None' if self.cli_avoid_json is True else 'null' self.cli_parse_none_str = cli_parse_none_str self.cli_enforce_required = ( @@ -1387,14 +1387,15 @@ def _metavar_format(self, obj: Any) -> str: def _help_format(self, field_info: FieldInfo) -> str: _help = field_info.description if field_info.description else '' - if field_info.is_required() and _CliPositionalArg not in field_info.metadata: - _help += ' (required)' if _help else '(required)' - elif field_info.default is not PydanticUndefined: - default = ( - f'(default: {field_info.default})' - if field_info.default is not None - else f'(default: {self.cli_parse_none_str})' - ) + if field_info.is_required(): + if _CliPositionalArg not in field_info.metadata: + _help += ' (required)' if _help else '(required)' + else: + default = f'(default: {self.cli_parse_none_str})' + if field_info.default not in (PydanticUndefined, None): + default = f'(default: {field_info.default})' + elif field_info.default_factory is not None: + default = f'(default: {field_info.default_factory})' _help += f' {default}' if _help else default return _help diff --git a/tests/test_settings.py b/tests/test_settings.py index d1b8514d..df89c821 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2,6 +2,7 @@ import dataclasses import json import os +import re import sys import typing import uuid @@ -2231,6 +2232,35 @@ class Cfg(BaseSettings): assert str(exc_info.value) == 'Case-insensitive matching is only supported on the internal root parser' +def test_cli_help_differentiation(capsys, monkeypatch): + class Cfg(BaseSettings): + foo: str + bar: int = 123 + boo: int = Field(default_factory=lambda: 456) + + argparse_options_text = 'options' if sys.version_info >= (3, 10) else 'optional arguments' + + with monkeypatch.context() as m: + m.setattr(sys, 'argv', ['example.py', '--help']) + + with pytest.raises(SystemExit): + Cfg(_cli_parse_args=True) + + assert ( + re.sub(r'0x\w+', '0xffffffff', capsys.readouterr().out, re.MULTILINE) + == f"""usage: example.py [-h] [--foo str] [--bar int] [--boo int] + +{argparse_options_text}: + -h, --help show this help message and exit + --foo str (required) + --bar int (default: 123) + --boo int (default: .Cfg. at + 0xffffffff>) +""" + ) + + def test_cli_nested_dataclass_arg(): @pydantic_dataclasses.dataclass class MyDataclass: