diff --git a/bison/__version__.py b/bison/__version__.py index eed5e0e..a6c7ca7 100644 --- a/bison/__version__.py +++ b/bison/__version__.py @@ -3,7 +3,7 @@ __title__ = 'bison' __description__ = 'Python application configuration' __url__ = 'https://github.com/edaniszewski/bison' -__version__ = '0.0.1' +__version__ = '0.0.2' __author__ = 'Erick Daniszewski' __author_email__ = 'edaniszewski@gmail.com' __license__ = 'MIT' diff --git a/bison/bison.py b/bison/bison.py index 442ab95..737bbc0 100644 --- a/bison/bison.py +++ b/bison/bison.py @@ -12,8 +12,7 @@ import os import yaml from bison.errors import BisonError -from bison.scheme import Option -from bison.utils import DotDict, cast +from bison.utils import DotDict logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -197,50 +196,9 @@ class Bison(object): # config if there is a scheme. if self.scheme: for k, v in self.scheme.flatten().items(): - # only Options can be bound to env variables currently. - if not isinstance(v, Option): - continue - - # we explicitly do not want to bind the option to env - if v.bind_env is False: - continue - - # we want to bind the option to env. in this case, bind_env is - # generated from the Option key. - elif v.bind_env is True: - env_key = k.replace('.', '_').upper() - - # if an env prefix exists, use it - if self.env_prefix: - env_key = self.env_prefix.upper() + env_key - - env = os.environ.get(env_key, None) - if env is not None: - env_cfg[k] = cast(v, env) - - # bind_env holds the env variable to use. since it is specified - # manually, we do not prepend the env prefix. - elif isinstance(v.bind_env, str): - env_key = v.bind_env - - env = os.environ.get(env_key, None) - if env is not None: - env_cfg[k] = cast(v, env) - - # bind_env is None - this is its default value. in this case, the - # option hasn't been explicitly set as False, so we can do env - # lookups if auto_env is set. - elif v.bind_env is None: - if self.auto_env: - env_key = k.replace('.', '_').upper() - - # if an env prefix exists, use it - if self.env_prefix: - env_key = self.env_prefix.upper() + env_key - - env = os.environ.get(env_key, None) - if env is not None: - env_cfg[k] = cast(v, env) + value = v.parse_env(k, self.env_prefix, self.auto_env) + if value is not None: + env_cfg[k] = value if len(env_cfg) > 0: # the configuration changes, so we invalidate the cached config diff --git a/bison/scheme.py b/bison/scheme.py index 572d73f..a8179ff 100644 --- a/bison/scheme.py +++ b/bison/scheme.py @@ -9,7 +9,9 @@ in order to do configuration defaults and validation. A here as well. """ -from bison import errors +import os + +from bison import errors, utils class NoDefault: @@ -61,9 +63,10 @@ class Scheme(object): # if we have a dict option, build the defaults for its scheme. # if any defaults exist, use them. if isinstance(arg, DictOption): - b = arg.scheme.build_defaults() - if b: - defaults[arg.name] = b + if arg.scheme: + b = arg.scheme.build_defaults() + if b: + defaults[arg.name] = b return defaults def flatten(self): @@ -84,9 +87,10 @@ class Scheme(object): flat[arg.name] = arg elif isinstance(arg, DictOption): - flat[arg.name] = DictOption - for k, v in arg.scheme.flatten().items(): - flat[arg.name + '.' + k] = v + flat[arg.name] = arg + if arg.scheme: + for k, v in arg.scheme.flatten().items(): + flat[arg.name + '.' + k] = v self._flat = flat return self._flat @@ -142,6 +146,25 @@ class _BaseOpt(object): """ raise NotImplementedError + def parse_env(self, key=None, prefix=None, auto_env=False): + """Parse the environment based on the option configuration. + + Args: + key (str|None): The full key (dot notation) to use for the option. + If None, this will use the option name. Otherwise, we expect to + have the full key (can be determined by flattening the base + Scheme). + prefix (str|None): The prefix to use for environment variables. + This is set in the `Bison` object and should be passed in + here. + auto_env (bool): The `Bison` setting for auto_env. + + Returns: + The value(s) for the option from the environment, if found. If + no values are found, None is returned. + """ + raise NotImplementedError + class Option(_BaseOpt): """Option represents a configuration option with a singular value. @@ -186,6 +209,87 @@ class Option(_BaseOpt): '{} is not in the valid options: {}'.format(value, self.choices) ) + def parse_env(self, key=None, prefix=None, auto_env=False): + if key is None: + key = self.name + + # we explicitly do not want to bind the option to env + if self.bind_env is False: + return None + + # we want to bind the option to env. in this case, bind_env is + # generated from the Option key. + elif self.bind_env is True: + env_key = key.replace('.', '_').upper() + + # if an env prefix exists, use it + if prefix: + env_key = prefix.upper() + env_key + + env = os.environ.get(env_key, None) + if env is not None: + return self.cast(env) + + # bind_env holds the env variable to use. since it is specified + # manually, we do not prepend the env prefix. + elif isinstance(self.bind_env, str): + env_key = self.bind_env + + env = os.environ.get(env_key, None) + if env is not None: + return self.cast(env) + + # bind_env is None - this is its default value. in this case, the + # option hasn't been explicitly set as False, so we can do env + # lookups if auto_env is set. + elif self.bind_env is None: + if auto_env: + env_key = key.replace('.', '_').upper() + + # if an env prefix exists, use it + if prefix: + env_key = prefix.upper() + env_key + + env = os.environ.get(env_key, None) + if env is not None: + return self.cast(env) + return None + + def cast(self, value): + """Cast a value to the type required by the option, if one is set. + + This is used to cast the string values gathered from environment + variable into their required type. + + Args: + value: The value to cast. + + Returns: + The value casted to the expected type for the option. + """ + # if there is no type set for the option, return the given + # value unchanged. + if self.type is None: + return value + + # cast directly + if self.type in (str, int, float): + try: + return self.type(value) + except Exception as e: + raise errors.BisonError( + 'Failed to cast {} to {}'.format(value, self.type) + ) from e + + # for bool, can't cast a string, since a string is truthy, + # so we need to check the value. + elif self.type == bool: + return value.lower() == 'true' + + # the option type is currently not supported + else: + raise errors.BisonError('Unsupported type for casting: {}'.format(self.type)) + class DictOption(_BaseOpt): """DictOption represents a configuration option with a dictionary value. @@ -212,7 +316,11 @@ class DictOption(_BaseOpt): not fail if it is not present in the config. If this is left at its default value of `_NoDefault`, then this option is considered required and will fail validation if not present. - bind_env (bool): Bind the option to an environment variable. + bind_env (bool): Bind the option to an environment variable. If False, + the option will not be bound to env. If True, the key for the this + DictOption will serve as an environment prefix. Any environment + variable matching that prefix will be added to the parsed result + as a string. """ def __init__(self, name, scheme, default=_no_default, bind_env=False): @@ -229,6 +337,34 @@ class DictOption(_BaseOpt): if isinstance(self.scheme, Scheme): self.scheme.validate(value) + def parse_env(self, key=None, prefix=None, auto_env=False): + if key is None: + key = self.name + + # we explicitly do not want to bind the option to env + if self.bind_env is False: + return None + + # we want to populate the dict from env. the dict option key + # will generate the prefix. anything after the prefix will be + # part of the populated value(s) + elif self.bind_env is True: + env_key = key.replace('.', '_').upper() + if prefix: + env_key = prefix.upper() + env_key + + if not env_key.endswith('_'): + env_key = env_key + '_' + + values = utils.DotDict() + for k, v in os.environ.items(): + if k.startswith(env_key): + dot_key = k[len(env_key):].replace('_', '.').lower() + values[dot_key] = v + if values: + return values + return None + class ListOption(_BaseOpt): """ListOption represents a configuration option with a list value. @@ -255,7 +391,11 @@ class ListOption(_BaseOpt): member_type: The type that all members of the list should have. member_scheme (Scheme): The `Scheme` that all members of the list should fulfil. This should be used when the list members are dictionaries/maps. - bind_env (bool): Bind the option to an environment variable. + bind_env (bool): Bind the option to an environment variable. If False, the + option will not be bound to env. If True, the option's key will be used + to create an env variable. The contents of that env variable will be used + to populate a list. This assumes that the env value is a string with the + items being comma separated. """ def __init__(self, name, default=_no_default, member_type=None, member_scheme=None, bind_env=False): @@ -290,3 +430,22 @@ class ListOption(_BaseOpt): for item in value: self.member_scheme.validate(item) + + def parse_env(self, key=None, prefix=None, auto_env=False): + if key is None: + key = self.name + + # we explicitly do not want to bind the option to env + if self.bind_env is False: + return None + + elif self.bind_env is True: + env_key = key.replace('.', '_').upper() + if prefix: + env_key = prefix.upper() + env_key + + value = os.environ.get(env_key, None) + if value: + return value.split(',') + + return None diff --git a/bison/utils.py b/bison/utils.py index 49de441..70db13e 100644 --- a/bison/utils.py +++ b/bison/utils.py @@ -6,49 +6,6 @@ bison.utils Utilities for `bison`. """ -from bison.errors import BisonError -from bison.scheme import Option - - -def cast(option, value): - """Cast a value to the type required by the option, if one is set. - - This is used to cast the string values gathered from environment - variable into their required type. - - Args: - option: The Option specifying the type. - value: The value to cast. - - Returns: - The value casted to the expected type for the option. - """ - if not isinstance(option, Option): - raise BisonError('Unable to cast - "{}" not an Option'.format(option)) - - # if there is no type set for the option, return the given - # value unchanged. - if option.type is None: - return value - - # cast directly - if option.type in (str, int, float): - try: - return option.type(value) - except Exception as e: - raise BisonError( - 'Failed to cast {} to {}'.format(value, option.type) - ) from e - - # for bool, can't cast a string, since a string is truthy, - # so we need to check the value. - elif option.type == bool: - return value.lower() == 'true' - - # the option type is currently not supported - else: - raise BisonError('Unsupported type for casting: {}'.format(option.type)) - def build_dot_value(key, value): """Build new dictionaries based off of the dot notation key. diff --git a/tests/conftest.py b/tests/conftest.py index 467a0ae..3400909 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,7 @@ def with_env(): """Set and cleanup environment variables for tests.""" os.environ['TEST_ENV_FOO'] = 'bar' os.environ['TEST_ENV_NESTED_ENV_KEY'] = 'test' + os.environ['TEST_ENV_BAR_LIST'] = 'a,b,c' os.environ['TEST_OTHER_ENV_BAR'] = 'baz' os.environ['FOO_INT'] = '1' os.environ['FOO_BOOL'] = 'False' @@ -47,6 +48,7 @@ def with_env(): del os.environ['TEST_ENV_FOO'] del os.environ['TEST_ENV_NESTED_ENV_KEY'] + del os.environ['TEST_ENV_BAR_LIST'] del os.environ['TEST_OTHER_ENV_BAR'] del os.environ['FOO_INT'] del os.environ['FOO_BOOL'] diff --git a/tests/test_bison.py b/tests/test_bison.py index f35bf48..a752a1f 100644 --- a/tests/test_bison.py +++ b/tests/test_bison.py @@ -186,6 +186,13 @@ class TestBison: assert b.config_file == os.path.join(yaml_config.dirname, yaml_config.basename) assert len(b._config) == 2 + assert b.config == { + 'foo': True, + 'bar': { + 'baz': 1, + 'test': 'value' + } + } def test_parse_config_fail(self, bad_yaml_config): """Parse the file config unsuccessfully.""" diff --git a/tests/test_scheme.py b/tests/test_scheme.py index fbe0b33..ac20cfb 100644 --- a/tests/test_scheme.py +++ b/tests/test_scheme.py @@ -5,13 +5,20 @@ import pytest from bison import errors, scheme -def test_base_opt(): +def test_base_opt_validate(): """Validate the base option, which should fail.""" opt = scheme._BaseOpt() with pytest.raises(NotImplementedError): opt.validate('test-data') +def test_base_opt_parse_env(): + """Parse env from the base option, which should fail.""" + opt = scheme._BaseOpt() + with pytest.raises(NotImplementedError): + opt.parse_env() + + class TestOption: """Tests for the `Option` class.""" @@ -143,6 +150,85 @@ class TestOption: with pytest.raises(errors.SchemeValidationError): opt.validate(value) + @pytest.mark.parametrize( + 'option,value,expected', [ + (scheme.Option('foo'), 'foo', 'foo'), + (scheme.Option('foo'), 1, 1), + (scheme.Option('foo'), None, None), + (scheme.Option('foo'), False, False), + (scheme.Option('foo', field_type=str), 'foo', 'foo'), + (scheme.Option('foo', field_type=str), 1, '1'), + (scheme.Option('foo', field_type=int), '1', 1), + (scheme.Option('foo', field_type=float), '1', 1.0), + (scheme.Option('foo', field_type=float), '1.23', 1.23), + (scheme.Option('foo', field_type=bool), 'false', False), + (scheme.Option('foo', field_type=bool), 'False', False), + (scheme.Option('foo', field_type=bool), 'FALSE', False), + (scheme.Option('foo', field_type=bool), 'true', True), + (scheme.Option('foo', field_type=bool), 'True', True), + (scheme.Option('foo', field_type=bool), 'TRUE', True), + ] + ) + def test_cast(self, option, value, expected): + """Cast values to the type set by the Option.""" + actual = option.cast(value) + assert actual == expected + + @pytest.mark.parametrize( + 'option,value', [ + (scheme.Option('foo', field_type=int), 'foo'), + (scheme.Option('foo', field_type=list), 'foo'), + (scheme.Option('foo', field_type=tuple), 'foo'), + ] + ) + def test_cast_fail(self, option, value): + """Cast values to the type set by the Option.""" + with pytest.raises(errors.BisonError): + option.cast(value) + + @pytest.mark.parametrize( + 'option,prefix,auto_env', [ + (scheme.Option('foo'), None, False), + (scheme.Option('foo', bind_env=False), None, False), + (scheme.Option('foo', bind_env=True), None, False), + (scheme.Option('foo', bind_env=True), 'TEST_ENV_', False), + + (scheme.Option('foo', bind_env='TEST_KEY'), None, False), + (scheme.Option('foo', bind_env='TEST_KEY'), 'TEST_ENV_', False), + + (scheme.Option('foo', bind_env=None), 'TEST_ENV_', False), + (scheme.Option('foo', bind_env=None), 'TEST_ENV_', True), + (scheme.Option('foo', bind_env=None), None, False), + (scheme.Option('foo', bind_env=None), None, True), + ] + ) + def test_parse_env_none(self, option, prefix, auto_env): + """Parse environment variables for the Option. All of theses tests + should result in None being returned because no environment variables + are actually set. + """ + actual = option.parse_env(prefix=prefix, auto_env=auto_env) + assert actual is None + + @pytest.mark.parametrize( + 'option,key,prefix,auto_env,expected', [ + (scheme.Option('foo', bind_env=True), 'foo', 'TEST_ENV_', False, 'bar'), + (scheme.Option('foo', bind_env=True), 'foo', 'TEST_ENV_', True, 'bar'), + + (scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', 'TEST_ENV_', False, 'bar'), + (scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', 'TEST_ENV_', True, 'bar'), + (scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', None, False, 'bar'), + (scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', None, True, 'bar'), + + (scheme.Option('foo', bind_env=None), 'foo', 'TEST_ENV_', True, 'bar'), + (scheme.Option('foo', bind_env=None), 'nested.env.key', 'TEST_ENV_', True, 'test'), + ] + ) + def test_parse_env_ok(self, option, key, prefix, auto_env, expected, with_env): + """Parse environment variables for the Option.""" + actual = option.parse_env(key=key, prefix=prefix, auto_env=auto_env) + assert actual == expected + class TestDictOption: """Tests for the `DictOption` class.""" @@ -204,6 +290,42 @@ class TestDictOption: )) opt.validate({'foo': 'bar'}) + @pytest.mark.parametrize( + 'option,prefix,auto_env', [ + (scheme.DictOption('foo', scheme=None, bind_env=False), None, False), + (scheme.DictOption('foo', scheme=None, bind_env=False), None, True), + (scheme.DictOption('foo', scheme=None, bind_env=False), 'TEST_ENV', False), + (scheme.DictOption('foo', scheme=None, bind_env=False), 'TEST_ENV', True), + + (scheme.DictOption('foo', scheme=None, bind_env=True), None, False), + (scheme.DictOption('foo', scheme=None, bind_env=True), None, True), + (scheme.DictOption('foo', scheme=None, bind_env=True), 'TEST_ENV', False), + (scheme.DictOption('foo', scheme=None, bind_env=True), 'TEST_ENV', True), + ] + ) + def test_parse_env_none(self, option, prefix, auto_env): + """Parse environment variables for the DictOption. All of theses tests + should result in None being returned because no environment variables + are actually set. + """ + actual = option.parse_env(prefix=prefix, auto_env=auto_env) + assert actual is None + + @pytest.mark.parametrize( + 'option,key,prefix,auto_env,expected', [ + (scheme.DictOption('foo', scheme=None, bind_env=True), 'foo', 'TEST_ENV_', False, None), + (scheme.DictOption('foo', scheme=None, bind_env=True), 'foo', 'TEST_ENV_', True, None), + (scheme.DictOption('foo', scheme=None, bind_env=True), 'nested', 'TEST_ENV_', False, {'env': {'key': 'test'}}), + (scheme.DictOption('foo', scheme=None, bind_env=True), 'nested', 'TEST_ENV_', True, {'env': {'key': 'test'}}), + (scheme.DictOption('foo', scheme=None, bind_env=True), 'nested.env', 'TEST_ENV_', False, {'key': 'test'}), + (scheme.DictOption('foo', scheme=None, bind_env=True), 'nested.env', 'TEST_ENV_', True, {'key': 'test'}), + ] + ) + def test_parse_env_ok(self, option, key, prefix, auto_env, expected, with_env): + """Parse environment variables for the DictOption.""" + actual = option.parse_env(key=key, prefix=prefix, auto_env=auto_env) + assert actual == expected + class TestListOption: """Tests for the `ListOption` class.""" @@ -357,6 +479,40 @@ class TestListOption: with pytest.raises(errors.SchemeValidationError): opt.validate(['a', 'b', 'c']) + @pytest.mark.parametrize( + 'option,prefix,auto_env', [ + (scheme.ListOption('foo', bind_env=False), None, False), + (scheme.ListOption('foo', bind_env=False), None, True), + (scheme.ListOption('foo', bind_env=False), 'TEST_ENV', False), + (scheme.ListOption('foo', bind_env=False), 'TEST_ENV', True), + + (scheme.ListOption('foo', bind_env=True), None, False), + (scheme.ListOption('foo', bind_env=True), None, True), + (scheme.ListOption('foo', bind_env=True), 'TEST_ENV', False), + (scheme.ListOption('foo', bind_env=True), 'TEST_ENV', True), + ] + ) + def test_parse_env_none(self, option, prefix, auto_env): + """Parse environment variables for the ListOption. All of theses tests + should result in None being returned because no environment variables + are actually set. + """ + actual = option.parse_env(prefix=prefix, auto_env=auto_env) + assert actual is None + + @pytest.mark.parametrize( + 'option,key,prefix,auto_env,expected', [ + (scheme.ListOption('foo', bind_env=True), 'foo', 'TEST_ENV_', False, ['bar']), + (scheme.ListOption('foo', bind_env=True), 'foo', 'TEST_ENV_', True, ['bar']), + (scheme.ListOption('foo', bind_env=True), 'bar.list', 'TEST_ENV_', False, ['a', 'b', 'c']), + (scheme.ListOption('foo', bind_env=True), 'bar.list', 'TEST_ENV_', True, ['a', 'b', 'c']), + ] + ) + def test_parse_env_ok(self, option, key, prefix, auto_env, expected, with_env): + """Parse environment variables for the ListOption.""" + actual = option.parse_env(key=key, prefix=prefix, auto_env=auto_env) + assert actual == expected + class TestScheme: """Tests for the `Scheme` class.""" @@ -409,6 +565,17 @@ class TestScheme: 'list': ['a', 'b'] } ), + ( + # args + ( + scheme.Option('foo', default='bar'), + scheme.DictOption('bar', scheme=None) + ), + # expected + { + 'foo': 'bar' + } + ), ( # args ( @@ -468,6 +635,10 @@ class TestScheme: (scheme.Option('foo'), scheme.Option('bar')), ['foo', 'bar'] ), + ( + (scheme.Option('foo'), scheme.DictOption('bar', scheme=None)), + ['foo', 'bar'] + ), ( ( scheme.Option('foo'), diff --git a/tests/test_utils.py b/tests/test_utils.py index 5dd9a2d..4b4d5ee 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,46 +2,7 @@ import pytest -from bison import errors, scheme, utils - - -@pytest.mark.parametrize( - 'option,value,expected', [ - (scheme.Option('foo'), 'foo', 'foo'), - (scheme.Option('foo'), 1, 1), - (scheme.Option('foo'), None, None), - (scheme.Option('foo'), False, False), - (scheme.Option('foo', field_type=str), 'foo', 'foo'), - (scheme.Option('foo', field_type=str), 1, '1'), - (scheme.Option('foo', field_type=int), '1', 1), - (scheme.Option('foo', field_type=float), '1', 1.0), - (scheme.Option('foo', field_type=float), '1.23', 1.23), - (scheme.Option('foo', field_type=bool), 'false', False), - (scheme.Option('foo', field_type=bool), 'False', False), - (scheme.Option('foo', field_type=bool), 'FALSE', False), - (scheme.Option('foo', field_type=bool), 'true', True), - (scheme.Option('foo', field_type=bool), 'True', True), - (scheme.Option('foo', field_type=bool), 'TRUE', True), - ] -) -def test_cast(option, value, expected): - """Cast values to the type set by the Option.""" - actual = utils.cast(option, value) - assert actual == expected - - -@pytest.mark.parametrize( - 'option,value', [ - (scheme.Option('foo', field_type=int), 'foo'), - (scheme.ListOption('foo'), 'foo'), - (scheme.Option('foo', field_type=list), 'foo'), - (scheme.Option('foo', field_type=tuple), 'foo'), - ] -) -def test_cast_fail(option, value): - """Cast values to the type set by the Option.""" - with pytest.raises(errors.BisonError): - utils.cast(option, value) +from bison import utils @pytest.mark.parametrize(