update env behavior for dict/list options

bison/__version__.py

@@ -3,7 +3,7 @@
 __title__ = 'bison'
 __description__ = 'Python application configuration'
 __url__ = 'https://github.com/edaniszewski/bison'
-__version__ = '0.0.1'
+__version__ = '0.0.2'
 __author__ = 'Erick Daniszewski'
 __author_email__ = 'edaniszewski@gmail.com'
 __license__ = 'MIT'

bison/bison.py

@@ -12,8 +12,7 @@ import os
 import yaml
 
 from bison.errors import BisonError
-from bison.scheme import Option
+from bison.utils import DotDict
-from bison.utils import DotDict, cast
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -197,50 +196,9 @@ class Bison(object):
         # config if there is a scheme.
         if self.scheme:
             for k, v in self.scheme.flatten().items():
-                # only Options can be bound to env variables currently.
+                value = v.parse_env(k, self.env_prefix, self.auto_env)
-                if not isinstance(v, Option):
+                if value is not None:
-                    continue
+                    env_cfg[k] = value
-
-                # we explicitly do not want to bind the option to env
-                if v.bind_env is False:
-                    continue
-
-                # we want to bind the option to env. in this case, bind_env is
-                # generated from the Option key.
-                elif v.bind_env is True:
-                    env_key = k.replace('.', '_').upper()
-
-                    # if an env prefix exists, use it
-                    if self.env_prefix:
-                        env_key = self.env_prefix.upper() + env_key
-
-                    env = os.environ.get(env_key, None)
-                    if env is not None:
-                        env_cfg[k] = cast(v, env)
-
-                # bind_env holds the env variable to use. since it is specified
-                # manually, we do not prepend the env prefix.
-                elif isinstance(v.bind_env, str):
-                    env_key = v.bind_env
-
-                    env = os.environ.get(env_key, None)
-                    if env is not None:
-                        env_cfg[k] = cast(v, env)
-
-                # bind_env is None - this is its default value. in this case, the
-                # option hasn't been explicitly set as False, so we can do env
-                # lookups if auto_env is set.
-                elif v.bind_env is None:
-                    if self.auto_env:
-                        env_key = k.replace('.', '_').upper()
-
-                        # if an env prefix exists, use it
-                        if self.env_prefix:
-                            env_key = self.env_prefix.upper() + env_key
-
-                        env = os.environ.get(env_key, None)
-                        if env is not None:
-                            env_cfg[k] = cast(v, env)
 
         if len(env_cfg) > 0:
             # the configuration changes, so we invalidate the cached config

bison/scheme.py

@@ -9,7 +9,9 @@ in order to do configuration defaults and validation. A
 here as well.
 """
 
-from bison import errors
+import os
+
+from bison import errors, utils
 
 
 class NoDefault:
@@ -61,6 +63,7 @@ class Scheme(object):
             # if we have a dict option, build the defaults for its scheme.
             # if any defaults exist, use them.
             if isinstance(arg, DictOption):
+                if arg.scheme:
                     b = arg.scheme.build_defaults()
                     if b:
                         defaults[arg.name] = b
@@ -84,7 +87,8 @@ class Scheme(object):
                     flat[arg.name] = arg
 
                 elif isinstance(arg, DictOption):
-                    flat[arg.name] = DictOption
+                    flat[arg.name] = arg
+                    if arg.scheme:
                         for k, v in arg.scheme.flatten().items():
                             flat[arg.name + '.' + k] = v
 
@@ -142,6 +146,25 @@ class _BaseOpt(object):
         """
         raise NotImplementedError
 
+    def parse_env(self, key=None, prefix=None, auto_env=False):
+        """Parse the environment based on the option configuration.
+
+        Args:
+            key (str|None): The full key (dot notation) to use for the option.
+                If None, this will use the option name. Otherwise, we expect to
+                have the full key (can be determined by flattening the base
+                Scheme).
+            prefix (str|None): The prefix to use for environment variables.
+                This is set in the `Bison` object and should be passed in
+                here.
+            auto_env (bool): The `Bison` setting for auto_env.
+
+        Returns:
+            The value(s) for the option from the environment, if found. If
+            no values are found, None is returned.
+        """
+        raise NotImplementedError
+
 
 class Option(_BaseOpt):
     """Option represents a configuration option with a singular value.
@@ -186,6 +209,87 @@ class Option(_BaseOpt):
                 '{} is not in the valid options: {}'.format(value, self.choices)
             )
 
+    def parse_env(self, key=None, prefix=None, auto_env=False):
+        if key is None:
+            key = self.name
+
+        # we explicitly do not want to bind the option to env
+        if self.bind_env is False:
+            return None
+
+        # we want to bind the option to env. in this case, bind_env is
+        # generated from the Option key.
+        elif self.bind_env is True:
+            env_key = key.replace('.', '_').upper()
+
+            # if an env prefix exists, use it
+            if prefix:
+                env_key = prefix.upper() + env_key
+
+            env = os.environ.get(env_key, None)
+            if env is not None:
+                return self.cast(env)
+
+        # bind_env holds the env variable to use. since it is specified
+        # manually, we do not prepend the env prefix.
+        elif isinstance(self.bind_env, str):
+            env_key = self.bind_env
+
+            env = os.environ.get(env_key, None)
+            if env is not None:
+                return self.cast(env)
+
+        # bind_env is None - this is its default value. in this case, the
+        # option hasn't been explicitly set as False, so we can do env
+        # lookups if auto_env is set.
+        elif self.bind_env is None:
+            if auto_env:
+                env_key = key.replace('.', '_').upper()
+
+                # if an env prefix exists, use it
+                if prefix:
+                    env_key = prefix.upper() + env_key
+
+                env = os.environ.get(env_key, None)
+                if env is not None:
+                    return self.cast(env)
+        return None
+
+    def cast(self, value):
+        """Cast a value to the type required by the option, if one is set.
+
+        This is used to cast the string values gathered from environment
+        variable into their required type.
+
+        Args:
+            value: The value to cast.
+
+        Returns:
+            The value casted to the expected type for the option.
+        """
+        # if there is no type set for the option, return the given
+        # value unchanged.
+        if self.type is None:
+            return value
+
+        # cast directly
+        if self.type in (str, int, float):
+            try:
+                return self.type(value)
+            except Exception as e:
+                raise errors.BisonError(
+                    'Failed to cast {} to {}'.format(value, self.type)
+                ) from e
+
+        # for bool, can't cast a string, since a string is truthy,
+        # so we need to check the value.
+        elif self.type == bool:
+            return value.lower() == 'true'
+
+        # the option type is currently not supported
+        else:
+            raise errors.BisonError('Unsupported type for casting: {}'.format(self.type))
+
 
 class DictOption(_BaseOpt):
     """DictOption represents a configuration option with a dictionary value.
@@ -212,7 +316,11 @@ class DictOption(_BaseOpt):
             not fail if it is not present in the config. If this is left at its
             default value of `_NoDefault`, then this option is considered required
             and will fail validation if not present.
-        bind_env (bool): Bind the option to an environment variable.
+        bind_env (bool): Bind the option to an environment variable. If False,
+            the option will not be bound to env. If True, the key for the this
+            DictOption will serve as an environment prefix. Any environment
+            variable matching that prefix will be added to the parsed result
+            as a string.
     """
 
     def __init__(self, name, scheme, default=_no_default, bind_env=False):
@@ -229,6 +337,34 @@ class DictOption(_BaseOpt):
         if isinstance(self.scheme, Scheme):
             self.scheme.validate(value)
 
+    def parse_env(self, key=None, prefix=None, auto_env=False):
+        if key is None:
+            key = self.name
+
+        # we explicitly do not want to bind the option to env
+        if self.bind_env is False:
+            return None
+
+        # we want to populate the dict from env. the dict option key
+        # will generate the prefix. anything after the prefix will be
+        # part of the populated value(s)
+        elif self.bind_env is True:
+            env_key = key.replace('.', '_').upper()
+            if prefix:
+                env_key = prefix.upper() + env_key
+
+            if not env_key.endswith('_'):
+                env_key = env_key + '_'
+
+            values = utils.DotDict()
+            for k, v in os.environ.items():
+                if k.startswith(env_key):
+                    dot_key = k[len(env_key):].replace('_', '.').lower()
+                    values[dot_key] = v
+            if values:
+                return values
+        return None
+
 
 class ListOption(_BaseOpt):
     """ListOption represents a configuration option with a list value.
@@ -255,7 +391,11 @@ class ListOption(_BaseOpt):
         member_type: The type that all members of the list should have.
         member_scheme (Scheme): The `Scheme` that all members of the list should
             fulfil. This should be used when the list members are dictionaries/maps.
-        bind_env (bool): Bind the option to an environment variable.
+        bind_env (bool): Bind the option to an environment variable. If False, the
+            option will not be bound to env. If True, the option's key will be used
+            to create an env variable. The contents of that env variable will be used
+            to populate a list. This assumes that the env value is a string with the
+            items being comma separated.
     """
 
     def __init__(self, name, default=_no_default, member_type=None, member_scheme=None, bind_env=False):
@@ -290,3 +430,22 @@ class ListOption(_BaseOpt):
 
             for item in value:
                 self.member_scheme.validate(item)
+
+    def parse_env(self, key=None, prefix=None, auto_env=False):
+        if key is None:
+            key = self.name
+
+        # we explicitly do not want to bind the option to env
+        if self.bind_env is False:
+            return None
+
+        elif self.bind_env is True:
+            env_key = key.replace('.', '_').upper()
+            if prefix:
+                env_key = prefix.upper() + env_key
+
+            value = os.environ.get(env_key, None)
+            if value:
+                return value.split(',')
+
+        return None

bison/utils.py

@@ -6,49 +6,6 @@ bison.utils
 Utilities for `bison`.
 """
 
-from bison.errors import BisonError
-from bison.scheme import Option
-
-
-def cast(option, value):
-    """Cast a value to the type required by the option, if one is set.
-
-    This is used to cast the string values gathered from environment
-    variable into their required type.
-
-    Args:
-        option: The Option specifying the type.
-        value: The value to cast.
-
-    Returns:
-        The value casted to the expected type for the option.
-    """
-    if not isinstance(option, Option):
-        raise BisonError('Unable to cast - "{}" not an Option'.format(option))
-
-    # if there is no type set for the option, return the given
-    # value unchanged.
-    if option.type is None:
-        return value
-
-    # cast directly
-    if option.type in (str, int, float):
-        try:
-            return option.type(value)
-        except Exception as e:
-            raise BisonError(
-                'Failed to cast {} to {}'.format(value, option.type)
-            ) from e
-
-    # for bool, can't cast a string, since a string is truthy,
-    # so we need to check the value.
-    elif option.type == bool:
-        return value.lower() == 'true'
-
-    # the option type is currently not supported
-    else:
-        raise BisonError('Unsupported type for casting: {}'.format(option.type))
-
 
 def build_dot_value(key, value):
     """Build new dictionaries based off of the dot notation key.

tests/conftest.py

@@ -39,6 +39,7 @@ def with_env():
     """Set and cleanup environment variables for tests."""
     os.environ['TEST_ENV_FOO'] = 'bar'
     os.environ['TEST_ENV_NESTED_ENV_KEY'] = 'test'
+    os.environ['TEST_ENV_BAR_LIST'] = 'a,b,c'
     os.environ['TEST_OTHER_ENV_BAR'] = 'baz'
     os.environ['FOO_INT'] = '1'
     os.environ['FOO_BOOL'] = 'False'
@@ -47,6 +48,7 @@ def with_env():
 
     del os.environ['TEST_ENV_FOO']
     del os.environ['TEST_ENV_NESTED_ENV_KEY']
+    del os.environ['TEST_ENV_BAR_LIST']
     del os.environ['TEST_OTHER_ENV_BAR']
     del os.environ['FOO_INT']
     del os.environ['FOO_BOOL']

tests/test_bison.py

@@ -186,6 +186,13 @@ class TestBison:
 
         assert b.config_file == os.path.join(yaml_config.dirname, yaml_config.basename)
         assert len(b._config) == 2
+        assert b.config == {
+            'foo': True,
+            'bar': {
+                'baz': 1,
+                'test': 'value'
+            }
+        }
 
     def test_parse_config_fail(self, bad_yaml_config):
         """Parse the file config unsuccessfully."""

tests/test_scheme.py

@@ -5,13 +5,20 @@ import pytest
 from bison import errors, scheme
 
 
-def test_base_opt():
+def test_base_opt_validate():
     """Validate the base option, which should fail."""
     opt = scheme._BaseOpt()
     with pytest.raises(NotImplementedError):
         opt.validate('test-data')
 
 
+def test_base_opt_parse_env():
+    """Parse env from the base option, which should fail."""
+    opt = scheme._BaseOpt()
+    with pytest.raises(NotImplementedError):
+        opt.parse_env()
+
+
 class TestOption:
     """Tests for the `Option` class."""
 
@@ -143,6 +150,85 @@ class TestOption:
         with pytest.raises(errors.SchemeValidationError):
             opt.validate(value)
 
+    @pytest.mark.parametrize(
+        'option,value,expected', [
+            (scheme.Option('foo'), 'foo', 'foo'),
+            (scheme.Option('foo'), 1, 1),
+            (scheme.Option('foo'), None, None),
+            (scheme.Option('foo'), False, False),
+            (scheme.Option('foo', field_type=str), 'foo', 'foo'),
+            (scheme.Option('foo', field_type=str), 1, '1'),
+            (scheme.Option('foo', field_type=int), '1', 1),
+            (scheme.Option('foo', field_type=float), '1', 1.0),
+            (scheme.Option('foo', field_type=float), '1.23', 1.23),
+            (scheme.Option('foo', field_type=bool), 'false', False),
+            (scheme.Option('foo', field_type=bool), 'False', False),
+            (scheme.Option('foo', field_type=bool), 'FALSE', False),
+            (scheme.Option('foo', field_type=bool), 'true', True),
+            (scheme.Option('foo', field_type=bool), 'True', True),
+            (scheme.Option('foo', field_type=bool), 'TRUE', True),
+        ]
+    )
+    def test_cast(self, option, value, expected):
+        """Cast values to the type set by the Option."""
+        actual = option.cast(value)
+        assert actual == expected
+
+    @pytest.mark.parametrize(
+        'option,value', [
+            (scheme.Option('foo', field_type=int), 'foo'),
+            (scheme.Option('foo', field_type=list), 'foo'),
+            (scheme.Option('foo', field_type=tuple), 'foo'),
+        ]
+    )
+    def test_cast_fail(self, option, value):
+        """Cast values to the type set by the Option."""
+        with pytest.raises(errors.BisonError):
+            option.cast(value)
+
+    @pytest.mark.parametrize(
+        'option,prefix,auto_env', [
+            (scheme.Option('foo'), None, False),
+            (scheme.Option('foo', bind_env=False), None, False),
+            (scheme.Option('foo', bind_env=True), None, False),
+            (scheme.Option('foo', bind_env=True), 'TEST_ENV_', False),
+
+            (scheme.Option('foo', bind_env='TEST_KEY'), None, False),
+            (scheme.Option('foo', bind_env='TEST_KEY'), 'TEST_ENV_', False),
+
+            (scheme.Option('foo', bind_env=None), 'TEST_ENV_', False),
+            (scheme.Option('foo', bind_env=None), 'TEST_ENV_', True),
+            (scheme.Option('foo', bind_env=None), None, False),
+            (scheme.Option('foo', bind_env=None), None, True),
+        ]
+    )
+    def test_parse_env_none(self, option, prefix, auto_env):
+        """Parse environment variables for the Option. All of theses tests
+        should result in None being returned because no environment variables
+        are actually set.
+        """
+        actual = option.parse_env(prefix=prefix, auto_env=auto_env)
+        assert actual is None
+
+    @pytest.mark.parametrize(
+        'option,key,prefix,auto_env,expected', [
+            (scheme.Option('foo', bind_env=True), 'foo', 'TEST_ENV_', False, 'bar'),
+            (scheme.Option('foo', bind_env=True), 'foo', 'TEST_ENV_', True, 'bar'),
+
+            (scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', 'TEST_ENV_', False, 'bar'),
+            (scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', 'TEST_ENV_', True, 'bar'),
+            (scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', None, False, 'bar'),
+            (scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', None, True, 'bar'),
+
+            (scheme.Option('foo', bind_env=None), 'foo', 'TEST_ENV_', True, 'bar'),
+            (scheme.Option('foo', bind_env=None), 'nested.env.key', 'TEST_ENV_', True, 'test'),
+        ]
+    )
+    def test_parse_env_ok(self, option, key, prefix, auto_env, expected, with_env):
+        """Parse environment variables for the Option."""
+        actual = option.parse_env(key=key, prefix=prefix, auto_env=auto_env)
+        assert actual == expected
+
 
 class TestDictOption:
     """Tests for the `DictOption` class."""
@@ -204,6 +290,42 @@ class TestDictOption:
         ))
         opt.validate({'foo': 'bar'})
 
+    @pytest.mark.parametrize(
+        'option,prefix,auto_env', [
+            (scheme.DictOption('foo', scheme=None, bind_env=False), None, False),
+            (scheme.DictOption('foo', scheme=None, bind_env=False), None, True),
+            (scheme.DictOption('foo', scheme=None, bind_env=False), 'TEST_ENV', False),
+            (scheme.DictOption('foo', scheme=None, bind_env=False), 'TEST_ENV', True),
+
+            (scheme.DictOption('foo', scheme=None, bind_env=True), None, False),
+            (scheme.DictOption('foo', scheme=None, bind_env=True), None, True),
+            (scheme.DictOption('foo', scheme=None, bind_env=True), 'TEST_ENV', False),
+            (scheme.DictOption('foo', scheme=None, bind_env=True), 'TEST_ENV', True),
+        ]
+    )
+    def test_parse_env_none(self, option, prefix, auto_env):
+        """Parse environment variables for the DictOption. All of theses tests
+        should result in None being returned because no environment variables
+        are actually set.
+        """
+        actual = option.parse_env(prefix=prefix, auto_env=auto_env)
+        assert actual is None
+
+    @pytest.mark.parametrize(
+        'option,key,prefix,auto_env,expected', [
+            (scheme.DictOption('foo', scheme=None, bind_env=True), 'foo', 'TEST_ENV_', False, None),
+            (scheme.DictOption('foo', scheme=None, bind_env=True), 'foo', 'TEST_ENV_', True, None),
+            (scheme.DictOption('foo', scheme=None, bind_env=True), 'nested', 'TEST_ENV_', False, {'env': {'key': 'test'}}),
+            (scheme.DictOption('foo', scheme=None, bind_env=True), 'nested', 'TEST_ENV_', True, {'env': {'key': 'test'}}),
+            (scheme.DictOption('foo', scheme=None, bind_env=True), 'nested.env', 'TEST_ENV_', False, {'key': 'test'}),
+            (scheme.DictOption('foo', scheme=None, bind_env=True), 'nested.env', 'TEST_ENV_', True, {'key': 'test'}),
+        ]
+    )
+    def test_parse_env_ok(self, option, key, prefix, auto_env, expected, with_env):
+        """Parse environment variables for the DictOption."""
+        actual = option.parse_env(key=key, prefix=prefix, auto_env=auto_env)
+        assert actual == expected
+
 
 class TestListOption:
     """Tests for the `ListOption` class."""
@@ -357,6 +479,40 @@ class TestListOption:
         with pytest.raises(errors.SchemeValidationError):
             opt.validate(['a', 'b', 'c'])
 
+    @pytest.mark.parametrize(
+        'option,prefix,auto_env', [
+            (scheme.ListOption('foo', bind_env=False), None, False),
+            (scheme.ListOption('foo', bind_env=False), None, True),
+            (scheme.ListOption('foo', bind_env=False), 'TEST_ENV', False),
+            (scheme.ListOption('foo', bind_env=False), 'TEST_ENV', True),
+
+            (scheme.ListOption('foo', bind_env=True), None, False),
+            (scheme.ListOption('foo', bind_env=True), None, True),
+            (scheme.ListOption('foo', bind_env=True), 'TEST_ENV', False),
+            (scheme.ListOption('foo', bind_env=True), 'TEST_ENV', True),
+        ]
+    )
+    def test_parse_env_none(self, option, prefix, auto_env):
+        """Parse environment variables for the ListOption. All of theses tests
+        should result in None being returned because no environment variables
+        are actually set.
+        """
+        actual = option.parse_env(prefix=prefix, auto_env=auto_env)
+        assert actual is None
+
+    @pytest.mark.parametrize(
+        'option,key,prefix,auto_env,expected', [
+            (scheme.ListOption('foo', bind_env=True), 'foo', 'TEST_ENV_', False, ['bar']),
+            (scheme.ListOption('foo', bind_env=True), 'foo', 'TEST_ENV_', True, ['bar']),
+            (scheme.ListOption('foo', bind_env=True), 'bar.list', 'TEST_ENV_', False, ['a', 'b', 'c']),
+            (scheme.ListOption('foo', bind_env=True), 'bar.list', 'TEST_ENV_', True, ['a', 'b', 'c']),
+        ]
+    )
+    def test_parse_env_ok(self, option, key, prefix, auto_env, expected, with_env):
+        """Parse environment variables for the ListOption."""
+        actual = option.parse_env(key=key, prefix=prefix, auto_env=auto_env)
+        assert actual == expected
+
 
 class TestScheme:
     """Tests for the `Scheme` class."""
@@ -409,6 +565,17 @@ class TestScheme:
                     'list': ['a', 'b']
                 }
             ),
+            (
+                # args
+                (
+                    scheme.Option('foo', default='bar'),
+                    scheme.DictOption('bar', scheme=None)
+                ),
+                # expected
+                {
+                    'foo': 'bar'
+                }
+            ),
             (
                 # args
                 (
@@ -468,6 +635,10 @@ class TestScheme:
                 (scheme.Option('foo'), scheme.Option('bar')),
                 ['foo', 'bar']
             ),
+            (
+                (scheme.Option('foo'), scheme.DictOption('bar', scheme=None)),
+                ['foo', 'bar']
+            ),
             (
                 (
                     scheme.Option('foo'),

tests/test_utils.py

@@ -2,46 +2,7 @@
 
 import pytest
 
-from bison import errors, scheme, utils
+from bison import utils
-
-
-@pytest.mark.parametrize(
-    'option,value,expected', [
-        (scheme.Option('foo'), 'foo', 'foo'),
-        (scheme.Option('foo'), 1, 1),
-        (scheme.Option('foo'), None, None),
-        (scheme.Option('foo'), False, False),
-        (scheme.Option('foo', field_type=str), 'foo', 'foo'),
-        (scheme.Option('foo', field_type=str), 1, '1'),
-        (scheme.Option('foo', field_type=int), '1', 1),
-        (scheme.Option('foo', field_type=float), '1', 1.0),
-        (scheme.Option('foo', field_type=float), '1.23', 1.23),
-        (scheme.Option('foo', field_type=bool), 'false', False),
-        (scheme.Option('foo', field_type=bool), 'False', False),
-        (scheme.Option('foo', field_type=bool), 'FALSE', False),
-        (scheme.Option('foo', field_type=bool), 'true', True),
-        (scheme.Option('foo', field_type=bool), 'True', True),
-        (scheme.Option('foo', field_type=bool), 'TRUE', True),
-    ]
-)
-def test_cast(option, value, expected):
-    """Cast values to the type set by the Option."""
-    actual = utils.cast(option, value)
-    assert actual == expected
-
-
-@pytest.mark.parametrize(
-    'option,value', [
-        (scheme.Option('foo', field_type=int), 'foo'),
-        (scheme.ListOption('foo'), 'foo'),
-        (scheme.Option('foo', field_type=list), 'foo'),
-        (scheme.Option('foo', field_type=tuple), 'foo'),
-    ]
-)
-def test_cast_fail(option, value):
-    """Cast values to the type set by the Option."""
-    with pytest.raises(errors.BisonError):
-        utils.cast(option, value)
 
 
 @pytest.mark.parametrize(