cyberes
/
bison
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update env behavior for dict/list options

This commit is contained in:
Erick Daniszewski 2018-03-08 12:25:18 -05:00
parent 69a5ce6975
commit 83809213f8
No known key found for this signature in database
GPG Key ID: DEA43A5D586F3E0E
8 changed files with 355 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bison/__version__.py
View File

@ -3,7 +3,7 @@
__title__ = 'bison'
__description__ = 'Python application configuration'
__url__ = 'https://github.com/edaniszewski/bison'
__version__ = '0.0.1'
__version__ = '0.0.2'
__author__ = 'Erick Daniszewski'
__author_email__ = 'edaniszewski@gmail.com'
__license__ = 'MIT'

50
bison/bison.py
View File

@ -12,8 +12,7 @@ import os
import yaml
from bison.errors import BisonError
from bison.scheme import Option
from bison.utils import DotDict, cast
from bison.utils import DotDict
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@ -197,50 +196,9 @@ class Bison(object):
# config if there is a scheme.
if self.scheme:
for k, v in self.scheme.flatten().items():
# only Options can be bound to env variables currently.
if not isinstance(v, Option):
continue
# we explicitly do not want to bind the option to env
if v.bind_env is False:
continue
# we want to bind the option to env. in this case, bind_env is
# generated from the Option key.
elif v.bind_env is True:
env_key = k.replace('.', '_').upper()
# if an env prefix exists, use it
if self.env_prefix:
env_key = self.env_prefix.upper() + env_key
env = os.environ.get(env_key, None)
if env is not None:
env_cfg[k] = cast(v, env)
# bind_env holds the env variable to use. since it is specified
# manually, we do not prepend the env prefix.
elif isinstance(v.bind_env, str):
env_key = v.bind_env
env = os.environ.get(env_key, None)
if env is not None:
env_cfg[k] = cast(v, env)
# bind_env is None - this is its default value. in this case, the
# option hasn't been explicitly set as False, so we can do env
# lookups if auto_env is set.
elif v.bind_env is None:
if self.auto_env:
env_key = k.replace('.', '_').upper()
# if an env prefix exists, use it
if self.env_prefix:
env_key = self.env_prefix.upper() + env_key
env = os.environ.get(env_key, None)
if env is not None:
env_cfg[k] = cast(v, env)
value = v.parse_env(k, self.env_prefix, self.auto_env)
if value is not None:
env_cfg[k] = value
if len(env_cfg) > 0:
# the configuration changes, so we invalidate the cached config

177
bison/scheme.py
View File

@ -9,7 +9,9 @@ in order to do configuration defaults and validation. A
here as well.
"""
from bison import errors
import os
from bison import errors, utils
class NoDefault:
@ -61,9 +63,10 @@ class Scheme(object):
# if we have a dict option, build the defaults for its scheme.
# if any defaults exist, use them.
if isinstance(arg, DictOption):
b = arg.scheme.build_defaults()
if b:
defaults[arg.name] = b
if arg.scheme:
b = arg.scheme.build_defaults()
if b:
defaults[arg.name] = b
return defaults
def flatten(self):
@ -84,9 +87,10 @@ class Scheme(object):
flat[arg.name] = arg
elif isinstance(arg, DictOption):
flat[arg.name] = DictOption
for k, v in arg.scheme.flatten().items():
flat[arg.name + '.' + k] = v
flat[arg.name] = arg
if arg.scheme:
for k, v in arg.scheme.flatten().items():
flat[arg.name + '.' + k] = v
self._flat = flat
return self._flat
@ -142,6 +146,25 @@ class _BaseOpt(object):
"""
raise NotImplementedError
def parse_env(self, key=None, prefix=None, auto_env=False):
"""Parse the environment based on the option configuration.
Args:
key (str|None): The full key (dot notation) to use for the option.
If None, this will use the option name. Otherwise, we expect to
have the full key (can be determined by flattening the base
Scheme).
prefix (str|None): The prefix to use for environment variables.
This is set in the `Bison` object and should be passed in
here.
auto_env (bool): The `Bison` setting for auto_env.
Returns:
The value(s) for the option from the environment, if found. If
no values are found, None is returned.
"""
raise NotImplementedError
class Option(_BaseOpt):
"""Option represents a configuration option with a singular value.
@ -186,6 +209,87 @@ class Option(_BaseOpt):
'{} is not in the valid options: {}'.format(value, self.choices)
)
def parse_env(self, key=None, prefix=None, auto_env=False):
if key is None:
key = self.name
# we explicitly do not want to bind the option to env
if self.bind_env is False:
return None
# we want to bind the option to env. in this case, bind_env is
# generated from the Option key.
elif self.bind_env is True:
env_key = key.replace('.', '_').upper()
# if an env prefix exists, use it
if prefix:
env_key = prefix.upper() + env_key
env = os.environ.get(env_key, None)
if env is not None:
return self.cast(env)
# bind_env holds the env variable to use. since it is specified
# manually, we do not prepend the env prefix.
elif isinstance(self.bind_env, str):
env_key = self.bind_env
env = os.environ.get(env_key, None)
if env is not None:
return self.cast(env)
# bind_env is None - this is its default value. in this case, the
# option hasn't been explicitly set as False, so we can do env
# lookups if auto_env is set.
elif self.bind_env is None:
if auto_env:
env_key = key.replace('.', '_').upper()
# if an env prefix exists, use it
if prefix:
env_key = prefix.upper() + env_key
env = os.environ.get(env_key, None)
if env is not None:
return self.cast(env)
return None
def cast(self, value):
"""Cast a value to the type required by the option, if one is set.
This is used to cast the string values gathered from environment
variable into their required type.
Args:
value: The value to cast.
Returns:
The value casted to the expected type for the option.
"""
# if there is no type set for the option, return the given
# value unchanged.
if self.type is None:
return value
# cast directly
if self.type in (str, int, float):
try:
return self.type(value)
except Exception as e:
raise errors.BisonError(
'Failed to cast {} to {}'.format(value, self.type)
) from e
# for bool, can't cast a string, since a string is truthy,
# so we need to check the value.
elif self.type == bool:
return value.lower() == 'true'
# the option type is currently not supported
else:
raise errors.BisonError('Unsupported type for casting: {}'.format(self.type))
class DictOption(_BaseOpt):
"""DictOption represents a configuration option with a dictionary value.
@ -212,7 +316,11 @@ class DictOption(_BaseOpt):
not fail if it is not present in the config. If this is left at its
default value of `_NoDefault`, then this option is considered required
and will fail validation if not present.
bind_env (bool): Bind the option to an environment variable.
bind_env (bool): Bind the option to an environment variable. If False,
the option will not be bound to env. If True, the key for the this
DictOption will serve as an environment prefix. Any environment
variable matching that prefix will be added to the parsed result
as a string.
"""
def __init__(self, name, scheme, default=_no_default, bind_env=False):
@ -229,6 +337,34 @@ class DictOption(_BaseOpt):
if isinstance(self.scheme, Scheme):
self.scheme.validate(value)
def parse_env(self, key=None, prefix=None, auto_env=False):
if key is None:
key = self.name
# we explicitly do not want to bind the option to env
if self.bind_env is False:
return None
# we want to populate the dict from env. the dict option key
# will generate the prefix. anything after the prefix will be
# part of the populated value(s)
elif self.bind_env is True:
env_key = key.replace('.', '_').upper()
if prefix:
env_key = prefix.upper() + env_key
if not env_key.endswith('_'):
env_key = env_key + '_'
values = utils.DotDict()
for k, v in os.environ.items():
if k.startswith(env_key):
dot_key = k[len(env_key):].replace('_', '.').lower()
values[dot_key] = v
if values:
return values
return None
class ListOption(_BaseOpt):
"""ListOption represents a configuration option with a list value.
@ -255,7 +391,11 @@ class ListOption(_BaseOpt):
member_type: The type that all members of the list should have.
member_scheme (Scheme): The `Scheme` that all members of the list should
fulfil. This should be used when the list members are dictionaries/maps.
bind_env (bool): Bind the option to an environment variable.
bind_env (bool): Bind the option to an environment variable. If False, the
option will not be bound to env. If True, the option's key will be used
to create an env variable. The contents of that env variable will be used
to populate a list. This assumes that the env value is a string with the
items being comma separated.
"""
def __init__(self, name, default=_no_default, member_type=None, member_scheme=None, bind_env=False):
@ -290,3 +430,22 @@ class ListOption(_BaseOpt):
for item in value:
self.member_scheme.validate(item)
def parse_env(self, key=None, prefix=None, auto_env=False):
if key is None:
key = self.name
# we explicitly do not want to bind the option to env
if self.bind_env is False:
return None
elif self.bind_env is True:
env_key = key.replace('.', '_').upper()
if prefix:
env_key = prefix.upper() + env_key
value = os.environ.get(env_key, None)
if value:
return value.split(',')
return None

43
bison/utils.py
View File

@ -6,49 +6,6 @@ bison.utils
Utilities for `bison`.
"""
from bison.errors import BisonError
from bison.scheme import Option
def cast(option, value):
"""Cast a value to the type required by the option, if one is set.
This is used to cast the string values gathered from environment
variable into their required type.
Args:
option: The Option specifying the type.
value: The value to cast.
Returns:
The value casted to the expected type for the option.
"""
if not isinstance(option, Option):
raise BisonError('Unable to cast - "{}" not an Option'.format(option))
# if there is no type set for the option, return the given
# value unchanged.
if option.type is None:
return value
# cast directly
if option.type in (str, int, float):
try:
return option.type(value)
except Exception as e:
raise BisonError(
'Failed to cast {} to {}'.format(value, option.type)
) from e
# for bool, can't cast a string, since a string is truthy,
# so we need to check the value.
elif option.type == bool:
return value.lower() == 'true'
# the option type is currently not supported
else:
raise BisonError('Unsupported type for casting: {}'.format(option.type))
def build_dot_value(key, value):
"""Build new dictionaries based off of the dot notation key.

2
tests/conftest.py
View File

@ -39,6 +39,7 @@ def with_env():
"""Set and cleanup environment variables for tests."""
os.environ['TEST_ENV_FOO'] = 'bar'
os.environ['TEST_ENV_NESTED_ENV_KEY'] = 'test'
os.environ['TEST_ENV_BAR_LIST'] = 'a,b,c'
os.environ['TEST_OTHER_ENV_BAR'] = 'baz'
os.environ['FOO_INT'] = '1'
os.environ['FOO_BOOL'] = 'False'
@ -47,6 +48,7 @@ def with_env():
del os.environ['TEST_ENV_FOO']
del os.environ['TEST_ENV_NESTED_ENV_KEY']
del os.environ['TEST_ENV_BAR_LIST']
del os.environ['TEST_OTHER_ENV_BAR']
del os.environ['FOO_INT']
del os.environ['FOO_BOOL']

7
tests/test_bison.py
View File

@ -186,6 +186,13 @@ class TestBison:
assert b.config_file == os.path.join(yaml_config.dirname, yaml_config.basename)
assert len(b._config) == 2
assert b.config == {
'foo': True,
'bar': {
'baz': 1,
'test': 'value'
}
}
def test_parse_config_fail(self, bad_yaml_config):
"""Parse the file config unsuccessfully."""

173
tests/test_scheme.py
View File

@ -5,13 +5,20 @@ import pytest
from bison import errors, scheme
def test_base_opt():
def test_base_opt_validate():
"""Validate the base option, which should fail."""
opt = scheme._BaseOpt()
with pytest.raises(NotImplementedError):
opt.validate('test-data')
def test_base_opt_parse_env():
"""Parse env from the base option, which should fail."""
opt = scheme._BaseOpt()
with pytest.raises(NotImplementedError):
opt.parse_env()
class TestOption:
"""Tests for the `Option` class."""
@ -143,6 +150,85 @@ class TestOption:
with pytest.raises(errors.SchemeValidationError):
opt.validate(value)
@pytest.mark.parametrize(
'option,value,expected', [
(scheme.Option('foo'), 'foo', 'foo'),
(scheme.Option('foo'), 1, 1),
(scheme.Option('foo'), None, None),
(scheme.Option('foo'), False, False),
(scheme.Option('foo', field_type=str), 'foo', 'foo'),
(scheme.Option('foo', field_type=str), 1, '1'),
(scheme.Option('foo', field_type=int), '1', 1),
(scheme.Option('foo', field_type=float), '1', 1.0),
(scheme.Option('foo', field_type=float), '1.23', 1.23),
(scheme.Option('foo', field_type=bool), 'false', False),
(scheme.Option('foo', field_type=bool), 'False', False),
(scheme.Option('foo', field_type=bool), 'FALSE', False),
(scheme.Option('foo', field_type=bool), 'true', True),
(scheme.Option('foo', field_type=bool), 'True', True),
(scheme.Option('foo', field_type=bool), 'TRUE', True),
]
)
def test_cast(self, option, value, expected):
"""Cast values to the type set by the Option."""
actual = option.cast(value)
assert actual == expected
@pytest.mark.parametrize(
'option,value', [
(scheme.Option('foo', field_type=int), 'foo'),
(scheme.Option('foo', field_type=list), 'foo'),
(scheme.Option('foo', field_type=tuple), 'foo'),
]
)
def test_cast_fail(self, option, value):
"""Cast values to the type set by the Option."""
with pytest.raises(errors.BisonError):
option.cast(value)
@pytest.mark.parametrize(
'option,prefix,auto_env', [
(scheme.Option('foo'), None, False),
(scheme.Option('foo', bind_env=False), None, False),
(scheme.Option('foo', bind_env=True), None, False),
(scheme.Option('foo', bind_env=True), 'TEST_ENV_', False),
(scheme.Option('foo', bind_env='TEST_KEY'), None, False),
(scheme.Option('foo', bind_env='TEST_KEY'), 'TEST_ENV_', False),
(scheme.Option('foo', bind_env=None), 'TEST_ENV_', False),
(scheme.Option('foo', bind_env=None), 'TEST_ENV_', True),
(scheme.Option('foo', bind_env=None), None, False),
(scheme.Option('foo', bind_env=None), None, True),
]
)
def test_parse_env_none(self, option, prefix, auto_env):
"""Parse environment variables for the Option. All of theses tests
should result in None being returned because no environment variables
are actually set.
"""
actual = option.parse_env(prefix=prefix, auto_env=auto_env)
assert actual is None
@pytest.mark.parametrize(
'option,key,prefix,auto_env,expected', [
(scheme.Option('foo', bind_env=True), 'foo', 'TEST_ENV_', False, 'bar'),
(scheme.Option('foo', bind_env=True), 'foo', 'TEST_ENV_', True, 'bar'),
(scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', 'TEST_ENV_', False, 'bar'),
(scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', 'TEST_ENV_', True, 'bar'),
(scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', None, False, 'bar'),
(scheme.Option('foo', bind_env='TEST_ENV_FOO'), 'foo', None, True, 'bar'),
(scheme.Option('foo', bind_env=None), 'foo', 'TEST_ENV_', True, 'bar'),
(scheme.Option('foo', bind_env=None), 'nested.env.key', 'TEST_ENV_', True, 'test'),
]
)
def test_parse_env_ok(self, option, key, prefix, auto_env, expected, with_env):
"""Parse environment variables for the Option."""
actual = option.parse_env(key=key, prefix=prefix, auto_env=auto_env)
assert actual == expected
class TestDictOption:
"""Tests for the `DictOption` class."""
@ -204,6 +290,42 @@ class TestDictOption:
))
opt.validate({'foo': 'bar'})
@pytest.mark.parametrize(
'option,prefix,auto_env', [
(scheme.DictOption('foo', scheme=None, bind_env=False), None, False),
(scheme.DictOption('foo', scheme=None, bind_env=False), None, True),
(scheme.DictOption('foo', scheme=None, bind_env=False), 'TEST_ENV', False),
(scheme.DictOption('foo', scheme=None, bind_env=False), 'TEST_ENV', True),
(scheme.DictOption('foo', scheme=None, bind_env=True), None, False),
(scheme.DictOption('foo', scheme=None, bind_env=True), None, True),
(scheme.DictOption('foo', scheme=None, bind_env=True), 'TEST_ENV', False),
(scheme.DictOption('foo', scheme=None, bind_env=True), 'TEST_ENV', True),
]
)
def test_parse_env_none(self, option, prefix, auto_env):
"""Parse environment variables for the DictOption. All of theses tests
should result in None being returned because no environment variables
are actually set.
"""
actual = option.parse_env(prefix=prefix, auto_env=auto_env)
assert actual is None
@pytest.mark.parametrize(
'option,key,prefix,auto_env,expected', [
(scheme.DictOption('foo', scheme=None, bind_env=True), 'foo', 'TEST_ENV_', False, None),
(scheme.DictOption('foo', scheme=None, bind_env=True), 'foo', 'TEST_ENV_', True, None),
(scheme.DictOption('foo', scheme=None, bind_env=True), 'nested', 'TEST_ENV_', False, {'env': {'key': 'test'}}),
(scheme.DictOption('foo', scheme=None, bind_env=True), 'nested', 'TEST_ENV_', True, {'env': {'key': 'test'}}),
(scheme.DictOption('foo', scheme=None, bind_env=True), 'nested.env', 'TEST_ENV_', False, {'key': 'test'}),
(scheme.DictOption('foo', scheme=None, bind_env=True), 'nested.env', 'TEST_ENV_', True, {'key': 'test'}),
]
)
def test_parse_env_ok(self, option, key, prefix, auto_env, expected, with_env):
"""Parse environment variables for the DictOption."""
actual = option.parse_env(key=key, prefix=prefix, auto_env=auto_env)
assert actual == expected
class TestListOption:
"""Tests for the `ListOption` class."""
@ -357,6 +479,40 @@ class TestListOption:
with pytest.raises(errors.SchemeValidationError):
opt.validate(['a', 'b', 'c'])
@pytest.mark.parametrize(
'option,prefix,auto_env', [
(scheme.ListOption('foo', bind_env=False), None, False),
(scheme.ListOption('foo', bind_env=False), None, True),
(scheme.ListOption('foo', bind_env=False), 'TEST_ENV', False),
(scheme.ListOption('foo', bind_env=False), 'TEST_ENV', True),
(scheme.ListOption('foo', bind_env=True), None, False),
(scheme.ListOption('foo', bind_env=True), None, True),
(scheme.ListOption('foo', bind_env=True), 'TEST_ENV', False),
(scheme.ListOption('foo', bind_env=True), 'TEST_ENV', True),
]
)
def test_parse_env_none(self, option, prefix, auto_env):
"""Parse environment variables for the ListOption. All of theses tests
should result in None being returned because no environment variables
are actually set.
"""
actual = option.parse_env(prefix=prefix, auto_env=auto_env)
assert actual is None
@pytest.mark.parametrize(
'option,key,prefix,auto_env,expected', [
(scheme.ListOption('foo', bind_env=True), 'foo', 'TEST_ENV_', False, ['bar']),
(scheme.ListOption('foo', bind_env=True), 'foo', 'TEST_ENV_', True, ['bar']),
(scheme.ListOption('foo', bind_env=True), 'bar.list', 'TEST_ENV_', False, ['a', 'b', 'c']),
(scheme.ListOption('foo', bind_env=True), 'bar.list', 'TEST_ENV_', True, ['a', 'b', 'c']),
]
)
def test_parse_env_ok(self, option, key, prefix, auto_env, expected, with_env):
"""Parse environment variables for the ListOption."""
actual = option.parse_env(key=key, prefix=prefix, auto_env=auto_env)
assert actual == expected
class TestScheme:
"""Tests for the `Scheme` class."""
@ -409,6 +565,17 @@ class TestScheme:
'list': ['a', 'b']
}
),
(
# args
(
scheme.Option('foo', default='bar'),
scheme.DictOption('bar', scheme=None)
),
# expected
{
'foo': 'bar'
}
),
(
# args
(
@ -468,6 +635,10 @@ class TestScheme:
(scheme.Option('foo'), scheme.Option('bar')),
['foo', 'bar']
),
(
(scheme.Option('foo'), scheme.DictOption('bar', scheme=None)),
['foo', 'bar']
),
(
(
scheme.Option('foo'),

41
tests/test_utils.py
View File

@ -2,46 +2,7 @@
import pytest
from bison import errors, scheme, utils
@pytest.mark.parametrize(
'option,value,expected', [
(scheme.Option('foo'), 'foo', 'foo'),
(scheme.Option('foo'), 1, 1),
(scheme.Option('foo'), None, None),
(scheme.Option('foo'), False, False),
(scheme.Option('foo', field_type=str), 'foo', 'foo'),
(scheme.Option('foo', field_type=str), 1, '1'),
(scheme.Option('foo', field_type=int), '1', 1),
(scheme.Option('foo', field_type=float), '1', 1.0),
(scheme.Option('foo', field_type=float), '1.23', 1.23),
(scheme.Option('foo', field_type=bool), 'false', False),
(scheme.Option('foo', field_type=bool), 'False', False),
(scheme.Option('foo', field_type=bool), 'FALSE', False),
(scheme.Option('foo', field_type=bool), 'true', True),
(scheme.Option('foo', field_type=bool), 'True', True),
(scheme.Option('foo', field_type=bool), 'TRUE', True),
]
)
def test_cast(option, value, expected):
"""Cast values to the type set by the Option."""
actual = utils.cast(option, value)
assert actual == expected
@pytest.mark.parametrize(
'option,value', [
(scheme.Option('foo', field_type=int), 'foo'),
(scheme.ListOption('foo'), 'foo'),
(scheme.Option('foo', field_type=list), 'foo'),
(scheme.Option('foo', field_type=tuple), 'foo'),
]
)
def test_cast_fail(option, value):
"""Cast values to the type set by the Option."""
with pytest.raises(errors.BisonError):
utils.cast(option, value)
from bison import utils
@pytest.mark.parametrize(