Configuration.__load_source() - Code Metrics - Inspection of "Convert readthedocs links for their .org -> .io mi..." - kalefranz/auxlib - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — develop (#12)

unknown

created 2016-10-10 07:15 UTC

Configuration.__load_source() D

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
cc	8
c	2
b	0
f	0
dl	0
loc	23
rs	4.7619

# -*- coding: utf-8 -*-
"""A specialized map implementation to manage configuration and context information.

Features:
  * Uses YAML configuration files
  * Use environment variables to override configs
  * Can pass a list of required parameters at initialization
  * Works with encrypted files
  * Accepts multiple config files
  * Can query information from consul
  * Does type coercion on strings
  * Composable configs
  * Ordered merge: downstream configs will override upstream
  * Reload from sources on SIGHUP

Available source types:
  * Environment variables
  * Yaml file on file system
  * Yaml file in S3
  * Consul
  * Hashicorp Vault

Notes:
  * Keys are case-insensitive.

"""
from __future__ import absolute_import, division, print_function

import inspect
import logging
import os
import signal

from .compat import string_types
from .decorators import memoize, memoizemethod
from .exceptions import AssignmentError, NotFoundError
from .path import PackageFile
from .type_coercion import listify
from .type_coercion import typify

log = logging.getLogger(__name__)


@memoize
def make_env_key(app_name, key):
    """Creates an environment key-equivalent for the given key"""
    key = key.replace('-', '_').replace(' ', '_')
    return "_".join((x.upper() for x in (app_name, key)))


@memoize
def reverse_env_key(app_name, key):
    app = app_name.upper() + '_'
    assert key.startswith(app), "{0} is not a(n) {1} environment key".format(key, app)
    return key[len(app):].lower()


class Configuration(object):
    """A map implementation to manage configuration and context information. Values
    can be accessed (read, not assigned) as either a dict lookup (e.g. `config[key]`) or as an
    attribute (e.g. `config.key`).

    This class makes the foundational assumption of a yaml configuration file, as values in yaml
    are typed.

    This class allows overriding configuration keys with environment variables. Given an app name
    `foo` and a config parameter `bar: 15`, setting a `FOO_BAR` environment variable to `22` will
    override the value of `bar`. The type of `22` remains `int` because the underlying value of
    `15` is used to infer the type of the `FOO_BAR` environment variable. When an underlying
    parameter does not exist in a config file, the type is intelligently guessed.

    Args:
        app_name (str)
        config_sources (str or list, optional)
        required_parameters (iter, optional)

    Raises:
        InitializationError: on instantiation, when `required_parameters` are not found
        warns: on instantiation, when a given `config_file` cannot be read
        NotFoundError: when requesting a key that does not exist

    Examples:
        >>> for (key, value) in [('FOO_BAR', 22), ('FOO_BAZ', 'yes'), ('FOO_BANG', 'monkey')]:
        ...     os.environ[key] = str(value)

        >>> context = Configuration('foo')
        >>> context.bar, context.bar.__class__
        (22, <... 'int'>)
        >>> context['baz'], type(context['baz'])
        (True, <... 'bool'>)
        >>> context.bang, type(context.bang)
        ('monkey', <... 'str'>)

    """

    def __init__(self, appname, config_sources=None, required_parameters=None, package=None):
        self.appname = appname
        self.package = package or inspect.getmodule(self).__package__

        # Indicates if sources are being loaded for the first time or are being reloaded.
        self.__initial_load = True

        # The ordered list of sources from which to load key, value pairs.
        self.__sources = list()

        # The object that holds the actual key, value pairs after they're loaded from sources.
        # Keys are stored as all lower-case. Access by key is provided in a case-insensitive way.
        self._config_map = dict()

        self._registered_env_keys = set()  # TODO: add explanation comments
        self._required_keys = set(listify(required_parameters))

        if hasattr(signal, 'SIGHUP'):  # Reload config on SIGHUP (UNIX only)
            self.__set_up_sighup_handler()

        self.__load_environment_keys()
        self.append_sources(config_sources)

    def append_sources(self, config_sources):
        force_reload = True
        for source in listify(config_sources):
            self.__append_source(source, force_reload)

    def append_required(self, required_parameters):
        self._required_keys.update(listify(required_parameters))

    def __append_source(self, source, force_reload=False, _parent_source=None):
        source.parent_config = self
        self.__load_source(source, force_reload)
        source.parent_source = _parent_source
        self.__sources.append(source)

    def verify(self):
        self.__ensure_required_keys()
        return self

    def set_env(self, key, value):
        """Sets environment variables by prepending the app_name to `key`. Also registers the
        environment variable with the instance object preventing an otherwise-required call to
        `reload()`.
        """
        os.environ[make_env_key(self.appname, key)] = str(value)  # must coerce to string
        self._registered_env_keys.add(key)
        self._clear_memoization()

    def unset_env(self, key):
        """Removes an environment variable using the prepended app_name convention with `key`."""
        os.environ.pop(make_env_key(self.appname, key), None)
        self._registered_env_keys.discard(key)
        self._clear_memoization()

    def _reload(self, force=False):
        """Reloads the configuration from the file and environment variables. Useful if using
        `os.environ` instead of this class' `set_env` method, or if the underlying configuration
        file is changed externally.
        """
        self._config_map = dict()
        self._registered_env_keys = set()
        self.__reload_sources(force)
        self.__load_environment_keys()
        self.verify()
        self._clear_memoization()

    @memoizemethod  # memoized for performance; always use self.set_env() instead of os.setenv()
    def __getitem__(self, key):
        key = key.lower()
        if key in self._registered_env_keys:
            from_env = os.getenv(make_env_key(self.appname, key))
            from_sources = self._config_map.get(key)
            return typify(from_env, type(from_sources) if from_sources is not None else None)
        else:
            try:
                return self._config_map[key]
            except KeyError as e:
                raise NotFoundError(e)

    def __getattr__(self, key):
        return self[key]

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __setitem__(self, *args):
        raise AssignmentError()

    def __iter__(self):
        for key in self._registered_env_keys | set(self._config_map.keys()):
            yield key

    def items(self):
        for key in self:
            yield key, self[key]

    def __load_source(self, source, force_reload=False):
        if force_reload and source.parent_source:
            # TODO: double-check case of reload without force reload for chained configs
            return

        items = source.dump(force_reload)
        if source.provides and not set(source.provides).issubset(items):
            raise NotImplementedError()  # TODO: fix this

        additional_requirements = items.pop('additional_requirements', None)
        if isinstance(additional_requirements, string_types):
            additional_requirements = additional_requirements.split(',')
        self._required_keys |= set(listify(additional_requirements))

        additional_sources = items.pop('additional_sources', None)

        self._config_map.update(items)

        if additional_sources:
            for src in additional_sources:
                class_name, kwargs = src.popitem()
                additional_source = globals()[class_name](**kwargs)
                self.__append_source(additional_source, force_reload, source)

    def __load_environment_keys(self):
        app_prefix = self.appname.upper() + '_'
        for env_key in os.environ:
            if env_key.startswith(app_prefix):
                self._registered_env_keys.add(reverse_env_key(self.appname, env_key))
                # We don't actually add values to _config_map here. Rather, they're pulled
                # directly from os.environ at __getitem__ time. This allows for type casting
                # environment variables if possible.

    def __ensure_required_keys(self):
        available_keys = self._registered_env_keys | set(self._config_map.keys())
        missing_keys = self._required_keys - available_keys
        if missing_keys:
            raise EnvironmentError("Required key(s) not found in environment\n"
                                   "  or configuration sources.\n"
                                   "  Missing Keys: {0}".format(list(missing_keys)))

    def _clear_memoization(self):
        self.__dict__.pop('_memoized_results', None)

    def __set_up_sighup_handler(self):
        def sighup_handler(signum, frame):
            if signum != signal.SIGHUP:
                return
            self._reload(True)
            if callable(self.__previous_sighup_handler):
                self.__previous_sighup_handler(signum, frame)
        self.__previous_sighup_handler = signal.getsignal(signal.SIGHUP)
        signal.signal(signal.SIGHUP, sighup_handler)


class Source(object):
    _items = None
    _provides = None
    _parent_source = None

    @property
    def provides(self):
        return self._provides

    @property
    def items(self):
        return self.dump()

    def load(self):
        """Must return a key, value dict"""
        raise NotImplementedError()  # pragma: no cover

    def dump(self, force_reload=False):
        if self._items is None or force_reload:
            self._items = self.load()
        return self._items

    @property
    def parent_config(self):
        return self._parent_config

    @parent_config.setter
    def parent_config(self, parent_config):
        self._parent_config = parent_config

    @property
    def parent_source(self):
        return self._parent_source

    @parent_source.setter
    def parent_source(self, parent_source):
        self._parent_source = parent_source


class YamlSource(Source):

    def __init__(self, location, provides=None):
        self._location = location
        self._provides = provides if provides else None

    def load(self):
        with PackageFile(self._location, self.parent_config.package) as fh:
            import yaml
            contents = yaml.load(fh)
            if self.provides is None:
                return contents
            else:
                return dict((key, contents[key]) for key in self.provides)


class EnvironmentMappedSource(Source):
    """Load a full Source object given the value of an environment variable."""

    def __init__(self, envvar, sourcemap):
        self._envvar = envvar
        self._sourcemap = sourcemap

    def load(self):
        mapped_source = self._sourcemap[self.parent_config[self._envvar]]
        mapped_source.parent_config = self.parent_config
        params = mapped_source.load()
        return params


1			# -- coding: utf-8 --
2			"""A specialized map implementation to manage configuration and context information.
3
4			Features:
5			* Uses YAML configuration files
6			* Use environment variables to override configs
7			* Can pass a list of required parameters at initialization
8			* Works with encrypted files
9			* Accepts multiple config files
10			* Can query information from consul
11			* Does type coercion on strings
12			* Composable configs
13			* Ordered merge: downstream configs will override upstream
14			* Reload from sources on SIGHUP
15
16			Available source types:
17			* Environment variables
18			* Yaml file on file system
19			* Yaml file in S3
20			* Consul
21			* Hashicorp Vault
22
23			Notes:
24			* Keys are case-insensitive.
25
26			"""
27			from __future__ import absolute_import, division, print_function
28
29			import inspect
30			import logging
31			import os
32			import signal
33
34			from .compat import string_types
35			from .decorators import memoize, memoizemethod
36			from .exceptions import AssignmentError, NotFoundError
37			from .path import PackageFile
38			from .type_coercion import listify
39			from .type_coercion import typify
40
41			log = logging.getLogger(__name__)
42
43
44			@memoize
45			def make_env_key(app_name, key):
46			"""Creates an environment key-equivalent for the given key"""
47			key = key.replace('-', '_').replace(' ', '_')
48			return "_".join((x.upper() for x in (app_name, key)))
49
50
51			@memoize
52			def reverse_env_key(app_name, key):
53			app = app_name.upper() + '_'
54			assert key.startswith(app), "{0} is not a(n) {1} environment key".format(key, app)
55			return key[len(app):].lower()
56
57
58			class Configuration(object):
59			"""A map implementation to manage configuration and context information. Values
60			can be accessed (read, not assigned) as either a dict lookup (e.g. `config[key]`) or as an
61			attribute (e.g. `config.key`).
62
63			This class makes the foundational assumption of a yaml configuration file, as values in yaml
64			are typed.
65
66			This class allows overriding configuration keys with environment variables. Given an app name
67			`foo` and a config parameter `bar: 15`, setting a `FOO_BAR` environment variable to `22` will
68			override the value of `bar`. The type of `22` remains `int` because the underlying value of
69			`15` is used to infer the type of the `FOO_BAR` environment variable. When an underlying
70			parameter does not exist in a config file, the type is intelligently guessed.
71
72			Args:
73			app_name (str)
74			config_sources (str or list, optional)
75			required_parameters (iter, optional)
76
77			Raises:
78			InitializationError: on instantiation, when `required_parameters` are not found
79			warns: on instantiation, when a given `config_file` cannot be read
80			NotFoundError: when requesting a key that does not exist
81
82			Examples:
83			>>> for (key, value) in [('FOO_BAR', 22), ('FOO_BAZ', 'yes'), ('FOO_BANG', 'monkey')]:
84			... os.environ[key] = str(value)
85
86			>>> context = Configuration('foo')
87			>>> context.bar, context.bar.__class__
88			(22, <... 'int'>)
89			>>> context['baz'], type(context['baz'])
90			(True, <... 'bool'>)
91			>>> context.bang, type(context.bang)
92			('monkey', <... 'str'>)
93
94			"""
95
96			def __init__(self, appname, config_sources=None, required_parameters=None, package=None):
97			self.appname = appname
98			self.package = package or inspect.getmodule(self).__package__
99
100			# Indicates if sources are being loaded for the first time or are being reloaded.
101			self.__initial_load = True
102
103			# The ordered list of sources from which to load key, value pairs.
104			self.__sources = list()
105
106			# The object that holds the actual key, value pairs after they're loaded from sources.
107			# Keys are stored as all lower-case. Access by key is provided in a case-insensitive way.
108			self._config_map = dict()
109
110			self._registered_env_keys = set() # TODO: add explanation comments
111			self._required_keys = set(listify(required_parameters))
112
113			if hasattr(signal, 'SIGHUP'): # Reload config on SIGHUP (UNIX only)
114			self.__set_up_sighup_handler()
115
116			self.__load_environment_keys()
117			self.append_sources(config_sources)
118
119			def append_sources(self, config_sources):
120			force_reload = True
121			for source in listify(config_sources):
122			self.__append_source(source, force_reload)
123
124			def append_required(self, required_parameters):
125			self._required_keys.update(listify(required_parameters))
126
127			def __append_source(self, source, force_reload=False, _parent_source=None):
128			source.parent_config = self
129			self.__load_source(source, force_reload)
130			source.parent_source = _parent_source
131			self.__sources.append(source)
132
133			def verify(self):
134			self.__ensure_required_keys()
135			return self
136
137			def set_env(self, key, value):
138			"""Sets environment variables by prepending the app_name to `key`. Also registers the
139			environment variable with the instance object preventing an otherwise-required call to
140			`reload()`.
141			"""
142			os.environ[make_env_key(self.appname, key)] = str(value) # must coerce to string
143			self._registered_env_keys.add(key)
144			self._clear_memoization()
145
146			def unset_env(self, key):
147			"""Removes an environment variable using the prepended app_name convention with `key`."""
148			os.environ.pop(make_env_key(self.appname, key), None)
149			self._registered_env_keys.discard(key)
150			self._clear_memoization()
151
152			def _reload(self, force=False):
153			"""Reloads the configuration from the file and environment variables. Useful if using
154			`os.environ` instead of this class' `set_env` method, or if the underlying configuration
155			file is changed externally.
156			"""
157			self._config_map = dict()
158			self._registered_env_keys = set()
159			self.__reload_sources(force)
160			self.__load_environment_keys()
161			self.verify()
162			self._clear_memoization()
163
164			@memoizemethod # memoized for performance; always use self.set_env() instead of os.setenv()
165			def __getitem__(self, key):
166			key = key.lower()
167			if key in self._registered_env_keys:
168			from_env = os.getenv(make_env_key(self.appname, key))
169			from_sources = self._config_map.get(key)
170			return typify(from_env, type(from_sources) if from_sources is not None else None)
171			else:
172			try:
173			return self._config_map[key]
174			except KeyError as e:
175			raise NotFoundError(e)
176
177			def __getattr__(self, key):
178			return self[key]
179
180			def get(self, key, default=None):
181			try:
182			return self[key]
183			except KeyError:
184			return default
185
186			def __setitem__(self, *args):
187			raise AssignmentError()
188
189			def __iter__(self):
190			for key in self._registered_env_keys \| set(self._config_map.keys()):
191			yield key
192
193			def items(self):
194			for key in self:
195			yield key, self[key]
196
197			def __load_source(self, source, force_reload=False):
198			if force_reload and source.parent_source:
199			# TODO: double-check case of reload without force reload for chained configs
200			return
201
202			items = source.dump(force_reload)
203			if source.provides and not set(source.provides).issubset(items):
204			raise NotImplementedError() # TODO: fix this
205
206			additional_requirements = items.pop('additional_requirements', None)
207			if isinstance(additional_requirements, string_types):
208			additional_requirements = additional_requirements.split(',')
209			self._required_keys \|= set(listify(additional_requirements))
210
211			additional_sources = items.pop('additional_sources', None)
212
213			self._config_map.update(items)
214
215			if additional_sources:
216			for src in additional_sources:
217			class_name, kwargs = src.popitem()
218			additional_source = globals()[class_name](**kwargs)
219			self.__append_source(additional_source, force_reload, source)
220
221			def __load_environment_keys(self):
222			app_prefix = self.appname.upper() + '_'
223			for env_key in os.environ:
224			if env_key.startswith(app_prefix):
225			self._registered_env_keys.add(reverse_env_key(self.appname, env_key))
226			# We don't actually add values to _config_map here. Rather, they're pulled
227			# directly from os.environ at __getitem__ time. This allows for type casting
228			# environment variables if possible.
229
230			def __ensure_required_keys(self):
231			available_keys = self._registered_env_keys \| set(self._config_map.keys())
232			missing_keys = self._required_keys - available_keys
233			if missing_keys:
234			raise EnvironmentError("Required key(s) not found in environment\n"
235			" or configuration sources.\n"
236			" Missing Keys: {0}".format(list(missing_keys)))
237
238			def _clear_memoization(self):
239			self.__dict__.pop('_memoized_results', None)
240
241			def __set_up_sighup_handler(self):
242			def sighup_handler(signum, frame):
243			if signum != signal.SIGHUP:
244			return
245			self._reload(True)
246			if callable(self.__previous_sighup_handler):
247			self.__previous_sighup_handler(signum, frame)
248			self.__previous_sighup_handler = signal.getsignal(signal.SIGHUP)
249			signal.signal(signal.SIGHUP, sighup_handler)
250
251
252			class Source(object):
253			_items = None
254			_provides = None
255			_parent_source = None
256
257			@property
258			def provides(self):
259			return self._provides
260
261			@property
262			def items(self):
263			return self.dump()
264
265			def load(self):
266			"""Must return a key, value dict"""
267			raise NotImplementedError() # pragma: no cover
268
269			def dump(self, force_reload=False):
270			if self._items is None or force_reload:
271			self._items = self.load()
272			return self._items
273
274			@property
275			def parent_config(self):
276			return self._parent_config
277
278			@parent_config.setter
279			def parent_config(self, parent_config):
280			self._parent_config = parent_config
281
282			@property
283			def parent_source(self):
284			return self._parent_source
285
286			@parent_source.setter
287			def parent_source(self, parent_source):
288			self._parent_source = parent_source
289
290
291			class YamlSource(Source):
292
293			def __init__(self, location, provides=None):
294			self._location = location
295			self._provides = provides if provides else None
296
297			def load(self):
298			with PackageFile(self._location, self.parent_config.package) as fh:
299			import yaml
300			contents = yaml.load(fh)
301			if self.provides is None:
302			return contents
303			else:
304			return dict((key, contents[key]) for key in self.provides)
305
306
307			class EnvironmentMappedSource(Source):
308			"""Load a full Source object given the value of an environment variable."""
309
310			def __init__(self, envvar, sourcemap):
311			self._envvar = envvar
312			self._sourcemap = sourcemap
313
314			def load(self):
315			mapped_source = self._sourcemap[self.parent_config[self._envvar]]
316			mapped_source.parent_config = self.parent_config
317			params = mapped_source.load()
318			return params
319

kalefranz / auxlib

Pull Request — develop (#12)

Configuration.__load_source() D

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like