auxlib.Configuration.sighup_handler() - Code Metrics - Inspection of "Reload config on SIGHUP (UNIX only)" - kalefranz/auxlib - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — develop ( 5780f7...1c0ef8 )

by Kale

created 2016-01-20 02:58 UTC

auxlib.Configuration.sighup_handler() A

↳ Parent: auxlib.Configuration.__set_up_sighup_handler()

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
cc	3
dl	0
loc	6
rs	9.4285

# -*- coding: utf-8 -*-
"""A specialized map implementation to manage configuration and context information.

Features:
  * Uses YAML configuration files
  * Use environment variables to override configs
  * Can pass a list of required parameters at initialization
  * Works with encrypted files
  * Accepts multiple config files
  * Can query information from consul
  * Does type coercion on strings
  * Composable configs
  * Ordered merge: downstream configs will override upstream
  * Reload from sources on SIGHUP

Available source types:
  * Environment variables
  * Yaml file on file system
  * Yaml file in S3
  * Consul
  * Hashicorp Vault

Notes:
  * Keys are case-insensitive.

"""
from __future__ import absolute_import, division, print_function
import inspect
import logging
import os
import signal

from ._vendor.six import string_types
from .type_coercion import listify
from .decorators import memoize, memoizemethod
from .exceptions import AssignmentError, NotFoundError
from .path import PackageFile
from .type_coercion import typify


log = logging.getLogger(__name__)


@memoize
def make_env_key(app_name, key):
    """Creates an environment key-equivalent for the given key"""
    key = key.replace('-', '_').replace(' ', '_')
    return "_".join((x.upper() for x in (app_name, key)))


@memoize
def reverse_env_key(app_name, key):
    app = app_name.upper() + '_'
    assert key.startswith(app), "{} is not a(n) {} environment key".format(key, app)
    return key[len(app):].lower()


class Configuration(object):
    """A map implementation to manage configuration and context information. Values
    can be accessed (read, not assigned) as either a dict lookup (e.g. `config[key]`) or as an
    attribute (e.g. `config.key`).

    This class makes the foundational assumption of a yaml configuration file, as values in yaml
    are typed.

    This class allows overriding configuration keys with environment variables. Given an app name
    `foo` and a config parameter `bar: 15`, setting a `FOO_BAR` environment variable to `22` will
    override the value of `bar`. The type of `22` remains `int` because the underlying value of
    `15` is used to infer the type of the `FOO_BAR` environment variable. When an underlying
    parameter does not exist in a config file, the type is intelligently guessed.

    Args:
        app_name (str)
        config_sources (str or list, optional)
        required_parameters (iter, optional)

    Raises:
        InitializationError: on instantiation, when `required_parameters` are not found
        warns: on instantiation, when a given `config_file` cannot be read
        NotFoundError: when requesting a key that does not exist

    Examples:
        >>> for (key, value) in [('FOO_BAR', 22), ('FOO_BAZ', 'yes'), ('FOO_BANG', 'monkey')]:
        ...     os.environ[key] = str(value)

        >>> context = Configuration('foo')
        >>> context.bar, context.bar.__class__
        (22, <... 'int'>)
        >>> context['baz'], type(context['baz'])
        (True, <... 'bool'>)
        >>> context.bang, type(context.bang)
        ('monkey', <... 'str'>)

    """

    def __init__(self, appname, config_sources=None, required_parameters=None):
        self.appname = appname
        self.package = inspect.getmodule(self).__package__

        # A private flag used to indicate if sources are being loaded for the first time or are
        # being reloaded.
        self.__initial_load = True

        # The ordered list of sources from which to load key, value pairs.
        self.__sources = list()

        # The object that holds the actual key, value pairs after they're loaded from sources.
        # Keys are stored as all lower-case. Access by key is provided in a case-insensitive way.
        self._config_map = dict()

        #
        self._registered_env_keys = set()

        self._required_keys = set(listify(required_parameters))

        self.__set_up_sighup_handler()
        self.__load_environment_keys()
        self.append_sources(config_sources)

    def append_sources(self, config_sources):
        force_reload = True
        for source in listify(config_sources):
            self.__append_source(source, force_reload)

    def append_required(self, required_parameters):
        self._required_keys.update(listify(required_parameters))

    def __append_source(self, source, force_reload=False, _parent_source=None):
        source.parent_config = self
        self.__load_source(source, force_reload)
        source.parent_source = _parent_source
        self.__sources.append(source)

    def verify(self):
        self.__ensure_required_keys()
        return self

    def set_env(self, key, value):
        """Sets environment variables by prepending the app_name to `key`. Also registers the
        environment variable with the instance object preventing an otherwise-required call to
        `reload()`.
        """
        os.environ[make_env_key(self.appname, key)] = str(value)  # must coerce to string
        self._registered_env_keys.add(key)
        self._clear_memoization()

    def unset_env(self, key):
        """Removes an environment variable using the prepended app_name convention with `key`."""
        os.environ.pop(make_env_key(self.appname, key), None)
        self._registered_env_keys.discard(key)
        self._clear_memoization()

    def _reload(self, force=False):
        """Reloads the configuration from the file and environment variables. Useful if using
        `os.environ` instead of this class' `set_env` method, or if the underlying configuration
        file is changed externally.
        """
        self._config_map = dict()
        self._registered_env_keys = set()
        self.__reload_sources(force)
        self.__load_environment_keys()
        self.verify()
        self._clear_memoization()

    @memoizemethod  # memoized for performance; always use self.set_env() instead of os.setenv()
    def __getitem__(self, key):
        key = key.lower()
        if key in self._registered_env_keys:
            from_env = os.getenv(make_env_key(self.appname, key))
            from_sources = self._config_map.get(key, None)
            return typify(from_env, type(from_sources) if from_sources is not None else None)
        else:
            try:
                return self._config_map[key]
            except KeyError as e:
                raise NotFoundError(e)

    def __getattr__(self, key):
        return self[key]

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __setitem__(self, key, value):
        raise AssignmentError()

    def __iter__(self):
        for key in self._registered_env_keys | set(self._config_map.keys()):
            yield key

    def items(self):
        for key in self:
            yield key, self[key]

    def __load_source(self, source, force_reload=False):
        if force_reload and source.parent_source:
            # TODO: double-check case of reload without force reload for chained configs
            return

        items = source.dump(force_reload)
        if source.provides and not set(source.provides).issubset(items):
            raise NotImplementedError()  # TODO: fix this

        additional_requirements = items.pop('additional_requirements', None)
        if isinstance(additional_requirements, string_types):
            additional_requirements = additional_requirements.split(',')
        self._required_keys |= set(listify(additional_requirements))

        additional_sources = items.pop('additional_sources', None)

        self._config_map.update(items)

        if additional_sources:
            for src in additional_sources:
                class_name, kwargs = src.popitem()
                additional_source = globals()[class_name](**kwargs)
                self.__append_source(additional_source, force_reload, source)

    def __load_environment_keys(self):
        app_prefix = self.appname.upper() + '_'
        for env_key in os.environ:
            if env_key.startswith(app_prefix):
                self._registered_env_keys.add(reverse_env_key(self.appname, env_key))
                # We don't actually add values to _config_map here. Rather, they're pulled
                # directly from os.environ at __getitem__ time. This allows for type casting
                # environment variables if possible.

    def __ensure_required_keys(self):
        available_keys = self._registered_env_keys | set(self._config_map.keys())
        missing_keys = self._required_keys - available_keys
        if missing_keys:
            raise EnvironmentError("Required key(s) not found in environment\n"
                                   "  or configuration sources.\n"
                                   "  Missing Keys: {}".format(list(missing_keys)))

    def _clear_memoization(self):
        self.__dict__.pop('_memoized_results', None)

    def __set_up_sighup_handler(self):
        # Reload config on SIGHUP (UNIX only)
        if hasattr(signal, 'SIGHUP'):
            def sighup_handler(signum, frame):
                if signum != signal.SIGHUP:
                    return
                self._reload(True)
                if callable(self.__previous_sighup_handler):
                    self.__previous_sighup_handler(signum, frame)
            self.__previous_sighup_handler = signal.getsignal(signal.SIGHUP)
            signal.signal(signal.SIGHUP, sighup_handler)



class Source(object):
    _items = None
    _provides = None
    _parent_source = None

    @property
    def provides(self):
        return self._provides

    @property
    def items(self):
        return self.dump()

    def load(self):
        """Must return a key, value dict"""
        raise NotImplementedError()  # pragma: no cover

    def dump(self, force_reload=False):
        if self._items is None or force_reload:
            self._items = self.load()
        return self._items

    @property
    def parent_config(self):
        return self._parent_config

    @parent_config.setter
    def parent_config(self, parent_config):
        self._parent_config = parent_config

    @property
    def parent_source(self):
        return self._parent_source

    @parent_source.setter
    def parent_source(self, parent_source):
        self._parent_source = parent_source


class YamlSource(Source):

    def __init__(self, location, provides=None):
        self._location = location
        self._provides = provides if provides else None

    def load(self):
        with PackageFile(self._location, self.parent_config.package) as fh:
            import yaml
            contents = yaml.load(fh)
            if self.provides is None:
                return contents
            else:
                return {key: contents[key] for key in self.provides}


class EnvironmentMappedSource(Source):
    """Load a full Source object given the value of an environment variable."""

    def __init__(self, envvar, sourcemap):
        self._envvar = envvar
        self._sourcemap = sourcemap

    def load(self):
        mapped_source = self._sourcemap[self.parent_config[self._envvar]]
        mapped_source.parent_config = self.parent_config
        params = mapped_source.load()
        return params


1			# -- coding: utf-8 --
2			"""A specialized map implementation to manage configuration and context information.
3
4			Features:
5			* Uses YAML configuration files
6			* Use environment variables to override configs
7			* Can pass a list of required parameters at initialization
8			* Works with encrypted files
9			* Accepts multiple config files
10			* Can query information from consul
11			* Does type coercion on strings
12			* Composable configs
13			* Ordered merge: downstream configs will override upstream
14			* Reload from sources on SIGHUP
15
16			Available source types:
17			* Environment variables
18			* Yaml file on file system
19			* Yaml file in S3
20			* Consul
21			* Hashicorp Vault
22
23			Notes:
24			* Keys are case-insensitive.
25
26			"""
27			from __future__ import absolute_import, division, print_function
28			import inspect
29			import logging
30			import os
31			import signal
32
33			from ._vendor.six import string_types
34			from .type_coercion import listify
35			from .decorators import memoize, memoizemethod
36			from .exceptions import AssignmentError, NotFoundError
37			from .path import PackageFile
38			from .type_coercion import typify
39
40
41			log = logging.getLogger(__name__)
42
43
44			@memoize
45			def make_env_key(app_name, key):
46			"""Creates an environment key-equivalent for the given key"""
47			key = key.replace('-', '_').replace(' ', '_')
48			return "_".join((x.upper() for x in (app_name, key)))
49
50
51			@memoize
52			def reverse_env_key(app_name, key):
53			app = app_name.upper() + '_'
54			assert key.startswith(app), "{} is not a(n) {} environment key".format(key, app)
55			return key[len(app):].lower()
56
57
58			class Configuration(object):
59			"""A map implementation to manage configuration and context information. Values
60			can be accessed (read, not assigned) as either a dict lookup (e.g. `config[key]`) or as an
61			attribute (e.g. `config.key`).
62
63			This class makes the foundational assumption of a yaml configuration file, as values in yaml
64			are typed.
65
66			This class allows overriding configuration keys with environment variables. Given an app name
67			`foo` and a config parameter `bar: 15`, setting a `FOO_BAR` environment variable to `22` will
68			override the value of `bar`. The type of `22` remains `int` because the underlying value of
69			`15` is used to infer the type of the `FOO_BAR` environment variable. When an underlying
70			parameter does not exist in a config file, the type is intelligently guessed.
71
72			Args:
73			app_name (str)
74			config_sources (str or list, optional)
75			required_parameters (iter, optional)
76
77			Raises:
78			InitializationError: on instantiation, when `required_parameters` are not found
79			warns: on instantiation, when a given `config_file` cannot be read
80			NotFoundError: when requesting a key that does not exist
81
82			Examples:
83			>>> for (key, value) in [('FOO_BAR', 22), ('FOO_BAZ', 'yes'), ('FOO_BANG', 'monkey')]:
84			... os.environ[key] = str(value)
85
86			>>> context = Configuration('foo')
87			>>> context.bar, context.bar.__class__
88			(22, <... 'int'>)
89			>>> context['baz'], type(context['baz'])
90			(True, <... 'bool'>)
91			>>> context.bang, type(context.bang)
92			('monkey', <... 'str'>)
93
94			"""
95
96			def __init__(self, appname, config_sources=None, required_parameters=None):
97			self.appname = appname
98			self.package = inspect.getmodule(self).__package__
99
100			# A private flag used to indicate if sources are being loaded for the first time or are
101			# being reloaded.
102			self.__initial_load = True
103
104			# The ordered list of sources from which to load key, value pairs.
105			self.__sources = list()
106
107			# The object that holds the actual key, value pairs after they're loaded from sources.
108			# Keys are stored as all lower-case. Access by key is provided in a case-insensitive way.
109			self._config_map = dict()
110
111			#
112			self._registered_env_keys = set()
113
114			self._required_keys = set(listify(required_parameters))
115
116			self.__set_up_sighup_handler()
117			self.__load_environment_keys()
118			self.append_sources(config_sources)
119
120			def append_sources(self, config_sources):
121			force_reload = True
122			for source in listify(config_sources):
123			self.__append_source(source, force_reload)
124
125			def append_required(self, required_parameters):
126			self._required_keys.update(listify(required_parameters))
127
128			def __append_source(self, source, force_reload=False, _parent_source=None):
129			source.parent_config = self
130			self.__load_source(source, force_reload)
131			source.parent_source = _parent_source
132			self.__sources.append(source)
133
134			def verify(self):
135			self.__ensure_required_keys()
136			return self
137
138			def set_env(self, key, value):
139			"""Sets environment variables by prepending the app_name to `key`. Also registers the
140			environment variable with the instance object preventing an otherwise-required call to
141			`reload()`.
142			"""
143			os.environ[make_env_key(self.appname, key)] = str(value) # must coerce to string
144			self._registered_env_keys.add(key)
145			self._clear_memoization()
146
147			def unset_env(self, key):
148			"""Removes an environment variable using the prepended app_name convention with `key`."""
149			os.environ.pop(make_env_key(self.appname, key), None)
150			self._registered_env_keys.discard(key)
151			self._clear_memoization()
152
153			def _reload(self, force=False):
154			"""Reloads the configuration from the file and environment variables. Useful if using
155			`os.environ` instead of this class' `set_env` method, or if the underlying configuration
156			file is changed externally.
157			"""
158			self._config_map = dict()
159			self._registered_env_keys = set()
160			self.__reload_sources(force)
161			self.__load_environment_keys()
162			self.verify()
163			self._clear_memoization()
164
165			@memoizemethod # memoized for performance; always use self.set_env() instead of os.setenv()
166			def __getitem__(self, key):
167			key = key.lower()
168			if key in self._registered_env_keys:
169			from_env = os.getenv(make_env_key(self.appname, key))
170			from_sources = self._config_map.get(key, None)
171			return typify(from_env, type(from_sources) if from_sources is not None else None)
172			else:
173			try:
174			return self._config_map[key]
175			except KeyError as e:
176			raise NotFoundError(e)
177
178			def __getattr__(self, key):
179			return self[key]
180
181			def get(self, key, default=None):
182			try:
183			return self[key]
184			except KeyError:
185			return default
186
187			def __setitem__(self, key, value):
188			raise AssignmentError()
189
190			def __iter__(self):
191			for key in self._registered_env_keys \| set(self._config_map.keys()):
192			yield key
193
194			def items(self):
195			for key in self:
196			yield key, self[key]
197
198			def __load_source(self, source, force_reload=False):
199			if force_reload and source.parent_source:
200			# TODO: double-check case of reload without force reload for chained configs
201			return
202
203			items = source.dump(force_reload)
204			if source.provides and not set(source.provides).issubset(items):
205			raise NotImplementedError() # TODO: fix this
206
207			additional_requirements = items.pop('additional_requirements', None)
208			if isinstance(additional_requirements, string_types):
209			additional_requirements = additional_requirements.split(',')
210			self._required_keys \|= set(listify(additional_requirements))
211
212			additional_sources = items.pop('additional_sources', None)
213
214			self._config_map.update(items)
215
216			if additional_sources:
217			for src in additional_sources:
218			class_name, kwargs = src.popitem()
219			additional_source = globals()[class_name](**kwargs)
220			self.__append_source(additional_source, force_reload, source)
221
222			def __load_environment_keys(self):
223			app_prefix = self.appname.upper() + '_'
224			for env_key in os.environ:
225			if env_key.startswith(app_prefix):
226			self._registered_env_keys.add(reverse_env_key(self.appname, env_key))
227			# We don't actually add values to _config_map here. Rather, they're pulled
228			# directly from os.environ at __getitem__ time. This allows for type casting
229			# environment variables if possible.
230
231			def __ensure_required_keys(self):
232			available_keys = self._registered_env_keys \| set(self._config_map.keys())
233			missing_keys = self._required_keys - available_keys
234			if missing_keys:
235			raise EnvironmentError("Required key(s) not found in environment\n"
236			" or configuration sources.\n"
237			" Missing Keys: {}".format(list(missing_keys)))
238
239			def _clear_memoization(self):
240			self.__dict__.pop('_memoized_results', None)
241
242			def __set_up_sighup_handler(self):
243			# Reload config on SIGHUP (UNIX only)
244			if hasattr(signal, 'SIGHUP'):
245			def sighup_handler(signum, frame):
246			if signum != signal.SIGHUP:
247			return
248			self._reload(True)
249			if callable(self.__previous_sighup_handler):
250			self.__previous_sighup_handler(signum, frame)
251			self.__previous_sighup_handler = signal.getsignal(signal.SIGHUP)
252			signal.signal(signal.SIGHUP, sighup_handler)
253
254
255
256			class Source(object):
257			_items = None
258			_provides = None
259			_parent_source = None
260
261			@property
262			def provides(self):
263			return self._provides
264
265			@property
266			def items(self):
267			return self.dump()
268
269			def load(self):
270			"""Must return a key, value dict"""
271			raise NotImplementedError() # pragma: no cover
272
273			def dump(self, force_reload=False):
274			if self._items is None or force_reload:
275			self._items = self.load()
276			return self._items
277
278			@property
279			def parent_config(self):
280			return self._parent_config
281
282			@parent_config.setter
283			def parent_config(self, parent_config):
284			self._parent_config = parent_config
285
286			@property
287			def parent_source(self):
288			return self._parent_source
289
290			@parent_source.setter
291			def parent_source(self, parent_source):
292			self._parent_source = parent_source
293
294
295			class YamlSource(Source):
296
297			def __init__(self, location, provides=None):
298			self._location = location
299			self._provides = provides if provides else None
300
301			def load(self):
302			with PackageFile(self._location, self.parent_config.package) as fh:
303			import yaml
304			contents = yaml.load(fh)
305			if self.provides is None:
306			return contents
307			else:
308			return {key: contents[key] for key in self.provides}
309
310
311			class EnvironmentMappedSource(Source):
312			"""Load a full Source object given the value of an environment variable."""
313
314			def __init__(self, envvar, sourcemap):
315			self._envvar = envvar
316			self._sourcemap = sourcemap
317
318			def load(self):
319			mapped_source = self._sourcemap[self.parent_config[self._envvar]]
320			mapped_source.parent_config = self.parent_config
321			params = mapped_source.load()
322			return params
323

kalefranz / auxlib

Push — develop ( 5780f7...1c0ef8 )

auxlib.Configuration.sighup_handler() A

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like