config_plugin_writers() - Code Metrics - Inspection of "added gauges, rates configuration values to proc r..." - s4z/plumd - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Branch — master (7e8cc2)

by Kenny

created 2016-08-18 13:55 UTC

config_plugin_writers() C

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	1	Features	0

Metric	Value
cc	7
c	1
b	1
f	0
dl	0
loc	28
rs	5.5

# -*- coding: utf-8 -*-
"""Utility classes to work with plugins - mainly plugin loading.

.. moduleauthor:: Kenny Freeman <[email protected]>

"""
__author__ = 'Kenny Freeman'
__email__ = '[email protected]'
__license__ = "ISCL"
__docformat__ = 'reStructuredText'

import sys
import imp
import inspect
import os.path
import traceback

import plumd
import plumd.util


def load_instance(log, cname, mod, cobj):
    """Returns an instance of the class cname from the specified module.

    :param log: A logger
    :type log: logging.RootLogger
    :param cname: The class name to load
    :type cname: str
    :param mod: A module object from eg. imp.load_source or __import__
    :type mod: module
    :param cobj: A conf configuration helper instance to pass to the class
    :type cobj: conf
    :rtype: Object -- an instance of the class requested
    :raises: PluginLoadError: many things can go wrong during loading
    """
    obj = None
    # find the class in the module
    for n, d in inspect.getmembers(mod, inspect.isclass):
        if n == cname:
            # try to instantiate the class
            try:
                obj = d(log, cobj)
            # plugins may raise pretty much any exception
            except Exception as e:
                tb = traceback.format_exc()
                estr = "class {0}, module {1} raised: {2}, trace:{3}"
                eargs = [cname, mod.__name__, e, tb]
                raise plumd.PluginLoadError(estr.format(*eargs))
            break
    # the module does not have the expected class in it
    if obj is None:
        estr = "{0} does not define class {1}".format(mod.__name__, cname)
        raise plumd.PluginLoadError(estr)
    return obj


def load_file_instance(log, cname, fname, cobj):
    """Imports the class cname from the python file fname and returns
    an instance of it.

    raises:
        PluginNotFoundError if the fname doesn't exist.

    :param log: A logger
    :type log: logging.RootLogger
    :param cname: The name of the class to load
    :type cname: str
    :param fname: The full path to the python file to load from
    :type fname: str
    :param cobj: A conf configuration helper instance to pass to the class
    :type cobj: conf
    :rtype: Object -- an instance of the class requested
    :raises: PluginNotFoundError
    """
    mod = None
    if os.path.isfile(fname):
        # load the module that has the same name as the file (minus .py)
        mod = imp.load_source(cname, fname)
    else:
        raise plumd.PluginNotFoundError("file {0} does not exist".format(fname))
    return load_instance(log, cname, mod, cobj)


def load_module_instance(log, cname, mname, cobj):
    """Imports the class cname from the python module mname and returns
    an instance of it.

    raises:
        PluginLoadError - so many things can go wrong loading plugins..

    :param log: A logger
    :type log: logging.RootLogger
    :param cname: The name of the class to load
    :type cname: str
    :param mname: The name of the module to load from
    :type fname: str
    :param cobj: A conf configuration helper instance to pass to the class
    :type cobj: conf
    :rtype: Object -- an instance of the class requested
    :raises: PluginLoadError
    """
    mod = None
    fname = None
    if "." in mname:
        fname = mname
    try:
        mod = __import__(mname, fromlist=fname)
    except Exception as e:
        msg = "module {0} import failed: {1}".format(mname, e)
        raise plumd.PluginLoadError(msg)
    return load_instance(log, cname, mod, cobj)



def load_plugin(log, pconf):
    """Loads the configured plugin and returns an instance of it.

    The configuration file for pconf must define at least:

    name: <name for this instance>          # a uniq name to give this instance
    module: <name of module to load from>   # the module name to load
    file: <full path to file to load from>  # or, the filename to load
    pclass: <name of class to load>         # name of the class to load

    plus any plugin specific configuration. You can instantiate multiple
    instances of a plugin with different configurations however must ensure
    that they are configured with uniq names.

    Also, if both file and module are defined in the configuration the file
    configuration is used and module is ignored.

    Raises:
        ConfigError if 'file' and 'module' sources are both configured.
        ConfigError if the configuration is missing 'name'.
        PluginLoadError if the configured file/module is not found.
        PluginLoadError if it was unable to instantiate the requested class.

    :param log: A logger
    :type log: logging.RootLogger
    :param pconf: a configuration object that defines the plugin to load
    :type pconf: plumd.config.conf
    :raises: PluginLoadError, ConfigError
    """
    fname = pconf.get('file')                   # full path to python file
    mname = pconf.get('module')                 # module name
    cname = pconf.get('class', exception=True)  # class name
    instance = None

    if fname:
        args = [ log, cname, fname, pconf ]
        instance = load_file_instance(*args)
    elif mname:
        try:
            __import__(mname)
        except ImportError as e:
            estr = "module not found: {0} : {1} : {2}"
            raise plumd.PluginLoadError(estr.format(mname, pconf.path, e))
        args = [ log, cname, mname, pconf ]
        instance = load_module_instance(*args)

    if instance is None:
        raise plumd.PluginLoadError("load failed for: {0}".format(pconf.path))

    return instance


def load_from_conf(log, config, pconf):
    """Load a plugin as defined by pconf configuration file and return an
    instance of it.

    raises:
        ConfigError if ptype has an incorrect configuration path set
        ConfigError if a plugin configuration is missing 'name'
        DuplicatePlugin if a plugin configuration has a duplicate 'name'
        PluginLoadError if there was an error loading the plugin

    :param log: A logger
    :type log: logging.RootLogger
    :param config: a :class:`plumd.config.Conf` instance
    :type config: plumd.config.Conf
    :param pconf: Full path to a plugin configuration.
    :type pconf: str
    :rtype: object
    :raises: ConfigError, DuplicatePlugin
    """
    # create a configuration and pass on select values from the main config
    defaults = {
        'poll.interval': config.get('poll.interval'),
        'delay.poll': config.get('delay.poll'),
        'meta': config.get('meta')
    }
    pconfig = plumd.config.Conf(pconf).defaults(defaults)
    # check to see if the plugin is disabled, if so return here
    if not pconfig.get('enabled', default=True):
        return

    pname = pconfig.get("name", exception=True)

    # load the plugin
    log.debug("loading {0} from {1}".format(pname, pconfig.path))
    return load_plugin(log, pconfig)


def get_plugins_dict(log, plugins, pclass):
    """Returns a dict of plugins that are of type pclass where the dict
    keys are the plugins name and the values are the plugin object.

    raises:
        DuplicatePlugin if a plugin has a duplicate 'name'

    :param log: A logger
    :type log: logging.RootLogger
    :param plugins: A list of plugin objects.
    :type plugins: list
    :param pclas: A class eg. :class:`plumd.Reader`, :class:`plumd.Writer`
    :type pclass: type
    :rtype: dict
    :raises: DuplicatePlugin
    """
    plugs = {}
    for pobj in plugins:
        # get the list of base/super classes for the plugin
        pclasses = inspect.getmro(pobj.__class__)

        # ensure the plugin is a subclass of the requested class
        if pclass in pclasses:
            pconf = pobj.config
            pname = pconf.get('name', exception=True)
            if pname in plugs:
                err = "duplicate plugin: {0} from {1}"
                raise plumd.DuplicatePlugin(err.format(pname, pconf.path))
            plugs[pname] = pobj
    return plugs


def load_all_plugins(log, config):
    """Returns a tuple of reader and writer plugin dicts.

    raises:
        ConfigError if ptype has an incorrect configuration path set
        ConfigError if a plugin configuration is missing 'name'
        DuplicatePlugin if a plugin configuration has a duplicate 'name'
        PluginLoadError if there was an error loading the plugin

    Returns a tuple of: (readers, writers) where each is a dict of:
    {'name': <plugin_instance>}. Name is the configured plugin name and
    <plugin_instance> is the corresponding instance of the plugin. Note plugins
    can be insantiated multiple times with different configurations.

    :param log: A logger
    :type log: logging.RootLogger
    :param config: a :class:`plumd.config.Conf` instance
    :type config: plumd.config.Conf
    :rtype: tuple
    :raises: ConfigError, DuplicatePlugin, PluginLoadError
    """
    plugins = []
    # get a list of configuration files
    # note the call to ls can raise an exception which we pass upwards
    for pconf in plumd.config.find(config.get('config.plugins'), 'yaml'):
        # load the plugin object
        pobj = load_from_conf(log, config, pconf)

        # check if the plugin has been disabled in configuration
        if pobj is None:
            msg = "skipping disabled plugin: {0}"
            log.info(msg.format(pconf))
            continue

        plugins.append(pobj)

    log.debug("plugins: {0}".format(" ".join([ p.__class__.__name__ for p in plugins])))

    # we iterate over the plugin list multiple times..
    # since the list is small and there are not many types this is ok.
    readers = get_plugins_dict(log, plugins, plumd.plugins.Reader)
    writers = get_plugins_dict(log, plugins, plumd.plugins.Writer)
    log.info("readers: {0}".format(" ".join(readers.keys())))
    log.info("writers: {0}".format(" ".join(writers.keys())))

    return (readers, writers)


def config_plugin_writers(lobj):
    """Update the list of writers for each reader in the PluginLoader.

    todo: change lobj to a tuple of (readers,writers)

    :param lobj: a :class:`plumd.plugins.load.PluginLoader` object
    :type lobj: plumd.plugins.load.PluginLoader

    :raises: ConfigError
    """
    # now set the PluginReaders writers
    for prname, probj in lobj.readers.items():
        wcfg = probj.pobj.config.get('writers')
        # if nothing is configured the reader writes to all
        if wcfg is None:
            probj.writers = lobj.writers.values()
            continue

        # get list of writer names
        wnames = [ w for w in wcfg if w in lobj.writers ]
        if wnames:
            probj.writers = [ lobj.writers[w] for w in wnames ]
            continue

        msg = "reader {0} from {1} has no valid writers configured"
        args = [ prname, probj.pobj.conf.path ]
        lobj.log.error(msg.format(*args))
        sys.exit(1)


1			# -- coding: utf-8 --
2			"""Utility classes to work with plugins - mainly plugin loading.
3
4			.. moduleauthor:: Kenny Freeman <[email protected]>
5
6			"""
7			__author__ = 'Kenny Freeman'
8			__email__ = '[email protected]'
9			__license__ = "ISCL"
10			__docformat__ = 'reStructuredText'
11
12			import sys
13			import imp
14			import inspect
15			import os.path
16			import traceback
17
18			import plumd
19			import plumd.util
20
21
22			def load_instance(log, cname, mod, cobj):
23			"""Returns an instance of the class cname from the specified module.
24
25			:param log: A logger
26			:type log: logging.RootLogger
27			:param cname: The class name to load
28			:type cname: str
29			:param mod: A module object from eg. imp.load_source or __import__
30			:type mod: module
31			:param cobj: A conf configuration helper instance to pass to the class
32			:type cobj: conf
33			:rtype: Object -- an instance of the class requested
34			:raises: PluginLoadError: many things can go wrong during loading
35			"""
36			obj = None
37			# find the class in the module
38			for n, d in inspect.getmembers(mod, inspect.isclass):
39			if n == cname:
40			# try to instantiate the class
41			try:
42			obj = d(log, cobj)
43			# plugins may raise pretty much any exception
44			except Exception as e:
45			tb = traceback.format_exc()
46			estr = "class {0}, module {1} raised: {2}, trace:{3}"
47			eargs = [cname, mod.__name__, e, tb]
48			raise plumd.PluginLoadError(estr.format(*eargs))
49			break
50			# the module does not have the expected class in it
51			if obj is None:
52			estr = "{0} does not define class {1}".format(mod.__name__, cname)
53			raise plumd.PluginLoadError(estr)
54			return obj
55
56
57			def load_file_instance(log, cname, fname, cobj):
58			"""Imports the class cname from the python file fname and returns
59			an instance of it.
60
61			raises:
62			PluginNotFoundError if the fname doesn't exist.
63
64			:param log: A logger
65			:type log: logging.RootLogger
66			:param cname: The name of the class to load
67			:type cname: str
68			:param fname: The full path to the python file to load from
69			:type fname: str
70			:param cobj: A conf configuration helper instance to pass to the class
71			:type cobj: conf
72			:rtype: Object -- an instance of the class requested
73			:raises: PluginNotFoundError
74			"""
75			mod = None
76			if os.path.isfile(fname):
77			# load the module that has the same name as the file (minus .py)
78			mod = imp.load_source(cname, fname)
79			else:
80			raise plumd.PluginNotFoundError("file {0} does not exist".format(fname))
81			return load_instance(log, cname, mod, cobj)
82
83
84			def load_module_instance(log, cname, mname, cobj):
85			"""Imports the class cname from the python module mname and returns
86			an instance of it.
87
88			raises:
89			PluginLoadError - so many things can go wrong loading plugins..
90
91			:param log: A logger
92			:type log: logging.RootLogger
93			:param cname: The name of the class to load
94			:type cname: str
95			:param mname: The name of the module to load from
96			:type fname: str
97			:param cobj: A conf configuration helper instance to pass to the class
98			:type cobj: conf
99			:rtype: Object -- an instance of the class requested
100			:raises: PluginLoadError
101			"""
102			mod = None
103			fname = None
104			if "." in mname:
105			fname = mname
106			try:
107			mod = __import__(mname, fromlist=fname)
108			except Exception as e:
109			msg = "module {0} import failed: {1}".format(mname, e)
110			raise plumd.PluginLoadError(msg)
111			return load_instance(log, cname, mod, cobj)
112
113
114
115			def load_plugin(log, pconf):
116			"""Loads the configured plugin and returns an instance of it.
117
118			The configuration file for pconf must define at least:
119
120			name: <name for this instance> # a uniq name to give this instance
121			module: <name of module to load from> # the module name to load
122			file: <full path to file to load from> # or, the filename to load
123			pclass: <name of class to load> # name of the class to load
124
125			plus any plugin specific configuration. You can instantiate multiple
126			instances of a plugin with different configurations however must ensure
127			that they are configured with uniq names.
128
129			Also, if both file and module are defined in the configuration the file
130			configuration is used and module is ignored.
131
132			Raises:
133			ConfigError if 'file' and 'module' sources are both configured.
134			ConfigError if the configuration is missing 'name'.
135			PluginLoadError if the configured file/module is not found.
136			PluginLoadError if it was unable to instantiate the requested class.
137
138			:param log: A logger
139			:type log: logging.RootLogger
140			:param pconf: a configuration object that defines the plugin to load
141			:type pconf: plumd.config.conf
142			:raises: PluginLoadError, ConfigError
143			"""
144			fname = pconf.get('file') # full path to python file
145			mname = pconf.get('module') # module name
146			cname = pconf.get('class', exception=True) # class name
147			instance = None
148
149			if fname:
150			args = [ log, cname, fname, pconf ]
151			instance = load_file_instance(*args)
152			elif mname:
153			try:
154			__import__(mname)
155			except ImportError as e:
156			estr = "module not found: {0} : {1} : {2}"
157			raise plumd.PluginLoadError(estr.format(mname, pconf.path, e))
158			args = [ log, cname, mname, pconf ]
159			instance = load_module_instance(*args)
160
161			if instance is None:
162			raise plumd.PluginLoadError("load failed for: {0}".format(pconf.path))
163
164			return instance
165
166
167			def load_from_conf(log, config, pconf):
168			"""Load a plugin as defined by pconf configuration file and return an
169			instance of it.
170
171			raises:
172			ConfigError if ptype has an incorrect configuration path set
173			ConfigError if a plugin configuration is missing 'name'
174			DuplicatePlugin if a plugin configuration has a duplicate 'name'
175			PluginLoadError if there was an error loading the plugin
176
177			:param log: A logger
178			:type log: logging.RootLogger
179			:param config: a :class:`plumd.config.Conf` instance
180			:type config: plumd.config.Conf
181			:param pconf: Full path to a plugin configuration.
182			:type pconf: str
183			:rtype: object
184			:raises: ConfigError, DuplicatePlugin
185			"""
186			# create a configuration and pass on select values from the main config
187			defaults = {
188			'poll.interval': config.get('poll.interval'),
189			'delay.poll': config.get('delay.poll'),
190			'meta': config.get('meta')
191			}
192			pconfig = plumd.config.Conf(pconf).defaults(defaults)
193			# check to see if the plugin is disabled, if so return here
194			if not pconfig.get('enabled', default=True):
195			return
196
197			pname = pconfig.get("name", exception=True)
198
199			# load the plugin
200			log.debug("loading {0} from {1}".format(pname, pconfig.path))
201			return load_plugin(log, pconfig)
202
203
204			def get_plugins_dict(log, plugins, pclass):
205			"""Returns a dict of plugins that are of type pclass where the dict
206			keys are the plugins name and the values are the plugin object.
207
208			raises:
209			DuplicatePlugin if a plugin has a duplicate 'name'
210
211			:param log: A logger
212			:type log: logging.RootLogger
213			:param plugins: A list of plugin objects.
214			:type plugins: list
215			:param pclas: A class eg. :class:`plumd.Reader`, :class:`plumd.Writer`
216			:type pclass: type
217			:rtype: dict
218			:raises: DuplicatePlugin
219			"""
220			plugs = {}
221			for pobj in plugins:
222			# get the list of base/super classes for the plugin
223			pclasses = inspect.getmro(pobj.__class__)
224
225			# ensure the plugin is a subclass of the requested class
226			if pclass in pclasses:
227			pconf = pobj.config
228			pname = pconf.get('name', exception=True)
229			if pname in plugs:
230			err = "duplicate plugin: {0} from {1}"
231			raise plumd.DuplicatePlugin(err.format(pname, pconf.path))
232			plugs[pname] = pobj
233			return plugs
234
235
236			def load_all_plugins(log, config):
237			"""Returns a tuple of reader and writer plugin dicts.
238
239			raises:
240			ConfigError if ptype has an incorrect configuration path set
241			ConfigError if a plugin configuration is missing 'name'
242			DuplicatePlugin if a plugin configuration has a duplicate 'name'
243			PluginLoadError if there was an error loading the plugin
244
245			Returns a tuple of: (readers, writers) where each is a dict of:
246			{'name': <plugin_instance>}. Name is the configured plugin name and
247			<plugin_instance> is the corresponding instance of the plugin. Note plugins
248			can be insantiated multiple times with different configurations.
249
250			:param log: A logger
251			:type log: logging.RootLogger
252			:param config: a :class:`plumd.config.Conf` instance
253			:type config: plumd.config.Conf
254			:rtype: tuple
255			:raises: ConfigError, DuplicatePlugin, PluginLoadError
256			"""
257			plugins = []
258			# get a list of configuration files
259			# note the call to ls can raise an exception which we pass upwards
260			for pconf in plumd.config.find(config.get('config.plugins'), 'yaml'):
261			# load the plugin object
262			pobj = load_from_conf(log, config, pconf)
263
264			# check if the plugin has been disabled in configuration
265			if pobj is None:
266			msg = "skipping disabled plugin: {0}"
267			log.info(msg.format(pconf))
268			continue
269
270			plugins.append(pobj)
271
272			log.debug("plugins: {0}".format(" ".join([ p.__class__.__name__ for p in plugins])))
273
274			# we iterate over the plugin list multiple times..
275			# since the list is small and there are not many types this is ok.
276			readers = get_plugins_dict(log, plugins, plumd.plugins.Reader)
277			writers = get_plugins_dict(log, plugins, plumd.plugins.Writer)
278			log.info("readers: {0}".format(" ".join(readers.keys())))
279			log.info("writers: {0}".format(" ".join(writers.keys())))
280
281			return (readers, writers)
282
283
284			def config_plugin_writers(lobj):
285			"""Update the list of writers for each reader in the PluginLoader.
286
287			todo: change lobj to a tuple of (readers,writers)
288
289			:param lobj: a :class:`plumd.plugins.load.PluginLoader` object
290			:type lobj: plumd.plugins.load.PluginLoader
291
292			:raises: ConfigError
293			"""
294			# now set the PluginReaders writers
295			for prname, probj in lobj.readers.items():
296			wcfg = probj.pobj.config.get('writers')
297			# if nothing is configured the reader writes to all
298			if wcfg is None:
299			probj.writers = lobj.writers.values()
300			continue
301
302			# get list of writer names
303			wnames = [ w for w in wcfg if w in lobj.writers ]
304			if wnames:
305			probj.writers = [ lobj.writers[w] for w in wnames ]
306			continue
307
308			msg = "reader {0} from {1} has no valid writers configured"
309			args = [ prname, probj.pobj.conf.path ]
310			lobj.log.error(msg.format(*args))
311			sys.exit(1)
312

s4z / plumd

Branch — master (7e8cc2)

config_plugin_writers() C

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like