config_plugin_writers() - Code Metrics - Inspection of "removed plumd.plugins.{reader,writer} proxy object..." - s4z/plumd - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 3a6f3a...76f5e2 )

by Kenny

created 2016-08-20 12:48 UTC

config_plugin_writers() D

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	1	Features	0

Metric	Value
cc	8
c	1
b	1
f	0
dl	0
loc	29
rs	4

# -*- coding: utf-8 -*-
"""Utility classes to work with plugins - mainly plugin loading.

.. moduleauthor:: Kenny Freeman <[email protected]>

"""
__author__ = 'Kenny Freeman'
__email__ = '[email protected]'
__license__ = "ISCL"
__docformat__ = 'reStructuredText'

import sys
import imp
import inspect
import os.path
import traceback

import plumd
import plumd.util


def load_instance(log, cname, mod, cobj):
    """Returns an instance of the class cname from the specified module.

    :param log: A logger
    :type log: logging.RootLogger
    :param cname: The class name to load
    :type cname: str
    :param mod: A module object from eg. imp.load_source or __import__
    :type mod: module
    :param cobj: A conf configuration helper instance to pass to the class
    :type cobj: conf
    :rtype: Object -- an instance of the class requested
    :raises: PluginLoadError: many things can go wrong during loading
    """
    obj = None
    # find the class in the module
    for n, d in inspect.getmembers(mod, inspect.isclass):
        if n == cname:
            # try to instantiate the class
            try:
                obj = d(log, cobj)
            # plugins may raise pretty much any exception
            except Exception as e:
                tb = traceback.format_exc()
                estr = "class {0}, module {1} raised: {2}, trace:{3}"
                eargs = [cname, mod.__name__, e, tb]
                raise plumd.PluginLoadError(estr.format(*eargs))
            break
    # the module does not have the expected class in it
    if obj is None:
        estr = "{0} does not define class {1}".format(mod.__name__, cname)
        raise plumd.PluginLoadError(estr)
    return obj


def load_file_instance(log, cname, fname, cobj):
    """Imports the class cname from the python file fname and returns
    an instance of it.

    raises:
        PluginNotFoundError if the fname doesn't exist.

    :param log: A logger
    :type log: logging.RootLogger
    :param cname: The name of the class to load
    :type cname: str
    :param fname: The full path to the python file to load from
    :type fname: str
    :param cobj: A conf configuration helper instance to pass to the class
    :type cobj: conf
    :rtype: Object -- an instance of the class requested
    :raises: PluginNotFoundError
    """
    mod = None
    if os.path.isfile(fname):
        # load the module that has the same name as the file (minus .py)
        mod = imp.load_source(cname, fname)
    else:
        raise plumd.PluginNotFoundError("file {0} does not exist".format(fname))
    return load_instance(log, cname, mod, cobj)


def load_module_instance(log, cname, mname, cobj):
    """Imports the class cname from the python module mname and returns
    an instance of it.

    raises:
        PluginLoadError - so many things can go wrong loading plugins..

    :param log: A logger
    :type log: logging.RootLogger
    :param cname: The name of the class to load
    :type cname: str
    :param mname: The name of the module to load from
    :type fname: str
    :param cobj: A conf configuration helper instance to pass to the class
    :type cobj: conf
    :rtype: Object -- an instance of the class requested
    :raises: PluginLoadError
    """
    mod = None
    fname = None
    if "." in mname:
        fname = mname
    try:
        mod = __import__(mname, fromlist=fname)
    except Exception as e:
        msg = "module {0} import failed: {1}".format(mname, e)
        raise plumd.PluginLoadError(msg)
    return load_instance(log, cname, mod, cobj)



def load_plugin(log, pconf):
    """Loads the configured plugin and returns an instance of it.

    The configuration file for pconf must define at least:

    name: <name for this instance>          # a uniq name to give this instance
    module: <name of module to load from>   # the module name to load
    file: <full path to file to load from>  # or, the filename to load
    pclass: <name of class to load>         # name of the class to load

    plus any plugin specific configuration. You can instantiate multiple
    instances of a plugin with different configurations however must ensure
    that they are configured with uniq names.

    Also, if both file and module are defined in the configuration the file
    configuration is used and module is ignored.

    Raises:
        ConfigError if 'file' and 'module' sources are both configured.
        ConfigError if the configuration is missing 'name'.
        PluginLoadError if the configured file/module is not found.
        PluginLoadError if it was unable to instantiate the requested class.

    :param log: A logger
    :type log: logging.RootLogger
    :param pconf: a configuration object that defines the plugin to load
    :type pconf: plumd.config.conf
    :raises: PluginLoadError, ConfigError
    """
    fname = pconf.get('file')                   # full path to python file
    mname = pconf.get('module')                 # module name
    cname = pconf.get('class', exception=True)  # class name
    instance = None

    if fname:
        instance = load_file_instance(log, cname, fname, pconf)
    elif mname:
        try:
            __import__(mname)
        except ImportError as e:
            estr = "module not found: {0} : {1} : {2}"
            raise plumd.PluginLoadError(estr.format(mname, pconf.path, e))
        instance = load_module_instance(log, cname, mname, pconf)

    if instance is None:
        raise plumd.PluginLoadError("load failed for: {0}".format(pconf.path))

    return instance


def load_from_conf(log, config, pconf):
    """Load a plugin as defined by pconf configuration file and return an
    instance of it.

    raises:
        ConfigError if ptype has an incorrect configuration path set
        ConfigError if a plugin configuration is missing 'name'
        DuplicatePlugin if a plugin configuration has a duplicate 'name'
        PluginLoadError if there was an error loading the plugin

    :param log: A logger
    :type log: logging.RootLogger
    :param config: a :class:`plumd.config.Conf` instance
    :type config: plumd.config.Conf
    :param pconf: Full path to a plugin configuration.
    :type pconf: str
    :rtype: object
    :raises: ConfigError, DuplicatePlugin
    """
    # create a configuration and pass on select values from the main config
    defaults = {
        'poll.interval': config.get('poll.interval'),
        'delay.poll': config.get('delay.poll'),
        'meta': config.get('meta')
    }
    pconfig = plumd.config.Conf(pconf).defaults(defaults)
    # check to see if the plugin is disabled, if so return here
    if not pconfig.get('enabled', default=True):
        return

    pname = pconfig.get("name", exception=True)

    # load the plugin
    log.debug("loading {0} from {1}".format(pname, pconfig.path))
    return load_plugin(log, pconfig)


def get_plugins_dict(log, plugins, pclass):
    """Returns a dict of plugins that are of type pclass where the dict
    keys are the plugins name and the values are the plugin object.

    raises:
        DuplicatePlugin if a plugin has a duplicate 'name'

    :param log: A logger
    :type log: logging.RootLogger
    :param plugins: A list of plugin objects.
    :type plugins: list
    :param pclas: A class eg. :class:`plumd.Reader`, :class:`plumd.Writer`
    :type pclass: type
    :rtype: dict
    :raises: DuplicatePlugin
    """
    plugs = {}
    for pobj in plugins:
        # get the list of base/super classes for the plugin
        pclasses = inspect.getmro(pobj.__class__)

        # ensure the plugin is a subclass of the requested class
        if pclass in pclasses:
            pconf = pobj.config
            pname = pconf.get('name', exception=True)
            if pname in plugs:
                err = "duplicate plugin: {0} from {1}"
                raise plumd.DuplicatePlugin(err.format(pname, pconf.path))
            plugs[pname] = pobj
    return plugs


def load_all_plugins(log, config):
    """Returns a tuple of reader and writer plugin dicts.

    raises:
        ConfigError if ptype has an incorrect configuration path set
        ConfigError if a plugin configuration is missing 'name'
        DuplicatePlugin if a plugin configuration has a duplicate 'name'
        PluginLoadError if there was an error loading the plugin

    Returns a tuple of: (readers, writers) where each is a dict of:
    {'name': <plugin_instance>}. Name is the configured plugin name and
    <plugin_instance> is the corresponding instance of the plugin. Note plugins
    can be insantiated multiple times with different configurations.

    :param log: A logger
    :type log: logging.RootLogger
    :param config: a :class:`plumd.config.Conf` instance
    :type config: plumd.config.Conf
    :rtype: tuple
    :raises: ConfigError, DuplicatePlugin, PluginLoadError
    """
    plugins = []
    # get a list of configuration files
    # note the call to ls can raise an exception which we pass upwards
    for pconf in plumd.config.find(config.get('config.plugins'), 'yaml'):
        # load the plugin object
        pobj = load_from_conf(log, config, pconf)

        # check if the plugin has been disabled in configuration
        if pobj is None:
            msg = "skipping disabled plugin: {0}"
            log.info(msg.format(pconf))
            continue

        plugins.append(pobj)

    log.debug("plugins: {0}".format(" ".join([ p.__class__.__name__ for p in plugins])))

    # we iterate over the plugin list multiple times..
    # since the list is small and there are not many types this is ok.
    readers = get_plugins_dict(log, plugins, plumd.plugins.Reader)
    writers = get_plugins_dict(log, plugins, plumd.plugins.Writer)
    log.info("readers: {0}".format(" ".join(readers.keys())))
    log.info("writers: {0}".format(" ".join(writers.keys())))

    return (readers, writers)


def config_plugin_writers(lobj):
    """Update the list of writers for each reader in the PluginLoader.

    todo: change lobj to a tuple of (readers,writers)

    :param lobj: a :class:`plumd.plugins.load.PluginLoader` object
    :type lobj: plumd.plugins.load.PluginLoader

    :raises: ConfigError
    """
    allw = [(wobj.queue_evt, wobj.queue) for wobj in lobj.writers.values()]
    # now set the PluginReaders writers
    for prname, probj in lobj.readers.items():
        wcfg = probj.config.get('writers')
        # if nothing is configured the reader writes to all
        if wcfg is None:
            probj.queues = allw
            continue

        # get list of writer names
        wnames = [ w for w in wcfg if w in lobj.writers ]
        if wnames:
            probj.queues = [ (lobj.writers[w].queue_evt,lobj.writers[w].queue)\
                              for w in wnames ]
            continue

        msg = "reader {0} from {1} has no valid writers configured"
        lobj.log.error(msg.format(prname, probj.conf.path))
        sys.exit(1)


1			# -- coding: utf-8 --
2			"""Utility classes to work with plugins - mainly plugin loading.
3
4			.. moduleauthor:: Kenny Freeman <[email protected]>
5
6			"""
7			__author__ = 'Kenny Freeman'
8			__email__ = '[email protected]'
9			__license__ = "ISCL"
10			__docformat__ = 'reStructuredText'
11
12			import sys
13			import imp
14			import inspect
15			import os.path
16			import traceback
17
18			import plumd
19			import plumd.util
20
21
22			def load_instance(log, cname, mod, cobj):
23			"""Returns an instance of the class cname from the specified module.
24
25			:param log: A logger
26			:type log: logging.RootLogger
27			:param cname: The class name to load
28			:type cname: str
29			:param mod: A module object from eg. imp.load_source or __import__
30			:type mod: module
31			:param cobj: A conf configuration helper instance to pass to the class
32			:type cobj: conf
33			:rtype: Object -- an instance of the class requested
34			:raises: PluginLoadError: many things can go wrong during loading
35			"""
36			obj = None
37			# find the class in the module
38			for n, d in inspect.getmembers(mod, inspect.isclass):
39			if n == cname:
40			# try to instantiate the class
41			try:
42			obj = d(log, cobj)
43			# plugins may raise pretty much any exception
44			except Exception as e:
45			tb = traceback.format_exc()
46			estr = "class {0}, module {1} raised: {2}, trace:{3}"
47			eargs = [cname, mod.__name__, e, tb]
48			raise plumd.PluginLoadError(estr.format(*eargs))
49			break
50			# the module does not have the expected class in it
51			if obj is None:
52			estr = "{0} does not define class {1}".format(mod.__name__, cname)
53			raise plumd.PluginLoadError(estr)
54			return obj
55
56
57			def load_file_instance(log, cname, fname, cobj):
58			"""Imports the class cname from the python file fname and returns
59			an instance of it.
60
61			raises:
62			PluginNotFoundError if the fname doesn't exist.
63
64			:param log: A logger
65			:type log: logging.RootLogger
66			:param cname: The name of the class to load
67			:type cname: str
68			:param fname: The full path to the python file to load from
69			:type fname: str
70			:param cobj: A conf configuration helper instance to pass to the class
71			:type cobj: conf
72			:rtype: Object -- an instance of the class requested
73			:raises: PluginNotFoundError
74			"""
75			mod = None
76			if os.path.isfile(fname):
77			# load the module that has the same name as the file (minus .py)
78			mod = imp.load_source(cname, fname)
79			else:
80			raise plumd.PluginNotFoundError("file {0} does not exist".format(fname))
81			return load_instance(log, cname, mod, cobj)
82
83
84			def load_module_instance(log, cname, mname, cobj):
85			"""Imports the class cname from the python module mname and returns
86			an instance of it.
87
88			raises:
89			PluginLoadError - so many things can go wrong loading plugins..
90
91			:param log: A logger
92			:type log: logging.RootLogger
93			:param cname: The name of the class to load
94			:type cname: str
95			:param mname: The name of the module to load from
96			:type fname: str
97			:param cobj: A conf configuration helper instance to pass to the class
98			:type cobj: conf
99			:rtype: Object -- an instance of the class requested
100			:raises: PluginLoadError
101			"""
102			mod = None
103			fname = None
104			if "." in mname:
105			fname = mname
106			try:
107			mod = __import__(mname, fromlist=fname)
108			except Exception as e:
109			msg = "module {0} import failed: {1}".format(mname, e)
110			raise plumd.PluginLoadError(msg)
111			return load_instance(log, cname, mod, cobj)
112
113
114
115			def load_plugin(log, pconf):
116			"""Loads the configured plugin and returns an instance of it.
117
118			The configuration file for pconf must define at least:
119
120			name: <name for this instance> # a uniq name to give this instance
121			module: <name of module to load from> # the module name to load
122			file: <full path to file to load from> # or, the filename to load
123			pclass: <name of class to load> # name of the class to load
124
125			plus any plugin specific configuration. You can instantiate multiple
126			instances of a plugin with different configurations however must ensure
127			that they are configured with uniq names.
128
129			Also, if both file and module are defined in the configuration the file
130			configuration is used and module is ignored.
131
132			Raises:
133			ConfigError if 'file' and 'module' sources are both configured.
134			ConfigError if the configuration is missing 'name'.
135			PluginLoadError if the configured file/module is not found.
136			PluginLoadError if it was unable to instantiate the requested class.
137
138			:param log: A logger
139			:type log: logging.RootLogger
140			:param pconf: a configuration object that defines the plugin to load
141			:type pconf: plumd.config.conf
142			:raises: PluginLoadError, ConfigError
143			"""
144			fname = pconf.get('file') # full path to python file
145			mname = pconf.get('module') # module name
146			cname = pconf.get('class', exception=True) # class name
147			instance = None
148
149			if fname:
150			instance = load_file_instance(log, cname, fname, pconf)
151			elif mname:
152			try:
153			__import__(mname)
154			except ImportError as e:
155			estr = "module not found: {0} : {1} : {2}"
156			raise plumd.PluginLoadError(estr.format(mname, pconf.path, e))
157			instance = load_module_instance(log, cname, mname, pconf)
158
159			if instance is None:
160			raise plumd.PluginLoadError("load failed for: {0}".format(pconf.path))
161
162			return instance
163
164
165			def load_from_conf(log, config, pconf):
166			"""Load a plugin as defined by pconf configuration file and return an
167			instance of it.
168
169			raises:
170			ConfigError if ptype has an incorrect configuration path set
171			ConfigError if a plugin configuration is missing 'name'
172			DuplicatePlugin if a plugin configuration has a duplicate 'name'
173			PluginLoadError if there was an error loading the plugin
174
175			:param log: A logger
176			:type log: logging.RootLogger
177			:param config: a :class:`plumd.config.Conf` instance
178			:type config: plumd.config.Conf
179			:param pconf: Full path to a plugin configuration.
180			:type pconf: str
181			:rtype: object
182			:raises: ConfigError, DuplicatePlugin
183			"""
184			# create a configuration and pass on select values from the main config
185			defaults = {
186			'poll.interval': config.get('poll.interval'),
187			'delay.poll': config.get('delay.poll'),
188			'meta': config.get('meta')
189			}
190			pconfig = plumd.config.Conf(pconf).defaults(defaults)
191			# check to see if the plugin is disabled, if so return here
192			if not pconfig.get('enabled', default=True):
193			return
194
195			pname = pconfig.get("name", exception=True)
196
197			# load the plugin
198			log.debug("loading {0} from {1}".format(pname, pconfig.path))
199			return load_plugin(log, pconfig)
200
201
202			def get_plugins_dict(log, plugins, pclass):
203			"""Returns a dict of plugins that are of type pclass where the dict
204			keys are the plugins name and the values are the plugin object.
205
206			raises:
207			DuplicatePlugin if a plugin has a duplicate 'name'
208
209			:param log: A logger
210			:type log: logging.RootLogger
211			:param plugins: A list of plugin objects.
212			:type plugins: list
213			:param pclas: A class eg. :class:`plumd.Reader`, :class:`plumd.Writer`
214			:type pclass: type
215			:rtype: dict
216			:raises: DuplicatePlugin
217			"""
218			plugs = {}
219			for pobj in plugins:
220			# get the list of base/super classes for the plugin
221			pclasses = inspect.getmro(pobj.__class__)
222
223			# ensure the plugin is a subclass of the requested class
224			if pclass in pclasses:
225			pconf = pobj.config
226			pname = pconf.get('name', exception=True)
227			if pname in plugs:
228			err = "duplicate plugin: {0} from {1}"
229			raise plumd.DuplicatePlugin(err.format(pname, pconf.path))
230			plugs[pname] = pobj
231			return plugs
232
233
234			def load_all_plugins(log, config):
235			"""Returns a tuple of reader and writer plugin dicts.
236
237			raises:
238			ConfigError if ptype has an incorrect configuration path set
239			ConfigError if a plugin configuration is missing 'name'
240			DuplicatePlugin if a plugin configuration has a duplicate 'name'
241			PluginLoadError if there was an error loading the plugin
242
243			Returns a tuple of: (readers, writers) where each is a dict of:
244			{'name': <plugin_instance>}. Name is the configured plugin name and
245			<plugin_instance> is the corresponding instance of the plugin. Note plugins
246			can be insantiated multiple times with different configurations.
247
248			:param log: A logger
249			:type log: logging.RootLogger
250			:param config: a :class:`plumd.config.Conf` instance
251			:type config: plumd.config.Conf
252			:rtype: tuple
253			:raises: ConfigError, DuplicatePlugin, PluginLoadError
254			"""
255			plugins = []
256			# get a list of configuration files
257			# note the call to ls can raise an exception which we pass upwards
258			for pconf in plumd.config.find(config.get('config.plugins'), 'yaml'):
259			# load the plugin object
260			pobj = load_from_conf(log, config, pconf)
261
262			# check if the plugin has been disabled in configuration
263			if pobj is None:
264			msg = "skipping disabled plugin: {0}"
265			log.info(msg.format(pconf))
266			continue
267
268			plugins.append(pobj)
269
270			log.debug("plugins: {0}".format(" ".join([ p.__class__.__name__ for p in plugins])))
271
272			# we iterate over the plugin list multiple times..
273			# since the list is small and there are not many types this is ok.
274			readers = get_plugins_dict(log, plugins, plumd.plugins.Reader)
275			writers = get_plugins_dict(log, plugins, plumd.plugins.Writer)
276			log.info("readers: {0}".format(" ".join(readers.keys())))
277			log.info("writers: {0}".format(" ".join(writers.keys())))
278
279			return (readers, writers)
280
281
282			def config_plugin_writers(lobj):
283			"""Update the list of writers for each reader in the PluginLoader.
284
285			todo: change lobj to a tuple of (readers,writers)
286
287			:param lobj: a :class:`plumd.plugins.load.PluginLoader` object
288			:type lobj: plumd.plugins.load.PluginLoader
289
290			:raises: ConfigError
291			"""
292			allw = [(wobj.queue_evt, wobj.queue) for wobj in lobj.writers.values()]
293			# now set the PluginReaders writers
294			for prname, probj in lobj.readers.items():
295			wcfg = probj.config.get('writers')
296			# if nothing is configured the reader writes to all
297			if wcfg is None:
298			probj.queues = allw
299			continue
300
301			# get list of writer names
302			wnames = [ w for w in wcfg if w in lobj.writers ]
303			if wnames:
304			probj.queues = [ (lobj.writers[w].queue_evt,lobj.writers[w].queue)\
305			for w in wnames ]
306			continue
307
308			msg = "reader {0} from {1} has no valid writers configured"
309			lobj.log.error(msg.format(prname, probj.conf.path))
310			sys.exit(1)
311

s4z / plumd

Push — master ( 3a6f3a...76f5e2 )

config_plugin_writers() D

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like