plumd.plugins.config_plugin_writers() - Code Metrics - Inspection of "misc code quality fixes (more to do yet), rework m..." - s4z/plumd - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( d058c9...ca4b9c )

by Kenny

created 2016-02-20 16:12 UTC

plumd.plugins.config_plugin_writers() C

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
cc	7
dl	0
loc	26
rs	5.5

# -*- coding: utf-8 -*-
"""Utility classes to work with plugins - mainly plugin loading.

.. moduleauthor:: Kenny Freeman <[email protected]>

"""
__author__ = 'Kenny Freeman'
__email__ = '[email protected]'
__license__ = "ISCL"
__docformat__ = 'reStructuredText'

from abc import ABCMeta, abstractmethod
import imp
import inspect
import os.path
import traceback

import structlog    # pip install structlog

import plumd
import plumd.util


def load_instance(log, cname, mod, cobj):
    """Returns an instance of the class cname from the specified module.

    :param cname: The class name to load
    :type cname: str
    :param mod: A module object from eg. imp.load_source or __import__
    :type mod: module
    :param cobj: A conf configuration helper instance to pass to the class
    :type cobj: conf
    :rtype: Object -- an instance of the class requested
    :raises: PluginLoadError: many things can go wrong during loading
    """
    obj = None
    # find the class in the module
    for n, d in inspect.getmembers(mod, inspect.isclass):
        if n == cname:
            # try to instantiate the class
            try:
                obj = d(log, cobj)
            # plugins may raise pretty much any exception
            except Exception as e:
                tb = traceback.format_exc()
                estr = "class {0}, module {1} raised: {2}, trace:{3}"
                eargs = [cname, mod.__name__, e, tb]
                raise plumd.PluginLoadError(estr.format(*eargs))
            break
    # the module does not have the expected class in it
    if obj is None:
        estr = "{0} does not define class {1}".format(mod.__name__, cname)
        raise plumd.PluginLoadError(estr)
    return obj


def load_file_instance(log, cname, fname, cobj):
    """Imports the class cname from the python file fname and returns
    an instance of it.

    raises:
        PluginNotFoundError if the fname doesn't exist.

    :param cname: The name of the class to load
    :type cname: str
    :param fname: The full path to the python file to load from
    :type fname: str
    :param cobj: A conf configuration helper instance to pass to the class
    :type cobj: conf
    :rtype: Object -- an instance of the class requested
    :raises: PluginNotFoundError
    """
    mod = None
    if os.path.isfile(fname):
        # load the module that has the same name as the file (minus .py)
        mod = imp.load_source(cname, fname)
    else:
        raise plumd.PluginNotFoundError("file {0} does not exist".format(fname))
    return load_instance(log, cname, mod, cobj)


def load_module_instance(log, cname, mname, cobj):
    """Imports the class cname from the python module mname and returns
    an instance of it.

    raises:
        PluginLoadError - so many things can go wrong loading plugins..

    :param cname: The name of the class to load
    :type cname: str
    :param mname: The name of the module to load from
    :type fname: str
    :param cobj: A conf configuration helper instance to pass to the class
    :type cobj: conf
    :rtype: Object -- an instance of the class requested
    :raises: PluginLoadError
    """
    mod = None
    fname = None
    if "." in mname:
        fname = mname
    try:
        mod = __import__(mname, fromlist=fname)
    except Exception as e:
        msg = "module {0} import failed: {1}".format(mname, e)
        raise plumd.PluginLoadError(msg)
    return load_instance(log, cname, mod, cobj)



def load_plugin(log, pconf):
    """Loads the configured plugin and returns an instance of it.

    The configuration file for pconf must define at least:

    name: <name for this instance>          # a uniq name to give this instance
    module: <name of module to load from>   # the module name to load
    file: <full path to file to load from>  # or, the filename to load
    pclass: <name of class to load>         # name of the class to load

    plus any plugin specific configuration. You can instantiate multiple
    instances of a plugin with different configurations however must ensure
    that they are configured with uniq names.

    Also, if both file and module are defined in the configuration the file
    configuration is used and module is ignored.

    Raises:
        ConfigError if 'file' and 'module' sources are both configured.
        ConfigError if the configuration is missing 'name'.
        PluginLoadError if the configured file/module is not found.
        PluginLoadError if it was unable to instantiate the requested class.

    :param pconf: a configuration object that defines the plugin to load
    :type pconf: plumd.config.conf
    :raises: PluginLoadError, ConfigError
    """
    fname = pconf.get('file')                   # full path to python file
    mname = pconf.get('module')                 # module name
    pname = pconf.get('name', exception=True)   # plugin identifier
    cname = pconf.get('class', exception=True)  # class name
    instance = None

    if fname:
        args = [ log.bind(src=pconf.get('name')), cname, fname, pconf ]
        instance = load_file_instance(*args)
    elif mname:
        try:
            __import__(mname)
        except ImportError as e:
            estr = "module not found: {0} : {1} : {2}"
            raise plumd.PluginLoadError(estr.format(mname, pconf.path, e))
        args = [log.bind(src=pconf.get('name')), cname, mname, pconf]
        instance = load_module_instance(*args)

    if instance is None:
        raise plumd.PluginLoadError("load failed for: {0}".format(pconf.path))

    return instance


def load_from_conf(log, config, pconf):
    """Load a plugin as defined by pconf configuration file and return an
    instance of it.

    raises:
        ConfigError if ptype has an incorrect configuration path set
        ConfigError if a plugin configuration is missing 'name'
        DuplicatePlugin if a plugin configuration has a duplicate 'name'
        PluginLoadError if there was an error loading the plugin

    :param log: A structlog logger instance
    :type log: structlog
    :param config: a :class:`plumd.config.Conf` instance
    :type config: plumd.config.Conf
    :param pconf: Full path to a plugin configuration.
    :type pconf: str
    :rtype: object
    :raises: ConfigError, DuplicatePlugin
    """
    # create a configuration and pass on select values from the main config
    defaults = {
        'poll.interval': config.get('poll.interval'),
        'delay.poll': config.get('delay.poll'),
        'meta': config.get('meta')
    }
    pconfig = plumd.config.Conf(pconf).defaults(defaults)
    pname = pconfig.get("name", exception=True)

    # check to see if the plugin is disabled, if so return here
    if not pconfig.get('enabled', default=True):
        return

    # create a logger for the plugin and bind the src to the plugin name
    plog = structlog.get_logger().bind(src=pname)

    # load the plugin
    log.info("loading", pname=pname, file=pconfig.path)
    return load_plugin(plog, pconfig)


def get_plugins_dict(log, plugins, pclass):
    """Returns a dict of plugins that are of type pclass where the dict
    keys are the plugins name and the values are the plugin object.

    raises:
        DuplicatePlugin if a plugin has a duplicate 'name'

    :param plugins: A list of plugin objects.
    :type plugins: list
    :param pclas: A class eg. :class:`plumd.Reader`, :class:`plumd.Writer`
    :type pclass: object
    :rtype: dict
    :raises: DuplicatePlugin
    """
    plugs = {}
    for pobj in plugins:
        cpclass = pobj.__class__.__base__
        if pobj.__class__.__base__ == pclass:
            log.debug('get_plugins_dict', matched=pobj, pclass=cpclass)
            pconf = pobj.config
            pname = pconf.get('name', exception=True)
            if pname in plugs:
                err = "duplicate plugin name: {0} from {1}"
                raise DuplicatePlugin(err.format(pname, pconf.path))
            plugs[pname] = pobj
    log.debug('get_plugins_dict', plugins=plugs, pclass=pclass)
    return plugs


def load_all_plugins(log, config):
    """Returns a tuple of reader and writer plugin dicts.

    raises:
        ConfigError if ptype has an incorrect configuration path set
        ConfigError if a plugin configuration is missing 'name'
        DuplicatePlugin if a plugin configuration has a duplicate 'name'
        PluginLoadError if there was an error loading the plugin

    Returns a tuple of: (readers, writers) where each is a dict of:
    {'name': <plugin_instance>}. Name is the configured plugin name and
    <plugin_instance> is the corresponding instance of the plugin. Note plugins
    can be insantiated multiple times with different configurations.

    :param log: A structlog logger instance
    :type log: structlog
    :param config: a :class:`plumd.config.Conf` instance
    :type config: plumd.config.Conf
    :rtype: tuple
    :raises: ConfigError, DuplicatePlugin, PluginLoadError
    """
    plugins = []
    # get a list of configuration files
    # note the call to ls can raise an exception which we pass upwards
    for pconf in plumd.config.ls(config.get('config.plugins'), 'yaml'):
        # load the plugin object
        pobj = load_from_conf(log, config, pconf)

        # check if the plugin has been disabled in configuration
        if pobj is None:
            log.info('load_all_plugins', skipping=pconf, reason="disabled")
            continue

        plugins.append(pobj)

    log.debug('load_all_plugins', plugins=plugins)

    # we iterate over the plugin list multiple times..
    # since the list is small and there are not many types this is ok.
    readers = get_plugins_dict(log, plugins, plumd.plugins.Reader)
    writers = get_plugins_dict(log, plugins, plumd.plugins.Writer)
    log.debug('load_all_plugins', readers=readers)
    log.debug('load_all_plugins', writers=writers)

    return (readers, writers)


def config_plugin_writers(lobj):
    """Update the list of writers for each reader in the PluginLoader.

    :param lobj: a :class:`plumd.plugins.load.PluginLoader` object
    :type lobj: plumd.plugins.load.PluginLoader

    :raises: ConfigError
    """
    # now set the PluginReaders writers
    for prname, probj in lobj.readers.items():
        wcfg = probj.pobj.config.get('writers')
        # if nothing is configured the reader writes to all
        if wcfg is None:
            probj.writers = lobj.writers.values()
            continue

        # get list of writer names
        wnames = [ w for w in wcfg if w in lobj.writers ]
        if len(wnames) > 0:
            probj.writers = [ lobj.writers[w] for w in wnames ]
            continue

        msg = 'reader {0} from {1} has no valid writers configured'
        args = [ prname, probj.pobj.conf.path ]
        lobj.log.error('invalid reader', msg=msg.format(*args))
        sys.exit(1)


1			# -- coding: utf-8 --
2			"""Utility classes to work with plugins - mainly plugin loading.
3
4			.. moduleauthor:: Kenny Freeman <[email protected]>
5
6			"""
7			__author__ = 'Kenny Freeman'
8			__email__ = '[email protected]'
9			__license__ = "ISCL"
10			__docformat__ = 'reStructuredText'
11
12			from abc import ABCMeta, abstractmethod
13			import imp
14			import inspect
15			import os.path
16			import traceback
17
18			import structlog # pip install structlog
19
20			import plumd
21			import plumd.util
22
23
24			def load_instance(log, cname, mod, cobj):
25			"""Returns an instance of the class cname from the specified module.
26
27			:param cname: The class name to load
28			:type cname: str
29			:param mod: A module object from eg. imp.load_source or __import__
30			:type mod: module
31			:param cobj: A conf configuration helper instance to pass to the class
32			:type cobj: conf
33			:rtype: Object -- an instance of the class requested
34			:raises: PluginLoadError: many things can go wrong during loading
35			"""
36			obj = None
37			# find the class in the module
38			for n, d in inspect.getmembers(mod, inspect.isclass):
39			if n == cname:
40			# try to instantiate the class
41			try:
42			obj = d(log, cobj)
43			# plugins may raise pretty much any exception
44			except Exception as e:
45			tb = traceback.format_exc()
46			estr = "class {0}, module {1} raised: {2}, trace:{3}"
47			eargs = [cname, mod.__name__, e, tb]
48			raise plumd.PluginLoadError(estr.format(*eargs))
49			break
50			# the module does not have the expected class in it
51			if obj is None:
52			estr = "{0} does not define class {1}".format(mod.__name__, cname)
53			raise plumd.PluginLoadError(estr)
54			return obj
55
56
57			def load_file_instance(log, cname, fname, cobj):
58			"""Imports the class cname from the python file fname and returns
59			an instance of it.
60
61			raises:
62			PluginNotFoundError if the fname doesn't exist.
63
64			:param cname: The name of the class to load
65			:type cname: str
66			:param fname: The full path to the python file to load from
67			:type fname: str
68			:param cobj: A conf configuration helper instance to pass to the class
69			:type cobj: conf
70			:rtype: Object -- an instance of the class requested
71			:raises: PluginNotFoundError
72			"""
73			mod = None
74			if os.path.isfile(fname):
75			# load the module that has the same name as the file (minus .py)
76			mod = imp.load_source(cname, fname)
77			else:
78			raise plumd.PluginNotFoundError("file {0} does not exist".format(fname))
79			return load_instance(log, cname, mod, cobj)
80
81
82			def load_module_instance(log, cname, mname, cobj):
83			"""Imports the class cname from the python module mname and returns
84			an instance of it.
85
86			raises:
87			PluginLoadError - so many things can go wrong loading plugins..
88
89			:param cname: The name of the class to load
90			:type cname: str
91			:param mname: The name of the module to load from
92			:type fname: str
93			:param cobj: A conf configuration helper instance to pass to the class
94			:type cobj: conf
95			:rtype: Object -- an instance of the class requested
96			:raises: PluginLoadError
97			"""
98			mod = None
99			fname = None
100			if "." in mname:
101			fname = mname
102			try:
103			mod = __import__(mname, fromlist=fname)
104			except Exception as e:
105			msg = "module {0} import failed: {1}".format(mname, e)
106			raise plumd.PluginLoadError(msg)
107			return load_instance(log, cname, mod, cobj)
108
109
110
111			def load_plugin(log, pconf):
112			"""Loads the configured plugin and returns an instance of it.
113
114			The configuration file for pconf must define at least:
115
116			name: <name for this instance> # a uniq name to give this instance
117			module: <name of module to load from> # the module name to load
118			file: <full path to file to load from> # or, the filename to load
119			pclass: <name of class to load> # name of the class to load
120
121			plus any plugin specific configuration. You can instantiate multiple
122			instances of a plugin with different configurations however must ensure
123			that they are configured with uniq names.
124
125			Also, if both file and module are defined in the configuration the file
126			configuration is used and module is ignored.
127
128			Raises:
129			ConfigError if 'file' and 'module' sources are both configured.
130			ConfigError if the configuration is missing 'name'.
131			PluginLoadError if the configured file/module is not found.
132			PluginLoadError if it was unable to instantiate the requested class.
133
134			:param pconf: a configuration object that defines the plugin to load
135			:type pconf: plumd.config.conf
136			:raises: PluginLoadError, ConfigError
137			"""
138			fname = pconf.get('file') # full path to python file
139			mname = pconf.get('module') # module name
140			pname = pconf.get('name', exception=True) # plugin identifier
141			cname = pconf.get('class', exception=True) # class name
142			instance = None
143
144			if fname:
145			args = [ log.bind(src=pconf.get('name')), cname, fname, pconf ]
146			instance = load_file_instance(*args)
147			elif mname:
148			try:
149			__import__(mname)
150			except ImportError as e:
151			estr = "module not found: {0} : {1} : {2}"
152			raise plumd.PluginLoadError(estr.format(mname, pconf.path, e))
153			args = [log.bind(src=pconf.get('name')), cname, mname, pconf]
154			instance = load_module_instance(*args)
155
156			if instance is None:
157			raise plumd.PluginLoadError("load failed for: {0}".format(pconf.path))
158
159			return instance
160
161
162			def load_from_conf(log, config, pconf):
163			"""Load a plugin as defined by pconf configuration file and return an
164			instance of it.
165
166			raises:
167			ConfigError if ptype has an incorrect configuration path set
168			ConfigError if a plugin configuration is missing 'name'
169			DuplicatePlugin if a plugin configuration has a duplicate 'name'
170			PluginLoadError if there was an error loading the plugin
171
172			:param log: A structlog logger instance
173			:type log: structlog
174			:param config: a :class:`plumd.config.Conf` instance
175			:type config: plumd.config.Conf
176			:param pconf: Full path to a plugin configuration.
177			:type pconf: str
178			:rtype: object
179			:raises: ConfigError, DuplicatePlugin
180			"""
181			# create a configuration and pass on select values from the main config
182			defaults = {
183			'poll.interval': config.get('poll.interval'),
184			'delay.poll': config.get('delay.poll'),
185			'meta': config.get('meta')
186			}
187			pconfig = plumd.config.Conf(pconf).defaults(defaults)
188			pname = pconfig.get("name", exception=True)
189
190			# check to see if the plugin is disabled, if so return here
191			if not pconfig.get('enabled', default=True):
192			return
193
194			# create a logger for the plugin and bind the src to the plugin name
195			plog = structlog.get_logger().bind(src=pname)
196
197			# load the plugin
198			log.info("loading", pname=pname, file=pconfig.path)
199			return load_plugin(plog, pconfig)
200
201
202			def get_plugins_dict(log, plugins, pclass):
203			"""Returns a dict of plugins that are of type pclass where the dict
204			keys are the plugins name and the values are the plugin object.
205
206			raises:
207			DuplicatePlugin if a plugin has a duplicate 'name'
208
209			:param plugins: A list of plugin objects.
210			:type plugins: list
211			:param pclas: A class eg. :class:`plumd.Reader`, :class:`plumd.Writer`
212			:type pclass: object
213			:rtype: dict
214			:raises: DuplicatePlugin
215			"""
216			plugs = {}
217			for pobj in plugins:
218			cpclass = pobj.__class__.__base__
219			if pobj.__class__.__base__ == pclass:
220			log.debug('get_plugins_dict', matched=pobj, pclass=cpclass)
221			pconf = pobj.config
222			pname = pconf.get('name', exception=True)
223			if pname in plugs:
224			err = "duplicate plugin name: {0} from {1}"
225			raise DuplicatePlugin(err.format(pname, pconf.path))
226			plugs[pname] = pobj
227			log.debug('get_plugins_dict', plugins=plugs, pclass=pclass)
228			return plugs
229
230
231			def load_all_plugins(log, config):
232			"""Returns a tuple of reader and writer plugin dicts.
233
234			raises:
235			ConfigError if ptype has an incorrect configuration path set
236			ConfigError if a plugin configuration is missing 'name'
237			DuplicatePlugin if a plugin configuration has a duplicate 'name'
238			PluginLoadError if there was an error loading the plugin
239
240			Returns a tuple of: (readers, writers) where each is a dict of:
241			{'name': <plugin_instance>}. Name is the configured plugin name and
242			<plugin_instance> is the corresponding instance of the plugin. Note plugins
243			can be insantiated multiple times with different configurations.
244
245			:param log: A structlog logger instance
246			:type log: structlog
247			:param config: a :class:`plumd.config.Conf` instance
248			:type config: plumd.config.Conf
249			:rtype: tuple
250			:raises: ConfigError, DuplicatePlugin, PluginLoadError
251			"""
252			plugins = []
253			# get a list of configuration files
254			# note the call to ls can raise an exception which we pass upwards
255			for pconf in plumd.config.ls(config.get('config.plugins'), 'yaml'):
256			# load the plugin object
257			pobj = load_from_conf(log, config, pconf)
258
259			# check if the plugin has been disabled in configuration
260			if pobj is None:
261			log.info('load_all_plugins', skipping=pconf, reason="disabled")
262			continue
263
264			plugins.append(pobj)
265
266			log.debug('load_all_plugins', plugins=plugins)
267
268			# we iterate over the plugin list multiple times..
269			# since the list is small and there are not many types this is ok.
270			readers = get_plugins_dict(log, plugins, plumd.plugins.Reader)
271			writers = get_plugins_dict(log, plugins, plumd.plugins.Writer)
272			log.debug('load_all_plugins', readers=readers)
273			log.debug('load_all_plugins', writers=writers)
274
275			return (readers, writers)
276
277
278			def config_plugin_writers(lobj):
279			"""Update the list of writers for each reader in the PluginLoader.
280
281			:param lobj: a :class:`plumd.plugins.load.PluginLoader` object
282			:type lobj: plumd.plugins.load.PluginLoader
283
284			:raises: ConfigError
285			"""
286			# now set the PluginReaders writers
287			for prname, probj in lobj.readers.items():
288			wcfg = probj.pobj.config.get('writers')
289			# if nothing is configured the reader writes to all
290			if wcfg is None:
291			probj.writers = lobj.writers.values()
292			continue
293
294			# get list of writer names
295			wnames = [ w for w in wcfg if w in lobj.writers ]
296			if len(wnames) > 0:
297			probj.writers = [ lobj.writers[w] for w in wnames ]
298			continue
299
300			msg = 'reader {0} from {1} has no valid writers configured'
301			args = [ prname, probj.pobj.conf.path ]
302			lobj.log.error('invalid reader', msg=msg.format(*args))
303			sys.exit(1)
304

s4z / plumd

Push — master ( d058c9...ca4b9c )

plumd.plugins.config_plugin_writers() C

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like