Differential.per_second() - Code Metrics - Inspection of "more prometheus related changes - refactoring for..." - s4z/plumd - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Branch — prom (d3fc9e)

by Kenny

created 2017-03-01 08:01 UTC

Differential.per_second() B

↳ Parent: Differential

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	1	Features	0

Metric	Value
c	2
b	1
f	0
dl	0
loc	30
rs	8.5806
cc	4

# -*- coding: utf-8 -*-
"""Various helper classes.

.. moduleauthor:: Kenny Freeman <[email protected]>

"""
import imp

import sys
import os
import os.path
import time
import fcntl
import socket
import signal
import string
import inspect

import platform
import traceback

from collections import deque

import plumd


__author__ = 'Kenny Freeman'
__email__ = '[email protected]'
__license__ = "ISCL"
__docformat__ = 'reStructuredText'

PY3 = sys.version_info > (3,)


# use CNT_LIMIT to detect counters wrapping over
# eg. platform.architecture() => ('64bit', 'ELF')
PLATFORM_ARCH = platform.architecture()[0]
if PLATFORM_ARCH == '64bit':
    MAX_CNT = 18446744073709551615
elif PLATFORM_ARCH == '32bit':
    MAX_CNT = 4294967295
else:
    MAX_CNT = long(sys.maxint)



def get_hostname():
    """Return the hostname of the system the code is running on. Eventually this
    may include more checks.

    :rtype: str
    """
    return socket.gethostname().split(".")[0]


def get_file_map(fname, col, skip):
    """Return a dict representation of a file - use on small files only.

    eg. file contents of:

    8       1 sda1 284 166 2673 1724 58 18 9056 369 0 1122 2091

    with a call of col=2, skip=0 returns:

    {"sda1": [8, 1, 284, 166, 2673, 1724, 58, 18, 9056, 369, 0, 1122, 2091]}

    :param fname: Full pathname to a file.
    :type fname: str
    :param col: Column number to use as the map
    :type col: int
    :param skip: Skip this many lines of the file.
    :type skip: int
    :rtype: dict
    """
    fdat = {}
    if os.access(fname, os.R_OK):
        with open(fname, 'r') as filed:
            dat = filed.read().split("\n")
            for line in dat[skip:]:
                vals = line.split()
                if len(vals) > col:
                    map_val = vals.pop(col)
                    fdat[map_val] = deque(vals)
    return fdat


def get_file_map_list(fname, col, skip):
    """Return a dict representation of a file - use on small files only.

    eg. file contents of:

    8       1 sda1 284 166 2673 1724 58 18 9056 369 0 1122 2091

    with a call of col=2, skip=0 returns:

    {"sda1": [8, 1, 284, 166, 2673, 1724, 58, 18, 9056, 369, 0, 1122, 2091]}

    :param fname: Full pathname to a file.
    :type fname: str
    :param col: Column number to use as the map
    :type col: int
    :param skip: Skip this many lines of the file.
    :type skip: int
    :rtype: dict
    """
    fdat = {}
    if os.access(fname, os.R_OK):
        with open(fname, 'r') as filed:
            dat = filed.read().split("\n")
            for line in dat[skip:]:
                vals = line.split()
                if len(vals) > col:
                    map_val = vals.pop(col)
                    fdat[map_val] = vals
    return fdat


def get_file_list(fname):
    """Return the file as a list - use on small files only.

    :param fname: Full pathname to a file.
    :type fname: str
    :rtype: dict
    """
    if os.access(fname, os.R_OK):
        with open(fname, 'r') as filed:
            return deque(filed.read().strip().split("\n"))
    return deque([''])


def get_file(fname):
    """Return the file as a string - use on small files only.

    :param fname: Full pathname to a file.
    :type fname: str
    :rtype: dict
    """
    if os.access(fname, os.R_OK):
        with open(fname, 'r') as filed:
            return filed.read()
    return ''


class Interval(object):
    """An interruptible time Interval for use with with statements.

    Provides a means to interupt the interval with a :class:`threading.Event`.

    :param ltime: The target duration of a loop
    :type ltime: int or float
    :param evt: An event object that will cause the loop to complete early if
        the event is triggered while waiting.
    :type evt: threading.Event
    :raises: ValueError if the passed loop time is not an int
    :raises: ValueError if the passed loop time is not a float
    :raises: ValueError if the passed loop time is < 0

    As an example, use it like this to loop on an interval:

    :Example:

        >>> import threading
        >>> evt = threading.Event()
        >>> while loop:
        >>>     with Interval(111.1, evt) as timer:
        >>>         do_things_here()

    .. note:: if the do_things_here() call returns before the loop timer
        completes the timer object will either evt.wait() if evt was defined or
        time.sleep() for the time remaining in the interval.
    """

    def __init__(self, ltime, evt=None):
        """Create an Interval helper instance.

        :param ltime: The target duration of a loop
        :type ltime: int or float
        :param evt: An event object that will cause the loop to complete early
            if the event is triggered while waiting.
        :type evt: threading.Event
        """
        if ltime is None or ltime < 0:
            raise ValueError("an interval must be defined")
        self.ltime = float(ltime)
        self.evt = evt
        self.lstart = None

    @property
    def remaining(self):
        """Return the remaining time in the loop.

        :rtype: float
        """
        if self.lstart is not None:
            return self.ltime - (time.time() - self.lstart)
        return 0.0

    @property
    def elapsed(self):
        """Return the elapsed time of the loop.

        :rtype: float
        """
        if self.lstart is not None:
            return time.time() - self.lstart
        return 0.0

    def __str__(self):
        """Return a nicely formatted string.

        :rtype: str
        """
        elapsed = None
        if self.lstart is not None:
            elapsed = time.time() - self.lstart
        sval = "Interval {0} seconds, {1} elapsed, {2} started"
        return sval.format(self.ltime, elapsed, self.lstart)

    def __repr__(self):
        """Return a nicely formatted string.

        :rtype: str
        """
        elapsed = None
        if self.lstart is not None:
            elapsed = time.time() - self.lstart
        sval = "Interval {0} seconds, {1} elapsed, {2} started"
        return sval.format(self.ltime, elapsed, self.lstart)

    def __enter__(self):
        """Called on entrance to with() statement - simply records loop start
        time.

        :rtype: Interval
        """
        self.lstart = time.time()
        return self

    def __exit__(self, *args):
        """Called on exit from a with() statement and either waits on the
        self.evt :class:`threading.Event` event if defined or time.sleep()'s
        the remaining interval time.
        """
        if self.lstart is None:
            self.lstart = time.time()
        # block on the event for the remaining time, if any
        wtime = self.ltime - (time.time() - self.lstart)
        if wtime > 0:
            if self.evt is not None:
                self.evt.wait(wtime)
            else:
                time.sleep(wtime)


class SignalWaiter(object):
    """A helper class that makes waiting on signals very easy.
    Define the list of signals to wait for and to ignore and then call wait().
    Instead of using signal handler functions it uses the signal.set_wakeup_fd()
    function and blocks on read().

    See http://www.pocketnix.org/doc/Fighting_set__wakeup__fd/ and others for
    more details.

    :param sig_wait: :class:`list` signals to wait for
    :type sig_wait: list
    :param sig_ignore: :class:`list` signals to ignore
    :type sig_ignore: list
    """

    def __init__(self, sig_wait, sig_ignore):
        """Create a :class:`SignalWaiter`"""
        self.sig_wait = sig_wait
        self.sig_ignore = sig_ignore

        # use a pipe to listen for signals
        self.rpipe, self.wpipe = os.pipe()
        # set non blocking mode on the write end of the pipe
        flags = fcntl.fcntl(self.wpipe, fcntl.F_GETFL, 0)
        flags = flags | os.O_NONBLOCK
        flags = fcntl.fcntl(self.wpipe, fcntl.F_SETFL, flags)

        # set the write end as a signal wakeup fd - we read signals from it
        signal.set_wakeup_fd(self.wpipe)

        # install dummy signal handlers that do nothing
        for signum in self.sig_wait:
            signal.signal(signum, lambda x, y: None)

        # install the ignore signal handler for sig_ignore signals
        for signum in self.sig_ignore:
            signal.signal(signum, signal.SIG_IGN)

        # now a call to wait() blocks the caller until a signal is received
        # only the signals we've registered handlers for will trigger

    def wait(self):
        """Block the calling thread until signal(s) are received.

        :rtype: :class:`int` or :class:`Exception`
        """
        try:
            return os.read(self.rpipe, 1)
        except Exception as exc:

            return exc


class Filter(object):
    """A helper class that efficiently filters characters from strings.

    :param keep_chars: A string containing all characters to keep in strings
    :type keep_chars: str
    """

    def __init__(self, keep_chars=None):
        """Create a :class:`Filter`"""
        if not keep_chars:
            # translate is different in python3, so is string
            if PY3:
                keep_chars = string.ascii_letters + string.digits + "_-"
            else:
                keep_chars = string.letters + string.digits + "_-"

        self.filter_chars = "".join(char for char in
                                    [chr(i) for i in range(256)]
                                    if char not in keep_chars)
        self.table = dict.fromkeys([ord(i) for i in self.filter_chars], None)
        if PY3:
            self.process = self.process_py3
        else:
            self.process = self.process_py2

    def process_py3(self, filter_str):
        """Return a filtered string, removes filter_char's from it.

        :rtype: str
        """
        return filter_str.translate(self.table)

    def process_py2(self, filter_str):
        """Return a filtered string, removes filter_char's from it.

        :rtype: str
        """
        return filter_str.translate(None, self.filter_chars)
        sys.exit(1)



class Differential(object):
    """Compute the rate of change of a set of metrics."""

    def __init__(self):
        """Compute the rate of change of a set of metrics."""
        self.metrics = {}

    def per_second(self, name, val, timestamp):
        """Record and return the rate of change of a metric in units/second.

        Assumes values will be < MAX_CNT and detects and corrects overflow.

        :param name: The name of a metric
        :type name: str
        :param val: The value of a metric
        :type val: int or float
        :param max: The maximum possible value for val
        :type val: int or float
        :rtype: int or float
        """
        ret = type(val)(0)
        if name not in self.metrics:
            self.metrics[name] = (val, timestamp)
        else:
            # get the previous value
            pval, ptime = self.metrics[name]
            self.metrics[name] = (val, timestamp)
            # check for counter wrap
            if val < pval:
                pval = pval - MAX_CNT
            dval = val - pval
            dtime = timestamp - ptime
            if dtime > 0:
                ret = float(dval) / float(dtime)
            else:
                ret = 0
        return ret


1		# -- coding: utf-8 --
2		"""Various helper classes.
3
4		.. moduleauthor:: Kenny Freeman <[email protected]>
5
6		"""
7		import imp
		0 ignored issues – show Unused Code introduced 2017-03-01 08:02 UTC by Report Bug Copy Issue Report The import `imp` seems to be unused. Loading history...
8		import sys
9		import os
10		import os.path
11		import time
12		import fcntl
13		import socket
14		import signal
15		import string
16		import inspect
		0 ignored issues – show Unused Code introduced 2017-03-01 08:02 UTC by Report Bug Copy Issue Report The import `inspect` seems to be unused. Loading history...
17		import platform
18		import traceback
		0 ignored issues – show Unused Code introduced 2017-03-01 08:02 UTC by Report Bug Copy Issue Report The import `traceback` seems to be unused. Loading history...
19		from collections import deque
20
21		import plumd
		0 ignored issues – show Unused Code introduced 2017-03-01 08:02 UTC by Report Bug Copy Issue Report The import `plumd` seems to be unused. Loading history...
22
23		__author__ = 'Kenny Freeman'
24		__email__ = '[email protected]'
25		__license__ = "ISCL"
26		__docformat__ = 'reStructuredText'
27
28		PY3 = sys.version_info > (3,)
29
30
31		# use CNT_LIMIT to detect counters wrapping over
32		# eg. platform.architecture() => ('64bit', 'ELF')
33		PLATFORM_ARCH = platform.architecture()[0]
34		if PLATFORM_ARCH == '64bit':
35		MAX_CNT = 18446744073709551615
36		elif PLATFORM_ARCH == '32bit':
37		MAX_CNT = 4294967295
38		else:
39		MAX_CNT = long(sys.maxint)
		0 ignored issues – show Comprehensibility Best Practice introduced 2016-11-13 07:47 UTC by Report Bug Copy Issue Report Undefined variable 'long' Loading history... Bug introduced 2016-09-10 09:25 UTC by Report Bug Copy Issue Report The Module `sys` does not seem to have a member named `maxint`. This check looks for calls to members that are non-existent. These calls will fail. The member could have been renamed or removed. Loading history...
40
41
42		def get_hostname():
43		"""Return the hostname of the system the code is running on. Eventually this
44		may include more checks.
45
46		:rtype: str
47		"""
48		return socket.gethostname().split(".")[0]
49
50
51		def get_file_map(fname, col, skip):
52		"""Return a dict representation of a file - use on small files only.
53
54		eg. file contents of:
55
56		8 1 sda1 284 166 2673 1724 58 18 9056 369 0 1122 2091
57
58		with a call of col=2, skip=0 returns:
59
60		{"sda1": [8, 1, 284, 166, 2673, 1724, 58, 18, 9056, 369, 0, 1122, 2091]}
61
62		:param fname: Full pathname to a file.
63		:type fname: str
64		:param col: Column number to use as the map
65		:type col: int
66		:param skip: Skip this many lines of the file.
67		:type skip: int
68		:rtype: dict
69		"""
70		fdat = {}
71		if os.access(fname, os.R_OK):
72		with open(fname, 'r') as filed:
73		dat = filed.read().split("\n")
74		for line in dat[skip:]:
75		vals = line.split()
76		if len(vals) > col:
77		map_val = vals.pop(col)
78		fdat[map_val] = deque(vals)
79		return fdat
80
81	View Code Duplication
82		def get_file_map_list(fname, col, skip):
83		"""Return a dict representation of a file - use on small files only.
84
85		eg. file contents of:
86
87		8 1 sda1 284 166 2673 1724 58 18 9056 369 0 1122 2091
88
89		with a call of col=2, skip=0 returns:
90
91		{"sda1": [8, 1, 284, 166, 2673, 1724, 58, 18, 9056, 369, 0, 1122, 2091]}
92
93		:param fname: Full pathname to a file.
94		:type fname: str
95		:param col: Column number to use as the map
96		:type col: int
97		:param skip: Skip this many lines of the file.
98		:type skip: int
99		:rtype: dict
100		"""
101		fdat = {}
102		if os.access(fname, os.R_OK):
103		with open(fname, 'r') as filed:
104		dat = filed.read().split("\n")
105		for line in dat[skip:]:
106		vals = line.split()
107		if len(vals) > col:
108		map_val = vals.pop(col)
109		fdat[map_val] = vals
110		return fdat
111
112	View Code Duplication
113		def get_file_list(fname):
114		"""Return the file as a list - use on small files only.
115
116		:param fname: Full pathname to a file.
117		:type fname: str
118		:rtype: dict
119		"""
120		if os.access(fname, os.R_OK):
121		with open(fname, 'r') as filed:
122		return deque(filed.read().strip().split("\n"))
123		return deque([''])
124
125
126		def get_file(fname):
127		"""Return the file as a string - use on small files only.
128
129		:param fname: Full pathname to a file.
130		:type fname: str
131		:rtype: dict
132		"""
133		if os.access(fname, os.R_OK):
134		with open(fname, 'r') as filed:
135		return filed.read()
136		return ''
137
138
139		class Interval(object):
140		"""An interruptible time Interval for use with with statements.
141
142		Provides a means to interupt the interval with a :class:`threading.Event`.
143
144		:param ltime: The target duration of a loop
145		:type ltime: int or float
146		:param evt: An event object that will cause the loop to complete early if
147		the event is triggered while waiting.
148		:type evt: threading.Event
149		:raises: ValueError if the passed loop time is not an int
150		:raises: ValueError if the passed loop time is not a float
151		:raises: ValueError if the passed loop time is < 0
152
153		As an example, use it like this to loop on an interval:
154
155		:Example:
156
157		>>> import threading
158		>>> evt = threading.Event()
159		>>> while loop:
160		>>> with Interval(111.1, evt) as timer:
161		>>> do_things_here()
162
163		.. note:: if the do_things_here() call returns before the loop timer
164		completes the timer object will either evt.wait() if evt was defined or
165		time.sleep() for the time remaining in the interval.
166		"""
167
168		def __init__(self, ltime, evt=None):
169		"""Create an Interval helper instance.
170
171		:param ltime: The target duration of a loop
172		:type ltime: int or float
173		:param evt: An event object that will cause the loop to complete early
174		if the event is triggered while waiting.
175		:type evt: threading.Event
176		"""
177		if ltime is None or ltime < 0:
178		raise ValueError("an interval must be defined")
179		self.ltime = float(ltime)
180		self.evt = evt
181		self.lstart = None
182
183		@property
184		def remaining(self):
185		"""Return the remaining time in the loop.
186
187		:rtype: float
188		"""
189		if self.lstart is not None:
190		return self.ltime - (time.time() - self.lstart)
191		return 0.0
192
193		@property
194		def elapsed(self):
195		"""Return the elapsed time of the loop.
196
197		:rtype: float
198		"""
199		if self.lstart is not None:
200		return time.time() - self.lstart
201		return 0.0
202
203		def __str__(self):
204		"""Return a nicely formatted string.
205
206		:rtype: str
207		"""
208		elapsed = None
209		if self.lstart is not None:
210		elapsed = time.time() - self.lstart
211		sval = "Interval {0} seconds, {1} elapsed, {2} started"
212		return sval.format(self.ltime, elapsed, self.lstart)
213
214		def __repr__(self):
215		"""Return a nicely formatted string.
216
217		:rtype: str
218		"""
219		elapsed = None
220		if self.lstart is not None:
221		elapsed = time.time() - self.lstart
222		sval = "Interval {0} seconds, {1} elapsed, {2} started"
223		return sval.format(self.ltime, elapsed, self.lstart)
224
225		def __enter__(self):
226		"""Called on entrance to with() statement - simply records loop start
227		time.
228
229		:rtype: Interval
230		"""
231		self.lstart = time.time()
232		return self
233
234		def __exit__(self, *args):
235		"""Called on exit from a with() statement and either waits on the
236		self.evt :class:`threading.Event` event if defined or time.sleep()'s
237		the remaining interval time.
238		"""
239		if self.lstart is None:
240		self.lstart = time.time()
241		# block on the event for the remaining time, if any
242		wtime = self.ltime - (time.time() - self.lstart)
243		if wtime > 0:
244		if self.evt is not None:
245		self.evt.wait(wtime)
246		else:
247		time.sleep(wtime)
248
249
250		class SignalWaiter(object):
251		"""A helper class that makes waiting on signals very easy.
252		Define the list of signals to wait for and to ignore and then call wait().
253		Instead of using signal handler functions it uses the signal.set_wakeup_fd()
254		function and blocks on read().
255
256		See http://www.pocketnix.org/doc/Fighting_set__wakeup__fd/ and others for
257		more details.
258
259		:param sig_wait: :class:`list` signals to wait for
260		:type sig_wait: list
261		:param sig_ignore: :class:`list` signals to ignore
262		:type sig_ignore: list
263		"""
264
265		def __init__(self, sig_wait, sig_ignore):
266		"""Create a :class:`SignalWaiter`"""
267		self.sig_wait = sig_wait
268		self.sig_ignore = sig_ignore
269
270		# use a pipe to listen for signals
271		self.rpipe, self.wpipe = os.pipe()
272		# set non blocking mode on the write end of the pipe
273		flags = fcntl.fcntl(self.wpipe, fcntl.F_GETFL, 0)
274		flags = flags \| os.O_NONBLOCK
275		flags = fcntl.fcntl(self.wpipe, fcntl.F_SETFL, flags)
276
277		# set the write end as a signal wakeup fd - we read signals from it
278		signal.set_wakeup_fd(self.wpipe)
279
280		# install dummy signal handlers that do nothing
281		for signum in self.sig_wait:
282		signal.signal(signum, lambda x, y: None)
283
284		# install the ignore signal handler for sig_ignore signals
285		for signum in self.sig_ignore:
286		signal.signal(signum, signal.SIG_IGN)
287
288		# now a call to wait() blocks the caller until a signal is received
289		# only the signals we've registered handlers for will trigger
290
291		def wait(self):
292		"""Block the calling thread until signal(s) are received.
293
294		:rtype: :class:`int` or :class:`Exception`
295		"""
296		try:
297		return os.read(self.rpipe, 1)
298		except Exception as exc:
		0 ignored issues – show Best Practice introduced 2016-09-10 09:25 UTC by Report Bug Copy Issue Report Catching very general exceptions such as `Exception` is usually not recommended. Generally, you would want to handle very specific errors in the exception handler. This ensure that you do not hide other types of errors which should be fixed. So, unless you specifically plan to handle any error, consider adding a more specific exception. Loading history...
299		return exc
300
301
302		class Filter(object):
303		"""A helper class that efficiently filters characters from strings.
304
305		:param keep_chars: A string containing all characters to keep in strings
306		:type keep_chars: str
307		"""
308
309		def __init__(self, keep_chars=None):
310		"""Create a :class:`Filter`"""
311		if not keep_chars:
312		# translate is different in python3, so is string
313		if PY3:
314		keep_chars = string.ascii_letters + string.digits + "_-"
315		else:
316		keep_chars = string.letters + string.digits + "_-"
		0 ignored issues – show Bug introduced 2016-09-10 09:25 UTC by Report Bug Copy Issue Report The Module `string` does not seem to have a member named `letters`. This check looks for calls to members that are non-existent. These calls will fail. The member could have been renamed or removed. Loading history...
317		self.filter_chars = "".join(char for char in
318		[chr(i) for i in range(256)]
319		if char not in keep_chars)
320		self.table = dict.fromkeys([ord(i) for i in self.filter_chars], None)
321		if PY3:
322		self.process = self.process_py3
323		else:
324		self.process = self.process_py2
325
326		def process_py3(self, filter_str):
327		"""Return a filtered string, removes filter_char's from it.
328
329		:rtype: str
330		"""
331		return filter_str.translate(self.table)
332
333		def process_py2(self, filter_str):
334		"""Return a filtered string, removes filter_char's from it.
335
336		:rtype: str
337		"""
338		return filter_str.translate(None, self.filter_chars)
339		sys.exit(1)
		0 ignored issues – show Unused Code introduced 2017-03-01 08:02 UTC by Report Bug Copy Issue Report This code does not seem to be reachable. Loading history...
340
341
342		class Differential(object):
343		"""Compute the rate of change of a set of metrics."""
344
345		def __init__(self):
346		"""Compute the rate of change of a set of metrics."""
347		self.metrics = {}
348
349		def per_second(self, name, val, timestamp):
350		"""Record and return the rate of change of a metric in units/second.
351
352		Assumes values will be < MAX_CNT and detects and corrects overflow.
353
354		:param name: The name of a metric
355		:type name: str
356		:param val: The value of a metric
357		:type val: int or float
358		:param max: The maximum possible value for val
359		:type val: int or float
360		:rtype: int or float
361		"""
362		ret = type(val)(0)
363		if name not in self.metrics:
364		self.metrics[name] = (val, timestamp)
365		else:
366		# get the previous value
367		pval, ptime = self.metrics[name]
368		self.metrics[name] = (val, timestamp)
369		# check for counter wrap
370		if val < pval:
371		pval = pval - MAX_CNT
372		dval = val - pval
373		dtime = timestamp - ptime
374		if dtime > 0:
375		ret = float(dval) / float(dtime)
376		else:
377		ret = 0
378		return ret
379

s4z / plumd

Branch — prom (d3fc9e)

Differential.per_second() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like