Proc

Complexity

Total Complexity

59

Size/Duplication

Total Lines	344
Duplicated Lines	15.7 %

Importance

Changes	1
Bugs	1	Features	0

14 Methods

Rating	Name	Duplication	Size	Complexity
B	proc_swap()	0	28	4
C	proc_stat()	0	26	7
B	proc_net_netstat()	0	20	5
A	proc_stat_cpu()	0	11	3
A	proc_loadavg()	0	17	3
A	proc_meminfo()	0	19	4
A	proc_uptime()	0	17	2
B	proc_net_sockstat()	0	20	5
B	proc_net_dev()	27	27	6
A	poll()	0	17	1
F	proc_net_snmp()	0	37	9
A	proc_stat_cpu_percent()	0	10	3
B	proc_diskstats()	27	27	6
A	__init__()	0	11	1

# -*- coding: utf-8 -*-

__author__ = 'Kenny Freeman'
__email__ = 'kenny.freeman@gmail.com'
__license__ = "ISCL"
__docformat__ = 'reStructuredText'

import os
import time
import os.path
import traceback
import multiprocessing
from collections import deque

import plumd
import plumd.plugins
from plumd.calc import Differential
from plumd.util import get_file_map, get_file_map_list, get_file_list, get_file


## todo: switch from list with pop(0) to deque
class Proc(plumd.plugins.Reader):
    """Plugin to measure various kernel metrics from /proc."""
    defaults = {
        'poll.interval': 10,
        'skip_proc_stat': ['btime'],
        'skip_proc_meminfo': [],
        'skip_proc_meminfo': [],
        'skip_proc_loadavg': [],
        'skip_proc_swap': [],
        'skip_proc_uptime': [],
        'skip_proc_diskstats': [],
        'skip_proc_net_dev': [],
        'skip_proc_net_snmp': [],
        'skip_proc_net_sockstat': [],
        'skip_proc_net_sockstat': [],
        'skip_proc_net_netstat': [],
        'cpu_metrics': ["user", "nice", "system", "idle", "iowait", "irq",
                        "softirq","steal", "guest", "guest_nice"],
        'per_cpu': False,
        'diskstats_cols': ["r", "r_merge", "r_sector", "r_time", "w", "w_merge",
                           "w_sector", "w_time", "io_inprog", "io_time",
                           "io_weighted_time"],
        'net_dev_cols': ["rx_bytes", "rx_pkt", "rx_errs", "rx_drop",
                         "rx_fifo_errs", "rx_frame_errs", "rx_compressed",
                         "rx_mcast", "tx_bytes", "tx_pkt", "tx_errs", "tx_drop",
                         "tx_fifo_errs", "collissions", "carrier",
                         "tx_compressed"],
        'net_snmp_items': ["Ip:", "Icmp:", "IcmpMsg:", "Tcp:", "Udp:",
                           "UdpLite:"]
    }

    def __init__(self, log, config):
        """Plugin to measure various kernel metrics from /proc.

        :param log: A logger
        :type log: logging.RootLogger
        :param config: a plumd.config.Conf configuration helper instance.
        :type config: plumd.config.Conf
        """
        super(Proc, self).__init__(log, config)
        self.config.defaults(Proc.defaults)
        self.calc = Differential()


    def poll(self):
        """Poll for kernel metrics under /proc.

        :rtype: ResultSet
        """
        ret = plumd.ResultSet([])
        ret.add(self.proc_stat())
        ret.add(self.proc_meminfo())
        ret.add(self.proc_loadavg())
        ret.add(self.proc_swap())
        ret.add(self.proc_uptime())
        ret.add(self.proc_diskstats())
        ret.add(self.proc_net_dev())
        ret.add(self.proc_net_snmp())
        ret.add(self.proc_net_sockstat())
        ret.add(self.proc_net_netstat())
        return ret


    def proc_stat_cpu_percent(self, key, val, ts):
        ret = []
        total = sum([ float(i) for i in val])
        cpu = self.config.get('cpu_metrics')
        for map_val in cpu:
            metric_val = float(val.popleft())
            mstr = "{0}.{1}".format(key, map_val)
            percent_val = metric_val / total * 100.00
            ret.append(plumd.Float(mstr, percent_val))
        return ret


    def proc_stat_cpu(self, key, val, ts):
        ret = []
        total = sum([ float(i) for i in val])
        cpu = self.config.get('cpu_metrics')
        for map_val in cpu:
            metric_val = float(val.popleft())
            mstr = "{0}.{1}".format(key, map_val)
            percent_val = float(metric_val / total) * 100.00
            mval = self.calc.per_second(key, percent_val, ts)
            ret.append(plumd.Float(mstr, mval))
        return ret


    def proc_stat(self):
        skip = self.config.get('skip_proc_stat')
        per_cpu = self.config.get('per_cpu')
        result = plumd.Result("stat")
        fname = "/proc/stat"
        # read and process /proc/stat
        dat = get_file_map(fname, 0, 0)
        ts = time.time()
        # parse
        for key, val in dat.items():
            # cpu is the only special metric
            if val is None:
                self.log.error("proc_stat: null value for {0}".format(key))
                continue
            elif key in skip:
                continue
            elif key == "cpu":
                result.add_list(self.proc_stat_cpu_percent(key, val, ts))
            elif key.startswith("cpu"):
                if not per_cpu:
                    continue
                result.add_list(self.proc_stat_cpu_percent(key, val, ts))
            else:
                mval = self.calc.per_second(key, float(val[0]), ts)
                result.add(plumd.Int(key, mval))
        return result


    def proc_meminfo(self):
        skip = self.config.get('skip_proc_meminfo')
        result = plumd.Result("mem")
        fname = "/proc/meminfo"
        # read and process /proc/stat
        dat = get_file_map(fname, 0, 0)
        ts = time.time()
        # parse
        for key, val in dat.items():
            # cpu is the only special metric
            if val is None:
                self.log.error("proc_meminfo: null value for {0}".format(key))
                continue
            elif key in skip:
                continue
            else:
                #mval = dcalc.per_second(key, float(val[0]), ts)
                result.add(plumd.Int(key, val[0]))
        return result


    def proc_loadavg(self):
        skip = self.config.get('skip_proc_loadavg')
        result = plumd.Result("loadavg")
        fname = "/proc/loadavg"
        dat = []
        # read and process /proc/stat
        try:
            dat = get_file(fname).split()
        except Exception as e:
            tb = traceback.format_exc()
            self.log.error("proc_loadavg: exception: {0} : {1}".format(e, tb))
            return result
        if len(dat) >= 3:
            result.add(plumd.Float("1", dat[0]))
            result.add(plumd.Float("5", dat[1]))
            result.add(plumd.Float("15", dat[2]))
        return result


    def proc_swap(self):
        skip = self.config.get('skip_proc_swap')
        result = plumd.Result("swap")
        fname = "/proc/swaps"
        dat = []
        # read and process /proc/stat
        dat = get_file_list(fname)
        # header: file, type, size, used, priority
        dat.popleft()
        for entry in dat:
            if not entry:
                continue
            #sfname, stype, ssize, sused, sprio = ("", None, 0, 0, 0)
            try:
                sfname, stype, ssize, sused, sprio = entry.split()
            except Exception as e:
                tb = traceback.format_exc()
                self.log.error("proc_swap: exception: {0}: {1}".format(e, tb))
                continue
            sname = os.path.basename(sfname)
            mstr = "{0}.used".format(sname)
            result.add(plumd.Float(mstr, sused))
            mstr = "{0}.size".format(sname)
            result.add(plumd.Float(mstr, ssize))
            sfree = float(ssize) - float(sused)
            mstr = "{0}.free".format(sname)
            result.add(plumd.Float(mstr, sfree))
        return result


    def proc_uptime(self):
        skip = self.config.get('skip_proc_uptime')
        result = plumd.Result("uptime")
        fname = "/proc/uptime"
        dat = []
        # read and process /proc/stat
        try:
            up, idle = get_file(fname).split()
        except Exception as e:
            tb = traceback.format_exc()
            self.log.error("proc_uptime: exception: {0}: {1}".format(e, tb))
            return result
        pidle = float(idle)/float(up) * 100 / multiprocessing.cpu_count()
        result.add(plumd.Float("up", up))
        result.add(plumd.Float("idle", idle))
        result.add(plumd.Float("idle_percent", pidle))
        return result


    def proc_diskstats(self):

This code seems to be duplicated in your project.

        skip = self.config.get('skip_proc_diskstats')
        # times in ms
        cols = self.config.get('diskstats_cols')
        result = plumd.Result("diskstats")
        fname = "/proc/diskstats"
        dat = {}
        # read and process /proc/stat
        try:
            dat = get_file_map(fname, 2, 0)
        except Exception as e:
            tb = traceback.format_exc()
            self.log.error("proc_diskstats: exception: {0}: {1}".format(e, tb))
            return result
        ts = time.time()
        for key, val in dat.items():
            if key in skip:
                continue
            if len(val) != 13:
                self.log.error("proc_diskstats: invalid entry: {0}".format(val))
                continue
            for mname in cols:
                mval = int(val.popleft())
                mstr = "{0}.{1}".format(key, mname)
                dval = self.calc.per_second(mstr, mval, ts)
                result.add(plumd.Int(mstr, dval))
        return result


    def proc_net_dev(self):

This code seems to be duplicated in your project.

        skip = self.config.get('skip_proc_net_dev')
        cols = self.config.get('net_dev_cols')
        result = plumd.Result("net")
        fname = "/proc/net/dev"
        dat = {}
        # read and process /proc/stat
        try:
            dat = get_file_map(fname, 0, 0)
        except Exception as e:
            tb = traceback.format_exc()
            self.log.error("proc_net_dev: exception: {0}: {1}".format(e, tb))
            return result
        ts = time.time()
        for key, val in dat.items():
            key = key.replace(":", "")
            if key in skip:
                continue
            if len(val) < len(cols):
                #self.log.error("proc_net_dev: invalid entry: {0}".format(val))
                continue
            for mname in cols:
                mval = int(val.popleft())
                mstr = "{0}.{1}".format(key, mname)
                dval = self.calc.per_second(mstr, mval, ts)
                result.add(plumd.Int(mstr, dval))
        return result


    def proc_net_snmp(self):
        skip = self.config.get('skip_proc_net_snmp')
        items = self.config.get('net_snmp_items')
        result = plumd.Result("net")
        fname = "/proc/net/snmp"
        dat = {}
        # read and process - dat is a list of lines from fname
        dat = get_file_list(fname)
        ts = time.time()

        # process each pair of lines
        if len(dat) != ((2 * len(items)) + 1):
            self.log.error("proc_net_dev: invalid entry: {0}: {1}".format(dat, items))
            return result
        for item in items:
            try:
                # first line is a list of: metric: header values
                header = deque(dat.popleft().split())
                # second line is a list of: <metric>: metric values
                vals = deque([ int(i) for i in dat.popleft().split()[1:] ])
            except Exception as e:
                tb = traceback.format_exc()
                self.log.error("proc_net_snmp: exception: {0}: {1}".format(e, tb))
                continue
            if len(header) < 2 or header[0] != item:
                self.log.error("proc_net_snmp: invalid entry: {0}: {1}".format(header, item))
                continue
            # first value is the name of the metric eg. Ip, Icmp, etc
            mheader = header.popleft().replace(":", "")
            for mname in header:
                mval = vals.popleft()
                if mname in skip:
                    continue
                mstr = "{0}.{1}".format(mheader, mname)
                dval = self.calc.per_second(mstr, mval, ts)
                result.add(plumd.Int(mstr, dval))
        return result


    def proc_net_sockstat(self):
        skip = self.config.get('skip_proc_net_sockstat')
        result = plumd.Result("sockstat")
        fname = "/proc/net/sockstat"
        dat = {}
        # read and process - dat is a list of lines from fname
        dat = get_file_map_list(fname, 0, 0)
        ts = time.time()

        for key, val in dat.items():
            mstr = key.replace(":", "")
            mnames = val[::2]
            mvals = deque([ int(i) for i in val[1::2] ])
            if len(mnames) != len(mvals):
                self.log.error("proc_net_sockstat: invalid entry: {0}".format(mnames))
                continue
            for mname in mnames:
                metric = "{0}.{1}".format(mstr, mname)
                result.add(plumd.Int(metric, mvals.popleft()))
        return result


    def proc_net_netstat(self):
        skip = self.config.get('skip_proc_net_netstat')
        result = plumd.Result("netstat")
        fname = "/proc/net/netstat"
        dat = {}
        # read and process - dat is a list of lines from fname
        dat = get_file_list(fname)
        ts = time.time()
        while len(dat) > 1:
            headers = deque(dat.popleft().split())
            mvals = deque([ int(i) for i in dat.popleft().split()[1:] ])
            mstr = headers.popleft().replace(":", "")
            if len(headers) != len(mvals):
                self.log.error("proc_net_netstat: invalid entry: {0}".format(headers))
                continue
            for mname in headers:
                metric = "{0}.{1}".format(mstr, mname)
                dval = self.calc.per_second(metric, mvals.popleft(), ts)
                result.add(plumd.Int(metric, dval))
        return result