Proc

Complexity

Total Complexity

77

Size/Duplication

Total Lines	536
Duplicated Lines	19.96 %

Importance

Changes	4
Bugs	2	Features	2

15 Methods

Rating	Name	Duplication	Size	Complexity
B	proc_swap()	0	32	5
C	proc_stat()	0	30	7
F	proc_net_netstat()	0	36	9
B	proc_stat_cpu()	17	22	4
A	proc_loadavg()	0	20	3
B	proc_meminfo()	0	24	4
A	proc_uptime()	3	20	2
D	proc_net_sockstat()	0	46	8
C	proc_net_dev()	30	35	7
A	poll()	3	9	2
F	proc_net_snmp()	0	49	10
A	proc_stat_cpu_percent()	18	20	4
B	proc_diskstats()	28	30	6
B	__init__()	0	27	3
B	proc_conntrack()	0	28	3

# -*- coding: utf-8 -*-

__author__ = 'Kenny Freeman'
__email__ = 'kenny.freeman@gmail.com'
__license__ = "ISCL"
__docformat__ = 'reStructuredText'

import re
import os
import time
import os.path
import traceback
import multiprocessing
from collections import deque

import plumd
import plumd.plugins
from plumd.calc import Differential
from plumd.util import get_file_map, get_file_map_list, get_file_list, get_file


## todo: switch from list with pop(0) to deque
class Proc(plumd.plugins.Reader):
    """Plugin to measure various kernel metrics from /proc."""
    defaults = {
        'poll.interval': 10,
        'proc_path': '/proc',
        'skip_funcs': [],
        'skip_proc_stat': ['btime'],
        'skip_proc_meminfo': ["Active(anon)","Active(file)",
            "AnonHugePages","AnonPages","Bounce","DirectMap2M","CommitLimit",
            "DirectMap4k","HugePages_Free","Hugepagesize",
            "HugePages_Rsvd","HugePages_Surp","HugePages_Total",
            "Inactive","Inactive(anon)","Inactive(file)",
            "KernelStack","NFS_Unstable","PageTables",
            "Shmem","Slab","SReclaimable","SUnreclaim",
            "SwapCached","SwapFree","SwapTotal","Writeback","WritebackTmp"],
        'skip_proc_net_snmp': ["Icmp.InAddrMaskReps","Icmp.InAddrMasks",
            "Icmp.InTimestampReps","Icmp.InTimestamps","Icmp.OutAddrMaskReps",
            "Icmp.OutAddrMasks","Icmp.OutTimestampReps","Icmp.OutTimestamps",
            "Ip.DefaultTTL","Ip.Forwarding","Tcp.MaxConn","Tcp.RtoAlgorithm",
            "Tcp.RtoMax","Tcp.RtoMin"],
        'skip_proc_net_sockstat': [],
        'skip_proc_net_netstat': ["TcpExt.ArpFilter","TcpExt.BusyPollRxPackets",
            "TcpExt.DelayedACKLocked","TcpExt.DelayedACKLost",
            "TcpExt.DelayedACKs","TcpExt.EmbryonicRsts",
            "TcpExt.IPReversePathFilter","TcpExt.LockDroppedIcmps",
            "TcpExt.OfoPruned","TcpExt.OutOfWindowIcmps",
            "TcpExt.PAWSActive","TcpExt.PAWSEstab", "TcpExt.PAWSPassive",
            "TcpExt.PruneCalled","TcpExt.RcvPruned",
            "TcpExt.TCPACKSkippedChallenge","TcpExt.TCPACKSkippedFinWait2",
            "TcpExt.TCPACKSkippedPAWS","TcpExt.TCPACKSkippedSeq",
            "TcpExt.TCPACKSkippedSynRecv","TcpExt.TCPACKSkippedTimeWait",
            "TcpExt.TCPAutoCorking","TcpExt.TCPChallengeACK",
            "TcpExt.TCPDirectCopyFromBacklog","TcpExt.TCPDirectCopyFromPrequeue",
            "TcpExt.TCPDSACKIgnoredNoUndo","TcpExt.TCPDSACKIgnoredOld",
            "TcpExt.TCPDSACKOfoRecv","TcpExt.TCPDSACKOfoSent",
            "TcpExt.TCPDSACKOldSent","TcpExt.TCPDSACKRecv","TcpExt.TCPDSACKUndo",
            "TcpExt.TCPFACKReorder","TcpExt.TCPFromZeroWindowAdv",
            "TcpExt.TCPFullUndo","TcpExt.TCPHPAcks","TcpExt.TCPHPHits",
            "TcpExt.TCPHPHitsToUser","TcpExt.TCPHystartDelayCwnd",
            "TcpExt.TCPHystartDelayDetect","TcpExt.TCPHystartTrainCwnd",
            "TcpExt.TCPHystartTrainDetect","TcpExt.TCPLossFailures",
            "TcpExt.TCPLossProbeRecovery","TcpExt.TCPLossProbes",
            "TcpExt.TCPLossUndo","TcpExt.TCPLostRetransmit",
            "TcpExt.TCPMD5NotFound","TcpExt.TCPMD5Unexpected",
            "TcpExt.TCPMTUPFail","TcpExt.TCPMTUPSuccess","TcpExt.TCPOFODrop",
            "TcpExt.TCPOFOMerge","TcpExt.TCPOFOQueue","TcpExt.TCPOrigDataSent",
            "TcpExt.TCPPartialUndo","TcpExt.TCPPrequeued",
            "TcpExt.TCPPrequeueDropped","TcpExt.TCPPureAcks",
            "TcpExt.TCPRcvCoalesce","TcpExt.TCPRcvCollapsed",
            "TcpExt.TCPRenoFailures","TcpExt.TCPRenoRecovery",
            "TcpExt.TCPRenoRecoveryFail","TcpExt.TCPRenoReorder",
            "TcpExt.TCPRetransFail","TcpExt.TCPSACKDiscard",
            "TcpExt.TCPSackFailures","TcpExt.TCPSackMerged",
            "TcpExt.TCPSackRecovery","TcpExt.TCPSackRecoveryFail",
            "TcpExt.TCPSACKReneging","TcpExt.TCPSACKReorder",
            "TcpExt.TCPSackShifted","TcpExt.TCPSackShiftFallback",
            "TcpExt.TCPSchedulerFailed","TcpExt.TCPSpuriousRTOs",
            "TcpExt.TCPSpuriousRtxHostQueues","TcpExt.TCPSYNChallenge",
            "TcpExt.TCPToZeroWindowAdv","TcpExt.TCPTSReorder",
            "TcpExt.TCPWantZeroWindowAdv","TcpExt.TCPWinProbe",
            "TcpExt.TCPKeepAlive", "TcpExt.TCPFastOpenCookieReqd",
            "IpExt.InNoECTPkts", "IpExt.InCEPkts"],
        'cpu_metrics': ["user", "nice", "system", "idle", "iowait", "irq",
                        "softirq","steal", "guest", "guest_nice"],
        'per_cpu': False,
        'diskstats_dev_re': "dm-\d",
        'diskstats_cols': ["r", "r_merge", "r_sector", "r_time", "w", "w_merge",
                           "w_sector", "w_time", "io_inprog", "io_time",
                           "io_weighted_time"],
        'net_dev_re': "(virbr\d+)|(vnet\d+)",
        'net_dev_cols': ["rx_bytes", "rx_pkt", "rx_errs", "rx_drop",
                         "rx_fifo_errs", "rx_frame_errs", "rx_compressed",
                         "rx_mcast", "tx_bytes", "tx_pkt", "tx_errs", "tx_drop",
                         "tx_fifo_errs", "collissions", "carrier",
                         "tx_compressed"],
        'net_snmp_items': ["Ip:", "Icmp:", "IcmpMsg:", "Tcp:", "Udp:",
                           "UdpLite:"]
    }

    def __init__(self, log, config):
        """Plugin to measure various kernel metrics from /proc.

        :param log: A logger
        :type log: logging.RootLogger
        :param config: a plumd.config.Conf configuration helper instance.
        :type config: plumd.config.Conf
        """
        super(Proc, self).__init__(log, config)
        self.config.defaults(Proc.defaults)
        self.calc = Differential()
        self.proc_path = config.get('proc_path')
        self.page_size = os.sysconf("SC_PAGESIZE")
        self.diskstats_dev_re = re.compile(config.get('diskstats_dev_re'))
        self.net_dev_re = re.compile(config.get('net_dev_re'))
        ## allow disabling of each collector via configuration
        all_functions = [self.proc_stat, self.proc_swap, self.proc_meminfo,
                         self.proc_loadavg, self.proc_uptime,
                         self.proc_net_dev, self.proc_diskstats,
                         self.proc_net_snmp, self.proc_net_netstat,
                         self.proc_conntrack, self.proc_net_sockstat]
        skip_functions = self.config.get('skip_funcs')
        self.funcs = []
        for func in all_functions:
            if func.__name__ in skip_functions:
                continue
            self.funcs.append(func)


    def poll(self):
        """Poll for kernel metrics under /proc.

        :rtype: ResultSet
        """
        ret = plumd.ResultSet([])
        for func in self.funcs:

This code seems to be duplicated in your project.

            ret.add(func())
        return ret


    def proc_stat_cpu_percent(self, key, val, ts):
        """Return cpu utilization metrics in percentage.

        :param key: The metric name (eg. cpu, cpu0, cpu1, etc)
        :type key: str
        :param val: A deque populated with the metric values from stat
        :type val: deque
        :rtype: list
        """
        ret = []
        total = sum([ float(i) for i in val])
        cpu = self.config.get('cpu_metrics')
        for map_val in cpu:
            if len(val) < 1:
                break
            metric_val = float(val.popleft())
            mstr = "{0}.{1}".format(key, map_val)
            percent_val = metric_val / total * 100.00

This code seems to be duplicated in your project.

            ret.append(plumd.Float(mstr, percent_val))
        return ret


    def proc_stat_cpu(self, key, val, ts):
        """Return cpu utilization metrics in USER_HZ or Jiffies
        (most likely units of 100Hz intervals ie. 100ms intervals).

        :param key: The metric name (eg. cpu, cpu0, cpu1, etc)
        :type key: str
        :param val: A deque populated with the metric values from stat
        :type val: deque
        :rtype: list
        """
        ret = []
        total = sum([ float(i) for i in val])
        cpu = self.config.get('cpu_metrics')
        for map_val in cpu:
            if len(val) < 1:
                break
            metric_val = float(val.popleft())
            mstr = "{0}.{1}".format(key, map_val)
            percent_val = float(metric_val / total) * 100.00
            mval = self.calc.per_second(key, percent_val, ts)
            ret.append(plumd.Float(mstr, mval))
        return ret


    def proc_stat(self):
        """Return cpu utilization and process metrics from proc file stat.

        :rtype: plumd.Result
        """
        skip = self.config.get('skip_proc_stat')
        per_cpu = self.config.get('per_cpu')
        result = plumd.Result("stat")
        fname = "{0}/stat".format(self.proc_path)
        # read and process /proc/stat
        dat = get_file_map(fname, 0, 0)
        ts = time.time()
        # parse
        for key, val in dat.items():
            # cpu is the only special metric
            if val is None:
                self.log.error("proc_stat: null value for {0}".format(key))
                continue
            elif key in skip:
                continue
            elif key == "cpu":
                result.add_list(self.proc_stat_cpu_percent(key, val, ts))
            elif key.startswith("cpu"):
                if not per_cpu:
                    continue
                result.add_list(self.proc_stat_cpu_percent(key, val, ts))
            else:
                mval = self.calc.per_second(key, float(val[0]), ts)
                result.add(plumd.Int(key, mval))
        return result


    def proc_meminfo(self):
        """Return memory utilization metrics from proc file mem.

        :rtype: plumd.Result
        """
        skip = self.config.get('skip_proc_meminfo')
        result = plumd.Result("mem")
        fname = "{0}/meminfo".format(self.proc_path)
        # read and process /proc/stat
        dat = get_file_map(fname, 0, 0)
        ts = time.time()
        # parse
        for key, val in dat.items():
            mstr = key.replace(":", "")
            # cpu is the only special metric
            if val is None:
                self.log.error("proc_meminfo: null value for {0}".format(mstr))
                continue
            elif mstr in skip:
                continue
            else:
                #mval = dcalc.per_second(key, float(val[0]), ts)
                result.add(plumd.Int(mstr, val[0]))
        return result


    def proc_loadavg(self):
        """Return 1, 5 and 15 minute load averages from proc file loadavg.

        :rtype: plumd.Result
        """
        result = plumd.Result("loadavg")
        fname = "{0}/loadavg".format(self.proc_path)
        dat = []
        # read and process /proc/stat
        try:
            dat = get_file(fname).split()
        except Exception as e:
            tb = traceback.format_exc()
            self.log.error("proc_loadavg: exception: {0} : {1}".format(e, tb))
            return result
        if len(dat) >= 3:
            result.add(plumd.Float("1", dat[0]))
            result.add(plumd.Float("5", dat[1]))
            result.add(plumd.Float("15", dat[2]))
        return result


    def proc_swap(self):
        """Return swap file usage metrics from proc file swap.

        :rtype: plumd.Result
        """
        result = plumd.Result("swap")
        fname = "{0}/swaps".format(self.proc_path)
        dat = []
        # read and process /proc/stat
        dat = get_file_list(fname)
        # header: file, type, size, used, priority
        if len(dat) > 1:
            dat.popleft()
        for entry in dat:
            if not entry:
                continue
            #sfname, stype, ssize, sused, sprio = ("", None, 0, 0, 0)
            try:
                sfname, stype, ssize, sused, sprio = entry.split()
            except Exception as e:
                tb = traceback.format_exc()
                self.log.error("proc_swap: exception: {0}: {1}".format(e, tb))
                continue
            sname = os.path.basename(sfname)
            mstr = "{0}.used".format(sname)
            result.add(plumd.Float(mstr, sused))
            mstr = "{0}.size".format(sname)
            result.add(plumd.Float(mstr, ssize))
            sfree = float(ssize) - float(sused)
            mstr = "{0}.free".format(sname)
            result.add(plumd.Float(mstr, sfree))
        return result


    def proc_uptime(self):
        """Return uptime from proc file swap.

        :rtype: plumd.Result
        """
        result = plumd.Result("uptime")
        fname = "{0}/uptime".format(self.proc_path)
        dat = []
        # read and process /proc/stat
        try:
            up, idle = get_file(fname).split()
        except Exception as e:
            tb = traceback.format_exc()
            self.log.error("proc_uptime: exception: {0}: {1}".format(e, tb))
            return result
        pidle = float(idle)/float(up) * 100 / multiprocessing.cpu_count()
        result.add(plumd.Float("up", up))
        result.add(plumd.Float("idle", idle))

This code seems to be duplicated in your project.

        result.add(plumd.Float("idle_percent", pidle))
        return result


    def proc_diskstats(self):
        """Return disk io metrics from proc file diskstats.

        :rtype: plumd.Result
        """
        # times in ms
        cols = self.config.get('diskstats_cols')
        result = plumd.Result("diskstats")
        fname = "{0}/diskstats".format(self.proc_path)
        dat = {}
        # read and process /proc/stat
        try:
            dat = get_file_map(fname, 2, 0)
        except Exception as e:
            tb = traceback.format_exc()
            self.log.error("proc_diskstats: exception: {0}: {1}".format(e, tb))
            return result
        ts = time.time()
        for key, val in dat.items():
            if self.diskstats_dev_re.match(key):
                continue
            if len(val) != 13:
                self.log.error("proc_diskstats: invalid entry: {0}".format(val))
                continue
            for mname in cols:
                mval = int(val.popleft())
                mstr = "{0}.{1}".format(key, mname)
                dval = self.calc.per_second(mstr, mval, ts)

This code seems to be duplicated in your project.

                result.add(plumd.Int(mstr, dval))
        return result


    def proc_net_dev(self):
        """Return network interface metrics from proc file net/dev.

        Add entries to the configuration value 'net_dev_re' to skip
        any network interfaces that match the regular expression.

        :rtype: plumd.Result
        """
        cols = self.config.get('net_dev_cols')
        result = plumd.Result("net")
        fname = "{0}/net/dev".format(self.proc_path)
        dat = {}
        # read and process /proc/stat
        try:
            dat = get_file_map(fname, 0, 0)
        except Exception as e:
            tb = traceback.format_exc()
            self.log.error("proc_net_dev: exception: {0}: {1}".format(e, tb))
            return result
        ts = time.time()
        for key, val in dat.items():
            key = key.replace(":", "")
            if self.net_dev_re.match(key):
                continue
            if len(val) < len(cols):
                #self.log.error("proc_net_dev: invalid entry: {0}".format(val))
                continue
            for mname in cols:
                if len(val) < 1:
                    break
                mval = int(val.popleft())
                mstr = "{0}.{1}".format(key, mname)
                dval = self.calc.per_second(mstr, mval, ts)
                result.add(plumd.Int(mstr, dval))
        return result


    def proc_net_snmp(self):
        """Return network protocol metrics from proc file net/snmp.

        Add entries to the configuration value 'skip_proc_net_snmp' to skip
        metrics.

        Add entries to the configuration value 'net_snmp_items' to match the
        format/order of the proc file net/snmp entries on the system.

        :rtype: plumd.Result
        """
        skip = self.config.get('skip_proc_net_snmp')
        items = self.config.get('net_snmp_items')
        result = plumd.Result("net_snmp")
        fname = "{0}/net/snmp".format(self.proc_path)
        dat = {}
        # read and process - dat is a list of lines from fname
        dat = get_file_list(fname)
        ts = time.time()

        # process each pair of lines
        for item in items:
            # older kernels may not have all items
            if len(dat) < 2:
                break
            try:
                # first line is a list of: metric: header values
                header = deque(dat.popleft().split())
                # second line is a list of: <metric>: metric values
                vals = deque([ int(i) for i in dat.popleft().split()[1:] ])
            except Exception as e:
                tb = traceback.format_exc()
                self.log.error("proc_net_snmp: exception: {0}: {1}".format(e, tb))
                continue
            if len(header) < 2 or header[0] != item:
                self.log.error("proc_net_snmp: invalid entry: {0}: {1}".format(header, item))
                continue
            # first value is the name of the metric eg. Ip, Icmp, etc
            mheader = header.popleft().replace(":", "")
            for mname in header:
                if len(vals) < 1:
                    break
                mval = vals.popleft()
                mstr = "{0}.{1}".format(mheader, mname)
                if mstr in skip:
                    continue
                dval = self.calc.per_second(mstr, mval, ts)
                result.add(plumd.Int(mstr, dval))
        return result


    def proc_net_sockstat(self):
        """Return network socket metrics from proc file net/sockstat.

        Note: sockstat.TCP.mem is measured in pages, you can get the system page
        size from os.sysconf("SC_PAGESIZE")

        Note: FRAG: ip fragmentation related

        :rtype: plumd.Result
        """
        skip = self.config.get('skip_proc_net_sockstat')
        result = plumd.Result("sockstat")
        fname = "{0}/net/sockstat".format(self.proc_path)
        # sys/net/ipv4/tcp_mem format: min, pressure, max
        fname_limits = "{0}/sys/net/ipv4/tcp_mem".format(self.proc_path)
        # orphan limit: /proc/sys/net/ipv4/tcp_max_orphans
        fname_orph = "{0}/sys/net/ipv4/tcp_max_orphans".format(self.proc_path)
        dat = {}
        # read and process - dat is a list of lines from fname
        dat = get_file_map_list(fname, 0, 0)
        ts = time.time()
        # each entry is a key: [metric, val, metric, val]
        for key, val in dat.items():
            if len(val) < 2:
                continue
            mstr = key.replace(":", "")
            if mstr in skip:
                continue
            mnames = val[::2]
            mvals = deque([ int(i) for i in val[1::2] ])
            if len(mnames) != len(mvals):
                self.log.error("proc_net_sockstat: invalid entry: {0}".format(mnames))
                continue
            for mname in mnames:
                metric = "{0}.{1}".format(mstr, mname)
                result.add(plumd.Int(metric, mvals.popleft()))
        # also record configured tcp mem limits
        dat = get_file(fname_limits).split()
        if len(dat) == 3:
            # eg. for alerting/dashboard on pages allocated vs max values
            result.add(plumd.Int("TCP.mem_min", dat[0]))
            result.add(plumd.Int("TCP.mem_pressure", dat[1]))
            result.add(plumd.Int("TCP.mem_max", dat[1]))
        dat = get_file(fname_orph)
        result.add(plumd.Int("TCP.orphan_max", dat))
        return result


    def proc_net_netstat(self):
        """Return detailed network statitistics proc file net/netstat.

        Note: add entries to the configuration value 'skip_proc_net_netstat' to
        skip metric names (eg. 'TcpExt.TCPMTUPSuccess'). Defaults should be
        reasonable however.

        Note: ECT1Pkts and ECT0Pkts relate to ECT congestion notifications.

        :rtype: plumd.Result
        """
        skip = self.config.get('skip_proc_net_netstat')
        result = plumd.Result("netstat")
        fname = "{0}/net/netstat".format(self.proc_path)
        dat = {}
        # read and process - dat is a list of lines from fname
        dat = get_file_list(fname)
        ts = time.time()
        while len(dat) > 1:
            headers = deque(dat.popleft().split())
            if len(dat) < 1 or len(headers) < 1:
                break
            mvals = deque([ int(i) for i in dat.popleft().split()[1:] ])
            mstr = headers.popleft().replace(":", "")
            if len(headers) != len(mvals):
                self.log.error("proc_net_netstat: invalid entry: {0}".format(headers))
                continue
            for mname in headers:
                if len(mvals) < 1:
                    break
                metric = "{0}.{1}".format(mstr, mname)
                if metric in skip:
                    continue
                dval = self.calc.per_second(metric, mvals.popleft(), ts)
                result.add(plumd.Int(metric, dval))
        return result



    def proc_conntrack(self):
        """Return current conntrack count and max.

        :rtype: plumd.Result
        """
        result = plumd.Result("conntrack")
        fname_cnt = "{0}/sys/net/netfilter/nf_conntrack_count"
        fname_cnt = fname_cnt.format(self.proc_path)
        fname_max = "{0}/sys/net/nf_conntrack_max".format(self.proc_path)
        if not os.path.isfile(fname_cnt):
            # try these - either no iptables or older kernel
            fname_cnt = "{0}/sys/net/ipv4/netfilter/ip_conntrack_count"
            fname_cnt = fname_cnt.format(self.proc_path)
            fname_max = "{0}/sys/net/ipv4/netfilter/ip_conntrack_max"
            fname_max = fname_max.format(self.proc_path)
        curr_val = 0
        max_val = 0
        # read and process /proc/stat
        try:
            curr_val = get_file(fname_cnt)
            max_val = get_file(fname_max)
        except Exception as e:
            tb = traceback.format_exc()
            self.log.error("proc_conntrack: exception: {0}: {1}".format(e, tb))
            return result
        result.add(plumd.Int("cur", curr_val))
        result.add(plumd.Int("max", max_val))
        return result