GlancesProcesses.__get_standard_stats() - Code Metrics - Inspection of "Generate documentation for Glances 2.8" - nicolargo/glances - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 2b80fa...6ea077 )

by Nicolas

created 2017-01-21 13:48 UTC

GlancesProcesses.__get_standard_stats() A

↳ Parent: GlancesProcesses

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	2
c	1
b	0
f	0
dl	0
loc	17
rs	9.4285

# -*- coding: utf-8 -*-
#
# This file is part of Glances.
#
# Copyright (C) 2017 Nicolargo <[email protected]>
#
# Glances is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Glances is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import operator
import os

from glances.compat import iteritems, itervalues, listitems
from glances.globals import BSD, LINUX, MACOS, WINDOWS
from glances.timer import Timer, getTimeSinceLastUpdate
from glances.processes_tree import ProcessTreeNode
from glances.filter import GlancesFilter
from glances.logger import logger

import psutil


def is_kernel_thread(proc):
    """Return True if proc is a kernel thread, False instead."""
    try:
        return os.getpgid(proc.pid) == 0
    # Python >= 3.3 raises ProcessLookupError, which inherits OSError
    except OSError:
        # return False is process is dead
        return False


class GlancesProcesses(object):

    """Get processed stats using the psutil library."""

    def __init__(self, cache_timeout=60):
        """Init the class to collect stats about processes."""
        # Add internals caches because PSUtil do not cache all the stats
        # See: https://code.google.com/p/psutil/issues/detail?id=462
        self.username_cache = {}
        self.cmdline_cache = {}

        # The internals caches will be cleaned each 'cache_timeout' seconds
        self.cache_timeout = cache_timeout
        self.cache_timer = Timer(self.cache_timeout)

        # Init the io dict
        # key = pid
        # value = [ read_bytes_old, write_bytes_old ]
        self.io_old = {}

        # Wether or not to enable process tree
        self._enable_tree = False
        self.process_tree = None

        # Init stats
        self.auto_sort = True
        self._sort_key = 'cpu_percent'
        self.allprocesslist = []
        self.processlist = []
        self.reset_processcount()

        # Tag to enable/disable the processes stats (to reduce the Glances CPU consumption)
        # Default is to enable the processes stats
        self.disable_tag = False

        # Extended stats for top process is enable by default
        self.disable_extended_tag = False

        # Maximum number of processes showed in the UI (None if no limit)
        self._max_processes = None

        # Process filter is a regular expression
        self._filter = GlancesFilter()

        # Whether or not to hide kernel threads
        self.no_kernel_threads = False

        # Store maximums values in a dict
        # Used in the UI to highlight the maximum value
        self._max_values_list = ('cpu_percent', 'memory_percent')
        # { 'cpu_percent': 0.0, 'memory_percent': 0.0 }
        self._max_values = {}
        self.reset_max_values()

    def reset_processcount(self):
        self.processcount = {'total': 0,
                             'running': 0,
                             'sleeping': 0,
                             'thread': 0,
                             'pid_max': None}

    def enable(self):
        """Enable process stats."""
        self.disable_tag = False
        self.update()

    def disable(self):
        """Disable process stats."""
        self.disable_tag = True

    def enable_extended(self):
        """Enable extended process stats."""
        self.disable_extended_tag = False
        self.update()

    def disable_extended(self):
        """Disable extended process stats."""
        self.disable_extended_tag = True

    @property
    def pid_max(self):
        """
        Get the maximum PID value.

        On Linux, the value is read from the `/proc/sys/kernel/pid_max` file.

        From `man 5 proc`:
        The default value for this file, 32768, results in the same range of
        PIDs as on earlier kernels. On 32-bit platfroms, 32768 is the maximum
        value for pid_max. On 64-bit systems, pid_max can be set to any value
        up to 2^22 (PID_MAX_LIMIT, approximately 4 million).

        If the file is unreadable or not available for whatever reason,
        returns None.

        Some other OSes:
        - On FreeBSD and macOS the maximum is 99999.
        - On OpenBSD >= 6.0 the maximum is 99999 (was 32766).
        - On NetBSD the maximum is 30000.

        :returns: int or None
        """
        if LINUX:
            # XXX: waiting for https://github.com/giampaolo/psutil/issues/720
            try:
                with open('/proc/sys/kernel/pid_max', 'rb') as f:
                    return int(f.read())
            except (OSError, IOError):
                return None

    @property
    def max_processes(self):
        """Get the maximum number of processes showed in the UI."""
        return self._max_processes

    @max_processes.setter
    def max_processes(self, value):
        """Set the maximum number of processes showed in the UI."""
        self._max_processes = value

    @property
    def process_filter_input(self):
        """Get the process filter (given by the user)."""
        return self._filter.filter_input

    @property
    def process_filter(self):
        """Get the process filter (current apply filter)."""
        return self._filter.filter

    @process_filter.setter
    def process_filter(self, value):
        """Set the process filter."""
        self._filter.filter = value

    @property
    def process_filter_key(self):
        """Get the process filter key."""
        return self._filter.filter_key

    @property
    def process_filter_re(self):
        """Get the process regular expression compiled."""
        return self._filter.filter_re

    def disable_kernel_threads(self):
        """Ignore kernel threads in process list."""
        self.no_kernel_threads = True

    def enable_tree(self):
        """Enable process tree."""
        self._enable_tree = True

    def is_tree_enabled(self):
        """Return True if process tree is enabled, False instead."""
        return self._enable_tree

    @property
    def sort_reverse(self):
        """Return True to sort processes in reverse 'key' order, False instead."""
        if self.sort_key == 'name' or self.sort_key == 'username':
            return False

        return True

    def max_values(self):
        """Return the max values dict."""
        return self._max_values

    def get_max_values(self, key):
        """Get the maximum values of the given stat (key)."""
        return self._max_values[key]

    def set_max_values(self, key, value):
        """Set the maximum value for a specific stat (key)."""
        self._max_values[key] = value

    def reset_max_values(self):
        """Reset the maximum values dict."""
        self._max_values = {}
        for k in self._max_values_list:
            self._max_values[k] = 0.0

    def __get_mandatory_stats(self, proc, procstat):
        """
        Get mandatory_stats: for all processes.
        Needed for the sorting/filter step.

        Stats grabbed inside this method:
        * 'name', 'cpu_times', 'status', 'ppid'
        * 'username', 'cpu_percent', 'memory_percent'
        """
        procstat['mandatory_stats'] = True

        # Name, cpu_times, status and ppid stats are in the same /proc file
        # Optimisation fir issue #958
        try:
            procstat.update(proc.as_dict(
                attrs=['name', 'cpu_times', 'status', 'ppid'],
                ad_value=''))
        except psutil.NoSuchProcess:
            # Try/catch for issue #432 (process no longer exist)
            return None
        else:
            procstat['status'] = str(procstat['status'])[:1].upper()

        try:
            procstat.update(proc.as_dict(
                attrs=['username', 'cpu_percent', 'memory_percent'],
                ad_value=''))
        except psutil.NoSuchProcess:
            # Try/catch for issue #432 (process no longer exist)
            return None

        if procstat['cpu_percent'] == '' or procstat['memory_percent'] == '':
            # Do not display process if we cannot get the basic
            # cpu_percent or memory_percent stats
            return None

        # Compute the maximum value for cpu_percent and memory_percent
        for k in self._max_values_list:
            if procstat[k] > self.get_max_values(k):
                self.set_max_values(k, procstat[k])

        # Process command line (cached with internal cache)
        if procstat['pid'] not in self.cmdline_cache:
            # Patch for issue #391
            try:
                self.cmdline_cache[procstat['pid']] = proc.cmdline()
            except (AttributeError, UnicodeDecodeError, psutil.AccessDenied, psutil.NoSuchProcess):
                self.cmdline_cache[procstat['pid']] = ""
        procstat['cmdline'] = self.cmdline_cache[procstat['pid']]

        # Process IO
        # procstat['io_counters'] is a list:
        # [read_bytes, write_bytes, read_bytes_old, write_bytes_old, io_tag]
        # If io_tag = 0 > Access denied (display "?")
        # If io_tag = 1 > No access denied (display the IO rate)
        # Availability: all platforms except macOS and Illumos/Solaris
        try:
            # Get the process IO counters
            proc_io = proc.io_counters()
            io_new = [proc_io.read_bytes, proc_io.write_bytes]
        except (psutil.AccessDenied, psutil.NoSuchProcess, NotImplementedError):
            # Access denied to process IO (no root account)
            # NoSuchProcess (process die between first and second grab)
            # Put 0 in all values (for sort) and io_tag = 0 (for display)
            procstat['io_counters'] = [0, 0] + [0, 0]
            io_tag = 0
        except AttributeError:
            return procstat
        else:
            # For IO rate computation
            # Append saved IO r/w bytes
            try:
                procstat['io_counters'] = io_new + self.io_old[procstat['pid']]
            except KeyError:
                procstat['io_counters'] = io_new + [0, 0]
            # then save the IO r/w bytes
            self.io_old[procstat['pid']] = io_new
            io_tag = 1

        # Append the IO tag (for display)
        procstat['io_counters'] += [io_tag]

        return procstat

    def __get_standard_stats(self, proc, procstat):
        """
        Get standard_stats: only for displayed processes.

        Stats grabbed inside this method:
        * nice and memory_info
        """
        procstat['standard_stats'] = True

        # Process nice and memory_info (issue #926)
        try:
            procstat.update(
                proc.as_dict(attrs=['nice', 'memory_info']))
        except psutil.NoSuchProcess:
            pass

        return procstat

    def __get_extended_stats(self, proc, procstat):
        """
        Get extended stats, only for top processes (see issue #403).

        - cpu_affinity (Linux, Windows, FreeBSD)
        - ionice (Linux and Windows > Vista)
        - memory_full_info (Linux)
        - num_ctx_switches (not available on Illumos/Solaris)
        - num_fds (Unix-like)
        - num_handles (Windows)
        - num_threads (not available on *BSD)
        - memory_maps (only swap, Linux)
          https://www.cyberciti.biz/faq/linux-which-process-is-using-swap/
        - connections (TCP and UDP)
        """
        procstat['extended_stats'] = True

        for stat in ['cpu_affinity', 'ionice', 'memory_full_info',
                     'num_ctx_switches', 'num_fds', 'num_handles',
                     'num_threads']:
            try:
                procstat.update(proc.as_dict(attrs=[stat]))
            except psutil.NoSuchProcess:
                pass
            # XXX: psutil>=4.3.1 raises ValueError while <4.3.1 raises AttributeError
            except (ValueError, AttributeError):
                procstat[stat] = None

        if LINUX:
            try:
                procstat['memory_swap'] = sum([v.swap for v in proc.memory_maps()])
            except psutil.NoSuchProcess:
                pass
            except (psutil.AccessDenied, TypeError, NotImplementedError):
                # NotImplementedError: /proc/${PID}/smaps file doesn't exist
                # on kernel < 2.6.14 or CONFIG_MMU kernel configuration option
                # is not enabled (see psutil #533/glances #413).
                # XXX: Remove TypeError once we'll drop psutil < 3.0.0.
                procstat['memory_swap'] = None

        try:
            procstat['tcp'] = len(proc.connections(kind="tcp"))
            procstat['udp'] = len(proc.connections(kind="udp"))
        except psutil.AccessDenied:
            procstat['tcp'] = None
            procstat['udp'] = None

        return procstat

    def __get_process_stats(self, proc,
                            mandatory_stats=True,
                            standard_stats=True,
                            extended_stats=False):
        """Get stats of a running processes."""
        # Process ID (always)
        procstat = proc.as_dict(attrs=['pid'])

        if mandatory_stats:
            procstat = self.__get_mandatory_stats(proc, procstat)

        if procstat is not None and standard_stats:
            procstat = self.__get_standard_stats(proc, procstat)

        if procstat is not None and extended_stats and not self.disable_extended_tag:
            procstat = self.__get_extended_stats(proc, procstat)

        return procstat

    def update(self):
        """Update the processes stats."""
        # Reset the stats
        self.processlist = []
        self.reset_processcount()

        # Do not process if disable tag is set
        if self.disable_tag:
            return

        # Get the time since last update
        time_since_update = getTimeSinceLastUpdate('process_disk')

        # Reset the max dict
        self.reset_max_values()

        # Update the maximum process ID (pid) number
        self.processcount['pid_max'] = self.pid_max

        # Build an internal dict with only mandatories stats (sort keys)
        processdict = {}
        excluded_processes = set()
        for proc in psutil.process_iter():
            # Ignore kernel threads if needed
            if self.no_kernel_threads and not WINDOWS and is_kernel_thread(proc):
                continue

            # If self.max_processes is None: Only retrieve mandatory stats
            # Else: retrieve mandatory and standard stats
            s = self.__get_process_stats(proc,
                                         mandatory_stats=True,
                                         standard_stats=self.max_processes is None)
            # Check if s is note None (issue #879)
            # ignore the 'idle' process on Windows and *BSD
            # ignore the 'kernel_task' process on macOS
            # waiting for upstream patch from psutil
            if (s is None or
                    BSD and s['name'] == 'idle' or
                    WINDOWS and s['name'] == 'System Idle Process' or
                    MACOS and s['name'] == 'kernel_task'):
                continue
            # Continue to the next process if it has to be filtered
            if self._filter.is_filtered(s):
                excluded_processes.add(proc)
                continue

            # Ok add the process to the list
            processdict[proc] = s
            # Update processcount (global statistics)
            try:
                self.processcount[str(proc.status())] += 1
            except KeyError:
                # Key did not exist, create it
                try:
                    self.processcount[str(proc.status())] = 1
                except psutil.NoSuchProcess:
                    pass
            except psutil.NoSuchProcess:
                pass
            else:
                self.processcount['total'] += 1
            # Update thread number (global statistics)
            try:
                self.processcount['thread'] += proc.num_threads()
            except Exception:
                pass

        if self._enable_tree:
            self.process_tree = ProcessTreeNode.build_tree(processdict,
                                                           self.sort_key,
                                                           self.sort_reverse,
                                                           self.no_kernel_threads,
                                                           excluded_processes)

            for i, node in enumerate(self.process_tree):
                # Only retreive stats for visible processes (max_processes)
                if self.max_processes is not None and i >= self.max_processes:
                    break

                # add standard stats
                new_stats = self.__get_process_stats(node.process,
                                                     mandatory_stats=False,
                                                     standard_stats=True,
                                                     extended_stats=False)
                if new_stats is not None:
                    node.stats.update(new_stats)

                # Add a specific time_since_update stats for bitrate
                node.stats['time_since_update'] = time_since_update

        else:
            # Process optimization
            # Only retreive stats for visible processes (max_processes)
            if self.max_processes is not None:
                # Sort the internal dict and cut the top N (Return a list of tuple)
                # tuple=key (proc), dict (returned by __get_process_stats)
                try:
                    processiter = sorted(iteritems(processdict),
                                         key=lambda x: x[1][self.sort_key],
                                         reverse=self.sort_reverse)
                except (KeyError, TypeError) as e:
                    logger.error("Cannot sort process list by {}: {}".format(self.sort_key, e))
                    logger.error('{}'.format(listitems(processdict)[0]))
                    # Fallback to all process (issue #423)
                    processloop = iteritems(processdict)
                    first = False
                else:
                    processloop = processiter[0:self.max_processes]
                    first = True
            else:
                # Get all processes stats
                processloop = iteritems(processdict)
                first = False

            for i in processloop:
                # Already existing mandatory stats
                procstat = i[1]
                if self.max_processes is not None:
                    # Update with standard stats
                    # and extended stats but only for TOP (first) process
                    s = self.__get_process_stats(i[0],
                                                 mandatory_stats=False,
                                                 standard_stats=True,
                                                 extended_stats=first)
                    if s is None:
                        continue
                    procstat.update(s)
                # Add a specific time_since_update stats for bitrate
                procstat['time_since_update'] = time_since_update
                # Update process list
                self.processlist.append(procstat)
                # Next...
                first = False

        # Build the all processes list used by the AMPs
        self.allprocesslist = [p for p in itervalues(processdict)]

        # Clean internals caches if timeout is reached
        if self.cache_timer.finished():
            self.username_cache = {}
            self.cmdline_cache = {}
            # Restart the timer
            self.cache_timer.reset()

    def getcount(self):
        """Get the number of processes."""
        return self.processcount

    def getalllist(self):
        """Get the allprocesslist."""
        return self.allprocesslist

    def getlist(self, sortedby=None):
        """Get the processlist."""
        return self.processlist

    def gettree(self):
        """Get the process tree."""
        return self.process_tree

    @property
    def sort_key(self):
        """Get the current sort key."""
        return self._sort_key

    @sort_key.setter
    def sort_key(self, key):
        """Set the current sort key."""
        self._sort_key = key


# TODO: move this global function (also used in glances_processlist
#       and logs) inside the GlancesProcesses class
def sort_stats(stats, sortedby=None, tree=False, reverse=True):
    """Return the stats (dict) sorted by (sortedby)
    Reverse the sort if reverse is True."""
    if sortedby is None:
        # No need to sort...
        return stats

    if sortedby == 'io_counters' and not tree:
        # Specific case for io_counters
        # Sum of io_r + io_w
        try:
            # Sort process by IO rate (sum IO read + IO write)
            stats.sort(key=lambda process: process[sortedby][0] -
                       process[sortedby][2] + process[sortedby][1] -
                       process[sortedby][3],
                       reverse=reverse)
        except Exception:
            stats.sort(key=operator.itemgetter('cpu_percent'),
                       reverse=reverse)
    else:
        # Others sorts
        if tree:
            stats.set_sorting(sortedby, reverse)
        else:
            try:
                stats.sort(key=operator.itemgetter(sortedby),
                           reverse=reverse)
            except (KeyError, TypeError):
                stats.sort(key=operator.itemgetter('name'),
                           reverse=False)

    return stats


glances_processes = GlancesProcesses()


1			# -- coding: utf-8 --
2			#
3			# This file is part of Glances.
4			#
5			# Copyright (C) 2017 Nicolargo <[email protected]>
6			#
7			# Glances is free software; you can redistribute it and/or modify
8			# it under the terms of the GNU Lesser General Public License as published by
9			# the Free Software Foundation, either version 3 of the License, or
10			# (at your option) any later version.
11			#
12			# Glances is distributed in the hope that it will be useful,
13			# but WITHOUT ANY WARRANTY; without even the implied warranty of
14			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15			# GNU Lesser General Public License for more details.
16			#
17			# You should have received a copy of the GNU Lesser General Public License
18			# along with this program. If not, see <http://www.gnu.org/licenses/>.
19
20			import operator
21			import os
22
23			from glances.compat import iteritems, itervalues, listitems
24			from glances.globals import BSD, LINUX, MACOS, WINDOWS
25			from glances.timer import Timer, getTimeSinceLastUpdate
26			from glances.processes_tree import ProcessTreeNode
27			from glances.filter import GlancesFilter
28			from glances.logger import logger
29
30			import psutil
31
32
33			def is_kernel_thread(proc):
34			"""Return True if proc is a kernel thread, False instead."""
35			try:
36			return os.getpgid(proc.pid) == 0
37			# Python >= 3.3 raises ProcessLookupError, which inherits OSError
38			except OSError:
39			# return False is process is dead
40			return False
41
42
43			class GlancesProcesses(object):
44
45			"""Get processed stats using the psutil library."""
46
47			def __init__(self, cache_timeout=60):
48			"""Init the class to collect stats about processes."""
49			# Add internals caches because PSUtil do not cache all the stats
50			# See: https://code.google.com/p/psutil/issues/detail?id=462
51			self.username_cache = {}
52			self.cmdline_cache = {}
53
54			# The internals caches will be cleaned each 'cache_timeout' seconds
55			self.cache_timeout = cache_timeout
56			self.cache_timer = Timer(self.cache_timeout)
57
58			# Init the io dict
59			# key = pid
60			# value = [ read_bytes_old, write_bytes_old ]
61			self.io_old = {}
62
63			# Wether or not to enable process tree
64			self._enable_tree = False
65			self.process_tree = None
66
67			# Init stats
68			self.auto_sort = True
69			self._sort_key = 'cpu_percent'
70			self.allprocesslist = []
71			self.processlist = []
72			self.reset_processcount()
73
74			# Tag to enable/disable the processes stats (to reduce the Glances CPU consumption)
75			# Default is to enable the processes stats
76			self.disable_tag = False
77
78			# Extended stats for top process is enable by default
79			self.disable_extended_tag = False
80
81			# Maximum number of processes showed in the UI (None if no limit)
82			self._max_processes = None
83
84			# Process filter is a regular expression
85			self._filter = GlancesFilter()
86
87			# Whether or not to hide kernel threads
88			self.no_kernel_threads = False
89
90			# Store maximums values in a dict
91			# Used in the UI to highlight the maximum value
92			self._max_values_list = ('cpu_percent', 'memory_percent')
93			# { 'cpu_percent': 0.0, 'memory_percent': 0.0 }
94			self._max_values = {}
95			self.reset_max_values()
96
97			def reset_processcount(self):
98			self.processcount = {'total': 0,
99			'running': 0,
100			'sleeping': 0,
101			'thread': 0,
102			'pid_max': None}
103
104			def enable(self):
105			"""Enable process stats."""
106			self.disable_tag = False
107			self.update()
108
109			def disable(self):
110			"""Disable process stats."""
111			self.disable_tag = True
112
113			def enable_extended(self):
114			"""Enable extended process stats."""
115			self.disable_extended_tag = False
116			self.update()
117
118			def disable_extended(self):
119			"""Disable extended process stats."""
120			self.disable_extended_tag = True
121
122			@property
123			def pid_max(self):
124			"""
125			Get the maximum PID value.
126
127			On Linux, the value is read from the `/proc/sys/kernel/pid_max` file.
128
129			From `man 5 proc`:
130			The default value for this file, 32768, results in the same range of
131			PIDs as on earlier kernels. On 32-bit platfroms, 32768 is the maximum
132			value for pid_max. On 64-bit systems, pid_max can be set to any value
133			up to 2^22 (PID_MAX_LIMIT, approximately 4 million).
134
135			If the file is unreadable or not available for whatever reason,
136			returns None.
137
138			Some other OSes:
139			- On FreeBSD and macOS the maximum is 99999.
140			- On OpenBSD >= 6.0 the maximum is 99999 (was 32766).
141			- On NetBSD the maximum is 30000.
142
143			:returns: int or None
144			"""
145			if LINUX:
146			# XXX: waiting for https://github.com/giampaolo/psutil/issues/720
147			try:
148			with open('/proc/sys/kernel/pid_max', 'rb') as f:
149			return int(f.read())
150			except (OSError, IOError):
151			return None
152
153			@property
154			def max_processes(self):
155			"""Get the maximum number of processes showed in the UI."""
156			return self._max_processes
157
158			@max_processes.setter
159			def max_processes(self, value):
160			"""Set the maximum number of processes showed in the UI."""
161			self._max_processes = value
162
163			@property
164			def process_filter_input(self):
165			"""Get the process filter (given by the user)."""
166			return self._filter.filter_input
167
168			@property
169			def process_filter(self):
170			"""Get the process filter (current apply filter)."""
171			return self._filter.filter
172
173			@process_filter.setter
174			def process_filter(self, value):
175			"""Set the process filter."""
176			self._filter.filter = value
177
178			@property
179			def process_filter_key(self):
180			"""Get the process filter key."""
181			return self._filter.filter_key
182
183			@property
184			def process_filter_re(self):
185			"""Get the process regular expression compiled."""
186			return self._filter.filter_re
187
188			def disable_kernel_threads(self):
189			"""Ignore kernel threads in process list."""
190			self.no_kernel_threads = True
191
192			def enable_tree(self):
193			"""Enable process tree."""
194			self._enable_tree = True
195
196			def is_tree_enabled(self):
197			"""Return True if process tree is enabled, False instead."""
198			return self._enable_tree
199
200			@property
201			def sort_reverse(self):
202			"""Return True to sort processes in reverse 'key' order, False instead."""
203			if self.sort_key == 'name' or self.sort_key == 'username':
204			return False
205
206			return True
207
208			def max_values(self):
209			"""Return the max values dict."""
210			return self._max_values
211
212			def get_max_values(self, key):
213			"""Get the maximum values of the given stat (key)."""
214			return self._max_values[key]
215
216			def set_max_values(self, key, value):
217			"""Set the maximum value for a specific stat (key)."""
218			self._max_values[key] = value
219
220			def reset_max_values(self):
221			"""Reset the maximum values dict."""
222			self._max_values = {}
223			for k in self._max_values_list:
224			self._max_values[k] = 0.0
225
226			def __get_mandatory_stats(self, proc, procstat):
227			"""
228			Get mandatory_stats: for all processes.
229			Needed for the sorting/filter step.
230
231			Stats grabbed inside this method:
232			* 'name', 'cpu_times', 'status', 'ppid'
233			* 'username', 'cpu_percent', 'memory_percent'
234			"""
235			procstat['mandatory_stats'] = True
236
237			# Name, cpu_times, status and ppid stats are in the same /proc file
238			# Optimisation fir issue #958
239			try:
240			procstat.update(proc.as_dict(
241			attrs=['name', 'cpu_times', 'status', 'ppid'],
242			ad_value=''))
243			except psutil.NoSuchProcess:
244			# Try/catch for issue #432 (process no longer exist)
245			return None
246			else:
247			procstat['status'] = str(procstat['status'])[:1].upper()
248
249			try:
250			procstat.update(proc.as_dict(
251			attrs=['username', 'cpu_percent', 'memory_percent'],
252			ad_value=''))
253			except psutil.NoSuchProcess:
254			# Try/catch for issue #432 (process no longer exist)
255			return None
256
257			if procstat['cpu_percent'] == '' or procstat['memory_percent'] == '':
258			# Do not display process if we cannot get the basic
259			# cpu_percent or memory_percent stats
260			return None
261
262			# Compute the maximum value for cpu_percent and memory_percent
263			for k in self._max_values_list:
264			if procstat[k] > self.get_max_values(k):
265			self.set_max_values(k, procstat[k])
266
267			# Process command line (cached with internal cache)
268			if procstat['pid'] not in self.cmdline_cache:
269			# Patch for issue #391
270			try:
271			self.cmdline_cache[procstat['pid']] = proc.cmdline()
272			except (AttributeError, UnicodeDecodeError, psutil.AccessDenied, psutil.NoSuchProcess):
273			self.cmdline_cache[procstat['pid']] = ""
274			procstat['cmdline'] = self.cmdline_cache[procstat['pid']]
275
276			# Process IO
277			# procstat['io_counters'] is a list:
278			# [read_bytes, write_bytes, read_bytes_old, write_bytes_old, io_tag]
279			# If io_tag = 0 > Access denied (display "?")
280			# If io_tag = 1 > No access denied (display the IO rate)
281			# Availability: all platforms except macOS and Illumos/Solaris
282			try:
283			# Get the process IO counters
284			proc_io = proc.io_counters()
285			io_new = [proc_io.read_bytes, proc_io.write_bytes]
286			except (psutil.AccessDenied, psutil.NoSuchProcess, NotImplementedError):
287			# Access denied to process IO (no root account)
288			# NoSuchProcess (process die between first and second grab)
289			# Put 0 in all values (for sort) and io_tag = 0 (for display)
290			procstat['io_counters'] = [0, 0] + [0, 0]
291			io_tag = 0
292			except AttributeError:
293			return procstat
294			else:
295			# For IO rate computation
296			# Append saved IO r/w bytes
297			try:
298			procstat['io_counters'] = io_new + self.io_old[procstat['pid']]
299			except KeyError:
300			procstat['io_counters'] = io_new + [0, 0]
301			# then save the IO r/w bytes
302			self.io_old[procstat['pid']] = io_new
303			io_tag = 1
304
305			# Append the IO tag (for display)
306			procstat['io_counters'] += [io_tag]
307
308			return procstat
309
310			def __get_standard_stats(self, proc, procstat):
311			"""
312			Get standard_stats: only for displayed processes.
313
314			Stats grabbed inside this method:
315			* nice and memory_info
316			"""
317			procstat['standard_stats'] = True
318
319			# Process nice and memory_info (issue #926)
320			try:
321			procstat.update(
322			proc.as_dict(attrs=['nice', 'memory_info']))
323			except psutil.NoSuchProcess:
324			pass
325
326			return procstat
327
328			def __get_extended_stats(self, proc, procstat):
329			"""
330			Get extended stats, only for top processes (see issue #403).
331
332			- cpu_affinity (Linux, Windows, FreeBSD)
333			- ionice (Linux and Windows > Vista)
334			- memory_full_info (Linux)
335			- num_ctx_switches (not available on Illumos/Solaris)
336			- num_fds (Unix-like)
337			- num_handles (Windows)
338			- num_threads (not available on *BSD)
339			- memory_maps (only swap, Linux)
340			https://www.cyberciti.biz/faq/linux-which-process-is-using-swap/
341			- connections (TCP and UDP)
342			"""
343			procstat['extended_stats'] = True
344
345			for stat in ['cpu_affinity', 'ionice', 'memory_full_info',
346			'num_ctx_switches', 'num_fds', 'num_handles',
347			'num_threads']:
348			try:
349			procstat.update(proc.as_dict(attrs=[stat]))
350			except psutil.NoSuchProcess:
351			pass
352			# XXX: psutil>=4.3.1 raises ValueError while <4.3.1 raises AttributeError
353			except (ValueError, AttributeError):
354			procstat[stat] = None
355
356			if LINUX:
357			try:
358			procstat['memory_swap'] = sum([v.swap for v in proc.memory_maps()])
359			except psutil.NoSuchProcess:
360			pass
361			except (psutil.AccessDenied, TypeError, NotImplementedError):
362			# NotImplementedError: /proc/${PID}/smaps file doesn't exist
363			# on kernel < 2.6.14 or CONFIG_MMU kernel configuration option
364			# is not enabled (see psutil #533/glances #413).
365			# XXX: Remove TypeError once we'll drop psutil < 3.0.0.
366			procstat['memory_swap'] = None
367
368			try:
369			procstat['tcp'] = len(proc.connections(kind="tcp"))
370			procstat['udp'] = len(proc.connections(kind="udp"))
371			except psutil.AccessDenied:
372			procstat['tcp'] = None
373			procstat['udp'] = None
374
375			return procstat
376
377			def __get_process_stats(self, proc,
378			mandatory_stats=True,
379			standard_stats=True,
380			extended_stats=False):
381			"""Get stats of a running processes."""
382			# Process ID (always)
383			procstat = proc.as_dict(attrs=['pid'])
384
385			if mandatory_stats:
386			procstat = self.__get_mandatory_stats(proc, procstat)
387
388			if procstat is not None and standard_stats:
389			procstat = self.__get_standard_stats(proc, procstat)
390
391			if procstat is not None and extended_stats and not self.disable_extended_tag:
392			procstat = self.__get_extended_stats(proc, procstat)
393
394			return procstat
395
396			def update(self):
397			"""Update the processes stats."""
398			# Reset the stats
399			self.processlist = []
400			self.reset_processcount()
401
402			# Do not process if disable tag is set
403			if self.disable_tag:
404			return
405
406			# Get the time since last update
407			time_since_update = getTimeSinceLastUpdate('process_disk')
408
409			# Reset the max dict
410			self.reset_max_values()
411
412			# Update the maximum process ID (pid) number
413			self.processcount['pid_max'] = self.pid_max
414
415			# Build an internal dict with only mandatories stats (sort keys)
416			processdict = {}
417			excluded_processes = set()
418			for proc in psutil.process_iter():
419			# Ignore kernel threads if needed
420			if self.no_kernel_threads and not WINDOWS and is_kernel_thread(proc):
421			continue
422
423			# If self.max_processes is None: Only retrieve mandatory stats
424			# Else: retrieve mandatory and standard stats
425			s = self.__get_process_stats(proc,
426			mandatory_stats=True,
427			standard_stats=self.max_processes is None)
428			# Check if s is note None (issue #879)
429			# ignore the 'idle' process on Windows and *BSD
430			# ignore the 'kernel_task' process on macOS
431			# waiting for upstream patch from psutil
432			if (s is None or
433			BSD and s['name'] == 'idle' or
434			WINDOWS and s['name'] == 'System Idle Process' or
435			MACOS and s['name'] == 'kernel_task'):
436			continue
437			# Continue to the next process if it has to be filtered
438			if self._filter.is_filtered(s):
439			excluded_processes.add(proc)
440			continue
441
442			# Ok add the process to the list
443			processdict[proc] = s
444			# Update processcount (global statistics)
445			try:
446			self.processcount[str(proc.status())] += 1
447			except KeyError:
448			# Key did not exist, create it
449			try:
450			self.processcount[str(proc.status())] = 1
451			except psutil.NoSuchProcess:
452			pass
453			except psutil.NoSuchProcess:
454			pass
455			else:
456			self.processcount['total'] += 1
457			# Update thread number (global statistics)
458			try:
459			self.processcount['thread'] += proc.num_threads()
460			except Exception:
461			pass
462
463			if self._enable_tree:
464			self.process_tree = ProcessTreeNode.build_tree(processdict,
465			self.sort_key,
466			self.sort_reverse,
467			self.no_kernel_threads,
468			excluded_processes)
469
470			for i, node in enumerate(self.process_tree):
471			# Only retreive stats for visible processes (max_processes)
472			if self.max_processes is not None and i >= self.max_processes:
473			break
474
475			# add standard stats
476			new_stats = self.__get_process_stats(node.process,
477			mandatory_stats=False,
478			standard_stats=True,
479			extended_stats=False)
480			if new_stats is not None:
481			node.stats.update(new_stats)
482
483			# Add a specific time_since_update stats for bitrate
484			node.stats['time_since_update'] = time_since_update
485
486			else:
487			# Process optimization
488			# Only retreive stats for visible processes (max_processes)
489			if self.max_processes is not None:
490			# Sort the internal dict and cut the top N (Return a list of tuple)
491			# tuple=key (proc), dict (returned by __get_process_stats)
492			try:
493			processiter = sorted(iteritems(processdict),
494			key=lambda x: x[1][self.sort_key],
495			reverse=self.sort_reverse)
496			except (KeyError, TypeError) as e:
497			logger.error("Cannot sort process list by {}: {}".format(self.sort_key, e))
498			logger.error('{}'.format(listitems(processdict)[0]))
499			# Fallback to all process (issue #423)
500			processloop = iteritems(processdict)
501			first = False
502			else:
503			processloop = processiter[0:self.max_processes]
504			first = True
505			else:
506			# Get all processes stats
507			processloop = iteritems(processdict)
508			first = False
509
510			for i in processloop:
511			# Already existing mandatory stats
512			procstat = i[1]
513			if self.max_processes is not None:
514			# Update with standard stats
515			# and extended stats but only for TOP (first) process
516			s = self.__get_process_stats(i[0],
517			mandatory_stats=False,
518			standard_stats=True,
519			extended_stats=first)
520			if s is None:
521			continue
522			procstat.update(s)
523			# Add a specific time_since_update stats for bitrate
524			procstat['time_since_update'] = time_since_update
525			# Update process list
526			self.processlist.append(procstat)
527			# Next...
528			first = False
529
530			# Build the all processes list used by the AMPs
531			self.allprocesslist = [p for p in itervalues(processdict)]
532
533			# Clean internals caches if timeout is reached
534			if self.cache_timer.finished():
535			self.username_cache = {}
536			self.cmdline_cache = {}
537			# Restart the timer
538			self.cache_timer.reset()
539
540			def getcount(self):
541			"""Get the number of processes."""
542			return self.processcount
543
544			def getalllist(self):
545			"""Get the allprocesslist."""
546			return self.allprocesslist
547
548			def getlist(self, sortedby=None):
549			"""Get the processlist."""
550			return self.processlist
551
552			def gettree(self):
553			"""Get the process tree."""
554			return self.process_tree
555
556			@property
557			def sort_key(self):
558			"""Get the current sort key."""
559			return self._sort_key
560
561			@sort_key.setter
562			def sort_key(self, key):
563			"""Set the current sort key."""
564			self._sort_key = key
565
566
567			# TODO: move this global function (also used in glances_processlist
568			# and logs) inside the GlancesProcesses class
569			def sort_stats(stats, sortedby=None, tree=False, reverse=True):
570			"""Return the stats (dict) sorted by (sortedby)
571			Reverse the sort if reverse is True."""
572			if sortedby is None:
573			# No need to sort...
574			return stats
575
576			if sortedby == 'io_counters' and not tree:
577			# Specific case for io_counters
578			# Sum of io_r + io_w
579			try:
580			# Sort process by IO rate (sum IO read + IO write)
581			stats.sort(key=lambda process: process[sortedby][0] -
582			process[sortedby][2] + process[sortedby][1] -
583			process[sortedby][3],
584			reverse=reverse)
585			except Exception:
586			stats.sort(key=operator.itemgetter('cpu_percent'),
587			reverse=reverse)
588			else:
589			# Others sorts
590			if tree:
591			stats.set_sorting(sortedby, reverse)
592			else:
593			try:
594			stats.sort(key=operator.itemgetter(sortedby),
595			reverse=reverse)
596			except (KeyError, TypeError):
597			stats.sort(key=operator.itemgetter('name'),
598			reverse=False)
599
600			return stats
601
602
603			glances_processes = GlancesProcesses()
604

nicolargo / glances

Push — master ( 2b80fa...6ea077 )

GlancesProcesses.__get_standard_stats() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like