Completed
Push — master ( 1806d1...053f07 )
by Nicolas
01:42
created

GlancesProcesses.process_filter()   A

Complexity

Conditions 3

Size

Total Lines 4

Duplication

Lines 0
Ratio 0 %
Metric Value
cc 3
dl 0
loc 4
rs 10
1
# -*- coding: utf-8 -*-
2
#
3
# This file is part of Glances.
4
#
5
# Copyright (C) 2016 Nicolargo <[email protected]>
6
#
7
# Glances is free software; you can redistribute it and/or modify
8
# it under the terms of the GNU Lesser General Public License as published by
9
# the Free Software Foundation, either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Glances is distributed in the hope that it will be useful,
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU Lesser General Public License for more details.
16
#
17
# You should have received a copy of the GNU Lesser General Public License
18
# along with this program. If not, see <http://www.gnu.org/licenses/>.
19
20
import os
21
import re
22
23
from glances.compat import iteritems, itervalues, listitems
24
from glances.globals import BSD, LINUX, OSX, WINDOWS
25
from glances.logger import logger
26
from glances.timer import Timer, getTimeSinceLastUpdate
27
from glances.processes_tree import ProcessTreeNode
28
29
import psutil
30
31
32
def is_kernel_thread(proc):
33
    """Return True if proc is a kernel thread, False instead."""
34
    try:
35
        return os.getpgid(proc.pid) == 0
36
    # Python >= 3.3 raises ProcessLookupError, which inherits OSError
37
    except OSError:
38
        # return False is process is dead
39
        return False
40
41
42
class GlancesProcesses(object):
43
44
    """Get processed stats using the psutil library."""
45
46
    def __init__(self, cache_timeout=60):
47
        """Init the class to collect stats about processes."""
48
        # Add internals caches because PSUtil do not cache all the stats
49
        # See: https://code.google.com/p/psutil/issues/detail?id=462
50
        self.username_cache = {}
51
        self.cmdline_cache = {}
52
53
        # The internals caches will be cleaned each 'cache_timeout' seconds
54
        self.cache_timeout = cache_timeout
55
        self.cache_timer = Timer(self.cache_timeout)
56
57
        # Init the io dict
58
        # key = pid
59
        # value = [ read_bytes_old, write_bytes_old ]
60
        self.io_old = {}
61
62
        # Wether or not to enable process tree
63
        self._enable_tree = False
64
        self.process_tree = None
65
66
        # Init stats
67
        self.auto_sort = True
68
        self._sort_key = 'cpu_percent'
69
        self.allprocesslist = []
70
        self.processlist = []
71
        self.processcount = {'total': 0, 'running': 0, 'sleeping': 0, 'thread': 0}
72
73
        # Tag to enable/disable the processes stats (to reduce the Glances CPU consumption)
74
        # Default is to enable the processes stats
75
        self.disable_tag = False
76
77
        # Extended stats for top process is enable by default
78
        self.disable_extended_tag = False
79
80
        # Maximum number of processes showed in the UI (None if no limit)
81
        self._max_processes = None
82
83
        # Process filter is a regular expression
84
        self._process_filter = None
85
        self._process_filter_re = None
86
87
        # Whether or not to hide kernel threads
88
        self.no_kernel_threads = False
89
90
    def enable(self):
91
        """Enable process stats."""
92
        self.disable_tag = False
93
        self.update()
94
95
    def disable(self):
96
        """Disable process stats."""
97
        self.disable_tag = True
98
99
    def enable_extended(self):
100
        """Enable extended process stats."""
101
        self.disable_extended_tag = False
102
        self.update()
103
104
    def disable_extended(self):
105
        """Disable extended process stats."""
106
        self.disable_extended_tag = True
107
108
    @property
109
    def max_processes(self):
110
        """Get the maximum number of processes showed in the UI."""
111
        return self._max_processes
112
113
    @max_processes.setter
114
    def max_processes(self, value):
115
        """Set the maximum number of processes showed in the UI."""
116
        self._max_processes = value
117
118
    @property
119
    def process_filter(self):
120
        """Get the process filter."""
121
        return self._process_filter
122
123
    @process_filter.setter
124
    def process_filter(self, value):
125
        """Set the process filter."""
126
        logger.info("Set process filter to {0}".format(value))
127
        self._process_filter = value
128
        if value is not None:
129
            try:
130
                self._process_filter_re = re.compile(value)
131
                logger.debug("Process filter regex compilation OK: {0}".format(self.process_filter))
132
            except Exception:
133
                logger.error("Cannot compile process filter regex: {0}".format(value))
134
                self._process_filter_re = None
135
        else:
136
            self._process_filter_re = None
137
138
    @property
139
    def process_filter_re(self):
140
        """Get the process regular expression compiled."""
141
        return self._process_filter_re
142
143
    def is_filtered(self, value):
144
        """Return True if the value should be filtered."""
145
        if self.process_filter is None:
146
            # No filter => Not filtered
147
            return False
148
        else:
149
            try:
150
                return self.process_filter_re.match(' '.join(value)) is None
151
            except AttributeError:
152
                #  Filter processes crashs with a bad regular expression pattern (issue #665)
153
                return False
154
155
    def disable_kernel_threads(self):
156
        """Ignore kernel threads in process list."""
157
        self.no_kernel_threads = True
158
159
    def enable_tree(self):
160
        """Enable process tree."""
161
        self._enable_tree = True
162
163
    def is_tree_enabled(self):
164
        """Return True if process tree is enabled, False instead."""
165
        return self._enable_tree
166
167
    @property
168
    def sort_reverse(self):
169
        """Return True to sort processes in reverse 'key' order, False instead."""
170
        if self.sort_key == 'name' or self.sort_key == 'username':
171
            return False
172
173
        return True
174
175
    def __get_mandatory_stats(self, proc, procstat):
176
        """
177
        Get mandatory_stats: need for the sorting/filter step.
178
179
        => cpu_percent, memory_percent, io_counters, name, cmdline
180
        """
181
        procstat['mandatory_stats'] = True
182
183
        # Process CPU, MEM percent and name
184
        try:
185
            procstat.update(proc.as_dict(
186
                attrs=['username', 'cpu_percent', 'memory_percent',
187
                       'name', 'cpu_times'], ad_value=''))
188
        except psutil.NoSuchProcess:
189
            # Try/catch for issue #432
190
            return None
191
        if procstat['cpu_percent'] == '' or procstat['memory_percent'] == '':
192
            # Do not display process if we cannot get the basic
193
            # cpu_percent or memory_percent stats
194
            return None
195
196
        # Process command line (cached with internal cache)
197
        try:
198
            self.cmdline_cache[procstat['pid']]
199
        except KeyError:
200
            # Patch for issue #391
201
            try:
202
                self.cmdline_cache[procstat['pid']] = proc.cmdline()
203
            except (AttributeError, UnicodeDecodeError, psutil.AccessDenied, psutil.NoSuchProcess):
204
                self.cmdline_cache[procstat['pid']] = ""
205
        procstat['cmdline'] = self.cmdline_cache[procstat['pid']]
206
207
        # Process IO
208
        # procstat['io_counters'] is a list:
209
        # [read_bytes, write_bytes, read_bytes_old, write_bytes_old, io_tag]
210
        # If io_tag = 0 > Access denied (display "?")
211
        # If io_tag = 1 > No access denied (display the IO rate)
212
        # Note Disk IO stat not available on Mac OS
213
        if not OSX:
214
            try:
215
                # Get the process IO counters
216
                proc_io = proc.io_counters()
217
                io_new = [proc_io.read_bytes, proc_io.write_bytes]
218
            except (psutil.AccessDenied, psutil.NoSuchProcess, NotImplementedError):
219
                # Access denied to process IO (no root account)
220
                # NoSuchProcess (process die between first and second grab)
221
                # Put 0 in all values (for sort) and io_tag = 0 (for
222
                # display)
223
                procstat['io_counters'] = [0, 0] + [0, 0]
224
                io_tag = 0
225
            else:
226
                # For IO rate computation
227
                # Append saved IO r/w bytes
228
                try:
229
                    procstat['io_counters'] = io_new + \
230
                        self.io_old[procstat['pid']]
231
                except KeyError:
232
                    procstat['io_counters'] = io_new + [0, 0]
233
                # then save the IO r/w bytes
234
                self.io_old[procstat['pid']] = io_new
235
                io_tag = 1
236
237
            # Append the IO tag (for display)
238
            procstat['io_counters'] += [io_tag]
239
240
        return procstat
241
242
    def __get_standard_stats(self, proc, procstat):
243
        """
244
        Get standard_stats: for all the displayed processes.
245
246
        => username, status, memory_info, cpu_times
247
        """
248
        procstat['standard_stats'] = True
249
250
        # Process username (cached with internal cache)
251
        try:
252
            self.username_cache[procstat['pid']]
253
        except KeyError:
254
            try:
255
                self.username_cache[procstat['pid']] = proc.username()
256
            except psutil.NoSuchProcess:
257
                self.username_cache[procstat['pid']] = "?"
258
            except (KeyError, psutil.AccessDenied):
259
                try:
260
                    self.username_cache[procstat['pid']] = proc.uids().real
261
                except (KeyError, AttributeError, psutil.AccessDenied):
262
                    self.username_cache[procstat['pid']] = "?"
263
        procstat['username'] = self.username_cache[procstat['pid']]
264
265
        # Process status, nice, memory_info and cpu_times
266
        try:
267
            procstat.update(
268
                proc.as_dict(attrs=['status', 'nice', 'memory_info', 'cpu_times']))
269
        except psutil.NoSuchProcess:
270
            pass
271
        else:
272
            procstat['status'] = str(procstat['status'])[:1].upper()
273
274
        return procstat
275
276
    def __get_extended_stats(self, proc, procstat):
277
        """
278
        Get extended_stats: only for top processes (see issue #403).
279
280
        => connections (UDP/TCP), memory_swap...
281
        """
282
        procstat['extended_stats'] = True
283
284
        # CPU affinity (Windows and Linux only)
285
        try:
286
            procstat.update(proc.as_dict(attrs=['cpu_affinity']))
287
        except psutil.NoSuchProcess:
288
            pass
289
        except AttributeError:
290
            procstat['cpu_affinity'] = None
291
        # Memory extended
292
        try:
293
            procstat.update(proc.as_dict(attrs=['memory_info_ex']))
294
        except psutil.NoSuchProcess:
295
            pass
296
        except AttributeError:
297
            procstat['memory_info_ex'] = None
298
        # Number of context switch
299
        try:
300
            procstat.update(proc.as_dict(attrs=['num_ctx_switches']))
301
        except psutil.NoSuchProcess:
302
            pass
303
        except AttributeError:
304
            procstat['num_ctx_switches'] = None
305
        # Number of file descriptors (Unix only)
306
        try:
307
            procstat.update(proc.as_dict(attrs=['num_fds']))
308
        except psutil.NoSuchProcess:
309
            pass
310
        except AttributeError:
311
            procstat['num_fds'] = None
312
        # Threads number
313
        try:
314
            procstat.update(proc.as_dict(attrs=['num_threads']))
315
        except psutil.NoSuchProcess:
316
            pass
317
        except AttributeError:
318
            procstat['num_threads'] = None
319
320
        # Number of handles (Windows only)
321
        if WINDOWS:
322
            try:
323
                procstat.update(proc.as_dict(attrs=['num_handles']))
324
            except psutil.NoSuchProcess:
325
                pass
326
        else:
327
            procstat['num_handles'] = None
328
329
        # SWAP memory (Only on Linux based OS)
330
        # http://www.cyberciti.biz/faq/linux-which-process-is-using-swap/
331
        if LINUX:
332
            try:
333
                procstat['memory_swap'] = sum(
334
                    [v.swap for v in proc.memory_maps()])
335
            except psutil.NoSuchProcess:
336
                pass
337
            except psutil.AccessDenied:
338
                procstat['memory_swap'] = None
339
            except Exception:
340
                # Add a dirty except to handle the PsUtil issue #413
341
                procstat['memory_swap'] = None
342
343
        # Process network connections (TCP and UDP)
344
        try:
345
            procstat['tcp'] = len(proc.connections(kind="tcp"))
346
            procstat['udp'] = len(proc.connections(kind="udp"))
347
        except Exception:
348
            procstat['tcp'] = None
349
            procstat['udp'] = None
350
351
        # IO Nice
352
        # http://pythonhosted.org/psutil/#psutil.Process.ionice
353
        if LINUX or WINDOWS:
354
            try:
355
                procstat.update(proc.as_dict(attrs=['ionice']))
356
            except psutil.NoSuchProcess:
357
                pass
358
        else:
359
            procstat['ionice'] = None
360
361
        return procstat
362
363
    def __get_process_stats(self, proc,
364
                            mandatory_stats=True,
365
                            standard_stats=True,
366
                            extended_stats=False):
367
        """Get stats of running processes."""
368
        # Process ID (always)
369
        procstat = proc.as_dict(attrs=['pid'])
370
371
        if mandatory_stats:
372
            procstat = self.__get_mandatory_stats(proc, procstat)
373
374
        if procstat is not None and standard_stats:
375
            procstat = self.__get_standard_stats(proc, procstat)
376
377
        if procstat is not None and extended_stats and not self.disable_extended_tag:
378
            procstat = self.__get_extended_stats(proc, procstat)
379
380
        return procstat
381
382
    def update(self):
383
        """Update the processes stats."""
384
        # Reset the stats
385
        self.processlist = []
386
        self.processcount = {'total': 0, 'running': 0, 'sleeping': 0, 'thread': 0}
387
388
        # Do not process if disable tag is set
389
        if self.disable_tag:
390
            return
391
392
        # Get the time since last update
393
        time_since_update = getTimeSinceLastUpdate('process_disk')
394
395
        # Build an internal dict with only mandatories stats (sort keys)
396
        processdict = {}
397
        excluded_processes = set()
398
        for proc in psutil.process_iter():
399
            # Ignore kernel threads if needed
400
            if self.no_kernel_threads and not WINDOWS and is_kernel_thread(proc):
401
                continue
402
403
            # If self.max_processes is None: Only retreive mandatory stats
404
            # Else: retreive mandatory and standard stats
405
            s = self.__get_process_stats(proc,
406
                                         mandatory_stats=True,
407
                                         standard_stats=self.max_processes is None)
408
            # Continue to the next process if it has to be filtered
409
            if s is None or (self.is_filtered(s['cmdline']) and self.is_filtered(s['name'])):
410
                excluded_processes.add(proc)
411
                continue
412
            # Ok add the process to the list
413
            processdict[proc] = s
414
            # ignore the 'idle' process on Windows and *BSD
415
            # ignore the 'kernel_task' process on OS X
416
            # waiting for upstream patch from psutil
417
            if (BSD and processdict[proc]['name'] == 'idle' or
418
                    WINDOWS and processdict[proc]['name'] == 'System Idle Process' or
419
                    OSX and processdict[proc]['name'] == 'kernel_task'):
420
                continue
421
            # Update processcount (global statistics)
422
            try:
423
                self.processcount[str(proc.status())] += 1
424
            except KeyError:
425
                # Key did not exist, create it
426
                try:
427
                    self.processcount[str(proc.status())] = 1
428
                except psutil.NoSuchProcess:
429
                    pass
430
            except psutil.NoSuchProcess:
431
                pass
432
            else:
433
                self.processcount['total'] += 1
434
            # Update thread number (global statistics)
435
            try:
436
                self.processcount['thread'] += proc.num_threads()
437
            except Exception:
438
                pass
439
440
        if self._enable_tree:
441
            self.process_tree = ProcessTreeNode.build_tree(processdict,
442
                                                           self.sort_key,
443
                                                           self.sort_reverse,
444
                                                           self.no_kernel_threads,
445
                                                           excluded_processes)
446
447
            for i, node in enumerate(self.process_tree):
448
                # Only retreive stats for visible processes (max_processes)
449
                if self.max_processes is not None and i >= self.max_processes:
450
                    break
451
452
                # add standard stats
453
                new_stats = self.__get_process_stats(node.process,
454
                                                     mandatory_stats=False,
455
                                                     standard_stats=True,
456
                                                     extended_stats=False)
457
                if new_stats is not None:
458
                    node.stats.update(new_stats)
459
460
                # Add a specific time_since_update stats for bitrate
461
                node.stats['time_since_update'] = time_since_update
462
463
        else:
464
            # Process optimization
465
            # Only retreive stats for visible processes (max_processes)
466
            if self.max_processes is not None:
467
                # Sort the internal dict and cut the top N (Return a list of tuple)
468
                # tuple=key (proc), dict (returned by __get_process_stats)
469
                try:
470
                    processiter = sorted(iteritems(processdict),
471
                                         key=lambda x: x[1][self.sort_key],
472
                                         reverse=self.sort_reverse)
473
                except (KeyError, TypeError) as e:
474
                    logger.error("Cannot sort process list by {0}: {1}".format(self.sort_key, e))
475
                    logger.error('{0}'.format(listitems(processdict)[0]))
476
                    # Fallback to all process (issue #423)
477
                    processloop = iteritems(processdict)
478
                    first = False
479
                else:
480
                    processloop = processiter[0:self.max_processes]
481
                    first = True
482
            else:
483
                # Get all processes stats
484
                processloop = iteritems(processdict)
485
                first = False
486
487
            for i in processloop:
488
                # Already existing mandatory stats
489
                procstat = i[1]
490
                if self.max_processes is not None:
491
                    # Update with standard stats
492
                    # and extended stats but only for TOP (first) process
493
                    s = self.__get_process_stats(i[0],
494
                                                 mandatory_stats=False,
495
                                                 standard_stats=True,
496
                                                 extended_stats=first)
497
                    if s is None:
498
                        continue
499
                    procstat.update(s)
500
                # Add a specific time_since_update stats for bitrate
501
                procstat['time_since_update'] = time_since_update
502
                # Update process list
503
                self.processlist.append(procstat)
504
                # Next...
505
                first = False
506
507
        # Build the all processes list used by the monitored list
508
        self.allprocesslist = itervalues(processdict)
509
510
        # Clean internals caches if timeout is reached
511
        if self.cache_timer.finished():
512
            self.username_cache = {}
513
            self.cmdline_cache = {}
514
            # Restart the timer
515
            self.cache_timer.reset()
516
517
    def getcount(self):
518
        """Get the number of processes."""
519
        return self.processcount
520
521
    def getalllist(self):
522
        """Get the allprocesslist."""
523
        return self.allprocesslist
524
525
    def getlist(self, sortedby=None):
526
        """Get the processlist."""
527
        return self.processlist
528
529
    def gettree(self):
530
        """Get the process tree."""
531
        return self.process_tree
532
533
    @property
534
    def sort_key(self):
535
        """Get the current sort key."""
536
        return self._sort_key
537
538
    @sort_key.setter
539
    def sort_key(self, key):
540
        """Set the current sort key."""
541
        self._sort_key = key
542
543
glances_processes = GlancesProcesses()
544