1
|
|
|
# -*- coding: utf-8 -*- |
2
|
|
|
|
3
|
|
|
__author__ = 'Kenny Freeman' |
4
|
|
|
__email__ = '[email protected]' |
5
|
|
|
__license__ = "ISCL" |
6
|
|
|
__docformat__ = 'reStructuredText' |
7
|
|
|
|
8
|
|
|
import os |
9
|
|
|
import time |
10
|
|
|
import os.path |
11
|
|
|
import traceback |
12
|
|
|
import multiprocessing |
13
|
|
|
from collections import deque |
14
|
|
|
|
15
|
|
|
import plumd |
16
|
|
|
import plumd.plugins |
17
|
|
|
from plumd.calc import Differential |
18
|
|
|
from plumd.util import get_file_map, get_file_map_list, get_file_list, get_file |
19
|
|
|
|
20
|
|
|
|
21
|
|
|
## todo: switch from list with pop(0) to deque |
22
|
|
|
class Proc(plumd.plugins.Reader): |
23
|
|
|
"""Plugin to measure various kernel metrics from /proc.""" |
24
|
|
|
defaults = { |
25
|
|
|
'poll.interval': 10, |
26
|
|
|
'skip_proc_stat': ['btime'], |
27
|
|
|
'skip_proc_meminfo': [], |
28
|
|
|
'skip_proc_meminfo': [], |
29
|
|
|
'skip_proc_loadavg': [], |
30
|
|
|
'skip_proc_swap': [], |
31
|
|
|
'skip_proc_uptime': [], |
32
|
|
|
'skip_proc_diskstats': [], |
33
|
|
|
'skip_proc_net_dev': [], |
34
|
|
|
'skip_proc_net_snmp': [], |
35
|
|
|
'skip_proc_net_sockstat': [], |
36
|
|
|
'skip_proc_net_sockstat': [], |
37
|
|
|
'skip_proc_net_netstat': [], |
38
|
|
|
'cpu_metrics': ["user", "nice", "system", "idle", "iowait", "irq", |
39
|
|
|
"softirq","steal", "guest", "guest_nice"], |
40
|
|
|
'per_cpu': False, |
41
|
|
|
'diskstats_cols': ["r", "r_merge", "r_sector", "r_time", "w", "w_merge", |
42
|
|
|
"w_sector", "w_time", "io_inprog", "io_time", |
43
|
|
|
"io_weighted_time"], |
44
|
|
|
'net_dev_cols': ["rx_bytes", "rx_pkt", "rx_errs", "rx_drop", |
45
|
|
|
"rx_fifo_errs", "rx_frame_errs", "rx_compressed", |
46
|
|
|
"rx_mcast", "tx_bytes", "tx_pkt", "tx_errs", "tx_drop", |
47
|
|
|
"tx_fifo_errs", "collissions", "carrier", |
48
|
|
|
"tx_compressed"], |
49
|
|
|
'net_snmp_items': ["Ip:", "Icmp:", "IcmpMsg:", "Tcp:", "Udp:", |
50
|
|
|
"UdpLite:"] |
51
|
|
|
} |
52
|
|
|
|
53
|
|
|
def __init__(self, log, config): |
54
|
|
|
"""Plugin to measure various kernel metrics from /proc. |
55
|
|
|
|
56
|
|
|
:param log: A logger |
57
|
|
|
:type log: logging.RootLogger |
58
|
|
|
:param config: a plumd.config.Conf configuration helper instance. |
59
|
|
|
:type config: plumd.config.Conf |
60
|
|
|
""" |
61
|
|
|
super(Proc, self).__init__(log, config) |
62
|
|
|
self.config.defaults(Proc.defaults) |
63
|
|
|
self.calc = Differential() |
64
|
|
|
|
65
|
|
|
|
66
|
|
|
def poll(self): |
67
|
|
|
"""Poll for kernel metrics under /proc. |
68
|
|
|
|
69
|
|
|
:rtype: ResultSet |
70
|
|
|
""" |
71
|
|
|
ret = plumd.ResultSet([]) |
72
|
|
|
ret.add(self.proc_stat()) |
73
|
|
|
ret.add(self.proc_meminfo()) |
74
|
|
|
ret.add(self.proc_loadavg()) |
75
|
|
|
ret.add(self.proc_swap()) |
76
|
|
|
ret.add(self.proc_uptime()) |
77
|
|
|
ret.add(self.proc_diskstats()) |
78
|
|
|
ret.add(self.proc_net_dev()) |
79
|
|
|
ret.add(self.proc_net_snmp()) |
80
|
|
|
ret.add(self.proc_net_sockstat()) |
81
|
|
|
ret.add(self.proc_net_netstat()) |
82
|
|
|
return ret |
83
|
|
|
|
84
|
|
|
|
85
|
|
|
def proc_stat_cpu_percent(self, key, val, ts): |
86
|
|
|
ret = [] |
87
|
|
|
total = sum([ float(i) for i in val]) |
88
|
|
|
cpu = self.config.get('cpu_metrics') |
89
|
|
|
for map_val in cpu: |
90
|
|
|
metric_val = float(val.popleft()) |
91
|
|
|
mstr = "{0}.{1}".format(key, map_val) |
92
|
|
|
percent_val = metric_val / total * 100.00 |
93
|
|
|
ret.append(plumd.Float(mstr, percent_val)) |
94
|
|
|
return ret |
95
|
|
|
|
96
|
|
|
|
97
|
|
|
def proc_stat_cpu(self, key, val, ts): |
98
|
|
|
ret = [] |
99
|
|
|
total = sum([ float(i) for i in val]) |
100
|
|
|
cpu = self.config.get('cpu_metrics') |
101
|
|
|
for map_val in cpu: |
102
|
|
|
metric_val = float(val.popleft()) |
103
|
|
|
mstr = "{0}.{1}".format(key, map_val) |
104
|
|
|
percent_val = float(metric_val / total) * 100.00 |
105
|
|
|
mval = self.calc.per_second(key, percent_val, ts) |
106
|
|
|
ret.append(plumd.Float(mstr, mval)) |
107
|
|
|
return ret |
108
|
|
|
|
109
|
|
|
|
110
|
|
|
def proc_stat(self): |
111
|
|
|
skip = self.config.get('skip_proc_stat') |
112
|
|
|
per_cpu = self.config.get('per_cpu') |
113
|
|
|
result = plumd.Result("stat") |
114
|
|
|
fname = "/proc/stat" |
115
|
|
|
# read and process /proc/stat |
116
|
|
|
dat = get_file_map(fname, 0, 0) |
117
|
|
|
ts = time.time() |
118
|
|
|
# parse |
119
|
|
|
for key, val in dat.items(): |
120
|
|
|
# cpu is the only special metric |
121
|
|
|
if val is None: |
122
|
|
|
self.log.error("proc_stat: null value for {0}".format(key)) |
123
|
|
|
continue |
124
|
|
|
elif key in skip: |
125
|
|
|
continue |
126
|
|
|
elif key == "cpu": |
127
|
|
|
result.add_list(self.proc_stat_cpu_percent(key, val, ts)) |
128
|
|
|
elif key.startswith("cpu"): |
129
|
|
|
if not per_cpu: |
130
|
|
|
continue |
131
|
|
|
result.add_list(self.proc_stat_cpu_percent(key, val, ts)) |
132
|
|
|
else: |
133
|
|
|
mval = self.calc.per_second(key, float(val[0]), ts) |
134
|
|
|
result.add(plumd.Int(key, mval)) |
135
|
|
|
return result |
136
|
|
|
|
137
|
|
|
|
138
|
|
|
def proc_meminfo(self): |
139
|
|
|
skip = self.config.get('skip_proc_meminfo') |
140
|
|
|
result = plumd.Result("mem") |
141
|
|
|
fname = "/proc/meminfo" |
142
|
|
|
# read and process /proc/stat |
143
|
|
|
dat = get_file_map(fname, 0, 0) |
144
|
|
|
ts = time.time() |
145
|
|
|
# parse |
146
|
|
|
for key, val in dat.items(): |
147
|
|
|
# cpu is the only special metric |
148
|
|
|
if val is None: |
149
|
|
|
self.log.error("proc_meminfo: null value for {0}".format(key)) |
150
|
|
|
continue |
151
|
|
|
elif key in skip: |
152
|
|
|
continue |
153
|
|
|
else: |
154
|
|
|
#mval = dcalc.per_second(key, float(val[0]), ts) |
155
|
|
|
result.add(plumd.Int(key, val[0])) |
156
|
|
|
return result |
157
|
|
|
|
158
|
|
|
|
159
|
|
|
def proc_loadavg(self): |
160
|
|
|
skip = self.config.get('skip_proc_loadavg') |
161
|
|
|
result = plumd.Result("loadavg") |
162
|
|
|
fname = "/proc/loadavg" |
163
|
|
|
dat = [] |
164
|
|
|
# read and process /proc/stat |
165
|
|
|
try: |
166
|
|
|
dat = get_file(fname).split() |
167
|
|
|
except Exception as e: |
168
|
|
|
tb = traceback.format_exc() |
169
|
|
|
self.log.error("proc_loadavg: exception: {0} : {1}".format(e, tb)) |
170
|
|
|
return result |
171
|
|
|
if len(dat) >= 3: |
172
|
|
|
result.add(plumd.Float("1", dat[0])) |
173
|
|
|
result.add(plumd.Float("5", dat[1])) |
174
|
|
|
result.add(plumd.Float("15", dat[2])) |
175
|
|
|
return result |
176
|
|
|
|
177
|
|
|
|
178
|
|
|
def proc_swap(self): |
179
|
|
|
skip = self.config.get('skip_proc_swap') |
180
|
|
|
result = plumd.Result("swap") |
181
|
|
|
fname = "/proc/swaps" |
182
|
|
|
dat = [] |
183
|
|
|
# read and process /proc/stat |
184
|
|
|
dat = get_file_list(fname) |
185
|
|
|
# header: file, type, size, used, priority |
186
|
|
|
dat.popleft() |
187
|
|
|
for entry in dat: |
188
|
|
|
if not entry: |
189
|
|
|
continue |
190
|
|
|
#sfname, stype, ssize, sused, sprio = ("", None, 0, 0, 0) |
191
|
|
|
try: |
192
|
|
|
sfname, stype, ssize, sused, sprio = entry.split() |
193
|
|
|
except Exception as e: |
194
|
|
|
tb = traceback.format_exc() |
195
|
|
|
self.log.error("proc_swap: exception: {0}: {1}".format(e, tb)) |
196
|
|
|
continue |
197
|
|
|
sname = os.path.basename(sfname) |
198
|
|
|
mstr = "{0}.used".format(sname) |
199
|
|
|
result.add(plumd.Float(mstr, sused)) |
200
|
|
|
mstr = "{0}.size".format(sname) |
201
|
|
|
result.add(plumd.Float(mstr, ssize)) |
202
|
|
|
sfree = float(ssize) - float(sused) |
203
|
|
|
mstr = "{0}.free".format(sname) |
204
|
|
|
result.add(plumd.Float(mstr, sfree)) |
205
|
|
|
return result |
206
|
|
|
|
207
|
|
|
|
208
|
|
|
def proc_uptime(self): |
209
|
|
|
skip = self.config.get('skip_proc_uptime') |
210
|
|
|
result = plumd.Result("uptime") |
211
|
|
|
fname = "/proc/uptime" |
212
|
|
|
dat = [] |
213
|
|
|
# read and process /proc/stat |
214
|
|
|
try: |
215
|
|
|
up, idle = get_file(fname).split() |
216
|
|
|
except Exception as e: |
217
|
|
|
tb = traceback.format_exc() |
218
|
|
|
self.log.error("proc_uptime: exception: {0}: {1}".format(e, tb)) |
219
|
|
|
return result |
220
|
|
|
pidle = float(idle)/float(up) * 100 / multiprocessing.cpu_count() |
221
|
|
|
result.add(plumd.Float("up", up)) |
222
|
|
|
result.add(plumd.Float("idle", idle)) |
223
|
|
|
result.add(plumd.Float("idle_percent", pidle)) |
224
|
|
|
return result |
225
|
|
|
|
226
|
|
|
|
227
|
|
View Code Duplication |
def proc_diskstats(self): |
|
|
|
|
228
|
|
|
skip = self.config.get('skip_proc_diskstats') |
229
|
|
|
# times in ms |
230
|
|
|
cols = self.config.get('diskstats_cols') |
231
|
|
|
result = plumd.Result("diskstats") |
232
|
|
|
fname = "/proc/diskstats" |
233
|
|
|
dat = {} |
234
|
|
|
# read and process /proc/stat |
235
|
|
|
try: |
236
|
|
|
dat = get_file_map(fname, 2, 0) |
237
|
|
|
except Exception as e: |
238
|
|
|
tb = traceback.format_exc() |
239
|
|
|
self.log.error("proc_diskstats: exception: {0}: {1}".format(e, tb)) |
240
|
|
|
return result |
241
|
|
|
ts = time.time() |
242
|
|
|
for key, val in dat.items(): |
243
|
|
|
if key in skip: |
244
|
|
|
continue |
245
|
|
|
if len(val) != 13: |
246
|
|
|
self.log.error("proc_diskstats: invalid entry: {0}".format(val)) |
247
|
|
|
continue |
248
|
|
|
for mname in cols: |
249
|
|
|
mval = int(val.popleft()) |
250
|
|
|
mstr = "{0}.{1}".format(key, mname) |
251
|
|
|
dval = self.calc.per_second(mstr, mval, ts) |
252
|
|
|
result.add(plumd.Int(mstr, dval)) |
253
|
|
|
return result |
254
|
|
|
|
255
|
|
|
|
256
|
|
View Code Duplication |
def proc_net_dev(self): |
|
|
|
|
257
|
|
|
skip = self.config.get('skip_proc_net_dev') |
258
|
|
|
cols = self.config.get('net_dev_cols') |
259
|
|
|
result = plumd.Result("net") |
260
|
|
|
fname = "/proc/net/dev" |
261
|
|
|
dat = {} |
262
|
|
|
# read and process /proc/stat |
263
|
|
|
try: |
264
|
|
|
dat = get_file_map(fname, 0, 0) |
265
|
|
|
except Exception as e: |
266
|
|
|
tb = traceback.format_exc() |
267
|
|
|
self.log.error("proc_net_dev: exception: {0}: {1}".format(e, tb)) |
268
|
|
|
return result |
269
|
|
|
ts = time.time() |
270
|
|
|
for key, val in dat.items(): |
271
|
|
|
key = key.replace(":", "") |
272
|
|
|
if key in skip: |
273
|
|
|
continue |
274
|
|
|
if len(val) < len(cols): |
275
|
|
|
#self.log.error("proc_net_dev: invalid entry: {0}".format(val)) |
276
|
|
|
continue |
277
|
|
|
for mname in cols: |
278
|
|
|
mval = int(val.popleft()) |
279
|
|
|
mstr = "{0}.{1}".format(key, mname) |
280
|
|
|
dval = self.calc.per_second(mstr, mval, ts) |
281
|
|
|
result.add(plumd.Int(mstr, dval)) |
282
|
|
|
return result |
283
|
|
|
|
284
|
|
|
|
285
|
|
|
def proc_net_snmp(self): |
286
|
|
|
skip = self.config.get('skip_proc_net_snmp') |
287
|
|
|
items = self.config.get('net_snmp_items') |
288
|
|
|
result = plumd.Result("net") |
289
|
|
|
fname = "/proc/net/snmp" |
290
|
|
|
dat = {} |
291
|
|
|
# read and process - dat is a list of lines from fname |
292
|
|
|
dat = get_file_list(fname) |
293
|
|
|
ts = time.time() |
294
|
|
|
|
295
|
|
|
# process each pair of lines |
296
|
|
|
if len(dat) != ((2 * len(items)) + 1): |
297
|
|
|
self.log.error("proc_net_dev: invalid entry: {0}: {1}".format(dat, items)) |
298
|
|
|
return result |
299
|
|
|
for item in items: |
300
|
|
|
try: |
301
|
|
|
# first line is a list of: metric: header values |
302
|
|
|
header = deque(dat.popleft().split()) |
303
|
|
|
# second line is a list of: <metric>: metric values |
304
|
|
|
vals = deque([ int(i) for i in dat.popleft().split()[1:] ]) |
305
|
|
|
except Exception as e: |
306
|
|
|
tb = traceback.format_exc() |
307
|
|
|
self.log.error("proc_net_snmp: exception: {0}: {1}".format(e, tb)) |
308
|
|
|
continue |
309
|
|
|
if len(header) < 2 or header[0] != item: |
310
|
|
|
self.log.error("proc_net_snmp: invalid entry: {0}: {1}".format(header, item)) |
311
|
|
|
continue |
312
|
|
|
# first value is the name of the metric eg. Ip, Icmp, etc |
313
|
|
|
mheader = header.popleft().replace(":", "") |
314
|
|
|
for mname in header: |
315
|
|
|
mval = vals.popleft() |
316
|
|
|
if mname in skip: |
317
|
|
|
continue |
318
|
|
|
mstr = "{0}.{1}".format(mheader, mname) |
319
|
|
|
dval = self.calc.per_second(mstr, mval, ts) |
320
|
|
|
result.add(plumd.Int(mstr, dval)) |
321
|
|
|
return result |
322
|
|
|
|
323
|
|
|
|
324
|
|
|
def proc_net_sockstat(self): |
325
|
|
|
skip = self.config.get('skip_proc_net_sockstat') |
326
|
|
|
result = plumd.Result("sockstat") |
327
|
|
|
fname = "/proc/net/sockstat" |
328
|
|
|
dat = {} |
329
|
|
|
# read and process - dat is a list of lines from fname |
330
|
|
|
dat = get_file_map_list(fname, 0, 0) |
331
|
|
|
ts = time.time() |
332
|
|
|
|
333
|
|
|
for key, val in dat.items(): |
334
|
|
|
mstr = key.replace(":", "") |
335
|
|
|
mnames = val[::2] |
336
|
|
|
mvals = deque([ int(i) for i in val[1::2] ]) |
337
|
|
|
if len(mnames) != len(mvals): |
338
|
|
|
self.log.error("proc_net_sockstat: invalid entry: {0}".format(mnames)) |
339
|
|
|
continue |
340
|
|
|
for mname in mnames: |
341
|
|
|
metric = "{0}.{1}".format(mstr, mname) |
342
|
|
|
result.add(plumd.Int(metric, mvals.popleft())) |
343
|
|
|
return result |
344
|
|
|
|
345
|
|
|
|
346
|
|
|
def proc_net_netstat(self): |
347
|
|
|
skip = self.config.get('skip_proc_net_netstat') |
348
|
|
|
result = plumd.Result("netstat") |
349
|
|
|
fname = "/proc/net/netstat" |
350
|
|
|
dat = {} |
351
|
|
|
# read and process - dat is a list of lines from fname |
352
|
|
|
dat = get_file_list(fname) |
353
|
|
|
ts = time.time() |
354
|
|
|
while len(dat) > 1: |
355
|
|
|
headers = deque(dat.popleft().split()) |
356
|
|
|
mvals = deque([ int(i) for i in dat.popleft().split()[1:] ]) |
357
|
|
|
mstr = headers.popleft().replace(":", "") |
358
|
|
|
if len(headers) != len(mvals): |
359
|
|
|
self.log.error("proc_net_netstat: invalid entry: {0}".format(headers)) |
360
|
|
|
continue |
361
|
|
|
for mname in headers: |
362
|
|
|
metric = "{0}.{1}".format(mstr, mname) |
363
|
|
|
dval = self.calc.per_second(metric, mvals.popleft(), ts) |
364
|
|
|
result.add(plumd.Int(metric, dval)) |
365
|
|
|
return result |
366
|
|
|
|