1
|
|
|
# -*- coding: utf-8 -*- |
2
|
|
|
|
3
|
|
|
__author__ = 'Kenny Freeman' |
4
|
|
|
__email__ = '[email protected]' |
5
|
|
|
__license__ = "ISCL" |
6
|
|
|
__docformat__ = 'reStructuredText' |
7
|
|
|
|
8
|
|
|
import os |
9
|
|
|
import time |
10
|
|
|
import os.path |
11
|
|
|
import traceback |
12
|
|
|
import multiprocessing |
13
|
|
|
from collections import deque |
14
|
|
|
|
15
|
|
|
import plumd |
16
|
|
|
import plumd.plugins |
17
|
|
|
from plumd.calc import Differential |
18
|
|
|
from plumd.util import get_file_map, get_file_map_list, get_file_list, get_file |
19
|
|
|
|
20
|
|
|
|
21
|
|
|
## todo: switch from list with pop(0) to deque |
22
|
|
|
class Proc(plumd.plugins.Reader): |
23
|
|
|
"""Plugin to measure various kernel metrics from /proc.""" |
24
|
|
|
defaults = { |
25
|
|
|
'poll.interval': 10, |
26
|
|
|
'skip_proc_stat': ['btime'], |
27
|
|
|
'skip_proc_meminfo': [], |
28
|
|
|
'skip_proc_meminfo': [], |
29
|
|
|
'skip_proc_loadavg': [], |
30
|
|
|
'skip_proc_swap': [], |
31
|
|
|
'skip_proc_uptime': [], |
32
|
|
|
'skip_proc_diskstats': [], |
33
|
|
|
'skip_proc_net_dev': [], |
34
|
|
|
'skip_proc_net_snmp': [], |
35
|
|
|
'skip_proc_net_sockstat': [], |
36
|
|
|
'skip_proc_net_sockstat': [], |
37
|
|
|
'skip_proc_net_netstat': [], |
38
|
|
|
'cpu_metrics': ["user", "nice", "system", "idle", "iowait", "irq", |
39
|
|
|
"softirq","steal", "guest", "guest_nice"], |
40
|
|
|
'per_cpu': False, |
41
|
|
|
'diskstats_cols': ["r", "r_merge", "r_sector", "r_time", "w", "w_merge", |
42
|
|
|
"w_sector", "w_time", "io_inprog", "io_time", |
43
|
|
|
"io_weighted_time"], |
44
|
|
|
'net_dev_cols': ["rx_bytes", "rx_pkt", "rx_errs", "rx_drop", |
45
|
|
|
"rx_fifo_errs", "rx_frame_errs", "rx_compressed", |
46
|
|
|
"rx_mcast", "tx_bytes", "tx_pkt", "tx_errs", "tx_drop", |
47
|
|
|
"tx_fifo_errs", "collissions", "carrier", |
48
|
|
|
"tx_compressed"], |
49
|
|
|
'net_snmp_items': ["Ip:", "Icmp:", "IcmpMsg:", "Tcp:", "Udp:", |
50
|
|
|
"UdpLite:"] |
51
|
|
|
} |
52
|
|
|
|
53
|
|
|
def __init__(self, log, config): |
54
|
|
|
"""Plugin to measure various kernel metrics from /proc. |
55
|
|
|
|
56
|
|
|
:param log: A logger |
57
|
|
|
:type log: logging.RootLogger |
58
|
|
|
:param config: a plumd.config.Conf configuration helper instance. |
59
|
|
|
:type config: plumd.config.Conf |
60
|
|
|
""" |
61
|
|
|
super(Proc, self).__init__(log, config) |
62
|
|
|
self.config.defaults(Proc.defaults) |
63
|
|
|
self.calc = Differential() |
64
|
|
|
|
65
|
|
|
|
66
|
|
|
def poll(self): |
67
|
|
|
"""Poll for kernel metrics under /proc. |
68
|
|
|
|
69
|
|
|
:rtype: ResultSet |
70
|
|
|
""" |
71
|
|
|
ret = plumd.ResultSet([]) |
72
|
|
|
ret.add(self.proc_stat()) |
73
|
|
|
ret.add(self.proc_meminfo()) |
74
|
|
|
ret.add(self.proc_loadavg()) |
75
|
|
|
ret.add(self.proc_swap()) |
76
|
|
|
ret.add(self.proc_uptime()) |
77
|
|
|
ret.add(self.proc_diskstats()) |
78
|
|
|
ret.add(self.proc_net_dev()) |
79
|
|
|
ret.add(self.proc_net_snmp()) |
80
|
|
|
ret.add(self.proc_net_sockstat()) |
81
|
|
|
ret.add(self.proc_net_netstat()) |
82
|
|
|
return ret |
83
|
|
|
|
84
|
|
|
|
85
|
|
View Code Duplication |
def proc_stat_cpu_percent(self, key, val, ts): |
|
|
|
|
86
|
|
|
ret = [] |
87
|
|
|
total = sum([ float(i) for i in val]) |
88
|
|
|
cpu = self.config.get('cpu_metrics') |
89
|
|
|
for map_val in cpu: |
90
|
|
|
if len(val) < 1: |
91
|
|
|
break |
92
|
|
|
metric_val = float(val.popleft()) |
93
|
|
|
mstr = "{0}.{1}".format(key, map_val) |
94
|
|
|
percent_val = metric_val / total * 100.00 |
95
|
|
|
ret.append(plumd.Float(mstr, percent_val)) |
96
|
|
|
return ret |
97
|
|
|
|
98
|
|
|
|
99
|
|
View Code Duplication |
def proc_stat_cpu(self, key, val, ts): |
|
|
|
|
100
|
|
|
ret = [] |
101
|
|
|
total = sum([ float(i) for i in val]) |
102
|
|
|
cpu = self.config.get('cpu_metrics') |
103
|
|
|
for map_val in cpu: |
104
|
|
|
if len(val) < 1: |
105
|
|
|
break |
106
|
|
|
metric_val = float(val.popleft()) |
107
|
|
|
mstr = "{0}.{1}".format(key, map_val) |
108
|
|
|
percent_val = float(metric_val / total) * 100.00 |
109
|
|
|
mval = self.calc.per_second(key, percent_val, ts) |
110
|
|
|
ret.append(plumd.Float(mstr, mval)) |
111
|
|
|
return ret |
112
|
|
|
|
113
|
|
|
|
114
|
|
|
def proc_stat(self): |
115
|
|
|
skip = self.config.get('skip_proc_stat') |
116
|
|
|
per_cpu = self.config.get('per_cpu') |
117
|
|
|
result = plumd.Result("stat") |
118
|
|
|
fname = "/proc/stat" |
119
|
|
|
# read and process /proc/stat |
120
|
|
|
dat = get_file_map(fname, 0, 0) |
121
|
|
|
ts = time.time() |
122
|
|
|
# parse |
123
|
|
|
for key, val in dat.items(): |
124
|
|
|
# cpu is the only special metric |
125
|
|
|
if val is None: |
126
|
|
|
self.log.error("proc_stat: null value for {0}".format(key)) |
127
|
|
|
continue |
128
|
|
|
elif key in skip: |
129
|
|
|
continue |
130
|
|
|
elif key == "cpu": |
131
|
|
|
result.add_list(self.proc_stat_cpu_percent(key, val, ts)) |
132
|
|
|
elif key.startswith("cpu"): |
133
|
|
|
if not per_cpu: |
134
|
|
|
continue |
135
|
|
|
result.add_list(self.proc_stat_cpu_percent(key, val, ts)) |
136
|
|
|
else: |
137
|
|
|
mval = self.calc.per_second(key, float(val[0]), ts) |
138
|
|
|
result.add(plumd.Int(key, mval)) |
139
|
|
|
return result |
140
|
|
|
|
141
|
|
|
|
142
|
|
|
def proc_meminfo(self): |
143
|
|
|
skip = self.config.get('skip_proc_meminfo') |
144
|
|
|
result = plumd.Result("mem") |
145
|
|
|
fname = "/proc/meminfo" |
146
|
|
|
# read and process /proc/stat |
147
|
|
|
dat = get_file_map(fname, 0, 0) |
148
|
|
|
ts = time.time() |
149
|
|
|
# parse |
150
|
|
|
for key, val in dat.items(): |
151
|
|
|
# cpu is the only special metric |
152
|
|
|
if val is None: |
153
|
|
|
self.log.error("proc_meminfo: null value for {0}".format(key)) |
154
|
|
|
continue |
155
|
|
|
elif key in skip: |
156
|
|
|
continue |
157
|
|
|
else: |
158
|
|
|
#mval = dcalc.per_second(key, float(val[0]), ts) |
159
|
|
|
result.add(plumd.Int(key, val[0])) |
160
|
|
|
return result |
161
|
|
|
|
162
|
|
|
|
163
|
|
|
def proc_loadavg(self): |
164
|
|
|
skip = self.config.get('skip_proc_loadavg') |
165
|
|
|
result = plumd.Result("loadavg") |
166
|
|
|
fname = "/proc/loadavg" |
167
|
|
|
dat = [] |
168
|
|
|
# read and process /proc/stat |
169
|
|
|
try: |
170
|
|
|
dat = get_file(fname).split() |
171
|
|
|
except Exception as e: |
172
|
|
|
tb = traceback.format_exc() |
173
|
|
|
self.log.error("proc_loadavg: exception: {0} : {1}".format(e, tb)) |
174
|
|
|
return result |
175
|
|
|
if len(dat) >= 3: |
176
|
|
|
result.add(plumd.Float("1", dat[0])) |
177
|
|
|
result.add(plumd.Float("5", dat[1])) |
178
|
|
|
result.add(plumd.Float("15", dat[2])) |
179
|
|
|
return result |
180
|
|
|
|
181
|
|
|
|
182
|
|
|
def proc_swap(self): |
183
|
|
|
skip = self.config.get('skip_proc_swap') |
184
|
|
|
result = plumd.Result("swap") |
185
|
|
|
fname = "/proc/swaps" |
186
|
|
|
dat = [] |
187
|
|
|
# read and process /proc/stat |
188
|
|
|
dat = get_file_list(fname) |
189
|
|
|
# header: file, type, size, used, priority |
190
|
|
|
if len(dat) > 1: |
191
|
|
|
dat.popleft() |
192
|
|
|
for entry in dat: |
193
|
|
|
if not entry: |
194
|
|
|
continue |
195
|
|
|
#sfname, stype, ssize, sused, sprio = ("", None, 0, 0, 0) |
196
|
|
|
try: |
197
|
|
|
sfname, stype, ssize, sused, sprio = entry.split() |
198
|
|
|
except Exception as e: |
199
|
|
|
tb = traceback.format_exc() |
200
|
|
|
self.log.error("proc_swap: exception: {0}: {1}".format(e, tb)) |
201
|
|
|
continue |
202
|
|
|
sname = os.path.basename(sfname) |
203
|
|
|
mstr = "{0}.used".format(sname) |
204
|
|
|
result.add(plumd.Float(mstr, sused)) |
205
|
|
|
mstr = "{0}.size".format(sname) |
206
|
|
|
result.add(plumd.Float(mstr, ssize)) |
207
|
|
|
sfree = float(ssize) - float(sused) |
208
|
|
|
mstr = "{0}.free".format(sname) |
209
|
|
|
result.add(plumd.Float(mstr, sfree)) |
210
|
|
|
return result |
211
|
|
|
|
212
|
|
|
|
213
|
|
|
def proc_uptime(self): |
214
|
|
|
skip = self.config.get('skip_proc_uptime') |
215
|
|
|
result = plumd.Result("uptime") |
216
|
|
|
fname = "/proc/uptime" |
217
|
|
|
dat = [] |
218
|
|
|
# read and process /proc/stat |
219
|
|
|
try: |
220
|
|
|
up, idle = get_file(fname).split() |
221
|
|
|
except Exception as e: |
222
|
|
|
tb = traceback.format_exc() |
223
|
|
|
self.log.error("proc_uptime: exception: {0}: {1}".format(e, tb)) |
224
|
|
|
return result |
225
|
|
|
pidle = float(idle)/float(up) * 100 / multiprocessing.cpu_count() |
226
|
|
|
result.add(plumd.Float("up", up)) |
227
|
|
|
result.add(plumd.Float("idle", idle)) |
228
|
|
|
result.add(plumd.Float("idle_percent", pidle)) |
229
|
|
|
return result |
230
|
|
|
|
231
|
|
|
|
232
|
|
View Code Duplication |
def proc_diskstats(self): |
|
|
|
|
233
|
|
|
skip = self.config.get('skip_proc_diskstats') |
234
|
|
|
# times in ms |
235
|
|
|
cols = self.config.get('diskstats_cols') |
236
|
|
|
result = plumd.Result("diskstats") |
237
|
|
|
fname = "/proc/diskstats" |
238
|
|
|
dat = {} |
239
|
|
|
# read and process /proc/stat |
240
|
|
|
try: |
241
|
|
|
dat = get_file_map(fname, 2, 0) |
242
|
|
|
except Exception as e: |
243
|
|
|
tb = traceback.format_exc() |
244
|
|
|
self.log.error("proc_diskstats: exception: {0}: {1}".format(e, tb)) |
245
|
|
|
return result |
246
|
|
|
ts = time.time() |
247
|
|
|
for key, val in dat.items(): |
248
|
|
|
if key in skip: |
249
|
|
|
continue |
250
|
|
|
if len(val) != 13: |
251
|
|
|
self.log.error("proc_diskstats: invalid entry: {0}".format(val)) |
252
|
|
|
continue |
253
|
|
|
for mname in cols: |
254
|
|
|
mval = int(val.popleft()) |
255
|
|
|
mstr = "{0}.{1}".format(key, mname) |
256
|
|
|
dval = self.calc.per_second(mstr, mval, ts) |
257
|
|
|
result.add(plumd.Int(mstr, dval)) |
258
|
|
|
return result |
259
|
|
|
|
260
|
|
|
|
261
|
|
View Code Duplication |
def proc_net_dev(self): |
|
|
|
|
262
|
|
|
skip = self.config.get('skip_proc_net_dev') |
263
|
|
|
cols = self.config.get('net_dev_cols') |
264
|
|
|
result = plumd.Result("net") |
265
|
|
|
fname = "/proc/net/dev" |
266
|
|
|
dat = {} |
267
|
|
|
# read and process /proc/stat |
268
|
|
|
try: |
269
|
|
|
dat = get_file_map(fname, 0, 0) |
270
|
|
|
except Exception as e: |
271
|
|
|
tb = traceback.format_exc() |
272
|
|
|
self.log.error("proc_net_dev: exception: {0}: {1}".format(e, tb)) |
273
|
|
|
return result |
274
|
|
|
ts = time.time() |
275
|
|
|
for key, val in dat.items(): |
276
|
|
|
key = key.replace(":", "") |
277
|
|
|
if key in skip: |
278
|
|
|
continue |
279
|
|
|
if len(val) < len(cols): |
280
|
|
|
#self.log.error("proc_net_dev: invalid entry: {0}".format(val)) |
281
|
|
|
continue |
282
|
|
|
for mname in cols: |
283
|
|
|
if len(val) < 1: |
284
|
|
|
break |
285
|
|
|
mval = int(val.popleft()) |
286
|
|
|
mstr = "{0}.{1}".format(key, mname) |
287
|
|
|
dval = self.calc.per_second(mstr, mval, ts) |
288
|
|
|
result.add(plumd.Int(mstr, dval)) |
289
|
|
|
return result |
290
|
|
|
|
291
|
|
|
|
292
|
|
|
def proc_net_snmp(self): |
293
|
|
|
skip = self.config.get('skip_proc_net_snmp') |
294
|
|
|
items = self.config.get('net_snmp_items') |
295
|
|
|
result = plumd.Result("net_snmp") |
296
|
|
|
fname = "/proc/net/snmp" |
297
|
|
|
dat = {} |
298
|
|
|
# read and process - dat is a list of lines from fname |
299
|
|
|
dat = get_file_list(fname) |
300
|
|
|
ts = time.time() |
301
|
|
|
|
302
|
|
|
# process each pair of lines |
303
|
|
|
for item in items: |
304
|
|
|
# older kernels may not have all items |
305
|
|
|
if len(dat) < 2: |
306
|
|
|
break |
307
|
|
|
try: |
308
|
|
|
# first line is a list of: metric: header values |
309
|
|
|
header = deque(dat.popleft().split()) |
310
|
|
|
# second line is a list of: <metric>: metric values |
311
|
|
|
vals = deque([ int(i) for i in dat.popleft().split()[1:] ]) |
312
|
|
|
except Exception as e: |
313
|
|
|
tb = traceback.format_exc() |
314
|
|
|
self.log.error("proc_net_snmp: exception: {0}: {1}".format(e, tb)) |
315
|
|
|
continue |
316
|
|
|
if len(header) < 2 or header[0] != item: |
317
|
|
|
self.log.error("proc_net_snmp: invalid entry: {0}: {1}".format(header, item)) |
318
|
|
|
continue |
319
|
|
|
# first value is the name of the metric eg. Ip, Icmp, etc |
320
|
|
|
mheader = header.popleft().replace(":", "") |
321
|
|
|
for mname in header: |
322
|
|
|
if len(vals) < 1: |
323
|
|
|
break |
324
|
|
|
mval = vals.popleft() |
325
|
|
|
if mname in skip: |
326
|
|
|
continue |
327
|
|
|
mstr = "{0}.{1}".format(mheader, mname) |
328
|
|
|
dval = self.calc.per_second(mstr, mval, ts) |
329
|
|
|
result.add(plumd.Int(mstr, dval)) |
330
|
|
|
return result |
331
|
|
|
|
332
|
|
|
|
333
|
|
|
def proc_net_sockstat(self): |
334
|
|
|
skip = self.config.get('skip_proc_net_sockstat') |
335
|
|
|
result = plumd.Result("sockstat") |
336
|
|
|
fname = "/proc/net/sockstat" |
337
|
|
|
dat = {} |
338
|
|
|
# read and process - dat is a list of lines from fname |
339
|
|
|
dat = get_file_map_list(fname, 0, 0) |
340
|
|
|
ts = time.time() |
341
|
|
|
|
342
|
|
|
for key, val in dat.items(): |
343
|
|
|
if len(val) < 2: |
344
|
|
|
continue |
345
|
|
|
mstr = key.replace(":", "") |
346
|
|
|
mnames = val[::2] |
347
|
|
|
mvals = deque([ int(i) for i in val[1::2] ]) |
348
|
|
|
if len(mnames) != len(mvals): |
349
|
|
|
self.log.error("proc_net_sockstat: invalid entry: {0}".format(mnames)) |
350
|
|
|
continue |
351
|
|
|
for mname in mnames: |
352
|
|
|
metric = "{0}.{1}".format(mstr, mname) |
353
|
|
|
result.add(plumd.Int(metric, mvals.popleft())) |
354
|
|
|
return result |
355
|
|
|
|
356
|
|
|
|
357
|
|
|
def proc_net_netstat(self): |
358
|
|
|
skip = self.config.get('skip_proc_net_netstat') |
359
|
|
|
result = plumd.Result("netstat") |
360
|
|
|
fname = "/proc/net/netstat" |
361
|
|
|
dat = {} |
362
|
|
|
# read and process - dat is a list of lines from fname |
363
|
|
|
dat = get_file_list(fname) |
364
|
|
|
ts = time.time() |
365
|
|
|
while len(dat) > 1: |
366
|
|
|
headers = deque(dat.popleft().split()) |
367
|
|
|
if len(dat) < 1: |
368
|
|
|
break |
369
|
|
|
mvals = deque([ int(i) for i in dat.popleft().split()[1:] ]) |
370
|
|
|
if len(headers) < 1: |
371
|
|
|
break |
372
|
|
|
mstr = headers.popleft().replace(":", "") |
373
|
|
|
if len(headers) != len(mvals): |
374
|
|
|
self.log.error("proc_net_netstat: invalid entry: {0}".format(headers)) |
375
|
|
|
continue |
376
|
|
|
for mname in headers: |
377
|
|
|
if len(mvals) < 1: |
378
|
|
|
break |
379
|
|
|
metric = "{0}.{1}".format(mstr, mname) |
380
|
|
|
dval = self.calc.per_second(metric, mvals.popleft(), ts) |
381
|
|
|
result.add(plumd.Int(metric, dval)) |
382
|
|
|
return result |
383
|
|
|
|