|
1
|
|
|
# -*- coding: utf-8 -*- |
|
2
|
|
|
|
|
3
|
|
|
__author__ = 'Kenny Freeman' |
|
4
|
|
|
__email__ = '[email protected]' |
|
5
|
|
|
__license__ = "ISCL" |
|
6
|
|
|
__docformat__ = 'reStructuredText' |
|
7
|
|
|
|
|
8
|
|
|
import os |
|
9
|
|
|
import time |
|
10
|
|
|
import os.path |
|
11
|
|
|
import traceback |
|
12
|
|
|
import multiprocessing |
|
13
|
|
|
from collections import deque |
|
14
|
|
|
|
|
15
|
|
|
import plumd |
|
16
|
|
|
import plumd.plugins |
|
17
|
|
|
from plumd.calc import Differential |
|
18
|
|
|
from plumd.util import get_file_map, get_file_map_list, get_file_list, get_file |
|
19
|
|
|
|
|
20
|
|
|
|
|
21
|
|
|
## todo: switch from list with pop(0) to deque |
|
22
|
|
|
class Proc(plumd.plugins.Reader): |
|
23
|
|
|
"""Plugin to measure various kernel metrics from /proc.""" |
|
24
|
|
|
defaults = { |
|
25
|
|
|
'poll.interval': 10, |
|
26
|
|
|
'skip_proc_stat': ['btime'], |
|
27
|
|
|
'skip_proc_meminfo': [], |
|
28
|
|
|
'skip_proc_meminfo': [], |
|
29
|
|
|
'skip_proc_loadavg': [], |
|
30
|
|
|
'skip_proc_swap': [], |
|
31
|
|
|
'skip_proc_uptime': [], |
|
32
|
|
|
'skip_proc_diskstats': [], |
|
33
|
|
|
'skip_proc_net_dev': [], |
|
34
|
|
|
'skip_proc_net_snmp': [], |
|
35
|
|
|
'skip_proc_net_sockstat': [], |
|
36
|
|
|
'skip_proc_net_sockstat': [], |
|
37
|
|
|
'skip_proc_net_netstat': [], |
|
38
|
|
|
'cpu_metrics': ["user", "nice", "system", "idle", "iowait", "irq", |
|
39
|
|
|
"softirq","steal", "guest", "guest_nice"], |
|
40
|
|
|
'per_cpu': False, |
|
41
|
|
|
'diskstats_cols': ["r", "r_merge", "r_sector", "r_time", "w", "w_merge", |
|
42
|
|
|
"w_sector", "w_time", "io_inprog", "io_time", |
|
43
|
|
|
"io_weighted_time"], |
|
44
|
|
|
'net_dev_cols': ["rx_bytes", "rx_pkt", "rx_errs", "rx_drop", |
|
45
|
|
|
"rx_fifo_errs", "rx_frame_errs", "rx_compressed", |
|
46
|
|
|
"rx_mcast", "tx_bytes", "tx_pkt", "tx_errs", "tx_drop", |
|
47
|
|
|
"tx_fifo_errs", "collissions", "carrier", |
|
48
|
|
|
"tx_compressed"], |
|
49
|
|
|
'net_snmp_items': ["Ip:", "Icmp:", "IcmpMsg:", "Tcp:", "Udp:", |
|
50
|
|
|
"UdpLite:"] |
|
51
|
|
|
} |
|
52
|
|
|
|
|
53
|
|
|
def __init__(self, log, config): |
|
54
|
|
|
"""Plugin to measure various kernel metrics from /proc. |
|
55
|
|
|
|
|
56
|
|
|
:param log: A logger |
|
57
|
|
|
:type log: logging.RootLogger |
|
58
|
|
|
:param config: a plumd.config.Conf configuration helper instance. |
|
59
|
|
|
:type config: plumd.config.Conf |
|
60
|
|
|
""" |
|
61
|
|
|
super(Proc, self).__init__(log, config) |
|
62
|
|
|
self.config.defaults(Proc.defaults) |
|
63
|
|
|
self.calc = Differential() |
|
64
|
|
|
|
|
65
|
|
|
|
|
66
|
|
|
def poll(self): |
|
67
|
|
|
"""Poll for kernel metrics under /proc. |
|
68
|
|
|
|
|
69
|
|
|
:rtype: ResultSet |
|
70
|
|
|
""" |
|
71
|
|
|
ret = plumd.ResultSet([]) |
|
72
|
|
|
ret.add(self.proc_stat()) |
|
73
|
|
|
ret.add(self.proc_meminfo()) |
|
74
|
|
|
ret.add(self.proc_loadavg()) |
|
75
|
|
|
ret.add(self.proc_swap()) |
|
76
|
|
|
ret.add(self.proc_uptime()) |
|
77
|
|
|
ret.add(self.proc_diskstats()) |
|
78
|
|
|
ret.add(self.proc_net_dev()) |
|
79
|
|
|
ret.add(self.proc_net_snmp()) |
|
80
|
|
|
ret.add(self.proc_net_sockstat()) |
|
81
|
|
|
ret.add(self.proc_net_netstat()) |
|
82
|
|
|
return ret |
|
83
|
|
|
|
|
84
|
|
|
|
|
85
|
|
|
def proc_stat_cpu_percent(self, key, val, ts): |
|
86
|
|
|
ret = [] |
|
87
|
|
|
total = sum([ float(i) for i in val]) |
|
88
|
|
|
cpu = self.config.get('cpu_metrics') |
|
89
|
|
|
for map_val in cpu: |
|
90
|
|
|
metric_val = float(val.popleft()) |
|
91
|
|
|
mstr = "{0}.{1}".format(key, map_val) |
|
92
|
|
|
percent_val = metric_val / total * 100.00 |
|
93
|
|
|
ret.append(plumd.Float(mstr, percent_val)) |
|
94
|
|
|
return ret |
|
95
|
|
|
|
|
96
|
|
|
|
|
97
|
|
|
def proc_stat_cpu(self, key, val, ts): |
|
98
|
|
|
ret = [] |
|
99
|
|
|
total = sum([ float(i) for i in val]) |
|
100
|
|
|
cpu = self.config.get('cpu_metrics') |
|
101
|
|
|
for map_val in cpu: |
|
102
|
|
|
metric_val = float(val.popleft()) |
|
103
|
|
|
mstr = "{0}.{1}".format(key, map_val) |
|
104
|
|
|
percent_val = float(metric_val / total) * 100.00 |
|
105
|
|
|
mval = self.calc.per_second(key, percent_val, ts) |
|
106
|
|
|
ret.append(plumd.Float(mstr, mval)) |
|
107
|
|
|
return ret |
|
108
|
|
|
|
|
109
|
|
|
|
|
110
|
|
|
def proc_stat(self): |
|
111
|
|
|
skip = self.config.get('skip_proc_stat') |
|
112
|
|
|
per_cpu = self.config.get('per_cpu') |
|
113
|
|
|
result = plumd.Result("stat") |
|
114
|
|
|
fname = "/proc/stat" |
|
115
|
|
|
# read and process /proc/stat |
|
116
|
|
|
dat = get_file_map(fname, 0, 0) |
|
117
|
|
|
ts = time.time() |
|
118
|
|
|
# parse |
|
119
|
|
|
for key, val in dat.items(): |
|
120
|
|
|
# cpu is the only special metric |
|
121
|
|
|
if val is None: |
|
122
|
|
|
self.log.error("proc_stat: null value for {0}".format(key)) |
|
123
|
|
|
continue |
|
124
|
|
|
elif key in skip: |
|
125
|
|
|
continue |
|
126
|
|
|
elif key == "cpu": |
|
127
|
|
|
result.add_list(self.proc_stat_cpu_percent(key, val, ts)) |
|
128
|
|
|
elif key.startswith("cpu"): |
|
129
|
|
|
if not per_cpu: |
|
130
|
|
|
continue |
|
131
|
|
|
result.add_list(self.proc_stat_cpu_percent(key, val, ts)) |
|
132
|
|
|
else: |
|
133
|
|
|
mval = self.calc.per_second(key, float(val[0]), ts) |
|
134
|
|
|
result.add(plumd.Int(key, mval)) |
|
135
|
|
|
return result |
|
136
|
|
|
|
|
137
|
|
|
|
|
138
|
|
|
def proc_meminfo(self): |
|
139
|
|
|
skip = self.config.get('skip_proc_meminfo') |
|
140
|
|
|
result = plumd.Result("mem") |
|
141
|
|
|
fname = "/proc/meminfo" |
|
142
|
|
|
# read and process /proc/stat |
|
143
|
|
|
dat = get_file_map(fname, 0, 0) |
|
144
|
|
|
ts = time.time() |
|
145
|
|
|
# parse |
|
146
|
|
|
for key, val in dat.items(): |
|
147
|
|
|
# cpu is the only special metric |
|
148
|
|
|
if val is None: |
|
149
|
|
|
self.log.error("proc_meminfo: null value for {0}".format(key)) |
|
150
|
|
|
continue |
|
151
|
|
|
elif key in skip: |
|
152
|
|
|
continue |
|
153
|
|
|
else: |
|
154
|
|
|
#mval = dcalc.per_second(key, float(val[0]), ts) |
|
155
|
|
|
result.add(plumd.Int(key, val[0])) |
|
156
|
|
|
return result |
|
157
|
|
|
|
|
158
|
|
|
|
|
159
|
|
|
def proc_loadavg(self): |
|
160
|
|
|
skip = self.config.get('skip_proc_loadavg') |
|
161
|
|
|
result = plumd.Result("loadavg") |
|
162
|
|
|
fname = "/proc/loadavg" |
|
163
|
|
|
dat = [] |
|
164
|
|
|
# read and process /proc/stat |
|
165
|
|
|
try: |
|
166
|
|
|
dat = get_file(fname).split() |
|
167
|
|
|
except Exception as e: |
|
168
|
|
|
tb = traceback.format_exc() |
|
169
|
|
|
self.log.error("proc_loadavg: exception: {0} : {1}".format(e, tb)) |
|
170
|
|
|
return result |
|
171
|
|
|
if len(dat) >= 3: |
|
172
|
|
|
result.add(plumd.Float("1", dat[0])) |
|
173
|
|
|
result.add(plumd.Float("5", dat[1])) |
|
174
|
|
|
result.add(plumd.Float("15", dat[2])) |
|
175
|
|
|
return result |
|
176
|
|
|
|
|
177
|
|
|
|
|
178
|
|
|
def proc_swap(self): |
|
179
|
|
|
skip = self.config.get('skip_proc_swap') |
|
180
|
|
|
result = plumd.Result("swap") |
|
181
|
|
|
fname = "/proc/swaps" |
|
182
|
|
|
dat = [] |
|
183
|
|
|
# read and process /proc/stat |
|
184
|
|
|
dat = get_file_list(fname) |
|
185
|
|
|
# header: file, type, size, used, priority |
|
186
|
|
|
dat.popleft() |
|
187
|
|
|
for entry in dat: |
|
188
|
|
|
if not entry: |
|
189
|
|
|
continue |
|
190
|
|
|
#sfname, stype, ssize, sused, sprio = ("", None, 0, 0, 0) |
|
191
|
|
|
try: |
|
192
|
|
|
sfname, stype, ssize, sused, sprio = entry.split() |
|
193
|
|
|
except Exception as e: |
|
194
|
|
|
tb = traceback.format_exc() |
|
195
|
|
|
self.log.error("proc_swap: exception: {0}: {1}".format(e, tb)) |
|
196
|
|
|
continue |
|
197
|
|
|
sname = os.path.basename(sfname) |
|
198
|
|
|
mstr = "{0}.used".format(sname) |
|
199
|
|
|
result.add(plumd.Float(mstr, sused)) |
|
200
|
|
|
mstr = "{0}.size".format(sname) |
|
201
|
|
|
result.add(plumd.Float(mstr, ssize)) |
|
202
|
|
|
sfree = float(ssize) - float(sused) |
|
203
|
|
|
mstr = "{0}.free".format(sname) |
|
204
|
|
|
result.add(plumd.Float(mstr, sfree)) |
|
205
|
|
|
return result |
|
206
|
|
|
|
|
207
|
|
|
|
|
208
|
|
|
def proc_uptime(self): |
|
209
|
|
|
skip = self.config.get('skip_proc_uptime') |
|
210
|
|
|
result = plumd.Result("uptime") |
|
211
|
|
|
fname = "/proc/uptime" |
|
212
|
|
|
dat = [] |
|
213
|
|
|
# read and process /proc/stat |
|
214
|
|
|
try: |
|
215
|
|
|
up, idle = get_file(fname).split() |
|
216
|
|
|
except Exception as e: |
|
217
|
|
|
tb = traceback.format_exc() |
|
218
|
|
|
self.log.error("proc_uptime: exception: {0}: {1}".format(e, tb)) |
|
219
|
|
|
return result |
|
220
|
|
|
pidle = float(idle)/float(up) * 100 / multiprocessing.cpu_count() |
|
221
|
|
|
result.add(plumd.Float("up", up)) |
|
222
|
|
|
result.add(plumd.Float("idle", idle)) |
|
223
|
|
|
result.add(plumd.Float("idle_percent", pidle)) |
|
224
|
|
|
return result |
|
225
|
|
|
|
|
226
|
|
|
|
|
227
|
|
View Code Duplication |
def proc_diskstats(self): |
|
|
|
|
|
|
228
|
|
|
skip = self.config.get('skip_proc_diskstats') |
|
229
|
|
|
# times in ms |
|
230
|
|
|
cols = self.config.get('diskstats_cols') |
|
231
|
|
|
result = plumd.Result("diskstats") |
|
232
|
|
|
fname = "/proc/diskstats" |
|
233
|
|
|
dat = {} |
|
234
|
|
|
# read and process /proc/stat |
|
235
|
|
|
try: |
|
236
|
|
|
dat = get_file_map(fname, 2, 0) |
|
237
|
|
|
except Exception as e: |
|
238
|
|
|
tb = traceback.format_exc() |
|
239
|
|
|
self.log.error("proc_diskstats: exception: {0}: {1}".format(e, tb)) |
|
240
|
|
|
return result |
|
241
|
|
|
ts = time.time() |
|
242
|
|
|
for key, val in dat.items(): |
|
243
|
|
|
if key in skip: |
|
244
|
|
|
continue |
|
245
|
|
|
if len(val) != 13: |
|
246
|
|
|
self.log.error("proc_diskstats: invalid entry: {0}".format(val)) |
|
247
|
|
|
continue |
|
248
|
|
|
for mname in cols: |
|
249
|
|
|
mval = int(val.popleft()) |
|
250
|
|
|
mstr = "{0}.{1}".format(key, mname) |
|
251
|
|
|
dval = self.calc.per_second(mstr, mval, ts) |
|
252
|
|
|
result.add(plumd.Int(mstr, dval)) |
|
253
|
|
|
return result |
|
254
|
|
|
|
|
255
|
|
|
|
|
256
|
|
View Code Duplication |
def proc_net_dev(self): |
|
|
|
|
|
|
257
|
|
|
skip = self.config.get('skip_proc_net_dev') |
|
258
|
|
|
cols = self.config.get('net_dev_cols') |
|
259
|
|
|
result = plumd.Result("net") |
|
260
|
|
|
fname = "/proc/net/dev" |
|
261
|
|
|
dat = {} |
|
262
|
|
|
# read and process /proc/stat |
|
263
|
|
|
try: |
|
264
|
|
|
dat = get_file_map(fname, 0, 0) |
|
265
|
|
|
except Exception as e: |
|
266
|
|
|
tb = traceback.format_exc() |
|
267
|
|
|
self.log.error("proc_net_dev: exception: {0}: {1}".format(e, tb)) |
|
268
|
|
|
return result |
|
269
|
|
|
ts = time.time() |
|
270
|
|
|
for key, val in dat.items(): |
|
271
|
|
|
key = key.replace(":", "") |
|
272
|
|
|
if key in skip: |
|
273
|
|
|
continue |
|
274
|
|
|
if len(val) < len(cols): |
|
275
|
|
|
#self.log.error("proc_net_dev: invalid entry: {0}".format(val)) |
|
276
|
|
|
continue |
|
277
|
|
|
for mname in cols: |
|
278
|
|
|
mval = int(val.popleft()) |
|
279
|
|
|
mstr = "{0}.{1}".format(key, mname) |
|
280
|
|
|
dval = self.calc.per_second(mstr, mval, ts) |
|
281
|
|
|
result.add(plumd.Int(mstr, dval)) |
|
282
|
|
|
return result |
|
283
|
|
|
|
|
284
|
|
|
|
|
285
|
|
|
def proc_net_snmp(self): |
|
286
|
|
|
skip = self.config.get('skip_proc_net_snmp') |
|
287
|
|
|
items = self.config.get('net_snmp_items') |
|
288
|
|
|
result = plumd.Result("net") |
|
289
|
|
|
fname = "/proc/net/snmp" |
|
290
|
|
|
dat = {} |
|
291
|
|
|
# read and process - dat is a list of lines from fname |
|
292
|
|
|
dat = get_file_list(fname) |
|
293
|
|
|
ts = time.time() |
|
294
|
|
|
|
|
295
|
|
|
# process each pair of lines |
|
296
|
|
|
if len(dat) != ((2 * len(items)) + 1): |
|
297
|
|
|
self.log.error("proc_net_dev: invalid entry: {0}: {1}".format(dat, items)) |
|
298
|
|
|
return result |
|
299
|
|
|
for item in items: |
|
300
|
|
|
try: |
|
301
|
|
|
# first line is a list of: metric: header values |
|
302
|
|
|
header = deque(dat.popleft().split()) |
|
303
|
|
|
# second line is a list of: <metric>: metric values |
|
304
|
|
|
vals = deque([ int(i) for i in dat.popleft().split()[1:] ]) |
|
305
|
|
|
except Exception as e: |
|
306
|
|
|
tb = traceback.format_exc() |
|
307
|
|
|
self.log.error("proc_net_snmp: exception: {0}: {1}".format(e, tb)) |
|
308
|
|
|
continue |
|
309
|
|
|
if len(header) < 2 or header[0] != item: |
|
310
|
|
|
self.log.error("proc_net_snmp: invalid entry: {0}: {1}".format(header, item)) |
|
311
|
|
|
continue |
|
312
|
|
|
# first value is the name of the metric eg. Ip, Icmp, etc |
|
313
|
|
|
mheader = header.popleft().replace(":", "") |
|
314
|
|
|
for mname in header: |
|
315
|
|
|
mval = vals.popleft() |
|
316
|
|
|
if mname in skip: |
|
317
|
|
|
continue |
|
318
|
|
|
mstr = "{0}.{1}".format(mheader, mname) |
|
319
|
|
|
dval = self.calc.per_second(mstr, mval, ts) |
|
320
|
|
|
result.add(plumd.Int(mstr, dval)) |
|
321
|
|
|
return result |
|
322
|
|
|
|
|
323
|
|
|
|
|
324
|
|
|
def proc_net_sockstat(self): |
|
325
|
|
|
skip = self.config.get('skip_proc_net_sockstat') |
|
326
|
|
|
result = plumd.Result("sockstat") |
|
327
|
|
|
fname = "/proc/net/sockstat" |
|
328
|
|
|
dat = {} |
|
329
|
|
|
# read and process - dat is a list of lines from fname |
|
330
|
|
|
dat = get_file_map_list(fname, 0, 0) |
|
331
|
|
|
ts = time.time() |
|
332
|
|
|
|
|
333
|
|
|
for key, val in dat.items(): |
|
334
|
|
|
mstr = key.replace(":", "") |
|
335
|
|
|
mnames = val[::2] |
|
336
|
|
|
mvals = deque([ int(i) for i in val[1::2] ]) |
|
337
|
|
|
if len(mnames) != len(mvals): |
|
338
|
|
|
self.log.error("proc_net_sockstat: invalid entry: {0}".format(mnames)) |
|
339
|
|
|
continue |
|
340
|
|
|
for mname in mnames: |
|
341
|
|
|
metric = "{0}.{1}".format(mstr, mname) |
|
342
|
|
|
result.add(plumd.Int(metric, mvals.popleft())) |
|
343
|
|
|
return result |
|
344
|
|
|
|
|
345
|
|
|
|
|
346
|
|
|
def proc_net_netstat(self): |
|
347
|
|
|
skip = self.config.get('skip_proc_net_netstat') |
|
348
|
|
|
result = plumd.Result("netstat") |
|
349
|
|
|
fname = "/proc/net/netstat" |
|
350
|
|
|
dat = {} |
|
351
|
|
|
# read and process - dat is a list of lines from fname |
|
352
|
|
|
dat = get_file_list(fname) |
|
353
|
|
|
ts = time.time() |
|
354
|
|
|
while len(dat) > 1: |
|
355
|
|
|
headers = deque(dat.popleft().split()) |
|
356
|
|
|
mvals = deque([ int(i) for i in dat.popleft().split()[1:] ]) |
|
357
|
|
|
mstr = headers.popleft().replace(":", "") |
|
358
|
|
|
if len(headers) != len(mvals): |
|
359
|
|
|
self.log.error("proc_net_netstat: invalid entry: {0}".format(headers)) |
|
360
|
|
|
continue |
|
361
|
|
|
for mname in headers: |
|
362
|
|
|
metric = "{0}.{1}".format(mstr, mname) |
|
363
|
|
|
dval = self.calc.per_second(metric, mvals.popleft(), ts) |
|
364
|
|
|
result.add(plumd.Int(metric, dval)) |
|
365
|
|
|
return result |
|
366
|
|
|
|