|
1
|
|
|
import unittest |
|
2
|
|
|
from kafka_influxdb.encoder import collectd_json_encoder |
|
3
|
|
|
import re |
|
4
|
|
|
|
|
5
|
|
|
|
|
6
|
|
|
class TestCollectdJsonEncoder(unittest.TestCase): |
|
7
|
|
|
def setUp(self): |
|
8
|
|
|
self.encoder = collectd_json_encoder.Encoder() |
|
9
|
|
|
|
|
10
|
|
View Code Duplication |
def test_encode(self): |
|
|
|
|
|
|
11
|
|
|
""" |
|
12
|
|
|
Test encoding of messages in collectd json format |
|
13
|
|
|
See https://github.com/mre/kafka-influxdb/issues/6 |
|
14
|
|
|
:return: |
|
15
|
|
|
""" |
|
16
|
|
|
msg = b""" |
|
17
|
|
|
[{"values":[0.6],"dstypes":["gauge"],"dsnames":["value"],"time":1444745144.824,"interval":10.000,"host":"xx.example.internal","plugin":"cpu","plugin_instance":"1","type":"percent","type_instance":"system"}] |
|
18
|
|
|
""" |
|
19
|
|
|
|
|
20
|
|
|
encoded_messages = self.encoder.encode(msg) |
|
21
|
|
|
|
|
22
|
|
|
# We've encoded exactly one message |
|
23
|
|
|
self.assertEqual(len(encoded_messages), 1) |
|
24
|
|
|
|
|
25
|
|
|
encoded_message = encoded_messages[0] |
|
26
|
|
|
|
|
27
|
|
|
expected = '^cpu_1_percent,host=xx\.example\.internal,type_instance=system value=(\d\.\d+) 1444745144$' |
|
28
|
|
|
result = re.match(expected, encoded_message) |
|
29
|
|
|
|
|
30
|
|
|
# Due to floating point precision there might be a tiny difference between the expected and the actual value. |
|
31
|
|
|
self.assertIsNotNone(result, "Unexpected message format") |
|
32
|
|
|
self.assertEqual(len(result.groups()), 1) |
|
33
|
|
|
self.assertAlmostEqual(float(result.group(1)), 0.6) |
|
34
|
|
|
|
|
35
|
|
|
def test_multiple_measurements(self): |
|
36
|
|
|
""" |
|
37
|
|
|
Test encoding of messages in collectd json format |
|
38
|
|
|
See https://github.com/mre/kafka-influxdb/issues/6 |
|
39
|
|
|
:return: |
|
40
|
|
|
""" |
|
41
|
|
|
msg = b""" |
|
42
|
|
|
[{"values":[0.6],"dstypes":["gauge"],"dsnames":["value"],"time":1444745144.824,"interval":10.000,"host":"xx.example.internal","plugin":"cpu","plugin_instance":"1","type":"percent","type_instance":"system"}] |
|
43
|
|
|
[{"values":[0.7],"dstypes":["gauge"],"dsnames":["value"],"time":1444745144.824,"interval":10.000,"host":"example.com","plugin":"cpu","plugin_instance":"1","type":"percent","type_instance":"user"}] |
|
44
|
|
|
[{"values":[37.7],"dstypes":["gauge"],"dsnames":["value"],"time":1444745144.824,"interval":10.000,"host":"myhost","plugin":"cpu","plugin_instance":"0","type":"percent","type_instance":"nice"}] |
|
45
|
|
|
[{"values":[0],"dstypes":["gauge"],"dsnames":["value"],"time":1444745145.824,"interval":10.000,"host":"myhost","plugin":"cpu","plugin_instance":"0","type":"percent","type_instance":"interrupt"}] |
|
46
|
|
|
[{"values":[1.1],"dstypes":["gauge"],"dsnames":["value"],"time":1444745136.182,"interval":10.000,"host":"myhost","plugin":"memory","plugin_instance":"","type":"percent","type_instance":"slab_recl"}] |
|
47
|
|
|
""" |
|
48
|
|
|
expected = [ |
|
49
|
|
|
('cpu_1_percent,host=xx.example.internal,type_instance=system value=(.*) 1444745144', 0.6), |
|
50
|
|
|
('cpu_1_percent,host=example.com,type_instance=user value=(.*) 1444745144', 0.7), |
|
51
|
|
|
('cpu_0_percent,host=myhost,type_instance=nice value=(.*) 1444745144', 37.7), |
|
52
|
|
|
('cpu_0_percent,host=myhost,type_instance=interrupt value=(.*) 1444745145', 0), |
|
53
|
|
|
('memory_percent,host=myhost,type_instance=slab_recl value=(.*) 1444745136', 1.1) |
|
54
|
|
|
] |
|
55
|
|
|
|
|
56
|
|
|
encoded_messages = self.encoder.encode(msg) |
|
57
|
|
|
|
|
58
|
|
|
# We've encoded exactly one message |
|
59
|
|
|
self.assertEqual(len(encoded_messages), 5) |
|
60
|
|
|
|
|
61
|
|
|
for encoded_message, expected in zip(encoded_messages, expected): |
|
62
|
|
|
expected_message, expected_value = expected |
|
63
|
|
|
result = re.match(expected_message, encoded_message) |
|
64
|
|
|
self.assertIsNotNone(result, "Unexpected message format") |
|
65
|
|
|
self.assertEqual(len(result.groups()), 1) |
|
66
|
|
|
self.assertAlmostEqual(float(result.group(1)), expected_value) |
|
67
|
|
|
|
|
68
|
|
|
def test_invalid_messages(self): |
|
69
|
|
|
invalid_messages = [b'', b'\n', b'bla', b'foo\nbar\nbaz'] |
|
70
|
|
|
for msg in invalid_messages: |
|
71
|
|
|
self.assertEqual(self.encoder.encode(msg), []) |
|
72
|
|
|
|
|
73
|
|
|
def test_documentation_examples(self): |
|
74
|
|
|
msg = b""" |
|
75
|
|
|
[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1436372292.412,"interval":10.000,"host":"26f2fc918f50","plugin":"cpu","plugin_instance":"1","type":"cpu","type_instance":"interrupt"}] |
|
76
|
|
|
""" |
|
77
|
|
|
expected = ['cpu_1_cpu,host=26f2fc918f50,type_instance=interrupt value=0 1436372292'] |
|
78
|
|
|
self.assertEqual(self.encoder.encode(msg), expected) |
|
79
|
|
|
|
|
80
|
|
View Code Duplication |
def test_multiple_fields(self): |
|
|
|
|
|
|
81
|
|
|
""" |
|
82
|
|
|
Test supporting multiple fields in a sample |
|
83
|
|
|
[{"values":[0.2, 0.3],"dstypes":["derive"],"dsnames":["cpu_usage", "mem_usage"],"time":1436372292.412,"interval":10.000,"host":"26f2fc918f50","plugin":"sys_usage","plugin_instance":"1","type":"percent"}] |
|
84
|
|
|
""" |
|
85
|
|
|
msg = b""" |
|
86
|
|
|
[{"values":[0.2, 0.3],"dstypes":["derive"],"dsnames":["cpu_usage", "mem_usage"],"time":1436372292.412,"interval":10.000,"host":"26f2fc918f50","plugin":"sys_usage","plugin_instance":"1","type":"percent"}] |
|
87
|
|
|
""" |
|
88
|
|
|
|
|
89
|
|
|
encoded_messages = self.encoder.encode(msg) |
|
90
|
|
|
|
|
91
|
|
|
self.assertEqual(len(encoded_messages), 1) |
|
92
|
|
|
|
|
93
|
|
|
encoded_message = encoded_messages[0] |
|
94
|
|
|
|
|
95
|
|
|
expected = '^sys_usage_1_percent,host=26f2fc918f50 cpu_usage=(\d\.\d+),mem_usage=(\d\.\d+) 1436372292$' |
|
96
|
|
|
result = re.match(expected, encoded_message) |
|
97
|
|
|
|
|
98
|
|
|
# Due to floating point precision there might be a tiny difference between the expected and the actual value. |
|
99
|
|
|
self.assertIsNotNone(result, "Unexpected message format") |
|
100
|
|
|
self.assertEqual(len(result.groups()), 2) |
|
101
|
|
|
self.assertAlmostEqual(float(result.group(1)), 0.2) |
|
102
|
|
|
self.assertAlmostEqual(float(result.group(2)), 0.3) |
|
103
|
|
|
|
|
104
|
|
|
|
|
105
|
|
|
""" |
|
106
|
|
|
[ |
|
107
|
|
|
{ |
|
108
|
|
|
"values": [1901474177], |
|
109
|
|
|
"dstypes": ["counter"], |
|
110
|
|
|
"dsnames": ["value"], |
|
111
|
|
|
"time": 1280959128, |
|
112
|
|
|
"interval": 10, |
|
113
|
|
|
"host": "leeloo.octo.it", |
|
114
|
|
|
"plugin": "cpu", |
|
115
|
|
|
"plugin_instance": "0", |
|
116
|
|
|
"type": "cpu", |
|
117
|
|
|
"type_instance": "idle" |
|
118
|
|
|
} |
|
119
|
|
|
] |
|
120
|
|
|
|
|
121
|
|
|
|
|
122
|
|
|
# See https://collectd.org/wiki/index.php/JSON |
|
123
|
|
|
|
|
124
|
|
|
[ |
|
125
|
|
|
{ |
|
126
|
|
|
"values": [1901474177], |
|
127
|
|
|
"dstypes": ["counter"], |
|
128
|
|
|
"dsnames": ["value"], |
|
129
|
|
|
"time": 1280959128, |
|
130
|
|
|
"interval": 10, |
|
131
|
|
|
"host": "leeloo.octo.it", |
|
132
|
|
|
"plugin": "cpu", |
|
133
|
|
|
"plugin_instance": "0", |
|
134
|
|
|
"type": "cpu", |
|
135
|
|
|
"type_instance": "idle" |
|
136
|
|
|
} |
|
137
|
|
|
] |
|
138
|
|
|
|
|
139
|
|
|
# See https://github.com/mjuenema/collectd-write_json |
|
140
|
|
|
|
|
141
|
|
|
[ |
|
142
|
|
|
{ |
|
143
|
|
|
"dsnames": ['shorttem', 'midterm', 'longterm'], |
|
144
|
|
|
"dstypes": ['gauge', 'gauge', 'gauge'], |
|
145
|
|
|
"host": "localhost", |
|
146
|
|
|
"interval": 5.0, |
|
147
|
|
|
"plugin": "load", |
|
148
|
|
|
"plugin_instance": "", |
|
149
|
|
|
"time": 1432086959.8153536, |
|
150
|
|
|
"type": "load", |
|
151
|
|
|
"type_instance": "", |
|
152
|
|
|
"values": [ |
|
153
|
|
|
0.0, |
|
154
|
|
|
0.01, |
|
155
|
|
|
0.050000000000000003 |
|
156
|
|
|
] |
|
157
|
|
|
} |
|
158
|
|
|
] |
|
159
|
|
|
""" |
|
160
|
|
|
|
mre /
kafka-influxdb
Loading history...
