1 | # |
||
2 | # This file is part of Glances. |
||
3 | # |
||
4 | # SPDX-FileCopyrightText: 2022 Nicolas Hennion <[email protected]> |
||
5 | # |
||
6 | # SPDX-License-Identifier: LGPL-3.0-only |
||
7 | # |
||
8 | |||
9 | """Containers plugin.""" |
||
10 | |||
11 | import os |
||
12 | from copy import deepcopy |
||
13 | |||
14 | from glances.logger import logger |
||
15 | from glances.plugins.containers.engines.docker import DockerContainersExtension, import_docker_error_tag |
||
16 | from glances.plugins.containers.engines.podman import PodmanContainersExtension, import_podman_error_tag |
||
17 | from glances.plugins.plugin.model import GlancesPluginModel |
||
18 | from glances.processes import glances_processes |
||
19 | from glances.processes import sort_stats as sort_stats_processes |
||
20 | |||
21 | # Fields description |
||
22 | # description: human readable description |
||
23 | # short_name: shortname to use un UI |
||
24 | # unit: unit type |
||
25 | # rate: is it a rate ? If yes, // by time_since_update when displayed, |
||
26 | # min_symbol: Auto unit should be used if value > than 1 'X' (K, M, G)... |
||
27 | fields_description = { |
||
28 | 'name': { |
||
29 | 'description': 'Container name', |
||
30 | }, |
||
31 | 'id': { |
||
32 | 'description': 'Container ID', |
||
33 | }, |
||
34 | 'image': { |
||
35 | 'description': 'Container image', |
||
36 | }, |
||
37 | 'status': { |
||
38 | 'description': 'Container status', |
||
39 | }, |
||
40 | 'created': { |
||
41 | 'description': 'Container creation date', |
||
42 | }, |
||
43 | 'command': { |
||
44 | 'description': 'Container command', |
||
45 | }, |
||
46 | 'cpu_percent': { |
||
47 | 'description': 'Container CPU consumption', |
||
48 | 'unit': 'percent', |
||
49 | }, |
||
50 | 'memory_usage': { |
||
51 | 'description': 'Container memory usage', |
||
52 | 'unit': 'byte', |
||
53 | }, |
||
54 | 'io_rx': { |
||
55 | 'description': 'Container IO bytes read rate', |
||
56 | 'unit': 'bytepersecond', |
||
57 | }, |
||
58 | 'io_wx': { |
||
59 | 'description': 'Container IO bytes write rate', |
||
60 | 'unit': 'bytepersecond', |
||
61 | }, |
||
62 | 'network_rx': { |
||
63 | 'description': 'Container network RX bitrate', |
||
64 | 'unit': 'bitpersecond', |
||
65 | }, |
||
66 | 'network_tx': { |
||
67 | 'description': 'Container network TX bitrate', |
||
68 | 'unit': 'bitpersecond', |
||
69 | }, |
||
70 | 'uptime': { |
||
71 | 'description': 'Container uptime', |
||
72 | }, |
||
73 | 'engine': { |
||
74 | 'description': 'Container engine (Docker and Podman are currently supported)', |
||
75 | }, |
||
76 | 'pod_name': { |
||
77 | 'description': 'Pod name (only with Podman)', |
||
78 | }, |
||
79 | 'pod_id': { |
||
80 | 'description': 'Pod ID (only with Podman)', |
||
81 | }, |
||
82 | } |
||
83 | |||
84 | # Define the items history list (list of items to add to history) |
||
85 | # TODO: For the moment limited to the CPU. Had to change the graph exports |
||
86 | # method to display one graph per container. |
||
87 | # items_history_list = [{'name': 'cpu_percent', |
||
88 | # 'description': 'Container CPU consumption in %', |
||
89 | # 'y_unit': '%'}, |
||
90 | # {'name': 'memory_usage', |
||
91 | # 'description': 'Container memory usage in bytes', |
||
92 | # 'y_unit': 'B'}, |
||
93 | # {'name': 'network_rx', |
||
94 | # 'description': 'Container network RX bitrate in bits per second', |
||
95 | # 'y_unit': 'bps'}, |
||
96 | # {'name': 'network_tx', |
||
97 | # 'description': 'Container network TX bitrate in bits per second', |
||
98 | # 'y_unit': 'bps'}, |
||
99 | # {'name': 'io_r', |
||
100 | # 'description': 'Container IO bytes read per second', |
||
101 | # 'y_unit': 'Bps'}, |
||
102 | # {'name': 'io_w', |
||
103 | # 'description': 'Container IO bytes write per second', |
||
104 | # 'y_unit': 'Bps'}] |
||
105 | items_history_list = [{'name': 'cpu_percent', 'description': 'Container CPU consumption in %', 'y_unit': '%'}] |
||
106 | |||
107 | # List of key to remove before export |
||
108 | export_exclude_list = ['cpu', 'io', 'memory', 'network'] |
||
109 | |||
110 | # Sort dictionary for human |
||
111 | sort_for_human = { |
||
112 | 'io_counters': 'disk IO', |
||
113 | 'cpu_percent': 'CPU consumption', |
||
114 | 'memory_usage': 'memory consumption', |
||
115 | 'cpu_times': 'uptime', |
||
116 | 'name': 'container name', |
||
117 | None: 'None', |
||
118 | } |
||
119 | |||
120 | |||
121 | class PluginModel(GlancesPluginModel): |
||
122 | """Glances Docker plugin. |
||
123 | |||
124 | stats is a dict: {'version': {...}, 'containers': [{}, {}]} |
||
125 | """ |
||
126 | |||
127 | def __init__(self, args=None, config=None): |
||
128 | """Init the plugin.""" |
||
129 | super().__init__( |
||
130 | args=args, config=config, items_history_list=items_history_list, fields_description=fields_description |
||
131 | ) |
||
132 | |||
133 | # The plugin can be disabled using: args.disable_docker |
||
134 | self.args = args |
||
135 | |||
136 | # Default config keys |
||
137 | self.config = config |
||
138 | |||
139 | # We want to display the stat in the curse interface |
||
140 | self.display_curse = True |
||
141 | |||
142 | # Init the Docker API |
||
143 | self.docker_extension = DockerContainersExtension() if not import_docker_error_tag else None |
||
144 | |||
145 | # Init the Podman API |
||
146 | if import_podman_error_tag: |
||
147 | self.podman_extension = None |
||
148 | else: |
||
149 | self.podman_extension = PodmanContainersExtension(podman_sock=self._podman_sock()) |
||
150 | |||
151 | # Sort key |
||
152 | self.sort_key = None |
||
153 | |||
154 | # Force a first update because we need two update to have the first stat |
||
155 | self.update() |
||
156 | self.refresh_timer.set(0) |
||
157 | |||
158 | def _podman_sock(self): |
||
159 | """Return the podman sock. |
||
160 | Could be desfined in the [docker] section thanks to the podman_sock option. |
||
161 | Default value: unix:///run/user/1000/podman/podman.sock |
||
162 | """ |
||
163 | conf_podman_sock = self.get_conf_value('podman_sock') |
||
164 | if len(conf_podman_sock) == 0: |
||
165 | return "unix:///run/user/1000/podman/podman.sock" |
||
166 | return conf_podman_sock[0] |
||
167 | |||
168 | def exit(self): |
||
169 | """Overwrite the exit method to close threads.""" |
||
170 | if self.docker_extension: |
||
171 | self.docker_extension.stop() |
||
172 | if self.podman_extension: |
||
173 | self.podman_extension.stop() |
||
174 | # Call the father class |
||
175 | super().exit() |
||
176 | |||
177 | def get_key(self): |
||
178 | """Return the key of the list.""" |
||
179 | return 'name' |
||
180 | |||
181 | def get_export(self): |
||
182 | """Overwrite the default export method. |
||
183 | |||
184 | - Only exports containers |
||
185 | - The key is the first container name |
||
186 | """ |
||
187 | try: |
||
188 | ret = deepcopy(self.stats) |
||
189 | except KeyError as e: |
||
190 | logger.debug(f"docker plugin - Docker export error {e}") |
||
191 | ret = [] |
||
192 | |||
193 | # Remove fields uses to compute rate |
||
194 | for container in ret: |
||
195 | for i in export_exclude_list: |
||
196 | container.pop(i) |
||
197 | |||
198 | return ret |
||
199 | |||
200 | def _all_tag(self): |
||
201 | """Return the all tag of the Glances/Docker configuration file. |
||
202 | |||
203 | # By default, Glances only display running containers |
||
204 | # Set the following key to True to display all containers |
||
205 | all=True |
||
206 | """ |
||
207 | all_tag = self.get_conf_value('all') |
||
208 | if len(all_tag) == 0: |
||
209 | return False |
||
210 | return all_tag[0].lower() == 'true' |
||
211 | |||
212 | @GlancesPluginModel._check_decorator |
||
213 | @GlancesPluginModel._log_result_decorator |
||
214 | def update(self): |
||
215 | """Update Docker and podman stats using the input method.""" |
||
216 | # Connection should be ok |
||
217 | if self.docker_extension is None and self.podman_extension is None: |
||
218 | return self.get_init_value() |
||
219 | |||
220 | if self.input_method == 'local': |
||
221 | # Update stats |
||
222 | stats_docker = self.update_docker() if self.docker_extension else {} |
||
223 | stats_podman = self.update_podman() if self.podman_extension else {} |
||
224 | stats = stats_docker.get('containers', []) + stats_podman.get('containers', []) |
||
225 | elif self.input_method == 'snmp': |
||
226 | # Update stats using SNMP |
||
227 | # Not available |
||
228 | pass |
||
229 | |||
230 | # Sort and update the stats |
||
231 | # @TODO: Have a look because sort did not work for the moment (need memory stats ?) |
||
232 | self.sort_key, self.stats = sort_docker_stats(stats) |
||
0 ignored issues
–
show
introduced
by
Loading history...
|
|||
233 | |||
234 | return self.stats |
||
235 | |||
236 | def update_docker(self): |
||
237 | """Update Docker stats using the input method.""" |
||
238 | version, containers = self.docker_extension.update(all_tag=self._all_tag()) |
||
239 | for container in containers: |
||
240 | container["engine"] = 'docker' |
||
241 | return {"version": version, "containers": containers} |
||
242 | |||
243 | def update_podman(self): |
||
244 | """Update Podman stats.""" |
||
245 | version, containers = self.podman_extension.update(all_tag=self._all_tag()) |
||
246 | for container in containers: |
||
247 | container["engine"] = 'podman' |
||
248 | return {"version": version, "containers": containers} |
||
249 | |||
250 | def get_user_ticks(self): |
||
251 | """Return the user ticks by reading the environment variable.""" |
||
252 | return os.sysconf(os.sysconf_names['SC_CLK_TCK']) |
||
253 | |||
254 | def memory_usage_no_cache(self, mem): |
||
255 | """Return the 'real' memory usage by removing inactive_file to usage""" |
||
256 | # Ref: https://github.com/docker/docker-py/issues/3210 |
||
257 | return mem['usage'] - (mem['inactive_file'] if 'inactive_file' in mem else 0) |
||
258 | |||
259 | def update_views(self): |
||
260 | """Update stats views.""" |
||
261 | # Call the father's method |
||
262 | super().update_views() |
||
263 | |||
264 | if not self.stats: |
||
265 | return False |
||
266 | |||
267 | # Add specifics information |
||
268 | # Alert |
||
269 | for i in self.stats: |
||
270 | # Init the views for the current container (key = container name) |
||
271 | self.views[i[self.get_key()]] = {'cpu': {}, 'mem': {}} |
||
272 | # CPU alert |
||
273 | if 'cpu' in i and 'total' in i['cpu']: |
||
274 | # Looking for specific CPU container threshold in the conf file |
||
275 | alert = self.get_alert(i['cpu']['total'], header=i['name'] + '_cpu', action_key=i['name']) |
||
276 | if alert == 'DEFAULT': |
||
277 | # Not found ? Get back to default CPU threshold value |
||
278 | alert = self.get_alert(i['cpu']['total'], header='cpu') |
||
279 | self.views[i[self.get_key()]]['cpu']['decoration'] = alert |
||
280 | # MEM alert |
||
281 | if 'memory' in i and 'usage' in i['memory']: |
||
282 | # Looking for specific MEM container threshold in the conf file |
||
283 | alert = self.get_alert( |
||
284 | self.memory_usage_no_cache(i['memory']), |
||
285 | maximum=i['memory']['limit'], |
||
286 | header=i['name'] + '_mem', |
||
287 | action_key=i['name'], |
||
288 | ) |
||
289 | if alert == 'DEFAULT': |
||
290 | # Not found ? Get back to default MEM threshold value |
||
291 | alert = self.get_alert( |
||
292 | self.memory_usage_no_cache(i['memory']), maximum=i['memory']['limit'], header='mem' |
||
293 | ) |
||
294 | self.views[i[self.get_key()]]['mem']['decoration'] = alert |
||
295 | |||
296 | # Display Engine and Pod name ? |
||
297 | show_pod_name = False |
||
298 | if any(ct.get("pod_name") for ct in self.stats): |
||
299 | show_pod_name = True |
||
300 | self.views['show_pod_name'] = show_pod_name |
||
301 | show_engine_name = False |
||
302 | if len({ct["engine"] for ct in self.stats}) > 1: |
||
303 | show_engine_name = True |
||
304 | self.views['show_engine_name'] = show_engine_name |
||
305 | |||
306 | return True |
||
307 | |||
308 | def msg_curse(self, args=None, max_width=None): |
||
309 | """Return the dict to display in the curse interface.""" |
||
310 | # Init the return message |
||
311 | ret = [] |
||
312 | |||
313 | # Only process if stats exist (and non null) and display plugin enable... |
||
314 | if not self.stats or len(self.stats) == 0 or self.is_disabled(): |
||
315 | return ret |
||
316 | |||
317 | # Build the string message |
||
318 | # Title |
||
319 | msg = '{}'.format('CONTAINERS') |
||
320 | ret.append(self.curse_add_line(msg, "TITLE")) |
||
321 | msg = f' {len(self.stats)}' |
||
322 | ret.append(self.curse_add_line(msg)) |
||
323 | msg = f' sorted by {sort_for_human[self.sort_key]}' |
||
324 | ret.append(self.curse_add_line(msg)) |
||
325 | ret.append(self.curse_new_line()) |
||
326 | # Header |
||
327 | ret.append(self.curse_new_line()) |
||
328 | # Get the maximum containers name |
||
329 | # Max size is configurable. See feature request #1723. |
||
330 | name_max_width = min( |
||
331 | self.config.get_int_value('containers', 'max_name_size', default=20) if self.config is not None else 20, |
||
332 | len(max(self.stats, key=lambda x: len(x['name']))['name']), |
||
333 | ) |
||
334 | |||
335 | if self.views['show_engine_name']: |
||
336 | msg = ' {:{width}}'.format('Engine', width=6) |
||
337 | ret.append(self.curse_add_line(msg)) |
||
338 | if self.views['show_pod_name']: |
||
339 | msg = ' {:{width}}'.format('Pod', width=12) |
||
340 | ret.append(self.curse_add_line(msg)) |
||
341 | msg = ' {:{width}}'.format('Name', width=name_max_width) |
||
342 | ret.append(self.curse_add_line(msg, 'SORT' if self.sort_key == 'name' else 'DEFAULT')) |
||
343 | msg = '{:>10}'.format('Status') |
||
344 | ret.append(self.curse_add_line(msg)) |
||
345 | msg = '{:>10}'.format('Uptime') |
||
346 | ret.append(self.curse_add_line(msg)) |
||
347 | msg = '{:>6}'.format('CPU%') |
||
348 | ret.append(self.curse_add_line(msg, 'SORT' if self.sort_key == 'cpu_percent' else 'DEFAULT')) |
||
349 | msg = '{:>7}'.format('MEM') |
||
350 | ret.append(self.curse_add_line(msg, 'SORT' if self.sort_key == 'memory_usage' else 'DEFAULT')) |
||
351 | msg = '/{:<7}'.format('MAX') |
||
352 | ret.append(self.curse_add_line(msg)) |
||
353 | msg = '{:>7}'.format('IOR/s') |
||
354 | ret.append(self.curse_add_line(msg)) |
||
355 | msg = ' {:<7}'.format('IOW/s') |
||
356 | ret.append(self.curse_add_line(msg)) |
||
357 | msg = '{:>7}'.format('Rx/s') |
||
358 | ret.append(self.curse_add_line(msg)) |
||
359 | msg = ' {:<7}'.format('Tx/s') |
||
360 | ret.append(self.curse_add_line(msg)) |
||
361 | msg = ' {:8}'.format('Command') |
||
362 | ret.append(self.curse_add_line(msg)) |
||
363 | |||
364 | # Data |
||
365 | for container in self.stats: |
||
366 | ret.append(self.curse_new_line()) |
||
367 | if self.views['show_engine_name']: |
||
368 | ret.append(self.curse_add_line(' {:{width}}'.format(container["engine"], width=6))) |
||
369 | if self.views['show_pod_name']: |
||
370 | ret.append(self.curse_add_line(' {:{width}}'.format(container.get("pod_id", "-"), width=12))) |
||
371 | # Name |
||
372 | ret.append(self.curse_add_line(self._msg_name(container=container, max_width=name_max_width))) |
||
373 | # Status |
||
374 | status = self.container_alert(container['status']) |
||
375 | msg = '{:>10}'.format(container['status'][0:10]) |
||
376 | ret.append(self.curse_add_line(msg, status)) |
||
377 | # Uptime |
||
378 | if container['uptime']: |
||
379 | msg = '{:>10}'.format(container['uptime']) |
||
380 | else: |
||
381 | msg = '{:>10}'.format('_') |
||
382 | ret.append(self.curse_add_line(msg)) |
||
383 | # CPU |
||
384 | try: |
||
385 | msg = '{:>6.1f}'.format(container['cpu']['total']) |
||
386 | except (KeyError, TypeError): |
||
387 | msg = '{:>6}'.format('_') |
||
388 | ret.append(self.curse_add_line(msg, self.get_views(item=container['name'], key='cpu', option='decoration'))) |
||
389 | # MEM |
||
390 | try: |
||
391 | msg = '{:>7}'.format(self.auto_unit(self.memory_usage_no_cache(container['memory']))) |
||
392 | except KeyError: |
||
393 | msg = '{:>7}'.format('_') |
||
394 | ret.append(self.curse_add_line(msg, self.get_views(item=container['name'], key='mem', option='decoration'))) |
||
395 | try: |
||
396 | msg = '/{:<7}'.format(self.auto_unit(container['memory']['limit'])) |
||
397 | except (KeyError, TypeError): |
||
398 | msg = '/{:<7}'.format('_') |
||
399 | ret.append(self.curse_add_line(msg)) |
||
400 | # IO R/W |
||
401 | unit = 'B' |
||
402 | try: |
||
403 | value = self.auto_unit(int(container['io_rx'])) + unit |
||
404 | msg = f'{value:>7}' |
||
405 | except (KeyError, TypeError): |
||
406 | msg = '{:>7}'.format('_') |
||
407 | ret.append(self.curse_add_line(msg)) |
||
408 | try: |
||
409 | value = self.auto_unit(int(container['io_wx'])) + unit |
||
410 | msg = f' {value:<7}' |
||
411 | except (KeyError, TypeError): |
||
412 | msg = ' {:<7}'.format('_') |
||
413 | ret.append(self.curse_add_line(msg)) |
||
414 | # NET RX/TX |
||
415 | if args.byte: |
||
416 | # Bytes per second (for dummy) |
||
417 | to_bit = 1 |
||
418 | unit = '' |
||
419 | else: |
||
420 | # Bits per second (for real network administrator | Default) |
||
421 | to_bit = 8 |
||
422 | unit = 'b' |
||
423 | try: |
||
424 | value = self.auto_unit(int(container['network_rx'] * to_bit)) + unit |
||
425 | msg = f'{value:>7}' |
||
426 | except (KeyError, TypeError): |
||
427 | msg = '{:>7}'.format('_') |
||
428 | ret.append(self.curse_add_line(msg)) |
||
429 | try: |
||
430 | value = self.auto_unit(int(container['network_tx'] * to_bit)) + unit |
||
431 | msg = f' {value:<7}' |
||
432 | except (KeyError, TypeError): |
||
433 | msg = ' {:<7}'.format('_') |
||
434 | ret.append(self.curse_add_line(msg)) |
||
435 | # Command |
||
436 | if container['command'] is not None: |
||
437 | msg = ' {}'.format(container['command']) |
||
438 | else: |
||
439 | msg = ' {}'.format('_') |
||
440 | ret.append(self.curse_add_line(msg, splittable=True)) |
||
441 | |||
442 | return ret |
||
443 | |||
444 | def _msg_name(self, container, max_width): |
||
445 | """Build the container name.""" |
||
446 | name = container['name'][:max_width] |
||
447 | return ' {:{width}}'.format(name, width=max_width) |
||
448 | |||
449 | def container_alert(self, status): |
||
450 | """Analyse the container status.""" |
||
451 | if status == 'running': |
||
452 | return 'OK' |
||
453 | if status == 'exited': |
||
454 | return 'WARNING' |
||
455 | if status == 'dead': |
||
456 | return 'CRITICAL' |
||
457 | return 'CAREFUL' |
||
458 | |||
459 | |||
460 | def sort_docker_stats(stats): |
||
461 | # Sort Docker stats using the same function than processes |
||
462 | sort_by = glances_processes.sort_key |
||
463 | sort_by_secondary = 'memory_usage' |
||
464 | if sort_by == 'memory_percent': |
||
465 | sort_by = 'memory_usage' |
||
466 | sort_by_secondary = 'cpu_percent' |
||
467 | elif sort_by in ['username', 'io_counters', 'cpu_times']: |
||
468 | sort_by = 'cpu_percent' |
||
469 | |||
470 | # Sort docker stats |
||
471 | sort_stats_processes( |
||
472 | stats, |
||
473 | sorted_by=sort_by, |
||
474 | sorted_by_secondary=sort_by_secondary, |
||
475 | # Reverse for all but name |
||
476 | reverse=glances_processes.sort_key != 'name', |
||
477 | ) |
||
478 | |||
479 | # Return the main sort key and the sorted stats |
||
480 | return sort_by, stats |
||
481 |