1
|
|
|
# |
2
|
|
|
# This file is part of Glances. |
3
|
|
|
# |
4
|
|
|
# SPDX-FileCopyrightText: 2022 Nicolas Hennion <[email protected]> |
5
|
|
|
# |
6
|
|
|
# SPDX-License-Identifier: LGPL-3.0-only |
7
|
|
|
|
8
|
|
|
"""Podman Extension unit for Glances' Containers plugin.""" |
9
|
|
|
|
10
|
|
|
from datetime import datetime |
11
|
|
|
|
12
|
|
|
from glances.globals import iterkeys, itervalues, nativestr, pretty_date, replace_special_chars, string_value_to_float |
13
|
|
|
from glances.logger import logger |
14
|
|
|
from glances.plugins.containers.stats_streamer import StatsStreamer |
15
|
|
|
|
16
|
|
|
# Podman library (optional and Linux-only) |
17
|
|
|
# https://pypi.org/project/podman/ |
18
|
|
|
try: |
19
|
|
|
from podman import PodmanClient |
20
|
|
|
except Exception as e: |
21
|
|
|
import_podman_error_tag = True |
22
|
|
|
# Display debug message if import KeyError |
23
|
|
|
logger.warning(f"Error loading Podman deps Lib. Podman feature in the Containers plugin is disabled ({e})") |
24
|
|
|
else: |
25
|
|
|
import_podman_error_tag = False |
26
|
|
|
|
27
|
|
|
|
28
|
|
|
class PodmanContainerStatsFetcher: |
29
|
|
|
MANDATORY_FIELDS = ["CPU", "MemUsage", "MemLimit", "NetInput", "NetOutput", "BlockInput", "BlockOutput"] |
30
|
|
|
|
31
|
|
|
def __init__(self, container): |
32
|
|
|
self._container = container |
33
|
|
|
|
34
|
|
|
# Threaded Streamer |
35
|
|
|
stats_iterable = container.stats(decode=True) |
36
|
|
|
self._streamer = StatsStreamer(stats_iterable, initial_stream_value={}) |
37
|
|
|
|
38
|
|
|
def _log_debug(self, msg, exception=None): |
39
|
|
|
logger.debug(f"containers (Podman) ID: {self._container.id} - {msg} ({exception})") |
40
|
|
|
logger.debug(self._streamer.stats) |
41
|
|
|
|
42
|
|
|
def stop(self): |
43
|
|
|
self._streamer.stop() |
44
|
|
|
|
45
|
|
|
@property |
46
|
|
|
def stats(self): |
47
|
|
|
stats = self._streamer.stats |
48
|
|
|
if stats["Error"]: |
49
|
|
|
self._log_debug("Stats fetching failed", stats["Error"]) |
50
|
|
|
|
51
|
|
|
return stats["Stats"][0] |
52
|
|
|
|
53
|
|
|
@property |
54
|
|
|
def activity_stats(self): |
55
|
|
|
result_stats = {"cpu": {}, "memory": {}, "io": {}, "network": {}} |
56
|
|
|
api_stats = self.stats |
57
|
|
|
|
58
|
|
|
if any(field not in api_stats for field in self.MANDATORY_FIELDS): |
59
|
|
|
self._log_debug("Missing mandatory fields") |
60
|
|
|
return result_stats |
61
|
|
|
|
62
|
|
|
try: |
63
|
|
|
cpu_usage = float(api_stats.get("CPU", 0)) |
64
|
|
|
|
65
|
|
|
mem_usage = float(api_stats["MemUsage"]) |
66
|
|
|
mem_limit = float(api_stats["MemLimit"]) |
67
|
|
|
|
68
|
|
|
rx = float(api_stats["NetInput"]) |
69
|
|
|
tx = float(api_stats["NetOutput"]) |
70
|
|
|
|
71
|
|
|
ior = float(api_stats["BlockInput"]) |
72
|
|
|
iow = float(api_stats["BlockOutput"]) |
73
|
|
|
|
74
|
|
|
# Hardcode `time_since_update` to 1 as podman |
75
|
|
|
# already sends the calculated rate per second |
76
|
|
|
result_stats = { |
77
|
|
|
"cpu": {"total": cpu_usage}, |
78
|
|
|
"memory": {"usage": mem_usage, "limit": mem_limit}, |
79
|
|
|
"io": {"ior": ior, "iow": iow, "time_since_update": 1}, |
80
|
|
|
"network": {"rx": rx, "tx": tx, "time_since_update": 1}, |
81
|
|
|
} |
82
|
|
|
except ValueError as e: |
83
|
|
|
self._log_debug("Non float stats values found", e) |
84
|
|
|
|
85
|
|
|
return result_stats |
86
|
|
|
|
87
|
|
|
|
88
|
|
|
class PodmanPodStatsFetcher: |
89
|
|
|
def __init__(self, pod_manager): |
90
|
|
|
self._pod_manager = pod_manager |
91
|
|
|
|
92
|
|
|
# Threaded Streamer |
93
|
|
|
# Temporary patch to get podman extension working |
94
|
|
|
stats_iterable = (pod_manager.stats(decode=True) for _ in iter(int, 1)) |
95
|
|
|
self._streamer = StatsStreamer(stats_iterable, initial_stream_value={}, sleep_duration=2) |
96
|
|
|
|
97
|
|
|
def _log_debug(self, msg, exception=None): |
98
|
|
|
logger.debug(f"containers (Podman): Pod Manager - {msg} ({exception})") |
99
|
|
|
logger.debug(self._streamer.stats) |
100
|
|
|
|
101
|
|
|
def stop(self): |
102
|
|
|
self._streamer.stop() |
103
|
|
|
|
104
|
|
|
@property |
105
|
|
|
def activity_stats(self): |
106
|
|
|
result_stats = {} |
107
|
|
|
container_stats = self._streamer.stats |
108
|
|
|
for stat in container_stats: |
109
|
|
|
io_stats = self._get_io_stats(stat) |
110
|
|
|
cpu_stats = self._get_cpu_stats(stat) |
111
|
|
|
memory_stats = self._get_memory_stats(stat) |
112
|
|
|
network_stats = self._get_network_stats(stat) |
113
|
|
|
|
114
|
|
|
computed_stats = { |
115
|
|
|
"name": stat["Name"], |
116
|
|
|
"cid": stat["CID"], |
117
|
|
|
"pod_id": stat["Pod"], |
118
|
|
|
"io": io_stats or {}, |
119
|
|
|
"memory": memory_stats or {}, |
120
|
|
|
"network": network_stats or {}, |
121
|
|
|
"cpu": cpu_stats or {"total": 0.0}, |
122
|
|
|
} |
123
|
|
|
result_stats[stat["CID"]] = computed_stats |
124
|
|
|
|
125
|
|
|
return result_stats |
126
|
|
|
|
127
|
|
|
def _get_cpu_stats(self, stats): |
128
|
|
|
"""Return the container CPU usage. |
129
|
|
|
|
130
|
|
|
Output: a dict {'total': 1.49} |
131
|
|
|
""" |
132
|
|
|
if "CPU" not in stats: |
133
|
|
|
self._log_debug("Missing CPU usage fields") |
134
|
|
|
return None |
135
|
|
|
|
136
|
|
|
cpu_usage = string_value_to_float(stats["CPU"].rstrip("%")) |
137
|
|
|
return {"total": cpu_usage} |
138
|
|
|
|
139
|
|
|
def _get_memory_stats(self, stats): |
140
|
|
|
"""Return the container MEMORY. |
141
|
|
|
|
142
|
|
|
Output: a dict {'usage': ..., 'limit': ...} |
143
|
|
|
""" |
144
|
|
|
if "MemUsage" not in stats or "/" not in stats["MemUsage"]: |
145
|
|
|
self._log_debug("Missing MEM usage fields") |
146
|
|
|
return None |
147
|
|
|
|
148
|
|
|
memory_usage_str = stats["MemUsage"] |
149
|
|
|
usage_str, limit_str = memory_usage_str.split("/") |
150
|
|
|
|
151
|
|
|
try: |
152
|
|
|
usage = string_value_to_float(usage_str) |
153
|
|
|
limit = string_value_to_float(limit_str) |
154
|
|
|
except ValueError as e: |
155
|
|
|
self._log_debug("Compute MEM usage failed", e) |
156
|
|
|
return None |
157
|
|
|
|
158
|
|
|
return {'usage': usage, 'limit': limit, 'inactive_file': 0} |
159
|
|
|
|
160
|
|
|
def _get_network_stats(self, stats): |
161
|
|
|
"""Return the container network usage using the Docker API (v1.0 or higher). |
162
|
|
|
|
163
|
|
|
Output: a dict {'time_since_update': 3000, 'rx': 10, 'tx': 65}. |
164
|
|
|
with: |
165
|
|
|
time_since_update: number of seconds elapsed between the latest grab |
166
|
|
|
rx: Number of bytes received |
167
|
|
|
tx: Number of bytes transmitted |
168
|
|
|
""" |
169
|
|
|
if "NetIO" not in stats or "/" not in stats["NetIO"]: |
170
|
|
|
self._log_debug("Compute MEM usage failed") |
171
|
|
|
return None |
172
|
|
|
|
173
|
|
|
net_io_str = stats["NetIO"] |
174
|
|
|
rx_str, tx_str = net_io_str.split("/") |
175
|
|
|
|
176
|
|
|
try: |
177
|
|
|
rx = string_value_to_float(rx_str) |
178
|
|
|
tx = string_value_to_float(tx_str) |
179
|
|
|
except ValueError as e: |
180
|
|
|
self._log_debug("Compute MEM usage failed", e) |
181
|
|
|
return None |
182
|
|
|
|
183
|
|
|
# Hardcode `time_since_update` to 1 as podman docs don't specify the rate calculated procedure |
184
|
|
|
return {"rx": rx, "tx": tx, "time_since_update": 1} |
185
|
|
|
|
186
|
|
|
def _get_io_stats(self, stats): |
187
|
|
|
"""Return the container IO usage using the Docker API (v1.0 or higher). |
188
|
|
|
|
189
|
|
|
Output: a dict {'time_since_update': 3000, 'ior': 10, 'iow': 65}. |
190
|
|
|
with: |
191
|
|
|
time_since_update: number of seconds elapsed between the latest grab |
192
|
|
|
ior: Number of bytes read |
193
|
|
|
iow: Number of bytes written |
194
|
|
|
""" |
195
|
|
|
if "BlockIO" not in stats or "/" not in stats["BlockIO"]: |
196
|
|
|
self._log_debug("Missing BlockIO usage fields") |
197
|
|
|
return None |
198
|
|
|
|
199
|
|
|
block_io_str = stats["BlockIO"] |
200
|
|
|
ior_str, iow_str = block_io_str.split("/") |
201
|
|
|
|
202
|
|
|
try: |
203
|
|
|
ior = string_value_to_float(ior_str) |
204
|
|
|
iow = string_value_to_float(iow_str) |
205
|
|
|
except ValueError as e: |
206
|
|
|
self._log_debug("Compute BlockIO usage failed", e) |
207
|
|
|
return None |
208
|
|
|
|
209
|
|
|
# Hardcode `time_since_update` to 1 as podman docs don't specify the rate calculated procedure |
210
|
|
|
return {"ior": ior, "iow": iow, "time_since_update": 1} |
211
|
|
|
|
212
|
|
|
|
213
|
|
|
class PodmanContainersExtension: |
214
|
|
|
"""Glances' Containers Plugin's Docker Extension unit""" |
215
|
|
|
|
216
|
|
|
CONTAINER_ACTIVE_STATUS = ['running', 'paused'] |
217
|
|
|
|
218
|
|
|
def __init__(self, podman_sock): |
219
|
|
|
if import_podman_error_tag: |
220
|
|
|
raise Exception("Missing libs required to run Podman Extension (Containers)") |
221
|
|
|
|
222
|
|
|
self.client = None |
223
|
|
|
self.ext_name = "containers (Podman)" |
224
|
|
|
self.podman_sock = podman_sock |
225
|
|
|
self.pods_stats_fetcher = None |
226
|
|
|
self.container_stats_fetchers = {} |
227
|
|
|
|
228
|
|
|
self.connect() |
229
|
|
|
|
230
|
|
|
def connect(self): |
231
|
|
|
"""Connect to Podman.""" |
232
|
|
|
try: |
233
|
|
|
self.client = PodmanClient(base_url=self.podman_sock) |
234
|
|
|
# PodmanClient works lazily, so make a ping to determine if socket is open |
235
|
|
|
self.client.ping() |
236
|
|
|
except Exception as e: |
237
|
|
|
logger.debug(f"{self.ext_name} plugin - Can't connect to Podman ({e})") |
238
|
|
|
self.client = None |
239
|
|
|
|
240
|
|
|
def update_version(self): |
241
|
|
|
# Long and not useful anymore because the information is no more displayed in UIs |
242
|
|
|
# return self.client.version() |
243
|
|
|
return {} |
244
|
|
|
|
245
|
|
|
def stop(self): |
246
|
|
|
# Stop all streaming threads |
247
|
|
|
for t in itervalues(self.container_stats_fetchers): |
248
|
|
|
t.stop() |
249
|
|
|
|
250
|
|
|
if self.pods_stats_fetcher: |
251
|
|
|
self.pods_stats_fetcher.stop() |
252
|
|
|
|
253
|
|
|
def update(self, all_tag): |
254
|
|
|
"""Update Podman stats using the input method.""" |
255
|
|
|
|
256
|
|
|
if not self.client: |
257
|
|
|
return {}, [] |
258
|
|
|
|
259
|
|
|
version_stats = self.update_version() |
260
|
|
|
|
261
|
|
|
# Update current containers list |
262
|
|
|
try: |
263
|
|
|
# Issue #1152: Podman module doesn't export details about stopped containers |
264
|
|
|
# The Containers/all key of the configuration file should be set to True |
265
|
|
|
containers = self.client.containers.list(all=all_tag) |
266
|
|
|
if not self.pods_stats_fetcher: |
267
|
|
|
self.pods_stats_fetcher = PodmanPodStatsFetcher(self.client.pods) |
268
|
|
|
except Exception as e: |
269
|
|
|
logger.error(f"{self.ext_name} plugin - Can't get containers list ({e})") |
270
|
|
|
return version_stats, [] |
271
|
|
|
|
272
|
|
|
# Start new thread for new container |
273
|
|
|
for container in containers: |
274
|
|
|
if container.id not in self.container_stats_fetchers: |
275
|
|
|
# StatsFetcher did not exist in the internal dict |
276
|
|
|
# Create it, add it to the internal dict |
277
|
|
|
logger.debug(f"{self.ext_name} plugin - Create thread for container {container.id[:12]}") |
278
|
|
|
self.container_stats_fetchers[container.id] = PodmanContainerStatsFetcher(container) |
279
|
|
|
|
280
|
|
|
# Stop threads for non-existing containers |
281
|
|
|
absent_containers = set(iterkeys(self.container_stats_fetchers)) - {c.id for c in containers} |
282
|
|
|
for container_id in absent_containers: |
283
|
|
|
# Stop the StatsFetcher |
284
|
|
|
logger.debug(f"{self.ext_name} plugin - Stop thread for old container {container_id[:12]}") |
285
|
|
|
self.container_stats_fetchers[container_id].stop() |
286
|
|
|
# Delete the StatsFetcher from the dict |
287
|
|
|
del self.container_stats_fetchers[container_id] |
288
|
|
|
|
289
|
|
|
# Get stats for all containers |
290
|
|
|
container_stats = [self.generate_stats(container) for container in containers] |
291
|
|
|
|
292
|
|
|
pod_stats = self.pods_stats_fetcher.activity_stats |
293
|
|
|
for stats in container_stats: |
294
|
|
|
if stats["id"][:12] in pod_stats: |
295
|
|
|
stats["pod_name"] = pod_stats[stats["id"][:12]]["name"] |
296
|
|
|
stats["pod_id"] = pod_stats[stats["id"][:12]]["pod_id"] |
297
|
|
|
|
298
|
|
|
return version_stats, container_stats |
299
|
|
|
|
300
|
|
|
@property |
301
|
|
|
def key(self): |
302
|
|
|
"""Return the key of the list.""" |
303
|
|
|
return 'name' |
304
|
|
|
|
305
|
|
|
def generate_stats(self, container): |
306
|
|
|
# Init the stats for the current container |
307
|
|
|
stats = { |
308
|
|
|
'key': self.key, |
309
|
|
|
# Export name |
310
|
|
|
'name': nativestr(container.name), |
311
|
|
|
# Container Id |
312
|
|
|
'id': container.id, |
313
|
|
|
# Container Image |
314
|
|
|
'image': ','.join(container.image.tags if container.image.tags else []), |
315
|
|
|
# Container Status (from attrs) |
316
|
|
|
'status': container.attrs['State'], |
317
|
|
|
'created': container.attrs['Created'], |
318
|
|
|
'command': container.attrs.get('Command') or [], |
319
|
|
|
} |
320
|
|
|
|
321
|
|
|
if stats['status'] in self.CONTAINER_ACTIVE_STATUS: |
322
|
|
|
started_at = datetime.fromtimestamp(container.attrs['StartedAt']) |
323
|
|
|
stats_fetcher = self.container_stats_fetchers[container.id] |
324
|
|
|
activity_stats = stats_fetcher.activity_stats |
325
|
|
|
stats.update(activity_stats) |
326
|
|
|
|
327
|
|
|
# Additional fields |
328
|
|
|
stats['cpu_percent'] = stats["cpu"]['total'] |
329
|
|
|
stats['memory_usage'] = stats["memory"].get('usage') |
330
|
|
|
if stats['memory'].get('cache') is not None: |
331
|
|
|
stats['memory_usage'] -= stats['memory']['cache'] |
332
|
|
|
stats['io_rx'] = stats['io'].get('ior') // stats['io'].get('time_since_update') |
333
|
|
|
stats['io_wx'] = stats['io'].get('iow') // stats['io'].get('time_since_update') |
334
|
|
|
stats['network_rx'] = stats['network'].get('rx') // stats['network'].get('time_since_update') |
335
|
|
|
stats['network_tx'] = stats['network'].get('tx') // stats['network'].get('time_since_update') |
336
|
|
|
stats['uptime'] = pretty_date(started_at) |
337
|
|
|
# Manage special chars in command (see isse#2733) |
338
|
|
|
stats['command'] = replace_special_chars(' '.join(stats['command'])) |
339
|
|
|
else: |
340
|
|
|
stats['io'] = {} |
341
|
|
|
stats['cpu'] = {} |
342
|
|
|
stats['memory'] = {} |
343
|
|
|
stats['network'] = {} |
344
|
|
|
stats['io_rx'] = None |
345
|
|
|
stats['io_wx'] = None |
346
|
|
|
stats['cpu_percent'] = None |
347
|
|
|
stats['memory_percent'] = None |
348
|
|
|
stats['network_rx'] = None |
349
|
|
|
stats['network_tx'] = None |
350
|
|
|
stats['uptime'] = None |
351
|
|
|
|
352
|
|
|
return stats |
353
|
|
|
|