glances.exports.glances_influxdb2.Export._normalize() - Code Metrics - Inspection of "Glances version 4.0.0" - nicolargo/glances - Measure and Improve Code Quality continuously with Scrutinizer

Test Failed

Push — master ( ee826a...d9056e )

by Nicolas

created 2024-05-12 08:45 UTC

Export._normalize() D

↳ Parent: glances.exports.glances_influxdb2

Complexity

Conditions

Size

Total Lines	61
Code Lines	35

Duplication

Lines	61
Ratio	100 %

Importance

Changes

Metric	Value
cc	13
eloc	35
nop	4
dl	61
loc	61
rs	4.2
c	0
b	0
f	0

How to fix Long Method Complexity

# -*- coding: utf-8 -*-
#
# This file is part of Glances.
#
# SPDX-FileCopyrightText: 2022 Nicolas Hennion <[email protected]>
#
# SPDX-License-Identifier: LGPL-3.0-only
#

"""InfluxDB (from to InfluxDB 1.8+) interface class."""

import sys
from platform import node

from glances.logger import logger
from glances.exports.export import GlancesExport

from influxdb_client import InfluxDBClient, WriteOptions

FIELD_TO_TAG = ['name', 'cmdline']


class Export(GlancesExport):
    """This class manages the InfluxDB export module."""

    def __init__(self, config=None, args=None):
        """Init the InfluxDB export IF."""
        super(Export, self).__init__(config=config, args=args)

        # Mandatory configuration keys (additional to host and port)
        self.org = None
        self.bucket = None
        self.token = None

        # Optional configuration keys
        self.protocol = 'http'
        self.prefix = None
        self.tags = None
        self.hostname = None
        self.interval = None

        # Load the InfluxDB configuration file
        self.export_enable = self.load_conf(
            'influxdb2',
            mandatories=['host', 'port', 'user', 'password', 'org', 'bucket', 'token'],
            options=['protocol', 'prefix', 'tags', 'interval'],
        )
        if not self.export_enable:
            exit('Missing influxdb2 config')

        # Interval between two exports (in seconds)
        if self.interval is None:
            self.interval = 0
        try:
            self.interval = int(self.interval)
        except ValueError:
            logger.warning("InfluxDB export interval is not an integer, use default value")
            self.interval = 0
        # and should be set to the Glances refresh time if the value is 0
        self.interval = self.interval if self.interval > 0 else self.args.time
        logger.debug("InfluxDB export interval is set to {} seconds".format(self.interval))

        # The hostname is always add as a tag
        self.hostname = node().split('.')[0]

        # Init the InfluxDB client
        self.client = self.init()

    def init(self):
        """Init the connection to the InfluxDB server."""
        if not self.export_enable:
            return None

        url = '{}://{}:{}'.format(self.protocol, self.host, self.port)
        try:
            # See docs: https://influxdb-client.readthedocs.io/en/stable/api.html#influxdbclient
            client = InfluxDBClient(url=url, enable_gzip=False, verify_ssl=False, org=self.org, token=self.token)
        except Exception as e:
            logger.critical("Cannot connect to InfluxDB server '%s' (%s)" % (url, e))
            sys.exit(2)
        else:
            logger.info(
                "Connected to InfluxDB server version {} ({})".format(client.health().version, client.health().message)
            )

        # Create the write client
        write_client = client.write_api(
            write_options=WriteOptions(
                batch_size=500,
                flush_interval=self.interval * 1000,
                jitter_interval=2000,
                retry_interval=5000,
                max_retries=5,
                max_retry_delay=30000,
                exponential_base=2,
            )
        )
        return write_client

    def _normalize(self, name, columns, points):

        """Normalize data for the InfluxDB's data model.

        :return: a list of measurements.
        """
        ret = []

        # Build initial dict by crossing columns and point
        data_dict = dict(zip(columns, points))

        # issue1871 - Check if a key exist. If a key exist, the value of
        # the key should be used as a tag to identify the measurement.
        keys_list = [k.split('.')[0] for k in columns if k.endswith('.key')]
        if len(keys_list) == 0:
            keys_list = [None]

        for measurement in keys_list:
            # Manage field
            if measurement is not None:
                fields = {
                    k.replace('{}.'.format(measurement), ''): data_dict[k]
                    for k in data_dict
                    if k.startswith('{}.'.format(measurement))
                }
            else:
                fields = data_dict
            # Transform to InfluxDB datamodel
            # https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/
            for k in fields:
                #  Do not export empty (None) value
                if fields[k] is None:
                    continue
                # Convert numerical to float
                try:
                    fields[k] = float(fields[k])
                except (TypeError, ValueError):
                    # Convert others to string
                    try:
                        fields[k] = str(fields[k])
                    except (TypeError, ValueError):
                        pass
            # Manage tags
            tags = self.parse_tags(self.tags)
            if 'key' in fields and fields['key'] in fields:
                # Create a tag from the key
                # Tag should be an string (see InfluxDB data model)
                tags[fields['key']] = str(fields[fields['key']])
                # Remove it from the field list (can not be a field and a tag)
                fields.pop(fields['key'])
            # Add the hostname as a tag
            tags['hostname'] = self.hostname
            # Add name as a tag (example for the process list)
            for k in FIELD_TO_TAG:
                if k in fields:
                    tags[k] = str(fields[k])
                    # Remove it from the field list (can not be a field and a tag)
                    if k in fields:
                        fields.pop(fields[k])
            # Add the measurement to the list
            ret.append({'measurement': name, 'tags': tags, 'fields': fields})
        return ret

    def export(self, name, columns, points):

        """Write the points to the InfluxDB server."""
        # Manage prefix
        if self.prefix is not None:
            name = self.prefix + '.' + name
        # Write input to the InfluxDB database
        if len(points) == 0:
            logger.debug("Cannot export empty {} stats to InfluxDB".format(name))
        else:
            try:
                self.client.write(self.bucket, self.org, self._normalize(name, columns, points), time_precision="s")
            except Exception as e:
                # Log level set to debug instead of error (see: issue #1561)
                logger.debug("Cannot export {} stats to InfluxDB ({})".format(name, e))
            else:
                logger.debug("Export {} stats to InfluxDB".format(name))


1		# -- coding: utf-8 --
2		#
3		# This file is part of Glances.
4		#
5		# SPDX-FileCopyrightText: 2022 Nicolas Hennion <[email protected]>
6		#
7		# SPDX-License-Identifier: LGPL-3.0-only
8		#
9
10		"""InfluxDB (from to InfluxDB 1.8+) interface class."""
11
12		import sys
13		from platform import node
14
15		from glances.logger import logger
16		from glances.exports.export import GlancesExport
17
18		from influxdb_client import InfluxDBClient, WriteOptions
19
20		FIELD_TO_TAG = ['name', 'cmdline']
21
22
23		class Export(GlancesExport):
24		"""This class manages the InfluxDB export module."""
25
26		def __init__(self, config=None, args=None):
27		"""Init the InfluxDB export IF."""
28		super(Export, self).__init__(config=config, args=args)
29
30		# Mandatory configuration keys (additional to host and port)
31		self.org = None
32		self.bucket = None
33		self.token = None
34
35		# Optional configuration keys
36		self.protocol = 'http'
37		self.prefix = None
38		self.tags = None
39		self.hostname = None
40		self.interval = None
41
42		# Load the InfluxDB configuration file
43		self.export_enable = self.load_conf(
44		'influxdb2',
45		mandatories=['host', 'port', 'user', 'password', 'org', 'bucket', 'token'],
46		options=['protocol', 'prefix', 'tags', 'interval'],
47		)
48		if not self.export_enable:
49		exit('Missing influxdb2 config')
50
51		# Interval between two exports (in seconds)
52		if self.interval is None:
53		self.interval = 0
54		try:
55		self.interval = int(self.interval)
56		except ValueError:
57		logger.warning("InfluxDB export interval is not an integer, use default value")
58		self.interval = 0
59		# and should be set to the Glances refresh time if the value is 0
60		self.interval = self.interval if self.interval > 0 else self.args.time
61		logger.debug("InfluxDB export interval is set to {} seconds".format(self.interval))
62
63		# The hostname is always add as a tag
64		self.hostname = node().split('.')[0]
65
66		# Init the InfluxDB client
67		self.client = self.init()
68
69		def init(self):
70		"""Init the connection to the InfluxDB server."""
71		if not self.export_enable:
72		return None
73
74		url = '{}://{}:{}'.format(self.protocol, self.host, self.port)
75		try:
76		# See docs: https://influxdb-client.readthedocs.io/en/stable/api.html#influxdbclient
77		client = InfluxDBClient(url=url, enable_gzip=False, verify_ssl=False, org=self.org, token=self.token)
78		except Exception as e:
79		logger.critical("Cannot connect to InfluxDB server '%s' (%s)" % (url, e))
80		sys.exit(2)
81		else:
82		logger.info(
83		"Connected to InfluxDB server version {} ({})".format(client.health().version, client.health().message)
84		)
85
86		# Create the write client
87		write_client = client.write_api(
88		write_options=WriteOptions(
89		batch_size=500,
90		flush_interval=self.interval * 1000,
91		jitter_interval=2000,
92		retry_interval=5000,
93		max_retries=5,
94		max_retry_delay=30000,
95		exponential_base=2,
96		)
97		)
98		return write_client
99
100	View Code Duplication	def _normalize(self, name, columns, points):
		0 ignored issues – show Duplication introduced 2020-12-24 08:57 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
101		"""Normalize data for the InfluxDB's data model.
102
103		:return: a list of measurements.
104		"""
105		ret = []
106
107		# Build initial dict by crossing columns and point
108		data_dict = dict(zip(columns, points))
109
110		# issue1871 - Check if a key exist. If a key exist, the value of
111		# the key should be used as a tag to identify the measurement.
112		keys_list = [k.split('.')[0] for k in columns if k.endswith('.key')]
113		if len(keys_list) == 0:
114		keys_list = [None]
115
116		for measurement in keys_list:
117		# Manage field
118		if measurement is not None:
119		fields = {
120		k.replace('{}.'.format(measurement), ''): data_dict[k]
121		for k in data_dict
122		if k.startswith('{}.'.format(measurement))
123		}
124		else:
125		fields = data_dict
126		# Transform to InfluxDB datamodel
127		# https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/
128		for k in fields:
129		# Do not export empty (None) value
130		if fields[k] is None:
131		continue
132		# Convert numerical to float
133		try:
134		fields[k] = float(fields[k])
135		except (TypeError, ValueError):
136		# Convert others to string
137		try:
138		fields[k] = str(fields[k])
139		except (TypeError, ValueError):
140		pass
141		# Manage tags
142		tags = self.parse_tags(self.tags)
143		if 'key' in fields and fields['key'] in fields:
144		# Create a tag from the key
145		# Tag should be an string (see InfluxDB data model)
146		tags[fields['key']] = str(fields[fields['key']])
147		# Remove it from the field list (can not be a field and a tag)
148		fields.pop(fields['key'])
149		# Add the hostname as a tag
150		tags['hostname'] = self.hostname
151		# Add name as a tag (example for the process list)
152		for k in FIELD_TO_TAG:
153		if k in fields:
154		tags[k] = str(fields[k])
155		# Remove it from the field list (can not be a field and a tag)
156		if k in fields:
157		fields.pop(fields[k])
158		# Add the measurement to the list
159		ret.append({'measurement': name, 'tags': tags, 'fields': fields})
160		return ret
161
162	View Code Duplication	def export(self, name, columns, points):
		0 ignored issues – show Duplication introduced 2020-12-24 08:57 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
163		"""Write the points to the InfluxDB server."""
164		# Manage prefix
165		if self.prefix is not None:
166		name = self.prefix + '.' + name
167		# Write input to the InfluxDB database
168		if len(points) == 0:
169		logger.debug("Cannot export empty {} stats to InfluxDB".format(name))
170		else:
171		try:
172		self.client.write(self.bucket, self.org, self._normalize(name, columns, points), time_precision="s")
173		except Exception as e:
174		# Log level set to debug instead of error (see: issue #1561)
175		logger.debug("Cannot export {} stats to InfluxDB ({})".format(name, e))
176		else:
177		logger.debug("Export {} stats to InfluxDB".format(name))
178

nicolargo / glances

Push — master ( ee826a...d9056e )

Export._normalize() D

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like