1 | # -*- coding: utf-8 -*- |
||
2 | # |
||
3 | # This file is part of Glances. |
||
4 | # |
||
5 | # Copyright (C) 2019 Nicolargo <[email protected]> |
||
6 | # |
||
7 | # Glances is free software; you can redistribute it and/or modify |
||
8 | # it under the terms of the GNU Lesser General Public License as published by |
||
9 | # the Free Software Foundation, either version 3 of the License, or |
||
10 | # (at your option) any later version. |
||
11 | # |
||
12 | # Glances is distributed in the hope that it will be useful, |
||
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
||
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||
15 | # GNU Lesser General Public License for more details. |
||
16 | # |
||
17 | # You should have received a copy of the GNU Lesser General Public License |
||
18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
||
19 | |||
20 | """ElasticSearch interface class.""" |
||
21 | |||
22 | import sys |
||
0 ignored issues
–
show
introduced
by
Loading history...
|
|||
23 | from datetime import datetime |
||
0 ignored issues
–
show
|
|||
24 | |||
25 | from glances.logger import logger |
||
0 ignored issues
–
show
|
|||
26 | from glances.exports.glances_export import GlancesExport |
||
0 ignored issues
–
show
|
|||
27 | |||
28 | from elasticsearch import Elasticsearch, helpers |
||
0 ignored issues
–
show
|
|||
29 | |||
30 | |||
31 | class Export(GlancesExport): |
||
32 | |||
33 | """This class manages the ElasticSearch (ES) export module.""" |
||
34 | |||
35 | def __init__(self, config=None, args=None): |
||
36 | """Init the ES export IF.""" |
||
37 | super(Export, self).__init__(config=config, args=args) |
||
38 | |||
39 | # Mandatories configuration keys (additional to host and port) |
||
40 | self.index = None |
||
41 | |||
42 | # Optionals configuration keys |
||
43 | # N/A |
||
44 | |||
45 | # Load the ES configuration file |
||
46 | self.export_enable = self.load_conf('elasticsearch', |
||
47 | mandatories=['host', 'port', 'index'], |
||
0 ignored issues
–
show
|
|||
48 | options=[]) |
||
49 | if not self.export_enable: |
||
50 | sys.exit(2) |
||
51 | |||
52 | # Init the ES client |
||
53 | self.client = self.init() |
||
54 | |||
55 | def init(self): |
||
56 | """Init the connection to the ES server.""" |
||
57 | if not self.export_enable: |
||
58 | return None |
||
59 | |||
60 | self.index='{}-{}'.format(self.index, datetime.utcnow().strftime("%Y.%m.%d")) |
||
0 ignored issues
–
show
|
|||
61 | template_body = { |
||
0 ignored issues
–
show
|
|||
62 | "mappings": { |
||
0 ignored issues
–
show
|
|||
63 | "glances": { |
||
0 ignored issues
–
show
|
|||
64 | "dynamic_templates": [ |
||
0 ignored issues
–
show
|
|||
65 | { |
||
0 ignored issues
–
show
|
|||
66 | "integers": { |
||
0 ignored issues
–
show
|
|||
67 | "match_mapping_type": "long", |
||
0 ignored issues
–
show
|
|||
68 | "mapping": { |
||
0 ignored issues
–
show
|
|||
69 | "type": "integer" |
||
0 ignored issues
–
show
|
|||
70 | } |
||
71 | } |
||
72 | }, |
||
73 | { |
||
0 ignored issues
–
show
|
|||
74 | "strings": { |
||
0 ignored issues
–
show
|
|||
75 | "match_mapping_type": "string", |
||
0 ignored issues
–
show
|
|||
76 | "mapping": { |
||
0 ignored issues
–
show
|
|||
77 | "type": "text", |
||
0 ignored issues
–
show
|
|||
78 | "fields": { |
||
0 ignored issues
–
show
|
|||
79 | "raw": { |
||
0 ignored issues
–
show
|
|||
80 | "type": "keyword", |
||
0 ignored issues
–
show
|
|||
81 | "ignore_above": 256 |
||
0 ignored issues
–
show
|
|||
82 | } |
||
83 | } |
||
84 | } |
||
85 | } |
||
86 | } |
||
87 | ] |
||
88 | } |
||
89 | } |
||
90 | } |
||
91 | |||
92 | try: |
||
93 | es = Elasticsearch(hosts=['{}:{}'.format(self.host, self.port)]) |
||
0 ignored issues
–
show
The name
es does not conform to the variable naming conventions ((([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ ).
This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site.
Loading history...
|
|||
94 | except Exception as e: |
||
0 ignored issues
–
show
Catching very general exceptions such as
Exception is usually not recommended.
Generally, you would want to handle very specific errors in the exception handler. This ensure that you do not hide other types of errors which should be fixed. So, unless you specifically plan to handle any error, consider adding a more specific exception.
Loading history...
The name
e does not conform to the variable naming conventions ((([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ ).
This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site.
Loading history...
|
|||
95 | logger.critical("Cannot connect to ElasticSearch server %s:%s (%s)" % (self.host, self.port, e)) |
||
0 ignored issues
–
show
|
|||
96 | sys.exit(2) |
||
97 | else: |
||
98 | logger.info("Connected to the ElasticSearch server %s:%s" % (self.host, self.port)) |
||
0 ignored issues
–
show
|
|||
99 | |||
100 | try: |
||
101 | index_count = es.count(index=self.index)['count'] |
||
102 | except Exception as e: |
||
0 ignored issues
–
show
Catching very general exceptions such as
Exception is usually not recommended.
Generally, you would want to handle very specific errors in the exception handler. This ensure that you do not hide other types of errors which should be fixed. So, unless you specifically plan to handle any error, consider adding a more specific exception.
Loading history...
The name
e does not conform to the variable naming conventions ((([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ ).
This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site.
Loading history...
|
|||
103 | # Index did not exist, it will be created at the first write |
||
104 | # Create it... |
||
105 | es.indices.create(index=self.index,body=template_body) |
||
0 ignored issues
–
show
|
|||
106 | else: |
||
107 | logger.info("The index %s exists and holds %s entries." % (self.index, index_count)) |
||
0 ignored issues
–
show
|
|||
108 | |||
109 | return es |
||
110 | |||
111 | def export(self, name, columns, points): |
||
112 | """Write the points to the ES server.""" |
||
113 | logger.debug("Export {} stats to ElasticSearch".format(name)) |
||
0 ignored issues
–
show
|
|||
114 | |||
115 | # Create DB input |
||
116 | # https://elasticsearch-py.readthedocs.io/en/master/helpers.html |
||
117 | actions = [] |
||
118 | for c, p in zip(columns, points): |
||
0 ignored issues
–
show
The name
c does not conform to the variable naming conventions ((([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ ).
This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site.
Loading history...
The name
p does not conform to the variable naming conventions ((([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ ).
This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site.
Loading history...
|
|||
119 | dtnow = datetime.utcnow() |
||
120 | action = { |
||
121 | "_index": self.index, |
||
122 | "_id": '{}.{}'.format(name,c), |
||
0 ignored issues
–
show
|
|||
123 | "_type": "glances", |
||
124 | "_source": { |
||
125 | "plugin": name, |
||
126 | "metric": c, |
||
127 | "value": str(p), |
||
128 | "timestamp": dtnow.isoformat('T') |
||
129 | } |
||
130 | } |
||
131 | logger.debug("Exporting the following object to elasticsearch: {}".format(action)) |
||
0 ignored issues
–
show
|
|||
132 | actions.append(action) |
||
133 | |||
134 | # Write input to the ES index |
||
135 | try: |
||
136 | helpers.bulk(self.client, actions) |
||
137 | except Exception as e: |
||
0 ignored issues
–
show
Catching very general exceptions such as
Exception is usually not recommended.
Generally, you would want to handle very specific errors in the exception handler. This ensure that you do not hide other types of errors which should be fixed. So, unless you specifically plan to handle any error, consider adding a more specific exception.
Loading history...
The name
e does not conform to the variable naming conventions ((([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ ).
This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site.
Loading history...
|
|||
138 | logger.error("Cannot export {} stats to ElasticSearch ({})".format(name, e)) |
||
0 ignored issues
–
show
|
|||
139 |