1 | # -*- coding: utf-8 -*- |
||
2 | # |
||
3 | # This file is part of Glances. |
||
4 | # |
||
5 | # Copyright (C) 2019 Nicolargo <[email protected]> |
||
6 | # |
||
7 | # Glances is free software; you can redistribute it and/or modify |
||
8 | # it under the terms of the GNU Lesser General Public License as published by |
||
9 | # the Free Software Foundation, either version 3 of the License, or |
||
10 | # (at your option) any later version. |
||
11 | # |
||
12 | # Glances is distributed in the hope that it will be useful, |
||
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
||
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||
15 | # GNU Lesser General Public License for more details. |
||
16 | # |
||
17 | # You should have received a copy of the GNU Lesser General Public License |
||
18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
||
19 | |||
20 | """ElasticSearch interface class.""" |
||
21 | |||
22 | import sys |
||
23 | from datetime import datetime |
||
24 | |||
25 | from glances.logger import logger |
||
26 | from glances.exports.glances_export import GlancesExport |
||
27 | |||
28 | from elasticsearch import Elasticsearch, helpers |
||
29 | |||
30 | |||
31 | class Export(GlancesExport): |
||
32 | |||
33 | """This class manages the ElasticSearch (ES) export module.""" |
||
34 | |||
35 | def __init__(self, config=None, args=None): |
||
36 | """Init the ES export IF.""" |
||
37 | super(Export, self).__init__(config=config, args=args) |
||
38 | |||
39 | # Mandatories configuration keys (additional to host and port) |
||
40 | self.index = None |
||
41 | |||
42 | # Optionals configuration keys |
||
43 | # N/A |
||
44 | |||
45 | # Load the ES configuration file |
||
46 | self.export_enable = self.load_conf('elasticsearch', |
||
47 | mandatories=['host', 'port', 'index'], |
||
48 | options=[]) |
||
49 | if not self.export_enable: |
||
50 | sys.exit(2) |
||
51 | |||
52 | # Init the ES client |
||
53 | self.client = self.init() |
||
54 | |||
55 | def init(self): |
||
56 | """Init the connection to the ES server.""" |
||
57 | if not self.export_enable: |
||
58 | return None |
||
59 | |||
60 | self.index='{}-{}'.format(self.index, datetime.utcnow().strftime("%Y.%m.%d")) |
||
61 | template_body = { |
||
62 | "mappings": { |
||
63 | "glances": { |
||
64 | "dynamic_templates": [ |
||
65 | { |
||
66 | "integers": { |
||
67 | "match_mapping_type": "long", |
||
68 | "mapping": { |
||
69 | "type": "integer" |
||
70 | } |
||
71 | } |
||
72 | }, |
||
73 | { |
||
74 | "strings": { |
||
75 | "match_mapping_type": "string", |
||
76 | "mapping": { |
||
77 | "type": "text", |
||
78 | "fields": { |
||
79 | "raw": { |
||
80 | "type": "keyword", |
||
81 | "ignore_above": 256 |
||
82 | } |
||
83 | } |
||
84 | } |
||
85 | } |
||
86 | } |
||
87 | ] |
||
88 | } |
||
89 | } |
||
90 | } |
||
91 | |||
92 | try: |
||
93 | es = Elasticsearch(hosts=['{}:{}'.format(self.host, self.port)]) |
||
94 | except Exception as e: |
||
0 ignored issues
–
show
|
|||
95 | logger.critical("Cannot connect to ElasticSearch server %s:%s (%s)" % (self.host, self.port, e)) |
||
0 ignored issues
–
show
|
|||
96 | sys.exit(2) |
||
97 | else: |
||
98 | logger.info("Connected to the ElasticSearch server %s:%s" % (self.host, self.port)) |
||
0 ignored issues
–
show
|
|||
99 | |||
100 | try: |
||
101 | index_count = es.count(index=self.index)['count'] |
||
102 | except Exception as e: |
||
0 ignored issues
–
show
Catching very general exceptions such as
Exception is usually not recommended.
Generally, you would want to handle very specific errors in the exception handler. This ensure that you do not hide other types of errors which should be fixed. So, unless you specifically plan to handle any error, consider adding a more specific exception.
Loading history...
|
|||
103 | # Index did not exist, it will be created at the first write |
||
104 | # Create it... |
||
105 | es.indices.create(index=self.index,body=template_body) |
||
106 | else: |
||
107 | logger.info("The index %s exists and holds %s entries." % (self.index, index_count)) |
||
0 ignored issues
–
show
|
|||
108 | |||
109 | return es |
||
110 | |||
111 | def export(self, name, columns, points): |
||
112 | """Write the points to the ES server.""" |
||
113 | logger.debug("Export {} stats to ElasticSearch".format(name)) |
||
114 | |||
115 | # Create DB input |
||
116 | # https://elasticsearch-py.readthedocs.io/en/master/helpers.html |
||
117 | actions = [] |
||
118 | for c, p in zip(columns, points): |
||
119 | dtnow = datetime.utcnow() |
||
120 | action = { |
||
121 | "_index": self.index, |
||
122 | "_id": '{}.{}'.format(name,c), |
||
123 | "_type": "glances", |
||
124 | "_source": { |
||
125 | "plugin": name, |
||
126 | "metric": c, |
||
127 | "value": str(p), |
||
128 | "timestamp": dtnow.isoformat('T') |
||
129 | } |
||
130 | } |
||
131 | logger.debug("Exporting the following object to elasticsearch: {}".format(action)) |
||
132 | actions.append(action) |
||
133 | |||
134 | # Write input to the ES index |
||
135 | try: |
||
136 | helpers.bulk(self.client, actions) |
||
137 | except Exception as e: |
||
0 ignored issues
–
show
Catching very general exceptions such as
Exception is usually not recommended.
Generally, you would want to handle very specific errors in the exception handler. This ensure that you do not hide other types of errors which should be fixed. So, unless you specifically plan to handle any error, consider adding a more specific exception.
Loading history...
|
|||
138 | logger.error("Cannot export {} stats to ElasticSearch ({})".format(name, e)) |
||
139 |
Generally, you would want to handle very specific errors in the exception handler. This ensure that you do not hide other types of errors which should be fixed.
So, unless you specifically plan to handle any error, consider adding a more specific exception.