1 | # Licensed to the StackStorm, Inc ('StackStorm') under one or more |
||
2 | # contributor license agreements. See the NOTICE file distributed with |
||
3 | # this work for additional information regarding copyright ownership. |
||
4 | # The ASF licenses this file to You under the Apache License, Version 2.0 |
||
5 | # (the "License"); you may not use this file except in compliance with |
||
6 | # the License. You may obtain a copy of the License at |
||
7 | # |
||
8 | # http://www.apache.org/licenses/LICENSE-2.0 |
||
9 | # |
||
10 | # Unless required by applicable law or agreed to in writing, software |
||
11 | # distributed under the License is distributed on an "AS IS" BASIS, |
||
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||
13 | # See the License for the specific language governing permissions and |
||
14 | # limitations under the License. |
||
15 | |||
16 | from __future__ import absolute_import |
||
17 | import os |
||
18 | import glob |
||
19 | |||
20 | import six |
||
21 | |||
22 | from st2common import log as logging |
||
23 | from st2common.constants.pack import CONFIG_SCHEMA_FILE_NAME |
||
24 | from st2common.content.loader import MetaLoader |
||
25 | from st2common.content.loader import ContentPackLoader |
||
26 | from st2common.content.loader import RunnersLoader |
||
27 | from st2common.models.api.pack import PackAPI |
||
28 | from st2common.models.api.pack import ConfigSchemaAPI |
||
29 | from st2common.persistence.pack import Pack |
||
30 | from st2common.persistence.pack import ConfigSchema |
||
31 | from st2common.util.file_system import get_file_list |
||
32 | from st2common.util.pack import get_pack_metadata |
||
33 | from st2common.util.pack import get_pack_ref_from_metadata |
||
34 | from st2common.exceptions.db import StackStormDBObjectNotFoundError |
||
35 | |||
36 | __all__ = [ |
||
37 | 'ResourceRegistrar' |
||
38 | ] |
||
39 | |||
40 | LOG = logging.getLogger(__name__) |
||
41 | |||
42 | # Note: We use the cache to avoid manipulating the DB object for the same pack multiple times |
||
43 | # during the same register-content run. |
||
44 | # This works fine since those classes are only uses from register-content which is a script and not |
||
45 | # a long running process. |
||
46 | REGISTERED_PACKS_CACHE = {} |
||
47 | |||
48 | EXCLUDE_FILE_PATTERNS = [ |
||
49 | '*.pyc', |
||
50 | '.git/*' |
||
51 | ] |
||
52 | |||
53 | |||
54 | class ResourceRegistrar(object): |
||
55 | ALLOWED_EXTENSIONS = [] |
||
56 | |||
57 | def __init__(self, use_pack_cache=True, fail_on_failure=False): |
||
58 | """ |
||
59 | :param use_pack_cache: True to cache which packs have been registered in memory and making |
||
60 | sure packs are only registered once. |
||
61 | :type use_pack_cache: ``bool`` |
||
62 | |||
63 | :param fail_on_failure: Throw an exception if resource registration fails. |
||
64 | :type fail_on_failure: ``bool`` |
||
65 | """ |
||
66 | self._use_pack_cache = use_pack_cache |
||
67 | self._fail_on_failure = fail_on_failure |
||
68 | |||
69 | self._meta_loader = MetaLoader() |
||
70 | self._pack_loader = ContentPackLoader() |
||
71 | self._runner_loader = RunnersLoader() |
||
72 | |||
73 | def get_resources_from_pack(self, resources_dir): |
||
74 | resources = [] |
||
75 | for ext in self.ALLOWED_EXTENSIONS: |
||
76 | resources_glob = resources_dir |
||
77 | |||
78 | if resources_dir.endswith('/'): |
||
79 | resources_glob = resources_dir + ext |
||
80 | else: |
||
81 | resources_glob = resources_dir + '/*' + ext |
||
82 | |||
83 | resource_files = glob.glob(resources_glob) |
||
84 | resources.extend(resource_files) |
||
85 | |||
86 | resources = sorted(resources) |
||
87 | return resources |
||
88 | |||
89 | def get_registered_packs(self): |
||
90 | """ |
||
91 | Return a list of registered packs. |
||
92 | |||
93 | :rype: ``list`` |
||
94 | """ |
||
95 | return list(REGISTERED_PACKS_CACHE.keys()) |
||
96 | |||
97 | def register_packs(self, base_dirs): |
||
98 | """ |
||
99 | Register packs in all the provided directories. |
||
100 | """ |
||
101 | packs = self._pack_loader.get_packs(base_dirs=base_dirs) |
||
102 | |||
103 | registered_count = 0 |
||
104 | for pack_name, pack_path in six.iteritems(packs): |
||
105 | self.register_pack(pack_name=pack_name, pack_dir=pack_path) |
||
106 | registered_count += 1 |
||
107 | |||
108 | return registered_count |
||
109 | |||
110 | def register_pack(self, pack_name, pack_dir): |
||
111 | """ |
||
112 | Register pack in the provided directory. |
||
113 | """ |
||
114 | if self._use_pack_cache and pack_name in REGISTERED_PACKS_CACHE: |
||
115 | # This pack has already been registered during this register content run |
||
116 | return |
||
117 | |||
118 | LOG.debug('Registering pack: %s' % (pack_name)) |
||
0 ignored issues
–
show
Coding Style
Best Practice
introduced
by
Loading history...
|
|||
119 | REGISTERED_PACKS_CACHE[pack_name] = True |
||
120 | |||
121 | try: |
||
122 | pack_db, _ = self._register_pack(pack_name=pack_name, pack_dir=pack_dir) |
||
123 | except Exception as e: |
||
124 | if self._fail_on_failure: |
||
125 | msg = 'Failed to register pack "%s": %s' % (pack_name, str(e)) |
||
126 | raise ValueError(msg) |
||
127 | |||
128 | LOG.exception('Failed to register pack "%s"' % (pack_name)) |
||
0 ignored issues
–
show
|
|||
129 | return None |
||
130 | |||
131 | return pack_db |
||
132 | |||
133 | def _register_pack(self, pack_name, pack_dir): |
||
134 | """ |
||
135 | Register a pack and corresponding pack config schema (create a DB object in the system). |
||
136 | |||
137 | Note: Pack registration now happens when registering the content and not when installing |
||
138 | a pack using packs.install. Eventually this will be moved to the pack management API. |
||
139 | """ |
||
140 | # 1. Register pack |
||
141 | pack_db = self._register_pack_db(pack_name=pack_name, pack_dir=pack_dir) |
||
142 | |||
143 | # Display a warning if pack contains deprecated config.yaml file. Support for those files |
||
144 | # will be fully removed in v2.4.0. |
||
145 | config_path = os.path.join(pack_dir, 'config.yaml') |
||
146 | if os.path.isfile(config_path): |
||
147 | LOG.error('Pack "%s" contains a deprecated config.yaml file (%s). ' |
||
0 ignored issues
–
show
|
|||
148 | 'Support for "config.yaml" files has been deprecated in StackStorm v1.6.0 ' |
||
149 | 'in favor of config.schema.yaml config schema files and config files in ' |
||
150 | '/opt/stackstorm/configs/ directory. Support for config.yaml files has ' |
||
151 | 'been removed in the release (v2.4.0) so please migrate. For more ' |
||
152 | 'information please refer to %s ' % (pack_db.name, config_path, |
||
153 | 'https://docs.stackstorm.com/reference/pack_configs.html')) |
||
154 | |||
155 | # 2. Register corresponding pack config schema |
||
156 | config_schema_db = self._register_pack_config_schema_db(pack_name=pack_name, |
||
157 | pack_dir=pack_dir) |
||
158 | |||
159 | return pack_db, config_schema_db |
||
160 | |||
161 | def _register_pack_db(self, pack_name, pack_dir): |
||
162 | content = get_pack_metadata(pack_dir=pack_dir) |
||
163 | |||
164 | # The rules for the pack ref are as follows: |
||
165 | # 1. If ref attribute is available, we used that |
||
166 | # 2. If pack_name is available we use that (this only applies to packs |
||
167 | # 2hich are in sub-directories) |
||
168 | # 2. If attribute is not available, but pack name is and pack name meets the valid name |
||
169 | # criteria, we use that |
||
170 | content['ref'] = get_pack_ref_from_metadata(metadata=content, |
||
171 | pack_directory_name=pack_name) |
||
172 | |||
173 | # Include a list of pack files |
||
174 | pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) |
||
175 | content['files'] = pack_file_list |
||
176 | content['path'] = pack_dir |
||
177 | |||
178 | pack_api = PackAPI(**content) |
||
179 | pack_api.validate() |
||
180 | pack_db = PackAPI.to_model(pack_api) |
||
181 | |||
182 | try: |
||
183 | pack_db.id = Pack.get_by_ref(content['ref']).id |
||
184 | except StackStormDBObjectNotFoundError: |
||
185 | LOG.debug('Pack %s not found. Creating new one.', pack_name) |
||
186 | |||
187 | pack_db = Pack.add_or_update(pack_db) |
||
188 | LOG.debug('Pack %s registered.' % (pack_name)) |
||
0 ignored issues
–
show
|
|||
189 | return pack_db |
||
190 | |||
191 | def _register_pack_config_schema_db(self, pack_name, pack_dir): |
||
192 | config_schema_path = os.path.join(pack_dir, CONFIG_SCHEMA_FILE_NAME) |
||
193 | |||
194 | if not os.path.isfile(config_schema_path): |
||
195 | # Note: Config schema is optional |
||
196 | return None |
||
197 | |||
198 | values = self._meta_loader.load(config_schema_path) |
||
199 | |||
200 | if not values: |
||
201 | raise ValueError('Config schema "%s" is empty and invalid.' % (config_schema_path)) |
||
202 | |||
203 | content = {} |
||
204 | content['pack'] = pack_name |
||
205 | content['attributes'] = values |
||
206 | |||
207 | config_schema_api = ConfigSchemaAPI(**content) |
||
208 | config_schema_api = config_schema_api.validate() |
||
209 | config_schema_db = ConfigSchemaAPI.to_model(config_schema_api) |
||
210 | |||
211 | try: |
||
212 | config_schema_db.id = ConfigSchema.get_by_pack(pack_name).id |
||
213 | except StackStormDBObjectNotFoundError: |
||
214 | LOG.debug('Config schema for pack %s not found. Creating new one.', pack_name) |
||
215 | |||
216 | config_schema_db = ConfigSchema.add_or_update(config_schema_db) |
||
217 | LOG.debug('Config schema for pack %s registered.' % (pack_name)) |
||
0 ignored issues
–
show
|
|||
218 | return config_schema_db |
||
219 | |||
220 | def register_runner(self): |
||
221 | pass |
||
222 |