Completed
Pull Request — master (#2304)
by Arma
07:07
created

st2tests.FixturesLoader   A

Complexity

Total Complexity 33

Size/Duplication

Total Lines 219
Duplicated Lines 0 %
Metric Value
wmc 33
dl 0
loc 219
rs 9.4
1
# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2
# contributor license agreements.  See the NOTICE file distributed with
3
# this work for additional information regarding copyright ownership.
4
# The ASF licenses this file to You under the Apache License, Version 2.0
5
# (the "License"); you may not use this file except in compliance with
6
# the License.  You may obtain a copy of the License at
7
#
8
#     http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15
16
import copy
17
import os
18
19
import six
20
21
from st2common.content.loader import MetaLoader
22
23
from st2common.models.api.action import (ActionAPI, LiveActionAPI, ActionExecutionStateAPI,
24
                                         RunnerTypeAPI, ActionAliasAPI)
25
from st2common.models.api.auth import ApiKeyAPI, UserAPI
26
from st2common.models.api.execution import (ActionExecutionAPI)
27
from st2common.models.api.policy import (PolicyTypeAPI, PolicyAPI)
28
from st2common.models.api.rule import (RuleAPI)
29
from st2common.models.api.rule_enforcement import RuleEnforcementAPI
30
from st2common.models.api.sensor import SensorTypeAPI
31
from st2common.models.api.trace import TraceAPI
32
from st2common.models.api.trigger import (TriggerAPI, TriggerTypeAPI, TriggerInstanceAPI)
33
34
from st2common.models.db.action import ActionDB
35
from st2common.models.db.actionalias import ActionAliasDB
36
from st2common.models.db.auth import ApiKeyDB, UserDB
37
from st2common.models.db.liveaction import LiveActionDB
38
from st2common.models.db.executionstate import ActionExecutionStateDB
39
from st2common.models.db.runner import RunnerTypeDB
40
from st2common.models.db.execution import (ActionExecutionDB)
41
from st2common.models.db.policy import (PolicyTypeDB, PolicyDB)
42
from st2common.models.db.rule import RuleDB
43
from st2common.models.db.rule_enforcement import RuleEnforcementDB
44
from st2common.models.db.sensor import SensorTypeDB
45
from st2common.models.db.trace import TraceDB
46
from st2common.models.db.trigger import (TriggerDB, TriggerTypeDB, TriggerInstanceDB)
47
from st2common.persistence.action import Action
48
from st2common.persistence.actionalias import ActionAlias
49
from st2common.persistence.execution import ActionExecution
50
from st2common.persistence.executionstate import ActionExecutionState
51
from st2common.persistence.auth import ApiKey, User
52
from st2common.persistence.liveaction import LiveAction
53
from st2common.persistence.runner import RunnerType
54
from st2common.persistence.policy import (PolicyType, Policy)
55
from st2common.persistence.rule import Rule
56
from st2common.persistence.rule_enforcement import RuleEnforcement
57
from st2common.persistence.sensor import SensorType
58
from st2common.persistence.trace import Trace
59
from st2common.persistence.trigger import (Trigger, TriggerType, TriggerInstance)
60
61
62
ALLOWED_DB_FIXTURES = ['actions', 'actionstates', 'aliases', 'executions', 'liveactions',
63
                       'policies', 'policytypes', 'rules', 'runners', 'sensors',
64
                       'triggertypes', 'triggers', 'triggerinstances', 'traces', 'apikeys',
65
                       'users', 'enforcements']
66
ALLOWED_FIXTURES = copy.copy(ALLOWED_DB_FIXTURES)
67
ALLOWED_FIXTURES.extend(['actionchains', 'workflows'])
68
69
FIXTURE_DB_MODEL = {
70
    'actions': ActionDB,
71
    'aliases': ActionAliasDB,
72
    'actionstates': ActionExecutionStateDB,
73
    'apikeys': ApiKeyDB,
74
    'enforcements': RuleEnforcementDB,
75
    'executions': ActionExecutionDB,
76
    'liveactions': LiveActionDB,
77
    'policies': PolicyDB,
78
    'policytypes': PolicyTypeDB,
79
    'rules': RuleDB,
80
    'runners': RunnerTypeDB,
81
    'sensors': SensorTypeDB,
82
    'traces': TraceDB,
83
    'triggertypes': TriggerTypeDB,
84
    'triggers': TriggerDB,
85
    'triggerinstances': TriggerInstanceDB,
86
    'users': UserDB
87
}
88
89
FIXTURE_API_MODEL = {
90
    'actions': ActionAPI,
91
    'aliases': ActionAliasAPI,
92
    'actionstates': ActionExecutionStateAPI,
93
    'apikeys': ApiKeyAPI,
94
    'enforcements': RuleEnforcementAPI,
95
    'executions': ActionExecutionAPI,
96
    'liveactions': LiveActionAPI,
97
    'policies': PolicyAPI,
98
    'policytypes': PolicyTypeAPI,
99
    'rules': RuleAPI,
100
    'runners': RunnerTypeAPI,
101
    'sensors': SensorTypeAPI,
102
    'traces': TraceAPI,
103
    'triggertypes': TriggerTypeAPI,
104
    'triggers': TriggerAPI,
105
    'triggerinstances': TriggerInstanceAPI,
106
    'users': UserAPI
107
}
108
109
110
FIXTURE_PERSISTENCE_MODEL = {
111
    'actions': Action,
112
    'aliases': ActionAlias,
113
    'actionstates': ActionExecutionState,
114
    'apikeys': ApiKey,
115
    'enforcements': RuleEnforcement,
116
    'executions': ActionExecution,
117
    'liveactions': LiveAction,
118
    'policies': Policy,
119
    'policytypes': PolicyType,
120
    'rules': Rule,
121
    'runners': RunnerType,
122
    'sensors': SensorType,
123
    'traces': Trace,
124
    'triggertypes': TriggerType,
125
    'triggers': Trigger,
126
    'triggerinstances': TriggerInstance,
127
    'users': User
128
}
129
130
131
def get_fixtures_base_path():
132
    return os.path.join(os.path.dirname(__file__), 'fixtures')
133
134
135
def get_resources_base_path():
136
    return os.path.join(os.path.dirname(__file__), 'resources')
137
138
139
class FixturesLoader(object):
140
    def __init__(self):
141
        self.meta_loader = MetaLoader()
142
143
    def save_fixtures_to_db(self, fixtures_pack='generic', fixtures_dict=None):
144
        """
145
        Loads fixtures specified in fixtures_dict into the database
146
        and returns DB models for the fixtures.
147
148
        fixtures_dict should be of the form:
149
        {
150
            'actions': ['action-1.yaml', 'action-2.yaml'],
151
            'rules': ['rule-1.yaml'],
152
            'liveactions': ['execution-1.yaml']
153
        }
154
155
        :param fixtures_pack: Name of the pack to load fixtures from.
156
        :type fixtures_pack: ``str``
157
158
        :param fixtures_dict: Dictionary specifying the fixtures to load for each type.
159
        :type fixtures_dict: ``dict``
160
161
        :rtype: ``dict``
162
        """
163
        if fixtures_dict is None:
164
            fixtures_dict = {}
165
        fixtures_pack_path = self._validate_fixtures_pack(fixtures_pack)
166
        self._validate_fixture_dict(fixtures_dict, allowed=ALLOWED_DB_FIXTURES)
167
168
        db_models = {}
169
        for fixture_type, fixtures in six.iteritems(fixtures_dict):
170
171
            API_MODEL = FIXTURE_API_MODEL.get(fixture_type, None)
172
            PERSISTENCE_MODEL = FIXTURE_PERSISTENCE_MODEL.get(fixture_type, None)
173
174
            loaded_fixtures = {}
175
            for fixture in fixtures:
176
                fixture_dict = self.meta_loader.load(
177
                    self._get_fixture_file_path_abs(fixtures_pack_path, fixture_type, fixture))
178
                api_model = API_MODEL(**fixture_dict)
179
                db_model = API_MODEL.to_model(api_model)
180
                db_model = PERSISTENCE_MODEL.add_or_update(db_model)
181
                loaded_fixtures[fixture] = db_model
182
183
            db_models[fixture_type] = loaded_fixtures
184
185
        return db_models
186
187
    def load_fixtures(self, fixtures_pack='generic', fixtures_dict=None):
188
        """
189
        Loads fixtures specified in fixtures_dict. We
190
        simply want to load the meta into dict objects.
191
192
        fixtures_dict should be of the form:
193
        {
194
            'actionchains': ['actionchain1.yaml', 'actionchain2.yaml'],
195
            'workflows': ['workflow.yaml']
196
        }
197
198
        :param fixtures_pack: Name of the pack to load fixtures from.
199
        :type fixtures_pack: ``str``
200
201
        :param fixtures_dict: Dictionary specifying the fixtures to load for each type.
202
        :type fixtures_dict: ``dict``
203
204
        :rtype: ``dict``
205
        """
206
        if not fixtures_dict:
207
            return {}
208
        fixtures_pack_path = self._validate_fixtures_pack(fixtures_pack)
209
        self._validate_fixture_dict(fixtures_dict)
210
211
        all_fixtures = {}
212
        for fixture_type, fixtures in six.iteritems(fixtures_dict):
213
            loaded_fixtures = {}
214
            for fixture in fixtures:
215
                fixture_dict = self.meta_loader.load(
216
                    self._get_fixture_file_path_abs(fixtures_pack_path, fixture_type, fixture))
217
                loaded_fixtures[fixture] = fixture_dict
218
            all_fixtures[fixture_type] = loaded_fixtures
219
220
        return all_fixtures
221
222
    def load_models(self, fixtures_pack='generic', fixtures_dict=None):
223
        """
224
        Loads fixtures specified in fixtures_dict as db models. This method must be
225
        used for fixtures that have associated DB models. We simply want to load the
226
        meta as DB models but don't want to save them to db.
227
228
        fixtures_dict should be of the form:
229
        {
230
            'actions': ['action-1.yaml', 'action-2.yaml'],
231
            'rules': ['rule-1.yaml'],
232
            'liveactions': ['execution-1.yaml']
233
        }
234
235
        :param fixtures_pack: Name of the pack to load fixtures from.
236
        :type fixtures_pack: ``str``
237
238
        :param fixtures_dict: Dictionary specifying the fixtures to load for each type.
239
        :type fixtures_dict: ``dict``
240
241
        :rtype: ``dict``
242
        """
243
        if not fixtures_dict:
244
            return {}
245
        fixtures_pack_path = self._validate_fixtures_pack(fixtures_pack)
246
        self._validate_fixture_dict(fixtures_dict, allowed=ALLOWED_DB_FIXTURES)
247
248
        all_fixtures = {}
249
        for fixture_type, fixtures in six.iteritems(fixtures_dict):
250
251
            API_MODEL = FIXTURE_API_MODEL.get(fixture_type, None)
252
253
            loaded_models = {}
254
            for fixture in fixtures:
255
                fixture_dict = self.meta_loader.load(
256
                    self._get_fixture_file_path_abs(fixtures_pack_path, fixture_type, fixture))
257
                api_model = API_MODEL(**fixture_dict)
258
                db_model = API_MODEL.to_model(api_model)
259
                loaded_models[fixture] = db_model
260
            all_fixtures[fixture_type] = loaded_models
261
262
        return all_fixtures
263
264
    def delete_fixtures_from_db(self, fixtures_pack='generic', fixtures_dict=None,
265
                                raise_on_fail=False):
266
        """
267
        Deletes fixtures specified in fixtures_dict from the database.
268
269
        fixtures_dict should be of the form:
270
        {
271
            'actions': ['action-1.yaml', 'action-2.yaml'],
272
            'rules': ['rule-1.yaml'],
273
            'liveactions': ['execution-1.yaml']
274
        }
275
276
        :param fixtures_pack: Name of the pack to delete fixtures from.
277
        :type fixtures_pack: ``str``
278
279
        :param fixtures_dict: Dictionary specifying the fixtures to delete for each type.
280
        :type fixtures_dict: ``dict``
281
282
        :param raise_on_fail: Optional If True, raises exception if delete fails on any fixture.
283
        :type raise_on_fail: ``boolean``
284
        """
285
        if not fixtures_dict:
286
            return
287
        fixtures_pack_path = self._validate_fixtures_pack(fixtures_pack)
288
        self._validate_fixture_dict(fixtures_dict)
289
290
        for fixture_type, fixtures in six.iteritems(fixtures_dict):
291
            API_MODEL = FIXTURE_API_MODEL.get(fixture_type, None)
292
            PERSISTENCE_MODEL = FIXTURE_PERSISTENCE_MODEL.get(fixture_type, None)
293
            for fixture in fixtures:
294
                fixture_dict = self.meta_loader.load(
295
                    self._get_fixture_file_path_abs(fixtures_pack_path, fixture_type, fixture))
296
                # Note that when we have a reference mechanism consistent for
297
                # every model, we can just do a get and delete the object. Until
298
                # then, this model conversions are necessary.
299
                api_model = API_MODEL(**fixture_dict)
300
                db_model = API_MODEL.to_model(api_model)
301
                try:
302
                    PERSISTENCE_MODEL.delete(db_model)
303
                except:
304
                    if raise_on_fail:
305
                        raise
306
307
    def delete_models_from_db(self, models_dict, raise_on_fail=False):
308
        """
309
        Deletes models specified in models_dict from the database.
310
311
        models_dict should be of the form:
312
        {
313
            'actions': [ACTION1, ACTION2],
314
            'rules': [RULE1],
315
            'liveactions': [EXECUTION]
316
        }
317
318
        :param fixtures_dict: Dictionary specifying the fixtures to delete for each type.
319
        :type fixtures_dict: ``dict``.
320
321
        :param raise_on_fail: Optional If True, raises exception if delete fails on any model.
322
        :type raise_on_fail: ``boolean``
323
        """
324
        for model_type, models in six.iteritems(models_dict):
325
            PERSISTENCE_MODEL = FIXTURE_PERSISTENCE_MODEL.get(model_type, None)
326
            for model in models:
327
                try:
328
                    PERSISTENCE_MODEL.delete(model)
329
                except:
330
                    if raise_on_fail:
331
                        raise
332
333
    def _validate_fixtures_pack(self, fixtures_pack):
334
        fixtures_pack_path = self._get_fixtures_pack_path(fixtures_pack)
335
336
        if not self._is_fixture_pack_exists(fixtures_pack_path):
337
            raise Exception('Fixtures pack not found ' +
338
                            'in fixtures path %s.' % get_fixtures_base_path())
339
        return fixtures_pack_path
340
341
    def _validate_fixture_dict(self, fixtures_dict, allowed=ALLOWED_FIXTURES):
342
        fixture_types = fixtures_dict.keys()
343
        for fixture_type in fixture_types:
344
            if fixture_type not in allowed:
345
                raise Exception('Disallowed fixture type: %s' % fixture_type)
346
347
    def _is_fixture_pack_exists(self, fixtures_pack_path):
348
        return os.path.exists(fixtures_pack_path)
349
350
    def _get_fixture_file_path_abs(self, fixtures_pack_path, fixtures_type, fixture_name):
351
        return os.path.join(fixtures_pack_path, fixtures_type, fixture_name)
352
353
    def _get_fixtures_pack_path(self, fixtures_pack_name):
354
        return os.path.join(get_fixtures_base_path(), fixtures_pack_name)
355
356
    def get_fixture_file_path_abs(self, fixtures_pack, fixtures_type, fixture_name):
357
        return os.path.join(get_fixtures_base_path(), fixtures_pack, fixtures_type, fixture_name)
358