Test Setup Failed
Pull Request — master (#4154)
by W
03:37
created

OrchestraErrorHandlingTest.setUpClass()   A

Complexity

Conditions 2

Size

Total Lines 15

Duplication

Lines 0
Ratio 0 %

Importance

Changes 1
Bugs 0 Features 0
Metric Value
cc 2
c 1
b 0
f 0
dl 0
loc 15
rs 9.4285
1
# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2
# contributor license agreements.  See the NOTICE file distributed with
3
# this work for additional information regarding copyright ownership.
4
# The ASF licenses this file to You under the Apache License, Version 2.0
5
# (the "License"); you may not use this file except in compliance with
6
# the License.  You may obtain a copy of the License at
7
#
8
#     http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15
16
from __future__ import absolute_import
17
18
import mock
19
20
from orchestra import states as wf_states
21
22
import st2tests
23
24
# XXX: actionsensor import depends on config being setup.
25
import st2tests.config as tests_config
26
tests_config.parse_args()
27
28
from tests.unit import base
29
30
from st2common.bootstrap import actionsregistrar
31
from st2common.bootstrap import runnersregistrar
32
from st2common.constants import action as ac_const
33
from st2common.models.db import liveaction as lv_db_models
34
from st2common.persistence import execution as ex_db_access
35
from st2common.persistence import liveaction as lv_db_access
36
from st2common.persistence import workflow as wf_db_access
37
from st2common.runners import base as runners
38
from st2common.services import action as ac_svc
39
from st2common.services import workflows as wf_svc
40
from st2common.transport import liveaction as lv_ac_xport
41
from st2common.transport import workflow as wf_ex_xport
42
from st2common.transport import publishers
43
from st2tests.mocks import liveaction as mock_lv_ac_xport
44
from st2tests.mocks import workflow as mock_wf_ex_xport
45
46
47
TEST_PACK = 'orchestra_tests'
48
TEST_PACK_PATH = st2tests.fixturesloader.get_fixtures_packs_base_path() + '/' + TEST_PACK
49
50
PACKS = [
51
    TEST_PACK_PATH,
52
    st2tests.fixturesloader.get_fixtures_packs_base_path() + '/core'
53
]
54
55
56
@mock.patch.object(
57
    publishers.CUDPublisher,
58
    'publish_update',
59
    mock.MagicMock(return_value=None))
60
@mock.patch.object(
61
    lv_ac_xport.LiveActionPublisher,
62
    'publish_create',
63
    mock.MagicMock(side_effect=mock_lv_ac_xport.MockLiveActionPublisher.publish_create))
64
@mock.patch.object(
65
    lv_ac_xport.LiveActionPublisher,
66
    'publish_state',
67
    mock.MagicMock(side_effect=mock_lv_ac_xport.MockLiveActionPublisher.publish_state))
68
@mock.patch.object(
69
    wf_ex_xport.WorkflowExecutionPublisher,
70
    'publish_create',
71
    mock.MagicMock(side_effect=mock_wf_ex_xport.MockWorkflowExecutionPublisher.publish_create))
72
@mock.patch.object(
73
    wf_ex_xport.WorkflowExecutionPublisher,
74
    'publish_state',
75
    mock.MagicMock(side_effect=mock_wf_ex_xport.MockWorkflowExecutionPublisher.publish_state))
76
class OrchestraErrorHandlingTest(st2tests.DbTestCase):
77
78
    @classmethod
79
    def setUpClass(cls):
80
        super(OrchestraErrorHandlingTest, cls).setUpClass()
81
82
        # Register runners.
83
        runnersregistrar.register_runners()
84
85
        # Register test pack(s).
86
        actions_registrar = actionsregistrar.ActionsRegistrar(
87
            use_pack_cache=False,
88
            fail_on_failure=True
89
        )
90
91
        for pack in PACKS:
92
            actions_registrar.register_from_pack(pack)
93
94
    @classmethod
95
    def get_runner_class(cls, runner_name):
96
        return runners.get_runner(runner_name, runner_name).__class__
97
98
    def sort_wf_runtime_errors(self, errors):
99
        return sorted(errors, key=lambda x: x.get('task_id', None))
100
101
    def test_fail_inspection(self):
102
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'fail-inspection.yaml')
103
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
104
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
105
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
106
107
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
108
        self.assertIn('errors', lv_ac_db.result)
109
        self.assertIn('expressions', lv_ac_db.result['errors'])
110
        self.assertGreater(len(lv_ac_db.result['errors']['expressions']), 0)
111
        self.assertIn('context', lv_ac_db.result['errors'])
112
        self.assertGreater(len(lv_ac_db.result['errors']['context']), 0)
113
        self.assertIn('syntax', lv_ac_db.result['errors'])
114
        self.assertGreater(len(lv_ac_db.result['errors']['syntax']), 0)
115
116
    def test_fail_input_rendering(self):
117
        expected_errors = [
118
            {
119
                'message': 'Unknown function "#property#value"'
120
            }
121
        ]
122
123
        expected_result = {'output': None, 'errors': expected_errors}
124
125
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'fail-input-rendering.yaml')
126
127
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
128
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
129
130
        # Assert action execution for task is not started and workflow failed.
131
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
132
        tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
133
        self.assertEqual(len(tk_ex_dbs), 0)
134
        self.assertEqual(wf_ex_db.status, wf_states.FAILED)
135
        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
136
137
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
138
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
139
        self.assertDictEqual(lv_ac_db.result, expected_result)
140
141
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
142
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
143
        self.assertDictEqual(ac_ex_db.result, expected_result)
144
145
    def test_fail_vars_rendering(self):
146
        expected_errors = [
147
            {
148
                'message': 'Unknown function "#property#value"'
149
            }
150
        ]
151
152
        expected_result = {'output': None, 'errors': expected_errors}
153
154
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'fail-vars-rendering.yaml')
155
156
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
157
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
158
159
        # Assert action execution for task is not started and workflow failed.
160
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
161
        tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
162
        self.assertEqual(len(tk_ex_dbs), 0)
163
        self.assertEqual(wf_ex_db.status, wf_states.FAILED)
164
        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
165
166
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
167
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
168
        self.assertDictEqual(lv_ac_db.result, expected_result)
169
170
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
171
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
172
        self.assertDictEqual(ac_ex_db.result, expected_result)
173
174
    def test_fail_start_task_action(self):
175
        expected_errors = [
176
            {
177
                'message': 'Unknown function "#property#value"',
178
                'task_id': 'task1'
179
            }
180
        ]
181
182
        expected_result = {'output': None, 'errors': expected_errors}
183
184
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'fail-start-task-action.yaml')
185
186
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
187
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
188
189
        # Assert action execution for task is not started and workflow failed.
190
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
191
        tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
192
        self.assertEqual(len(tk_ex_dbs), 0)
193
        self.assertEqual(wf_ex_db.status, wf_states.FAILED)
194
        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
195
196
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
197
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
198
        self.assertDictEqual(lv_ac_db.result, expected_result)
199
200
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
201
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
202
        self.assertDictEqual(ac_ex_db.result, expected_result)
203
204
    def test_fail_start_task_input(self):
205
        expected_errors = [
206
            {
207
                'message': 'Unknown function "#property#value"',
208
                'task_id': 'task1'
209
            }
210
        ]
211
212
        expected_result = {'output': None, 'errors': expected_errors}
213
214
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'fail-start-task-input.yaml')
215
216
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
217
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
218
219
        # Assert action execution for task is not started and workflow failed.
220
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
221
        tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
222
        self.assertEqual(len(tk_ex_dbs), 0)
223
        self.assertEqual(wf_ex_db.status, wf_states.FAILED)
224
        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
225
226
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
227
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
228
        self.assertDictEqual(lv_ac_db.result, expected_result)
229
230
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
231
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
232
        self.assertDictEqual(ac_ex_db.result, expected_result)
233
234
    def test_fail_next_task_action(self):
235
        expected_errors = [
236
            {
237
                'message': 'Unknown function "#property#value"',
238
                'task_id': 'task2'
239
            }
240
        ]
241
242
        expected_result = {'output': None, 'errors': expected_errors}
243
244
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'fail-task-action.yaml')
245
246
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
247
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
248
249
        # Assert task1 is already completed.
250
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
251
        tk_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))[0]
252
        tk_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_db.id))[0]
253
        tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction['id'])
254
        self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
255
256
        # Manually handle action execution completion for task1 which has an error in publish.
257
        wf_svc.handle_action_execution_completion(tk_ac_ex_db)
258
259
        # Assert task1 succeeded but workflow failed.
260
        tk_ex_db = wf_db_access.TaskExecution.get_by_id(tk_ex_db.id)
261
        self.assertEqual(tk_ex_db.status, wf_states.SUCCEEDED)
262
        wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
263
        self.assertEqual(wf_ex_db.status, wf_states.FAILED)
264
        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
265
266
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
267
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
268
        self.assertDictEqual(lv_ac_db.result, expected_result)
269
270
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
271
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
272
        self.assertDictEqual(ac_ex_db.result, expected_result)
273
274
    def test_fail_next_task_input(self):
275
        expected_errors = [
276
            {
277
                'message': 'Unknown function "#property#value"',
278
                'task_id': 'task2'
279
            }
280
        ]
281
282
        expected_result = {'output': None, 'errors': expected_errors}
283
284
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'fail-task-input.yaml')
285
286
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
287
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
288
289
        # Assert task1 is already completed.
290
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
291
        tk_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))[0]
292
        tk_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_db.id))[0]
293
        tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction['id'])
294
        self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
295
296
        # Manually handle action execution completion for task1 which has an error in publish.
297
        wf_svc.handle_action_execution_completion(tk_ac_ex_db)
298
299
        # Assert task1 succeeded but workflow failed.
300
        tk_ex_db = wf_db_access.TaskExecution.get_by_id(tk_ex_db.id)
301
        self.assertEqual(tk_ex_db.status, wf_states.SUCCEEDED)
302
        wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
303
        self.assertEqual(wf_ex_db.status, wf_states.FAILED)
304
        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
305
306
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
307
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
308
        self.assertDictEqual(lv_ac_db.result, expected_result)
309
310
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
311
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
312
        self.assertDictEqual(ac_ex_db.result, expected_result)
313
314
    def test_fail_task_transition(self):
315
        expected_errors = [
316
            {
317
                'message': 'Unable to resolve key \'foobar\' in expression '
318
                           '\'<% succeeded() and result().foobar %>\' from context.',
319
                'task_transition_id': 'task2__0',
320
                'task_id': 'task1'
321
            }
322
        ]
323
324
        expected_result = {'output': None, 'errors': expected_errors}
325
326
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'fail-task-transition.yaml')
327
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
328
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
329
330
        # Assert task1 is already completed.
331
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
332
        tk_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))[0]
333
        tk_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_db.id))[0]
334
        tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction['id'])
335
        self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
336
337
        # Manually handle action execution completion for task1 which has an error in publish.
338
        wf_svc.handle_action_execution_completion(tk_ac_ex_db)
339
340
        # Assert task1 succeeded but workflow failed.
341
        tk_ex_db = wf_db_access.TaskExecution.get_by_id(tk_ex_db.id)
342
        self.assertEqual(tk_ex_db.status, wf_states.SUCCEEDED)
343
        wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
344
        self.assertEqual(wf_ex_db.status, wf_states.FAILED)
345
        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
346
347
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
348
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
349
        self.assertDictEqual(lv_ac_db.result, expected_result)
350
351
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
352
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
353
        self.assertDictEqual(ac_ex_db.result, expected_result)
354
355
    def test_fail_task_publish(self):
356
        expected_errors = [
357
            {
358
                'message': 'Unknown function "foobar"',
359
                'task_transition_id': 'task2__0',
360
                'task_id': 'task1'
361
            }
362
        ]
363
364
        expected_result = {'output': None, 'errors': expected_errors}
365
366
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'fail-task-publish.yaml')
367
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
368
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
369
370
        # Assert task1 is already completed.
371
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
372
        tk_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))[0]
373
        tk_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_db.id))[0]
374
        tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction['id'])
375
        self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
376
377
        # Manually handle action execution completion for task1 which has an error in publish.
378
        wf_svc.handle_action_execution_completion(tk_ac_ex_db)
379
380
        # Assert task1 succeeded but workflow failed.
381
        tk_ex_db = wf_db_access.TaskExecution.get_by_id(tk_ex_db.id)
382
        self.assertEqual(tk_ex_db.status, wf_states.SUCCEEDED)
383
        wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
384
        self.assertEqual(wf_ex_db.status, wf_states.FAILED)
385
        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
386
387
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
388
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
389
        self.assertDictEqual(lv_ac_db.result, expected_result)
390
391
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
392
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
393
        self.assertDictEqual(ac_ex_db.result, expected_result)
394
395
    def test_fail_output_rendering(self):
396
        expected_errors = [
397
            {
398
                'message': 'Unknown function "#property#value"'
399
            }
400
        ]
401
402
        expected_result = {'output': None, 'errors': expected_errors}
403
404
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'fail-output-rendering.yaml')
405
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
406
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
407
408
        # Assert task1 is already completed.
409
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
410
        tk_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))[0]
411
        tk_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_db.id))[0]
412
        tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction['id'])
413
        self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
414
415
        # Manually handle action execution completion for task1 which has an error in publish.
416
        wf_svc.handle_action_execution_completion(tk_ac_ex_db)
417
418
        # Assert task1 succeeded but workflow failed.
419
        tk_ex_db = wf_db_access.TaskExecution.get_by_id(tk_ex_db.id)
420
        self.assertEqual(tk_ex_db.status, wf_states.SUCCEEDED)
421
        wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
422
        self.assertEqual(wf_ex_db.status, wf_states.FAILED)
423
        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
424
425
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
426
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
427
        self.assertDictEqual(lv_ac_db.result, expected_result)
428
429
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
430
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
431
        self.assertDictEqual(ac_ex_db.result, expected_result)
432