|
1
|
|
|
"""Module for handling the scheduled execution of maintenance windows.""" |
|
2
|
1 |
|
import pytz |
|
3
|
1 |
|
from dataclasses import dataclass |
|
4
|
|
|
|
|
5
|
1 |
|
from apscheduler.jobstores.base import JobLookupError |
|
6
|
1 |
|
from apscheduler.schedulers.background import BackgroundScheduler |
|
7
|
1 |
|
from apscheduler.schedulers.base import BaseScheduler |
|
8
|
|
|
|
|
9
|
|
|
|
|
10
|
1 |
|
from .deployer import MaintenanceDeployer |
|
11
|
1 |
|
from ..controllers import MaintenanceController |
|
12
|
1 |
|
from ..models import ( |
|
13
|
|
|
MaintenanceID, |
|
14
|
|
|
MaintenanceWindow, |
|
15
|
|
|
MaintenanceWindows, |
|
16
|
|
|
OverlapError, |
|
17
|
|
|
Status, |
|
18
|
|
|
) |
|
19
|
|
|
|
|
20
|
1 |
|
from kytos.core import log |
|
21
|
|
|
|
|
22
|
1 |
|
@dataclass |
|
23
|
1 |
|
class MaintenanceStart: |
|
24
|
|
|
""" |
|
25
|
|
|
Callable used for starting maintenance windows |
|
26
|
|
|
""" |
|
27
|
1 |
|
maintenance_scheduler: 'MaintenanceScheduler' |
|
28
|
1 |
|
mw_id: MaintenanceID |
|
29
|
|
|
|
|
30
|
1 |
|
def __call__(self): |
|
31
|
1 |
|
self.maintenance_scheduler.start_maintenance(self.mw_id) |
|
32
|
|
|
|
|
33
|
|
|
|
|
34
|
1 |
|
@dataclass |
|
35
|
1 |
|
class MaintenanceEnd: |
|
36
|
|
|
""" |
|
37
|
|
|
Callable used for ending maintenance windows |
|
38
|
|
|
""" |
|
39
|
1 |
|
maintenance_scheduler: 'MaintenanceScheduler' |
|
40
|
1 |
|
mw_id: MaintenanceID |
|
41
|
|
|
|
|
42
|
1 |
|
def __call__(self): |
|
43
|
1 |
|
self.maintenance_scheduler.end_maintenance(self.mw_id) |
|
44
|
|
|
|
|
45
|
1 |
|
@dataclass |
|
46
|
1 |
|
class MaintenanceScheduler: |
|
47
|
|
|
"""Class for scheduling maintenance windows.""" |
|
48
|
1 |
|
deployer: MaintenanceDeployer |
|
49
|
1 |
|
db_controller: MaintenanceController |
|
50
|
1 |
|
scheduler: BaseScheduler |
|
51
|
|
|
|
|
52
|
1 |
|
@classmethod |
|
53
|
1 |
|
def new_scheduler(cls, deployer: MaintenanceDeployer): |
|
54
|
|
|
""" |
|
55
|
|
|
Creates a new scheduler from the given MaintenanceDeployer |
|
56
|
|
|
""" |
|
57
|
|
|
scheduler = BackgroundScheduler(timezone=pytz.utc) |
|
58
|
|
|
db_controller = MaintenanceController() |
|
59
|
|
|
db_controller.bootstrap_indexes() |
|
60
|
|
|
instance = cls(deployer, db_controller, scheduler) |
|
61
|
|
|
return instance |
|
62
|
|
|
|
|
63
|
1 |
|
def start(self): |
|
64
|
|
|
""" |
|
65
|
|
|
Begin running the scheduler. |
|
66
|
|
|
""" |
|
67
|
1 |
|
self.db_controller.prepare_start() |
|
68
|
|
|
|
|
69
|
|
|
# Populate the scheduler with all pending tasks |
|
70
|
1 |
|
windows = self.db_controller.get_unfinished_windows() |
|
71
|
1 |
|
for window in windows: |
|
72
|
1 |
|
if window.status == Status.RUNNING: |
|
73
|
1 |
|
self.deployer.start_mw(window) |
|
74
|
1 |
|
self._schedule(window) |
|
75
|
|
|
|
|
76
|
|
|
# Start the scheduler |
|
77
|
1 |
|
self.scheduler.start() |
|
78
|
|
|
|
|
79
|
1 |
|
def shutdown(self): |
|
80
|
|
|
""" |
|
81
|
|
|
Stop running the scheduler. |
|
82
|
|
|
""" |
|
83
|
1 |
|
windows = self.db_controller.get_windows() |
|
84
|
|
|
|
|
85
|
|
|
# Depopulate the scheduler |
|
86
|
1 |
|
for window in windows: |
|
87
|
1 |
|
self._unschedule(window) |
|
88
|
|
|
|
|
89
|
1 |
|
self.scheduler.remove_all_jobs() |
|
90
|
1 |
|
self.scheduler.shutdown() |
|
91
|
|
|
|
|
92
|
1 |
|
def start_maintenance(self, mw_id: MaintenanceID): |
|
93
|
|
|
"""Begins executing the maintenance window |
|
94
|
|
|
""" |
|
95
|
|
|
# Get Maintenance from DB and Update |
|
96
|
1 |
|
window = self.db_controller.start_window(mw_id) |
|
97
|
|
|
|
|
98
|
|
|
# Activate Running |
|
99
|
1 |
|
self.deployer.start_mw(window) |
|
100
|
|
|
|
|
101
|
|
|
# Schedule next task |
|
102
|
1 |
|
self._schedule(window) |
|
103
|
|
|
|
|
104
|
1 |
|
def end_maintenance(self, mw_id: MaintenanceID): |
|
105
|
|
|
"""Ends execution of the maintenance window |
|
106
|
|
|
""" |
|
107
|
|
|
# Get Maintenance from DB |
|
108
|
1 |
|
window = self.db_controller.end_window(mw_id) |
|
109
|
|
|
|
|
110
|
|
|
# Set to Ending |
|
111
|
1 |
|
self.deployer.end_mw(window) |
|
112
|
|
|
|
|
113
|
1 |
|
def end_maintenance_early(self, mw_id: MaintenanceID): |
|
114
|
|
|
"""Ends execution of the maintenance window early |
|
115
|
|
|
""" |
|
116
|
|
|
# Get Maintenance from DB |
|
117
|
|
|
window = self.db_controller.end_window(mw_id) |
|
118
|
|
|
|
|
119
|
|
|
# Unschedule tasks |
|
120
|
|
|
self._unschedule(window) |
|
121
|
|
|
|
|
122
|
1 |
|
def add(self, window: MaintenanceWindow, force=False): |
|
123
|
|
|
"""Add jobs to start and end a maintenance window.""" |
|
124
|
|
|
|
|
125
|
|
|
if force is False: |
|
126
|
|
|
overlapping_windows = self.db_controller.check_overlap(window) |
|
127
|
|
|
if overlapping_windows: |
|
128
|
|
|
raise OverlapError(window, overlapping_windows) |
|
129
|
|
|
|
|
130
|
|
|
# Add window to DB |
|
131
|
|
|
self.db_controller.insert_window(window) |
|
132
|
|
|
|
|
133
|
|
|
# Schedule next task |
|
134
|
|
|
self._schedule(window) |
|
135
|
|
|
|
|
136
|
1 |
|
def update(self, window: MaintenanceWindow): |
|
137
|
|
|
"""Update an existing Maintenance Window.""" |
|
138
|
|
|
|
|
139
|
|
|
# Update window |
|
140
|
1 |
|
self.db_controller.update_window(window) |
|
141
|
|
|
|
|
142
|
|
|
# Reschedule any pending tasks |
|
143
|
1 |
|
self._reschedule(window) |
|
144
|
|
|
|
|
145
|
1 |
|
def remove(self, mw_id: MaintenanceID): |
|
146
|
|
|
"""Remove jobs that start and end a maintenance window.""" |
|
147
|
|
|
# Get Maintenance from DB |
|
148
|
|
|
window = self.db_controller.get_window(mw_id) |
|
149
|
|
|
|
|
150
|
|
|
# Remove from schedule |
|
151
|
|
|
self._unschedule(window) |
|
152
|
|
|
|
|
153
|
|
|
# Remove from DB |
|
154
|
|
|
self.db_controller.remove_window(mw_id) |
|
155
|
|
|
|
|
156
|
1 |
|
def _schedule(self, window: MaintenanceWindow): |
|
157
|
1 |
|
log.info(f'Scheduling "{window.id}"') |
|
158
|
1 |
|
if window.status == Status.PENDING: |
|
159
|
1 |
|
self.scheduler.add_job( |
|
160
|
|
|
MaintenanceStart(self, window.id), |
|
161
|
|
|
'date', |
|
162
|
|
|
id=f'{window.id}-start', |
|
163
|
|
|
run_date=window.start |
|
164
|
|
|
) |
|
165
|
1 |
|
log.info(f'Scheduled "{window.id}" start at {window.start}') |
|
166
|
1 |
|
if window.status == Status.RUNNING: |
|
167
|
1 |
|
self.scheduler.add_job( |
|
168
|
|
|
MaintenanceEnd(self, window.id), |
|
169
|
|
|
'date', |
|
170
|
|
|
id=f'{window.id}-end', |
|
171
|
|
|
run_date=window.end |
|
172
|
|
|
) |
|
173
|
1 |
|
log.info(f'Scheduled "{window.id}" end at {window.end}') |
|
174
|
|
|
|
|
175
|
1 |
|
def _reschedule(self, window: MaintenanceWindow): |
|
176
|
1 |
|
log.info(f'Rescheduling "{window.id}"') |
|
177
|
1 |
|
try: |
|
178
|
1 |
|
self.scheduler.remove_job( |
|
179
|
|
|
f'{window.id}-start', |
|
180
|
|
|
) |
|
181
|
1 |
|
self.scheduler.add_job( |
|
182
|
|
|
MaintenanceStart(self, window.id), |
|
183
|
|
|
'date', |
|
184
|
|
|
id=f'{window.id}-start', |
|
185
|
|
|
run_date=window.start |
|
186
|
|
|
) |
|
187
|
1 |
|
log.info(f'Rescheduled "{window.id}" start to {window.start}') |
|
188
|
|
|
except JobLookupError: |
|
189
|
|
|
log.info(f'Could not reschedule "{window.id}" start, no start job') |
|
190
|
1 |
|
try: |
|
191
|
1 |
|
self.scheduler.remove_job( |
|
192
|
|
|
f'{window.id}-end', |
|
193
|
|
|
) |
|
194
|
1 |
|
self.scheduler.add_job( |
|
195
|
|
|
MaintenanceEnd(self, window.id), |
|
196
|
|
|
'date', |
|
197
|
|
|
id=f'{window.id}-end', |
|
198
|
|
|
run_date=window.end |
|
199
|
|
|
) |
|
200
|
1 |
|
log.info(f'Rescheduled "{window.id}" end to {window.end}') |
|
201
|
|
|
except JobLookupError: |
|
202
|
|
|
log.info(f'Could not reschedule "{window.id}" end, no end job') |
|
203
|
|
|
|
|
204
|
1 |
|
def _unschedule(self, window: MaintenanceWindow): |
|
205
|
|
|
"""Remove maintenance events from scheduler. |
|
206
|
|
|
Does not update DB, due to being |
|
207
|
|
|
primarily for shutdown startup cases. |
|
208
|
|
|
""" |
|
209
|
1 |
|
started = False |
|
210
|
1 |
|
ended = False |
|
211
|
1 |
|
try: |
|
212
|
1 |
|
self.scheduler.remove_job(f'{window.id}-start') |
|
213
|
1 |
|
except JobLookupError: |
|
214
|
1 |
|
started = True |
|
215
|
1 |
|
log.info(f'Job to start "{window.id}" already removed.') |
|
216
|
1 |
|
try: |
|
217
|
1 |
|
self.scheduler.remove_job(f'{window.id}-end') |
|
218
|
1 |
|
except JobLookupError: |
|
219
|
1 |
|
ended = True |
|
220
|
1 |
|
log.info(f'Job to end "{window.id}" already removed.') |
|
221
|
1 |
|
if started and not ended: |
|
222
|
1 |
|
self.deployer.end_mw(window) |
|
223
|
|
|
|
|
224
|
1 |
|
def get_maintenance(self, mw_id: MaintenanceID) -> MaintenanceWindow: |
|
225
|
|
|
"""Get a single maintenance by id""" |
|
226
|
|
|
return self.db_controller.get_window(mw_id) |
|
227
|
|
|
|
|
228
|
1 |
|
def list_maintenances(self) -> MaintenanceWindows: |
|
229
|
|
|
"""Returns a list of all maintenances""" |
|
230
|
|
|
return self.db_controller.get_windows() |
|
231
|
|
|
|
kytos-ng /
maintenance
