1
|
|
|
"""Classes used in the main application.""" # pylint: disable=too-many-lines |
2
|
1 |
|
from datetime import datetime |
3
|
1 |
|
from threading import Lock |
4
|
1 |
|
from uuid import uuid4 |
5
|
|
|
|
6
|
1 |
|
import requests |
7
|
1 |
|
from glom import glom |
8
|
|
|
|
9
|
1 |
|
from kytos.core import log |
10
|
1 |
|
from kytos.core.common import EntityStatus, GenericEntity |
11
|
1 |
|
from kytos.core.exceptions import KytosNoTagAvailableError |
12
|
1 |
|
from kytos.core.helpers import get_time, now |
13
|
1 |
|
from kytos.core.interface import UNI |
14
|
1 |
|
from napps.kytos.mef_eline import settings |
15
|
1 |
|
from napps.kytos.mef_eline.exceptions import FlowModException, InvalidPath |
16
|
1 |
|
from napps.kytos.mef_eline.storehouse import StoreHouse |
17
|
1 |
|
from napps.kytos.mef_eline.utils import ( |
18
|
|
|
compare_endpoint_trace, |
19
|
|
|
emit_event, |
20
|
|
|
notify_link_available_tags, |
21
|
|
|
) |
22
|
1 |
|
from .path import Path, DynamicPathManager |
23
|
|
|
|
24
|
|
|
|
25
|
1 |
|
class EVCBase(GenericEntity): |
26
|
|
|
"""Class to represent a circuit.""" |
27
|
|
|
|
28
|
1 |
|
read_only_attributes = [ |
29
|
|
|
"creation_time", |
30
|
|
|
"active", |
31
|
|
|
"current_path", |
32
|
|
|
"_id", |
33
|
|
|
"archived", |
34
|
|
|
] |
35
|
1 |
|
attributes_requiring_redeploy = [ |
36
|
|
|
"primary_path", |
37
|
|
|
"backup_path", |
38
|
|
|
"dynamic_backup_path", |
39
|
|
|
"queue_id", |
40
|
|
|
"priority", |
41
|
|
|
] |
42
|
1 |
|
required_attributes = ["name", "uni_a", "uni_z"] |
43
|
|
|
|
44
|
1 |
|
def __init__(self, controller, **kwargs): |
45
|
|
|
"""Create an EVC instance with the provided parameters. |
46
|
|
|
|
47
|
|
|
Args: |
48
|
|
|
id(str): EVC identifier. Whether it's None an ID will be genereted. |
49
|
|
|
Only the first 14 bytes passed will be used. |
50
|
|
|
name: represents an EVC name.(Required) |
51
|
|
|
uni_a (UNI): Endpoint A for User Network Interface.(Required) |
52
|
|
|
uni_z (UNI): Endpoint Z for User Network Interface.(Required) |
53
|
|
|
start_date(datetime|str): Date when the EVC was registred. |
54
|
|
|
Default is now(). |
55
|
|
|
end_date(datetime|str): Final date that the EVC will be fineshed. |
56
|
|
|
Default is None. |
57
|
|
|
bandwidth(int): Bandwidth used by EVC instance. Default is 0. |
58
|
|
|
primary_links(list): Primary links used by evc. Default is [] |
59
|
|
|
backup_links(list): Backups links used by evc. Default is [] |
60
|
|
|
current_path(list): Circuit being used at the moment if this is an |
61
|
|
|
active circuit. Default is []. |
62
|
|
|
primary_path(list): primary circuit offered to user IF one or more |
63
|
|
|
links were provided. Default is []. |
64
|
|
|
backup_path(list): backup circuit offered to the user IF one or |
65
|
|
|
more links were provided. Default is []. |
66
|
|
|
dynamic_backup_path(bool): Enable computer backup path dynamically. |
67
|
|
|
Dafault is False. |
68
|
|
|
creation_time(datetime|str): datetime when the circuit should be |
69
|
|
|
activated. default is now(). |
70
|
|
|
enabled(Boolean): attribute to indicate the administrative state; |
71
|
|
|
default is False. |
72
|
|
|
active(Boolean): attribute to indicate the operational state; |
73
|
|
|
default is False. |
74
|
|
|
archived(Boolean): indicate the EVC has been deleted and is |
75
|
|
|
archived; default is False. |
76
|
|
|
owner(str): The EVC owner. Default is None. |
77
|
|
|
priority(int): Service level provided in the request. Default is 0. |
78
|
|
|
|
79
|
|
|
Raises: |
80
|
|
|
ValueError: raised when object attributes are invalid. |
81
|
|
|
|
82
|
|
|
""" |
83
|
1 |
|
self._validate(**kwargs) |
84
|
1 |
|
super().__init__() |
85
|
|
|
|
86
|
|
|
# required attributes |
87
|
1 |
|
self._id = kwargs.get("id", uuid4().hex)[:14] |
88
|
1 |
|
self.uni_a = kwargs.get("uni_a") |
89
|
1 |
|
self.uni_z = kwargs.get("uni_z") |
90
|
1 |
|
self.name = kwargs.get("name") |
91
|
|
|
|
92
|
|
|
# optional attributes |
93
|
1 |
|
self.start_date = get_time(kwargs.get("start_date")) or now() |
94
|
1 |
|
self.end_date = get_time(kwargs.get("end_date")) or None |
95
|
1 |
|
self.queue_id = kwargs.get("queue_id", None) |
96
|
|
|
|
97
|
1 |
|
self.bandwidth = kwargs.get("bandwidth", 0) |
98
|
1 |
|
self.primary_links = Path(kwargs.get("primary_links", [])) |
99
|
1 |
|
self.backup_links = Path(kwargs.get("backup_links", [])) |
100
|
1 |
|
self.current_path = Path(kwargs.get("current_path", [])) |
101
|
1 |
|
self.primary_path = Path(kwargs.get("primary_path", [])) |
102
|
1 |
|
self.backup_path = Path(kwargs.get("backup_path", [])) |
103
|
1 |
|
self.dynamic_backup_path = kwargs.get("dynamic_backup_path", False) |
104
|
1 |
|
self.creation_time = get_time(kwargs.get("creation_time")) or now() |
105
|
1 |
|
self.owner = kwargs.get("owner", None) |
106
|
1 |
|
self.priority = kwargs.get("priority", -1) |
107
|
1 |
|
self.circuit_scheduler = kwargs.get("circuit_scheduler", []) |
108
|
|
|
|
109
|
1 |
|
self.current_links_cache = set() |
110
|
1 |
|
self.primary_links_cache = set() |
111
|
1 |
|
self.backup_links_cache = set() |
112
|
|
|
|
113
|
1 |
|
self.lock = Lock() |
114
|
|
|
|
115
|
1 |
|
self.archived = kwargs.get("archived", False) |
116
|
|
|
|
117
|
1 |
|
self.metadata = kwargs.get("metadata", {}) |
118
|
|
|
|
119
|
1 |
|
self._storehouse = StoreHouse(controller) |
120
|
1 |
|
self._controller = controller |
121
|
|
|
|
122
|
1 |
|
if kwargs.get("active", False): |
123
|
1 |
|
self.activate() |
124
|
|
|
else: |
125
|
1 |
|
self.deactivate() |
126
|
|
|
|
127
|
1 |
|
if kwargs.get("enabled", False): |
128
|
1 |
|
self.enable() |
129
|
|
|
else: |
130
|
1 |
|
self.disable() |
131
|
|
|
|
132
|
|
|
# datetime of user request for a EVC (or datetime when object was |
133
|
|
|
# created) |
134
|
1 |
|
self.request_time = kwargs.get("request_time", now()) |
135
|
|
|
# dict with the user original request (input) |
136
|
1 |
|
self._requested = kwargs |
137
|
|
|
|
138
|
1 |
|
def sync(self): |
139
|
|
|
"""Sync this EVC in the storehouse.""" |
140
|
1 |
|
self._storehouse.save_evc(self) |
141
|
1 |
|
log.info(f"EVC {self.id} was synced to the storehouse.") |
142
|
|
|
|
143
|
1 |
|
def update(self, **kwargs): |
144
|
|
|
"""Update evc attributes. |
145
|
|
|
|
146
|
|
|
This method will raises an error trying to change the following |
147
|
|
|
attributes: [name, uni_a and uni_z] |
148
|
|
|
|
149
|
|
|
Returns: |
150
|
|
|
the values for enable and a redeploy attribute, if exists and None |
151
|
|
|
otherwise |
152
|
|
|
Raises: |
153
|
|
|
ValueError: message with error detail. |
154
|
|
|
|
155
|
|
|
""" |
156
|
1 |
|
enable, redeploy = (None, None) |
157
|
1 |
|
uni_a = kwargs.get("uni_a") or self.uni_a |
158
|
1 |
|
uni_z = kwargs.get("uni_z") or self.uni_z |
159
|
1 |
|
for attribute, value in kwargs.items(): |
160
|
1 |
|
if attribute in self.read_only_attributes: |
161
|
1 |
|
raise ValueError(f"{attribute} can't be updated.") |
162
|
1 |
|
if not hasattr(self, attribute): |
163
|
|
|
raise ValueError(f'The attribute "{attribute}" is invalid.') |
164
|
1 |
|
if attribute in ("primary_path", "backup_path"): |
165
|
1 |
|
try: |
166
|
1 |
|
value.is_valid( |
167
|
|
|
uni_a.interface.switch, uni_z.interface.switch |
168
|
|
|
) |
169
|
1 |
|
except InvalidPath as exception: |
170
|
1 |
|
raise ValueError( |
171
|
|
|
f"{attribute} is not a " f"valid path: {exception}" |
172
|
|
|
) |
173
|
1 |
|
for attribute, value in kwargs.items(): |
174
|
1 |
|
if attribute in ("enable", "enabled"): |
175
|
1 |
|
if value: |
176
|
1 |
|
self.enable() |
177
|
|
|
else: |
178
|
1 |
|
self.disable() |
179
|
1 |
|
enable = value |
180
|
|
|
else: |
181
|
1 |
|
setattr(self, attribute, value) |
182
|
1 |
|
if attribute in self.attributes_requiring_redeploy: |
183
|
1 |
|
redeploy = value |
184
|
1 |
|
self.sync() |
185
|
1 |
|
return enable, redeploy |
186
|
|
|
|
187
|
1 |
|
def __repr__(self): |
188
|
|
|
"""Repr method.""" |
189
|
1 |
|
return f"EVC({self._id}, {self.name})" |
190
|
|
|
|
191
|
1 |
|
def _validate(self, **kwargs): |
192
|
|
|
"""Do Basic validations. |
193
|
|
|
|
194
|
|
|
Verify required attributes: name, uni_a, uni_z |
195
|
|
|
Verify if the attributes uni_a and uni_z are valid. |
196
|
|
|
|
197
|
|
|
Raises: |
198
|
|
|
ValueError: message with error detail. |
199
|
|
|
|
200
|
|
|
""" |
201
|
1 |
|
for attribute in self.required_attributes: |
202
|
|
|
|
203
|
1 |
|
if attribute not in kwargs: |
204
|
1 |
|
raise ValueError(f"{attribute} is required.") |
205
|
|
|
|
206
|
1 |
|
if "uni" in attribute: |
207
|
1 |
|
uni = kwargs.get(attribute) |
208
|
1 |
|
if not isinstance(uni, UNI): |
209
|
|
|
raise ValueError(f"{attribute} is an invalid UNI.") |
210
|
|
|
|
211
|
1 |
|
if not uni.is_valid(): |
212
|
1 |
|
tag = uni.user_tag.value |
213
|
1 |
|
message = f"VLAN tag {tag} is not available in {attribute}" |
214
|
1 |
|
raise ValueError(message) |
215
|
|
|
|
216
|
1 |
|
def __eq__(self, other): |
217
|
|
|
"""Override the default implementation.""" |
218
|
1 |
|
if not isinstance(other, EVC): |
219
|
|
|
return False |
220
|
|
|
|
221
|
1 |
|
attrs_to_compare = ["name", "uni_a", "uni_z", "owner", "bandwidth"] |
222
|
1 |
|
for attribute in attrs_to_compare: |
223
|
1 |
|
if getattr(other, attribute) != getattr(self, attribute): |
224
|
1 |
|
return False |
225
|
1 |
|
return True |
226
|
|
|
|
227
|
1 |
|
def shares_uni(self, other): |
228
|
|
|
"""Check if two EVCs share an UNI.""" |
229
|
1 |
|
if other.uni_a in (self.uni_a, self.uni_z) or other.uni_z in ( |
230
|
|
|
self.uni_a, |
231
|
|
|
self.uni_z, |
232
|
|
|
): |
233
|
1 |
|
return True |
234
|
|
|
return False |
235
|
|
|
|
236
|
1 |
|
def as_dict(self): |
237
|
|
|
"""Return a dictionary representing an EVC object.""" |
238
|
1 |
|
evc_dict = { |
239
|
|
|
"id": self.id, |
240
|
|
|
"name": self.name, |
241
|
|
|
"uni_a": self.uni_a.as_dict(), |
242
|
|
|
"uni_z": self.uni_z.as_dict(), |
243
|
|
|
} |
244
|
|
|
|
245
|
1 |
|
time_fmt = "%Y-%m-%dT%H:%M:%S" |
246
|
|
|
|
247
|
1 |
|
evc_dict["start_date"] = self.start_date |
248
|
1 |
|
if isinstance(self.start_date, datetime): |
249
|
1 |
|
evc_dict["start_date"] = self.start_date.strftime(time_fmt) |
250
|
|
|
|
251
|
1 |
|
evc_dict["end_date"] = self.end_date |
252
|
1 |
|
if isinstance(self.end_date, datetime): |
253
|
1 |
|
evc_dict["end_date"] = self.end_date.strftime(time_fmt) |
254
|
|
|
|
255
|
1 |
|
evc_dict["queue_id"] = self.queue_id |
256
|
1 |
|
evc_dict["bandwidth"] = self.bandwidth |
257
|
1 |
|
evc_dict["primary_links"] = self.primary_links.as_dict() |
258
|
1 |
|
evc_dict["backup_links"] = self.backup_links.as_dict() |
259
|
1 |
|
evc_dict["current_path"] = self.current_path.as_dict() |
260
|
1 |
|
evc_dict["primary_path"] = self.primary_path.as_dict() |
261
|
1 |
|
evc_dict["backup_path"] = self.backup_path.as_dict() |
262
|
1 |
|
evc_dict["dynamic_backup_path"] = self.dynamic_backup_path |
263
|
1 |
|
evc_dict["metadata"] = self.metadata |
264
|
|
|
|
265
|
|
|
# if self._requested: |
266
|
|
|
# request_dict = self._requested.copy() |
267
|
|
|
# request_dict['uni_a'] = request_dict['uni_a'].as_dict() |
268
|
|
|
# request_dict['uni_z'] = request_dict['uni_z'].as_dict() |
269
|
|
|
# request_dict['circuit_scheduler'] = self.circuit_scheduler |
270
|
|
|
# evc_dict['_requested'] = request_dict |
271
|
|
|
|
272
|
1 |
|
evc_dict["request_time"] = self.request_time |
273
|
1 |
|
if isinstance(self.request_time, datetime): |
274
|
1 |
|
evc_dict["request_time"] = self.request_time.strftime(time_fmt) |
275
|
|
|
|
276
|
1 |
|
time = self.creation_time.strftime(time_fmt) |
277
|
1 |
|
evc_dict["creation_time"] = time |
278
|
|
|
|
279
|
1 |
|
evc_dict["owner"] = self.owner |
280
|
1 |
|
evc_dict["circuit_scheduler"] = [ |
281
|
|
|
sc.as_dict() for sc in self.circuit_scheduler |
282
|
|
|
] |
283
|
|
|
|
284
|
1 |
|
evc_dict["active"] = self.is_active() |
285
|
1 |
|
evc_dict["enabled"] = self.is_enabled() |
286
|
1 |
|
evc_dict["archived"] = self.archived |
287
|
1 |
|
evc_dict["priority"] = self.priority |
288
|
|
|
|
289
|
1 |
|
return evc_dict |
290
|
|
|
|
291
|
1 |
|
@property |
292
|
|
|
def id(self): # pylint: disable=invalid-name |
293
|
|
|
"""Return this EVC's ID.""" |
294
|
1 |
|
return self._id |
295
|
|
|
|
296
|
1 |
|
def archive(self): |
297
|
|
|
"""Archive this EVC on deletion.""" |
298
|
1 |
|
self.archived = True |
299
|
|
|
|
300
|
|
|
|
301
|
|
|
# pylint: disable=fixme, too-many-public-methods |
302
|
1 |
|
class EVCDeploy(EVCBase): |
303
|
|
|
"""Class to handle the deploy procedures.""" |
304
|
|
|
|
305
|
1 |
|
def create(self): |
306
|
|
|
"""Create a EVC.""" |
307
|
|
|
|
308
|
1 |
|
def discover_new_paths(self): |
309
|
|
|
"""Discover new paths to satisfy this circuit and deploy it.""" |
310
|
|
|
return DynamicPathManager.get_best_paths(self) |
311
|
|
|
|
312
|
1 |
|
def change_path(self): |
313
|
|
|
"""Change EVC path.""" |
314
|
|
|
|
315
|
1 |
|
def reprovision(self): |
316
|
|
|
"""Force the EVC (re-)provisioning.""" |
317
|
|
|
|
318
|
1 |
|
def is_affected_by_link(self, link): |
319
|
|
|
"""Return True if this EVC has the given link on its current path.""" |
320
|
|
|
return link in self.current_path |
321
|
|
|
|
322
|
1 |
|
def link_affected_by_interface(self, interface): |
323
|
|
|
"""Return True if this EVC has the given link on its current path.""" |
324
|
|
|
return self.current_path.link_affected_by_interface(interface) |
325
|
|
|
|
326
|
1 |
|
def is_backup_path_affected_by_link(self, link): |
327
|
|
|
"""Return True if the backup path of this EVC uses the given link.""" |
328
|
|
|
return link in self.backup_path |
329
|
|
|
|
330
|
|
|
# pylint: disable=invalid-name |
331
|
1 |
|
def is_primary_path_affected_by_link(self, link): |
332
|
|
|
"""Return True if the primary path of this EVC uses the given link.""" |
333
|
|
|
return link in self.primary_path |
334
|
|
|
|
335
|
1 |
|
def is_using_primary_path(self): |
336
|
|
|
"""Verify if the current deployed path is self.primary_path.""" |
337
|
|
|
return self.primary_path and (self.current_path == self.primary_path) |
338
|
|
|
|
339
|
1 |
|
def is_using_backup_path(self): |
340
|
|
|
"""Verify if the current deployed path is self.backup_path.""" |
341
|
|
|
return self.backup_path and (self.current_path == self.backup_path) |
342
|
|
|
|
343
|
1 |
|
def is_using_dynamic_path(self): |
344
|
|
|
"""Verify if the current deployed path is a dynamic path.""" |
345
|
|
|
if ( |
346
|
|
|
self.current_path |
347
|
|
|
and not self.is_using_primary_path() |
348
|
|
|
and not self.is_using_backup_path() |
349
|
|
|
and self.current_path.status == EntityStatus.UP |
350
|
|
|
): |
351
|
|
|
return True |
352
|
|
|
return False |
353
|
|
|
|
354
|
1 |
|
def deploy_to_backup_path(self): |
355
|
|
|
"""Deploy the backup path into the datapaths of this circuit. |
356
|
|
|
|
357
|
|
|
If the backup_path attribute is valid and up, this method will try to |
358
|
|
|
deploy this backup_path. |
359
|
|
|
|
360
|
|
|
If everything fails and dynamic_backup_path is True, then tries to |
361
|
|
|
deploy a dynamic path. |
362
|
|
|
""" |
363
|
|
|
# TODO: Remove flows from current (cookies) |
364
|
1 |
|
if self.is_using_backup_path(): |
365
|
|
|
# TODO: Log to say that cannot move backup to backup |
366
|
|
|
return True |
367
|
|
|
|
368
|
1 |
|
success = False |
369
|
1 |
|
if self.backup_path.status is EntityStatus.UP: |
370
|
1 |
|
success = self.deploy_to_path(self.backup_path) |
371
|
|
|
|
372
|
1 |
|
if success: |
373
|
1 |
|
return True |
374
|
|
|
|
375
|
1 |
|
if ( |
376
|
|
|
self.dynamic_backup_path |
377
|
|
|
or self.uni_a.interface.switch == self.uni_z.interface.switch |
378
|
|
|
): |
379
|
1 |
|
return self.deploy_to_path() |
380
|
|
|
|
381
|
|
|
return False |
382
|
|
|
|
383
|
1 |
|
def deploy_to_primary_path(self): |
384
|
|
|
"""Deploy the primary path into the datapaths of this circuit. |
385
|
|
|
|
386
|
|
|
If the primary_path attribute is valid and up, this method will try to |
387
|
|
|
deploy this primary_path. |
388
|
|
|
""" |
389
|
|
|
# TODO: Remove flows from current (cookies) |
390
|
1 |
|
if self.is_using_primary_path(): |
391
|
|
|
# TODO: Log to say that cannot move primary to primary |
392
|
|
|
return True |
393
|
|
|
|
394
|
1 |
|
if self.primary_path.status is EntityStatus.UP: |
395
|
1 |
|
return self.deploy_to_path(self.primary_path) |
396
|
|
|
return False |
397
|
|
|
|
398
|
1 |
|
def deploy(self): |
399
|
|
|
"""Deploy EVC to best path. |
400
|
|
|
|
401
|
|
|
Best path can be the primary path, if available. If not, the backup |
402
|
|
|
path, and, if it is also not available, a dynamic path. |
403
|
|
|
""" |
404
|
|
|
if self.archived: |
405
|
|
|
return False |
406
|
|
|
self.enable() |
407
|
|
|
success = self.deploy_to_primary_path() |
408
|
|
|
if not success: |
409
|
|
|
success = self.deploy_to_backup_path() |
410
|
|
|
|
411
|
|
|
if success: |
412
|
|
|
emit_event(self._controller, "deployed", evc_id=self.id) |
413
|
|
|
return success |
414
|
|
|
|
415
|
1 |
|
@staticmethod |
416
|
|
|
def get_path_status(path): |
417
|
|
|
"""Check for the current status of a path. |
418
|
|
|
|
419
|
|
|
If any link in this path is down, the path is considered down. |
420
|
|
|
""" |
421
|
|
|
if not path: |
422
|
|
|
return EntityStatus.DISABLED |
423
|
|
|
|
424
|
|
|
for link in path: |
425
|
|
|
if link.status is not EntityStatus.UP: |
426
|
|
|
return link.status |
427
|
|
|
return EntityStatus.UP |
428
|
|
|
|
429
|
|
|
# def discover_new_path(self): |
430
|
|
|
# # TODO: discover a new path to satisfy this circuit and deploy |
431
|
|
|
|
432
|
1 |
|
def remove(self): |
433
|
|
|
"""Remove EVC path and disable it.""" |
434
|
|
|
self.remove_current_flows() |
435
|
|
|
self.disable() |
436
|
|
|
self.sync() |
437
|
|
|
emit_event(self._controller, "undeployed", evc_id=self.id) |
438
|
|
|
|
439
|
1 |
|
def remove_current_flows(self, current_path=None, force=True): |
440
|
|
|
"""Remove all flows from current path.""" |
441
|
1 |
|
switches = set() |
442
|
|
|
|
443
|
1 |
|
switches.add(self.uni_a.interface.switch) |
444
|
1 |
|
switches.add(self.uni_z.interface.switch) |
445
|
1 |
|
if not current_path: |
446
|
1 |
|
current_path = self.current_path |
447
|
1 |
|
for link in current_path: |
448
|
1 |
|
switches.add(link.endpoint_a.switch) |
449
|
1 |
|
switches.add(link.endpoint_b.switch) |
450
|
|
|
|
451
|
1 |
|
match = { |
452
|
|
|
"cookie": self.get_cookie(), |
453
|
|
|
"cookie_mask": 18446744073709551615, |
454
|
|
|
} |
455
|
|
|
|
456
|
1 |
|
for switch in switches: |
457
|
1 |
|
try: |
458
|
1 |
|
self._send_flow_mods(switch, [match], 'delete', force=force) |
459
|
|
|
except FlowModException: |
460
|
|
|
log.error( |
461
|
|
|
f"Error removing flows from switch {switch.id} for" |
462
|
|
|
f"EVC {self}" |
463
|
|
|
) |
464
|
|
|
|
465
|
1 |
|
current_path.make_vlans_available() |
466
|
1 |
|
for link in current_path: |
467
|
1 |
|
notify_link_available_tags(self._controller, link) |
468
|
1 |
|
self.current_path = Path([]) |
469
|
1 |
|
self.deactivate() |
470
|
1 |
|
self.sync() |
471
|
|
|
|
472
|
1 |
|
@staticmethod |
473
|
1 |
|
def links_zipped(path=None): |
474
|
|
|
"""Return an iterator which yields pairs of links in order.""" |
475
|
1 |
|
if not path: |
476
|
|
|
return [] |
477
|
1 |
|
return zip(path[:-1], path[1:]) |
478
|
|
|
|
479
|
1 |
|
def should_deploy(self, path=None): |
480
|
|
|
"""Verify if the circuit should be deployed.""" |
481
|
1 |
|
if not path: |
482
|
1 |
|
log.debug("Path is empty.") |
483
|
1 |
|
return False |
484
|
|
|
|
485
|
1 |
|
if not self.is_enabled(): |
486
|
1 |
|
log.debug(f"{self} is disabled.") |
487
|
1 |
|
return False |
488
|
|
|
|
489
|
1 |
|
if not self.is_active(): |
490
|
1 |
|
log.debug(f"{self} will be deployed.") |
491
|
1 |
|
return True |
492
|
|
|
|
493
|
1 |
|
return False |
494
|
|
|
|
495
|
1 |
|
def deploy_to_path(self, path=None): |
496
|
|
|
"""Install the flows for this circuit. |
497
|
|
|
|
498
|
|
|
Procedures to deploy: |
499
|
|
|
|
500
|
|
|
0. Remove current flows installed |
501
|
|
|
1. Decide if will deploy "path" or discover a new path |
502
|
|
|
2. Choose vlan |
503
|
|
|
3. Install NNI flows |
504
|
|
|
4. Install UNI flows |
505
|
|
|
5. Activate |
506
|
|
|
6. Update current_path |
507
|
|
|
7. Update links caches(primary, current, backup) |
508
|
|
|
|
509
|
|
|
""" |
510
|
1 |
|
self.remove_current_flows() |
511
|
1 |
|
use_path = path |
512
|
1 |
|
if self.should_deploy(use_path): |
513
|
1 |
|
try: |
514
|
1 |
|
use_path.choose_vlans() |
515
|
1 |
|
for link in use_path: |
516
|
1 |
|
notify_link_available_tags(self._controller, link) |
517
|
|
|
except KytosNoTagAvailableError: |
518
|
|
|
use_path = None |
519
|
|
|
else: |
520
|
1 |
|
for use_path in self.discover_new_paths(): |
521
|
1 |
|
if use_path is None: |
522
|
|
|
continue |
523
|
1 |
|
try: |
524
|
1 |
|
use_path.choose_vlans() |
525
|
1 |
|
for link in use_path: |
526
|
1 |
|
notify_link_available_tags(self._controller, link) |
527
|
1 |
|
break |
528
|
|
|
except KytosNoTagAvailableError: |
529
|
|
|
pass |
530
|
|
|
else: |
531
|
1 |
|
use_path = None |
532
|
|
|
|
533
|
1 |
|
try: |
534
|
1 |
|
if use_path: |
535
|
1 |
|
self._install_nni_flows(use_path) |
536
|
1 |
|
self._install_uni_flows(use_path) |
537
|
1 |
|
elif self.uni_a.interface.switch == self.uni_z.interface.switch: |
538
|
|
|
use_path = Path() |
539
|
|
|
self._install_direct_uni_flows() |
540
|
|
|
else: |
541
|
1 |
|
log.warn( |
542
|
|
|
f"{self} was not deployed. " "No available path was found." |
543
|
|
|
) |
544
|
1 |
|
return False |
545
|
1 |
|
except FlowModException: |
546
|
1 |
|
log.error(f"Error deploying EVC {self} when calling flow_manager.") |
547
|
1 |
|
self.remove_current_flows(use_path) |
548
|
1 |
|
return False |
549
|
1 |
|
self.activate() |
550
|
1 |
|
self.current_path = use_path |
551
|
1 |
|
self.sync() |
552
|
1 |
|
log.info(f"{self} was deployed.") |
553
|
1 |
|
return True |
554
|
|
|
|
555
|
1 |
|
def _install_direct_uni_flows(self): |
556
|
|
|
"""Install flows connecting two UNIs. |
557
|
|
|
|
558
|
|
|
This case happens when the circuit is between UNIs in the |
559
|
|
|
same switch. |
560
|
|
|
""" |
561
|
|
|
vlan_a = self.uni_a.user_tag.value if self.uni_a.user_tag else None |
562
|
|
|
vlan_z = self.uni_z.user_tag.value if self.uni_z.user_tag else None |
563
|
|
|
|
564
|
|
|
flow_mod_az = self._prepare_flow_mod( |
565
|
|
|
self.uni_a.interface, self.uni_z.interface, self.queue_id |
566
|
|
|
) |
567
|
|
|
flow_mod_za = self._prepare_flow_mod( |
568
|
|
|
self.uni_z.interface, self.uni_a.interface, self.queue_id |
569
|
|
|
) |
570
|
|
|
|
571
|
|
|
if vlan_a and vlan_z: |
572
|
|
|
flow_mod_az["match"]["dl_vlan"] = vlan_a |
573
|
|
|
flow_mod_za["match"]["dl_vlan"] = vlan_z |
574
|
|
|
flow_mod_az["actions"].insert( |
575
|
|
|
0, {"action_type": "set_vlan", "vlan_id": vlan_z} |
576
|
|
|
) |
577
|
|
|
flow_mod_za["actions"].insert( |
578
|
|
|
0, {"action_type": "set_vlan", "vlan_id": vlan_a} |
579
|
|
|
) |
580
|
|
|
elif vlan_a: |
581
|
|
|
flow_mod_az["match"]["dl_vlan"] = vlan_a |
582
|
|
|
flow_mod_az["actions"].insert(0, {"action_type": "pop_vlan"}) |
583
|
|
|
flow_mod_za["actions"].insert( |
584
|
|
|
0, {"action_type": "set_vlan", "vlan_id": vlan_a} |
585
|
|
|
) |
586
|
|
|
elif vlan_z: |
587
|
|
|
flow_mod_za["match"]["dl_vlan"] = vlan_z |
588
|
|
|
flow_mod_za["actions"].insert(0, {"action_type": "pop_vlan"}) |
589
|
|
|
flow_mod_az["actions"].insert( |
590
|
|
|
0, {"action_type": "set_vlan", "vlan_id": vlan_z} |
591
|
|
|
) |
592
|
|
|
self._send_flow_mods( |
593
|
|
|
self.uni_a.interface.switch, [flow_mod_az, flow_mod_za] |
594
|
|
|
) |
595
|
|
|
|
596
|
1 |
|
def _install_nni_flows(self, path=None): |
597
|
|
|
"""Install NNI flows.""" |
598
|
1 |
|
for incoming, outcoming in self.links_zipped(path): |
599
|
1 |
|
in_vlan = incoming.get_metadata("s_vlan").value |
600
|
1 |
|
out_vlan = outcoming.get_metadata("s_vlan").value |
601
|
|
|
|
602
|
1 |
|
flows = [] |
603
|
|
|
# Flow for one direction |
604
|
1 |
|
flows.append( |
605
|
|
|
self._prepare_nni_flow( |
606
|
|
|
incoming.endpoint_b, |
607
|
|
|
outcoming.endpoint_a, |
608
|
|
|
in_vlan, |
609
|
|
|
out_vlan, |
610
|
|
|
queue_id=self.queue_id, |
611
|
|
|
) |
612
|
|
|
) |
613
|
|
|
|
614
|
|
|
# Flow for the other direction |
615
|
1 |
|
flows.append( |
616
|
|
|
self._prepare_nni_flow( |
617
|
|
|
outcoming.endpoint_a, |
618
|
|
|
incoming.endpoint_b, |
619
|
|
|
out_vlan, |
620
|
|
|
in_vlan, |
621
|
|
|
queue_id=self.queue_id, |
622
|
|
|
) |
623
|
|
|
) |
624
|
1 |
|
self._send_flow_mods(incoming.endpoint_b.switch, flows) |
625
|
|
|
|
626
|
1 |
|
def _install_uni_flows(self, path=None): |
627
|
|
|
"""Install UNI flows.""" |
628
|
1 |
|
if not path: |
629
|
|
|
log.info("install uni flows without path.") |
630
|
|
|
return |
631
|
|
|
|
632
|
|
|
# Determine VLANs |
633
|
1 |
|
in_vlan_a = self.uni_a.user_tag.value if self.uni_a.user_tag else None |
634
|
1 |
|
out_vlan_a = path[0].get_metadata("s_vlan").value |
635
|
|
|
|
636
|
1 |
|
in_vlan_z = self.uni_z.user_tag.value if self.uni_z.user_tag else None |
637
|
1 |
|
out_vlan_z = path[-1].get_metadata("s_vlan").value |
638
|
|
|
|
639
|
|
|
# Flows for the first UNI |
640
|
1 |
|
flows_a = [] |
641
|
|
|
|
642
|
|
|
# Flow for one direction, pushing the service tag |
643
|
1 |
|
push_flow = self._prepare_push_flow( |
644
|
|
|
self.uni_a.interface, |
645
|
|
|
path[0].endpoint_a, |
646
|
|
|
in_vlan_a, |
647
|
|
|
out_vlan_a, |
648
|
|
|
queue_id=self.queue_id, |
649
|
|
|
) |
650
|
1 |
|
flows_a.append(push_flow) |
651
|
|
|
|
652
|
|
|
# Flow for the other direction, popping the service tag |
653
|
1 |
|
pop_flow = self._prepare_pop_flow( |
654
|
|
|
path[0].endpoint_a, |
655
|
|
|
self.uni_a.interface, |
656
|
|
|
in_vlan_a, |
657
|
|
|
out_vlan_a, |
658
|
|
|
queue_id=self.queue_id, |
659
|
|
|
) |
660
|
1 |
|
flows_a.append(pop_flow) |
661
|
|
|
|
662
|
1 |
|
self._send_flow_mods(self.uni_a.interface.switch, flows_a) |
663
|
|
|
|
664
|
|
|
# Flows for the second UNI |
665
|
1 |
|
flows_z = [] |
666
|
|
|
|
667
|
|
|
# Flow for one direction, pushing the service tag |
668
|
1 |
|
push_flow = self._prepare_push_flow( |
669
|
|
|
self.uni_z.interface, |
670
|
|
|
path[-1].endpoint_b, |
671
|
|
|
in_vlan_z, |
672
|
|
|
out_vlan_z, |
673
|
|
|
queue_id=self.queue_id, |
674
|
|
|
) |
675
|
1 |
|
flows_z.append(push_flow) |
676
|
|
|
|
677
|
|
|
# Flow for the other direction, popping the service tag |
678
|
1 |
|
pop_flow = self._prepare_pop_flow( |
679
|
|
|
path[-1].endpoint_b, |
680
|
|
|
self.uni_z.interface, |
681
|
|
|
in_vlan_z, |
682
|
|
|
out_vlan_z, |
683
|
|
|
queue_id=self.queue_id, |
684
|
|
|
) |
685
|
1 |
|
flows_z.append(pop_flow) |
686
|
|
|
|
687
|
1 |
|
self._send_flow_mods(self.uni_z.interface.switch, flows_z) |
688
|
|
|
|
689
|
1 |
|
@staticmethod |
690
|
1 |
|
def _send_flow_mods(switch, flow_mods, command='flows', force=False): |
691
|
|
|
"""Send a flow_mod list to a specific switch. |
692
|
|
|
|
693
|
|
|
Args: |
694
|
|
|
switch(Switch): The target of flows. |
695
|
|
|
flow_mods(dict): Python dictionary with flow_mods. |
696
|
|
|
command(str): By default is 'flows'. To remove a flow is 'remove'. |
697
|
|
|
force(bool): True to send via consistency check in case of conn errors |
698
|
|
|
|
699
|
|
|
""" |
700
|
1 |
|
endpoint = f"{settings.MANAGER_URL}/{command}/{switch.id}" |
701
|
|
|
|
702
|
1 |
|
data = {"flows": flow_mods, "force": force} |
703
|
1 |
|
response = requests.post(endpoint, json=data) |
704
|
1 |
|
if response.status_code >= 400: |
705
|
|
|
raise FlowModException |
706
|
|
|
|
707
|
1 |
|
def get_cookie(self): |
708
|
|
|
"""Return the cookie integer from evc id.""" |
709
|
1 |
|
return int(self.id, 16) + (settings.COOKIE_PREFIX << 56) |
710
|
|
|
|
711
|
1 |
|
@staticmethod |
712
|
|
|
def get_id_from_cookie(cookie): |
713
|
|
|
"""Return the evc id given a cookie value.""" |
714
|
1 |
|
evc_id = cookie - (settings.COOKIE_PREFIX << 56) |
715
|
1 |
|
return f"{evc_id:x}".zfill(14) |
716
|
|
|
|
717
|
1 |
|
def _prepare_flow_mod(self, in_interface, out_interface, queue_id=None): |
718
|
|
|
"""Prepare a common flow mod.""" |
719
|
1 |
|
default_actions = [ |
720
|
|
|
{"action_type": "output", "port": out_interface.port_number} |
721
|
|
|
] |
722
|
1 |
|
if queue_id: |
723
|
|
|
default_actions.append( |
724
|
|
|
{"action_type": "set_queue", "queue_id": queue_id} |
725
|
|
|
) |
726
|
|
|
|
727
|
1 |
|
flow_mod = { |
728
|
|
|
"match": {"in_port": in_interface.port_number}, |
729
|
|
|
"cookie": self.get_cookie(), |
730
|
|
|
"actions": default_actions, |
731
|
|
|
} |
732
|
1 |
|
if self.priority > -1: |
733
|
|
|
flow_mod["priority"] = self.priority |
734
|
|
|
|
735
|
1 |
|
return flow_mod |
736
|
|
|
|
737
|
1 |
|
def _prepare_nni_flow(self, *args, queue_id=None): |
738
|
|
|
"""Create NNI flows.""" |
739
|
1 |
|
in_interface, out_interface, in_vlan, out_vlan = args |
740
|
1 |
|
flow_mod = self._prepare_flow_mod( |
741
|
|
|
in_interface, out_interface, queue_id |
742
|
|
|
) |
743
|
1 |
|
flow_mod["match"]["dl_vlan"] = in_vlan |
744
|
|
|
|
745
|
1 |
|
new_action = {"action_type": "set_vlan", "vlan_id": out_vlan} |
746
|
1 |
|
flow_mod["actions"].insert(0, new_action) |
747
|
|
|
|
748
|
1 |
|
return flow_mod |
749
|
|
|
|
750
|
1 |
View Code Duplication |
def _prepare_push_flow(self, *args, queue_id=None): |
|
|
|
|
751
|
|
|
"""Prepare push flow. |
752
|
|
|
|
753
|
|
|
Arguments: |
754
|
|
|
in_interface(str): Interface input. |
755
|
|
|
out_interface(str): Interface output. |
756
|
|
|
in_vlan(str): Vlan input. |
757
|
|
|
out_vlan(str): Vlan output. |
758
|
|
|
|
759
|
|
|
Return: |
760
|
|
|
dict: An python dictionary representing a FlowMod |
761
|
|
|
|
762
|
|
|
""" |
763
|
|
|
# assign all arguments |
764
|
1 |
|
in_interface, out_interface, in_vlan, out_vlan = args |
765
|
|
|
|
766
|
1 |
|
flow_mod = self._prepare_flow_mod( |
767
|
|
|
in_interface, out_interface, queue_id |
768
|
|
|
) |
769
|
|
|
|
770
|
|
|
# the service tag must be always pushed |
771
|
1 |
|
new_action = {"action_type": "set_vlan", "vlan_id": out_vlan} |
772
|
1 |
|
flow_mod["actions"].insert(0, new_action) |
773
|
|
|
|
774
|
1 |
|
new_action = {"action_type": "push_vlan", "tag_type": "s"} |
775
|
1 |
|
flow_mod["actions"].insert(0, new_action) |
776
|
|
|
|
777
|
1 |
|
if in_vlan: |
778
|
|
|
# if in_vlan is set, it must be included in the match |
779
|
1 |
|
flow_mod["match"]["dl_vlan"] = in_vlan |
780
|
1 |
|
new_action = {"action_type": "pop_vlan"} |
781
|
1 |
|
flow_mod["actions"].insert(0, new_action) |
782
|
1 |
|
return flow_mod |
783
|
|
|
|
784
|
1 |
View Code Duplication |
def _prepare_pop_flow( |
|
|
|
|
785
|
|
|
self, in_interface, out_interface, in_vlan, out_vlan, queue_id=None |
786
|
|
|
): |
787
|
|
|
# pylint: disable=too-many-arguments |
788
|
|
|
"""Prepare pop flow.""" |
789
|
1 |
|
flow_mod = self._prepare_flow_mod( |
790
|
|
|
in_interface, out_interface, queue_id |
791
|
|
|
) |
792
|
1 |
|
flow_mod["match"]["dl_vlan"] = out_vlan |
793
|
1 |
|
if in_vlan: |
794
|
1 |
|
new_action = {"action_type": "set_vlan", "vlan_id": in_vlan} |
795
|
1 |
|
flow_mod["actions"].insert(0, new_action) |
796
|
1 |
|
new_action = {"action_type": "push_vlan", "tag_type": "c"} |
797
|
1 |
|
flow_mod["actions"].insert(0, new_action) |
798
|
1 |
|
new_action = {"action_type": "pop_vlan"} |
799
|
1 |
|
flow_mod["actions"].insert(0, new_action) |
800
|
1 |
|
return flow_mod |
801
|
|
|
|
802
|
1 |
|
@staticmethod |
803
|
|
|
def run_sdntrace(uni): |
804
|
|
|
"""Run SDN trace on control plane starting from EVC UNIs.""" |
805
|
|
|
endpoint = f"{settings.SDN_TRACE_CP_URL}/trace" |
806
|
|
|
data_uni = { |
807
|
|
|
"trace": { |
808
|
|
|
"switch": { |
809
|
|
|
"dpid": uni.interface.switch.dpid, |
810
|
|
|
"in_port": uni.interface.port_number, |
811
|
|
|
} |
812
|
|
|
} |
813
|
|
|
} |
814
|
|
|
if uni.user_tag: |
815
|
|
|
data_uni["trace"]["eth"] = { |
816
|
|
|
"dl_type": 0x8100, |
817
|
|
|
"dl_vlan": uni.user_tag.value, |
818
|
|
|
} |
819
|
|
|
response = requests.put(endpoint, json=data_uni) |
820
|
|
|
if response.status_code >= 400: |
821
|
|
|
log.error(f"Failed to run sdntrace-cp: {response.text}") |
822
|
|
|
return [] |
823
|
|
|
return response.json().get('result', []) |
824
|
|
|
|
825
|
1 |
|
def check_traces(self): |
826
|
|
|
"""Check if current_path is deployed comparing with SDN traces.""" |
827
|
|
|
trace_a = self.run_sdntrace(self.uni_a) |
828
|
|
|
if len(trace_a) != len(self.current_path) + 1: |
829
|
|
|
log.warn(f"Invalid trace from uni_a: {trace_a}") |
830
|
|
|
return False |
831
|
|
|
trace_z = self.run_sdntrace(self.uni_z) |
832
|
|
|
if len(trace_z) != len(self.current_path) + 1: |
833
|
|
|
log.warn(f"Invalid trace from uni_z: {trace_z}") |
834
|
|
|
return False |
835
|
|
|
|
836
|
|
|
for link, trace1, trace2 in zip(self.current_path, |
837
|
|
|
trace_a[1:], |
838
|
|
|
trace_z[:0:-1]): |
839
|
|
|
if compare_endpoint_trace( |
840
|
|
|
link.endpoint_a, |
841
|
|
|
glom(link.metadata, 's_vlan.value'), trace2) is False: |
842
|
|
|
log.warn(f"Invalid trace from uni_a: {trace_a}") |
843
|
|
|
return False |
844
|
|
|
if compare_endpoint_trace( |
845
|
|
|
link.endpoint_b, |
846
|
|
|
glom(link.metadata, 's_vlan.value'), trace1) is False: |
847
|
|
|
log.warn(f"Invalid trace from uni_z: {trace_z}") |
848
|
|
|
return False |
849
|
|
|
|
850
|
|
|
return True |
851
|
|
|
|
852
|
|
|
|
853
|
1 |
|
class LinkProtection(EVCDeploy): |
854
|
|
|
"""Class to handle link protection.""" |
855
|
|
|
|
856
|
1 |
|
def is_affected_by_link(self, link=None): |
857
|
|
|
"""Verify if the current path is affected by link down event.""" |
858
|
|
|
return self.current_path.is_affected_by_link(link) |
859
|
|
|
|
860
|
1 |
|
def is_using_primary_path(self): |
861
|
|
|
"""Verify if the current deployed path is self.primary_path.""" |
862
|
1 |
|
return self.current_path == self.primary_path |
863
|
|
|
|
864
|
1 |
|
def is_using_backup_path(self): |
865
|
|
|
"""Verify if the current deployed path is self.backup_path.""" |
866
|
1 |
|
return self.current_path == self.backup_path |
867
|
|
|
|
868
|
1 |
|
def is_using_dynamic_path(self): |
869
|
|
|
"""Verify if the current deployed path is dynamic.""" |
870
|
1 |
|
if ( |
871
|
|
|
self.current_path |
872
|
|
|
and not self.is_using_primary_path() |
873
|
|
|
and not self.is_using_backup_path() |
874
|
|
|
and self.current_path.status is EntityStatus.UP |
875
|
|
|
): |
876
|
|
|
return True |
877
|
1 |
|
return False |
878
|
|
|
|
879
|
1 |
|
def deploy_to(self, path_name=None, path=None): |
880
|
|
|
"""Create a deploy to path.""" |
881
|
1 |
|
if self.current_path == path: |
882
|
1 |
|
log.debug(f"{path_name} is equal to current_path.") |
883
|
1 |
|
return True |
884
|
|
|
|
885
|
1 |
|
if path.status is EntityStatus.UP: |
886
|
1 |
|
return self.deploy_to_path(path) |
887
|
|
|
|
888
|
1 |
|
return False |
889
|
|
|
|
890
|
1 |
|
def handle_link_up(self, link): |
891
|
|
|
"""Handle circuit when link down. |
892
|
|
|
|
893
|
|
|
Args: |
894
|
|
|
link(Link): Link affected by link.down event. |
895
|
|
|
|
896
|
|
|
""" |
897
|
1 |
|
if self.is_using_primary_path(): |
898
|
1 |
|
return True |
899
|
|
|
|
900
|
1 |
|
success = False |
901
|
1 |
|
if self.primary_path.is_affected_by_link(link): |
902
|
1 |
|
success = self.deploy_to_primary_path() |
903
|
|
|
|
904
|
1 |
|
if success: |
905
|
1 |
|
return True |
906
|
|
|
|
907
|
|
|
# We tried to deploy(primary_path) without success. |
908
|
|
|
# And in this case is up by some how. Nothing to do. |
909
|
1 |
|
if self.is_using_backup_path() or self.is_using_dynamic_path(): |
910
|
|
|
return True |
911
|
|
|
|
912
|
|
|
# In this case, probably the circuit is not being used and |
913
|
|
|
# we can move to backup |
914
|
1 |
|
if self.backup_path.is_affected_by_link(link): |
915
|
1 |
|
success = self.deploy_to_backup_path() |
916
|
|
|
|
917
|
1 |
|
if success: |
918
|
1 |
|
return True |
919
|
|
|
|
920
|
|
|
# In this case, the circuit is not being used and we should |
921
|
|
|
# try a dynamic path |
922
|
|
|
if self.dynamic_backup_path: |
923
|
|
|
return self.deploy_to_path() |
924
|
|
|
|
925
|
|
|
return True |
926
|
|
|
|
927
|
1 |
|
def handle_link_down(self): |
928
|
|
|
"""Handle circuit when link down. |
929
|
|
|
|
930
|
|
|
Returns: |
931
|
|
|
bool: True if the re-deploy was successly otherwise False. |
932
|
|
|
|
933
|
|
|
""" |
934
|
1 |
|
success = False |
935
|
1 |
|
if self.is_using_primary_path(): |
936
|
1 |
|
success = self.deploy_to_backup_path() |
937
|
1 |
|
elif self.is_using_backup_path(): |
938
|
1 |
|
success = self.deploy_to_primary_path() |
939
|
|
|
|
940
|
1 |
|
if not success and self.dynamic_backup_path: |
941
|
1 |
|
success = self.deploy_to_path() |
942
|
|
|
|
943
|
1 |
|
if success: |
944
|
1 |
|
log.debug(f"{self} deployed after link down.") |
945
|
|
|
else: |
946
|
1 |
|
self.deactivate() |
947
|
1 |
|
self.current_path = Path([]) |
948
|
1 |
|
self.sync() |
949
|
1 |
|
log.debug(f"Failed to re-deploy {self} after link down.") |
950
|
|
|
|
951
|
1 |
|
return success |
952
|
|
|
|
953
|
|
|
|
954
|
1 |
|
class EVC(LinkProtection): |
955
|
|
|
"""Class that represents a E-Line Virtual Connection.""" |
956
|
|
|
|