|
1
|
|
|
"""Utilities functions used in Kytos.""" |
|
2
|
|
|
import functools |
|
3
|
|
|
import logging |
|
4
|
|
|
import traceback |
|
5
|
|
|
from asyncio import AbstractEventLoop |
|
6
|
|
|
from collections import defaultdict |
|
7
|
|
|
from concurrent.futures import ThreadPoolExecutor |
|
8
|
|
|
from datetime import datetime, timezone |
|
9
|
|
|
from pathlib import Path |
|
10
|
|
|
from threading import Thread |
|
11
|
|
|
|
|
12
|
|
|
from openapi_core import Spec, unmarshal_request |
|
13
|
|
|
from openapi_core.exceptions import OpenAPIError |
|
14
|
|
|
from openapi_spec_validator import validate_spec |
|
15
|
|
|
from openapi_spec_validator.readers import read_from_filename |
|
16
|
|
|
|
|
17
|
|
|
from kytos.core.apm import ElasticAPM |
|
18
|
|
|
from kytos.core.config import KytosConfig |
|
19
|
|
|
from kytos.core.rest_api import (AStarletteOpenAPIRequest, HTTPException, |
|
20
|
|
|
Request, StarletteOpenAPIRequest, |
|
21
|
|
|
content_type_json_or_415, get_body) |
|
22
|
|
|
|
|
23
|
|
|
__all__ = ['listen_to', 'now', 'run_on_thread', 'get_time'] |
|
24
|
|
|
|
|
25
|
|
|
LOG = logging.getLogger(__name__) |
|
26
|
|
|
|
|
27
|
|
|
|
|
28
|
|
|
def get_thread_pool_max_workers(): |
|
29
|
|
|
"""Get the number of thread pool max workers.""" |
|
30
|
|
|
return KytosConfig().options["daemon"].thread_pool_max_workers |
|
31
|
|
|
|
|
32
|
|
|
|
|
33
|
|
|
def get_apm_name(): |
|
34
|
|
|
"""Get apm backend name.""" |
|
35
|
|
|
return KytosConfig().options["daemon"].apm |
|
36
|
|
|
|
|
37
|
|
|
|
|
38
|
|
|
# pylint: disable=invalid-name |
|
39
|
|
|
executors = {name: ThreadPoolExecutor(max_workers=max_workers, |
|
40
|
|
|
thread_name_prefix=f"thread_pool_{name}") |
|
41
|
|
|
for name, max_workers in get_thread_pool_max_workers().items()} |
|
42
|
|
|
|
|
43
|
|
|
ds_executors = defaultdict(lambda: ThreadPoolExecutor(max_workers=1, |
|
44
|
|
|
thread_name_prefix="dynamic_single")) |
|
45
|
|
|
|
|
46
|
|
|
|
|
47
|
|
|
def listen_to(event, *events, pool=None): |
|
48
|
|
|
"""Decorate Event Listener methods. |
|
49
|
|
|
|
|
50
|
|
|
This decorator was built to be used on NAPPs methods to define which |
|
51
|
|
|
type of event the method will handle. With this, we will be able to |
|
52
|
|
|
'schedule' the app/method to receive an event when a new event is |
|
53
|
|
|
registered on the controller buffers. |
|
54
|
|
|
By using the run_on_thread decorator, we also guarantee that the method |
|
55
|
|
|
(handler) will be called from inside a new thread, avoiding this method to |
|
56
|
|
|
block its caller. |
|
57
|
|
|
|
|
58
|
|
|
The decorator will add an attribute to the method called 'events', that |
|
59
|
|
|
will be a list of the events that the method will handle. |
|
60
|
|
|
|
|
61
|
|
|
The event that will be listened to is always a string, but it can represent |
|
62
|
|
|
a regular expression to match against multiple Event Types. All listened |
|
63
|
|
|
events are documented in :doc:`/developer/listened_events` section. |
|
64
|
|
|
|
|
65
|
|
|
The pool option gives you control on which ThreadPoolExecutor that will |
|
66
|
|
|
execute the decorated handler. This knob is meant for prioritization, |
|
67
|
|
|
allowing executions on different pools depending on the handler primary |
|
68
|
|
|
responsibility and importance to avoid potential scheduling starvation. |
|
69
|
|
|
|
|
70
|
|
|
Example of usage: |
|
71
|
|
|
|
|
72
|
|
|
.. code-block:: python3 |
|
73
|
|
|
|
|
74
|
|
|
class MyAppClass(KytosApp): |
|
75
|
|
|
@listen_to('kytos/of_core.messages.in') |
|
76
|
|
|
def my_handler_of_message_in(self, event): |
|
77
|
|
|
# Do stuff here... |
|
78
|
|
|
|
|
79
|
|
|
@listen_to('kytos/of_core.messages.out') |
|
80
|
|
|
def my_handler_of_message_out(self, event): |
|
81
|
|
|
# Do stuff here... |
|
82
|
|
|
|
|
83
|
|
|
@listen_to('kytos/of_core.messages.in.ofpt_hello', |
|
84
|
|
|
'kytos/of_core.messages.out.ofpt_hello') |
|
85
|
|
|
def my_handler_of_hello_messages(self, event): |
|
86
|
|
|
# Do stuff here... |
|
87
|
|
|
|
|
88
|
|
|
@listen_to('kytos/of_core.message.*.hello') |
|
89
|
|
|
def my_other_handler_of_hello_messages(self, event): |
|
90
|
|
|
# Do stuff here... |
|
91
|
|
|
|
|
92
|
|
|
@listen_to('kytos/of_core.message.*.hello') |
|
93
|
|
|
def my_handler_of_hello_messages(self, event): |
|
94
|
|
|
# Do stuff here... |
|
95
|
|
|
|
|
96
|
|
|
@listen_to('kytos/of_core.message.*') |
|
97
|
|
|
def my_stats_handler_of_any_message(self, event): |
|
98
|
|
|
# Do stuff here... |
|
99
|
|
|
|
|
100
|
|
|
@listen_to("some_db_oriented_event", pool="db") |
|
101
|
|
|
def db_update(self, event): |
|
102
|
|
|
# Do stuff here... |
|
103
|
|
|
""" |
|
104
|
|
|
def thread_decorator(handler): |
|
105
|
|
|
"""Decorate the handler method. |
|
106
|
|
|
|
|
107
|
|
|
Returns: |
|
108
|
|
|
A method with an `events` attribute (list of events to be listened) |
|
109
|
|
|
and also decorated to run on a new thread. |
|
110
|
|
|
|
|
111
|
|
|
""" |
|
112
|
|
|
@run_on_thread |
|
113
|
|
|
def threaded_handler(*args): |
|
114
|
|
|
"""Decorate the handler to run from a new thread.""" |
|
115
|
|
|
handler(*args) |
|
116
|
|
|
|
|
117
|
|
|
threaded_handler.events = [event] |
|
118
|
|
|
threaded_handler.events.extend(events) |
|
119
|
|
|
return threaded_handler |
|
120
|
|
|
|
|
121
|
|
|
# pylint: disable=broad-except |
|
122
|
|
|
def thread_pool_decorator(handler): |
|
123
|
|
|
"""Decorate the handler method. |
|
124
|
|
|
|
|
125
|
|
|
Returns: |
|
126
|
|
|
A method with an `events` attribute (list of events to be listened) |
|
127
|
|
|
and also decorated to run on in the thread pool |
|
128
|
|
|
|
|
129
|
|
|
""" |
|
130
|
|
|
def done_callback(future): |
|
131
|
|
|
"""Done callback.""" |
|
132
|
|
|
if not future.exception(): |
|
133
|
|
|
_ = future.result() |
|
134
|
|
|
return |
|
135
|
|
|
|
|
136
|
|
|
# pylint: disable=unused-argument |
|
137
|
|
View Code Duplication |
def handler_context(*args, **kwargs): |
|
|
|
|
|
|
138
|
|
|
"""Handler's context for ThreadPool.""" |
|
139
|
|
|
cls, kytos_event = args[0], args[1] |
|
140
|
|
|
try: |
|
141
|
|
|
result = handler(*args) |
|
142
|
|
|
except Exception: |
|
143
|
|
|
result = None |
|
144
|
|
|
traceback_str = traceback.format_exc() |
|
145
|
|
|
LOG.error(f"listen_to handler: {handler}, " |
|
146
|
|
|
f"args: {args} traceback: {traceback_str}") |
|
147
|
|
|
if hasattr(cls, "controller"): |
|
148
|
|
|
cls.controller.dead_letter.add_event(kytos_event) |
|
149
|
|
|
return result |
|
150
|
|
|
|
|
151
|
|
View Code Duplication |
def handler_context_apm(*args, apm_client=None): |
|
|
|
|
|
|
152
|
|
|
"""Handler's context for ThreadPool APM instrumentation.""" |
|
153
|
|
|
cls, kytos_event = args[0], args[1] |
|
154
|
|
|
trace_parent = kytos_event.trace_parent |
|
155
|
|
|
tx_type = "kytos_event" |
|
156
|
|
|
tx = apm_client.begin_transaction(transaction_type=tx_type, |
|
157
|
|
|
trace_parent=trace_parent) |
|
158
|
|
|
kytos_event.trace_parent = tx.trace_parent |
|
159
|
|
|
tx.name = f"{kytos_event.name}@{cls.napp_id}" |
|
160
|
|
|
try: |
|
161
|
|
|
result = handler(*args) |
|
162
|
|
|
tx.result = result |
|
163
|
|
|
except Exception as exc: |
|
164
|
|
|
result = None |
|
165
|
|
|
traceback_str = traceback.format_exc() |
|
166
|
|
|
LOG.error(f"listen_to handler: {handler}, " |
|
167
|
|
|
f"args: {args} traceback: {traceback_str}") |
|
168
|
|
|
if hasattr(cls, "controller"): |
|
169
|
|
|
cls.controller.dead_letter.add_event(kytos_event) |
|
170
|
|
|
apm_client.capture_exception( |
|
171
|
|
|
exc_info=(type(exc), exc, exc.__traceback__), |
|
172
|
|
|
context={"args": args}, |
|
173
|
|
|
handled=False, |
|
174
|
|
|
) |
|
175
|
|
|
tx.end() |
|
176
|
|
|
apm_client.tracer.queue_func("transaction", tx.to_dict()) |
|
177
|
|
|
return result |
|
178
|
|
|
|
|
179
|
|
|
handler_func, kwargs = handler_context, {} |
|
180
|
|
|
if get_apm_name() == "es": |
|
181
|
|
|
handler_func = handler_context_apm |
|
182
|
|
|
kwargs = {"apm_client": ElasticAPM.get_client()} |
|
183
|
|
|
|
|
184
|
|
|
def get_executor(pool, event, default_pool="app", handler=handler): |
|
185
|
|
|
"""Get executor.""" |
|
186
|
|
|
if pool == "dynamic_single": |
|
187
|
|
|
return ds_executors[handler] |
|
188
|
|
|
if pool and pool in executors: |
|
189
|
|
|
return executors[pool] |
|
190
|
|
|
if not event: |
|
191
|
|
|
return executors[default_pool] |
|
192
|
|
|
|
|
193
|
|
|
if event.name.startswith("kytos/of_core") and "sb" in executors: |
|
194
|
|
|
return executors["sb"] |
|
195
|
|
|
core_of = "kytos/core.openflow" |
|
196
|
|
|
if event.name.startswith(core_of) and "sb" in executors: |
|
197
|
|
|
return executors["sb"] |
|
198
|
|
|
return executors[default_pool] |
|
199
|
|
|
|
|
200
|
|
|
def inner(*args): |
|
201
|
|
|
"""Decorate the handler to run in the thread pool.""" |
|
202
|
|
|
event = args[1] if len(args) > 1 else None |
|
203
|
|
|
executor = get_executor(pool, event) |
|
204
|
|
|
future = executor.submit(handler_func, *args, **kwargs) |
|
205
|
|
|
future.add_done_callback(done_callback) |
|
206
|
|
|
|
|
207
|
|
|
inner.events = [event] |
|
208
|
|
|
inner.events.extend(events) |
|
209
|
|
|
return inner |
|
210
|
|
|
|
|
211
|
|
|
if executors: |
|
212
|
|
|
return thread_pool_decorator |
|
213
|
|
|
return thread_decorator |
|
214
|
|
|
|
|
215
|
|
|
|
|
216
|
|
|
def alisten_to(event, *events): |
|
217
|
|
|
"""Decorate async subscribing methods.""" |
|
218
|
|
|
|
|
219
|
|
|
def decorator(handler): |
|
220
|
|
|
"""Decorate the handler method. |
|
221
|
|
|
|
|
222
|
|
|
Returns: |
|
223
|
|
|
A method with an `events` attribute (list of events to be listened) |
|
224
|
|
|
and also decorated as an asyncio task. |
|
225
|
|
|
""" |
|
226
|
|
|
# pylint: disable=unused-argument,broad-except |
|
227
|
|
View Code Duplication |
async def handler_context(*args, **kwargs): |
|
|
|
|
|
|
228
|
|
|
"""Async handler's execution context.""" |
|
229
|
|
|
cls, kytos_event = args[0], args[1] |
|
230
|
|
|
try: |
|
231
|
|
|
result = await handler(*args) |
|
232
|
|
|
except Exception: |
|
233
|
|
|
result = None |
|
234
|
|
|
traceback_str = traceback.format_exc() |
|
235
|
|
|
LOG.error(f"alisten_to handler: {handler}, " |
|
236
|
|
|
f"args: {args} traceback: {traceback_str}") |
|
237
|
|
|
if hasattr(cls, "controller"): |
|
238
|
|
|
cls.controller.dead_letter.add_event(kytos_event) |
|
239
|
|
|
return result |
|
240
|
|
|
|
|
241
|
|
View Code Duplication |
async def handler_context_apm(*args, apm_client=None): |
|
|
|
|
|
|
242
|
|
|
"""Async handler's execution context with APM instrumentation.""" |
|
243
|
|
|
cls, kytos_event = args[0], args[1] |
|
244
|
|
|
trace_parent = kytos_event.trace_parent |
|
245
|
|
|
tx_type = "kytos_event" |
|
246
|
|
|
tx = apm_client.begin_transaction(transaction_type=tx_type, |
|
247
|
|
|
trace_parent=trace_parent) |
|
248
|
|
|
kytos_event.trace_parent = tx.trace_parent |
|
249
|
|
|
tx.name = f"{kytos_event.name}@{cls.napp_id}" |
|
250
|
|
|
try: |
|
251
|
|
|
result = await handler(*args) |
|
252
|
|
|
tx.result = result |
|
253
|
|
|
except Exception as exc: |
|
254
|
|
|
result = None |
|
255
|
|
|
traceback_str = traceback.format_exc() |
|
256
|
|
|
LOG.error(f"alisten_to handler: {handler}, " |
|
257
|
|
|
f"args: {args} traceback: {traceback_str}") |
|
258
|
|
|
if hasattr(cls, "controller"): |
|
259
|
|
|
cls.controller.dead_letter.add_event(kytos_event) |
|
260
|
|
|
apm_client.capture_exception( |
|
261
|
|
|
exc_info=(type(exc), exc, exc.__traceback__), |
|
262
|
|
|
context={"args": args}, |
|
263
|
|
|
handled=False, |
|
264
|
|
|
) |
|
265
|
|
|
tx.end() |
|
266
|
|
|
apm_client.tracer.queue_func("transaction", tx.to_dict()) |
|
267
|
|
|
return result |
|
268
|
|
|
|
|
269
|
|
|
handler_func, kwargs = handler_context, {} |
|
270
|
|
|
if get_apm_name() == "es": |
|
271
|
|
|
handler_func = handler_context_apm |
|
272
|
|
|
kwargs = {"apm_client": ElasticAPM.get_client()} |
|
273
|
|
|
|
|
274
|
|
|
async def inner(*args): |
|
275
|
|
|
"""Inner decorated with events attribute.""" |
|
276
|
|
|
return await handler_func(*args, **kwargs) |
|
277
|
|
|
inner.events = [event] |
|
278
|
|
|
inner.events.extend(events) |
|
279
|
|
|
return inner |
|
280
|
|
|
|
|
281
|
|
|
return decorator |
|
282
|
|
|
|
|
283
|
|
|
|
|
284
|
|
|
def now(tzone=timezone.utc): |
|
285
|
|
|
"""Return the current datetime (default to UTC). |
|
286
|
|
|
|
|
287
|
|
|
Args: |
|
288
|
|
|
tzone (datetime.timezone): Specific time zone used in datetime. |
|
289
|
|
|
|
|
290
|
|
|
Returns: |
|
291
|
|
|
datetime.datetime.now: Date time with specific time zone. |
|
292
|
|
|
|
|
293
|
|
|
""" |
|
294
|
|
|
return datetime.now(tzone) |
|
295
|
|
|
|
|
296
|
|
|
|
|
297
|
|
|
def run_on_thread(method): |
|
298
|
|
|
"""Decorate to run the decorated method inside a new thread. |
|
299
|
|
|
|
|
300
|
|
|
Args: |
|
301
|
|
|
method (function): function used to run as a new thread. |
|
302
|
|
|
|
|
303
|
|
|
Returns: |
|
304
|
|
|
Decorated method that will run inside a new thread. |
|
305
|
|
|
When the decorated method is called, it will not return the created |
|
306
|
|
|
thread to the caller. |
|
307
|
|
|
|
|
308
|
|
|
""" |
|
309
|
|
|
def threaded_method(*args): |
|
310
|
|
|
"""Ensure the handler method runs inside a new thread.""" |
|
311
|
|
|
thread = Thread(target=method, args=args) |
|
312
|
|
|
|
|
313
|
|
|
# Set daemon mode so that we don't have to wait for these threads |
|
314
|
|
|
# to finish when exiting Kytos |
|
315
|
|
|
thread.daemon = True |
|
316
|
|
|
thread.start() |
|
317
|
|
|
return threaded_method |
|
318
|
|
|
|
|
319
|
|
|
|
|
320
|
|
|
def get_time(data=None): |
|
321
|
|
|
"""Receive a dictionary or a string and return a datatime instance. |
|
322
|
|
|
|
|
323
|
|
|
data = {"year": 2006, |
|
324
|
|
|
"month": 11, |
|
325
|
|
|
"day": 21, |
|
326
|
|
|
"hour": 16, |
|
327
|
|
|
"minute": 30 , |
|
328
|
|
|
"second": 00} |
|
329
|
|
|
|
|
330
|
|
|
or |
|
331
|
|
|
|
|
332
|
|
|
data = "21/11/06 16:30:00" |
|
333
|
|
|
|
|
334
|
|
|
2018-04-17T17:13:50Z |
|
335
|
|
|
|
|
336
|
|
|
Args: |
|
337
|
|
|
data (str, dict): python dict or string to be converted to datetime |
|
338
|
|
|
|
|
339
|
|
|
Returns: |
|
340
|
|
|
datetime: datetime instance. |
|
341
|
|
|
|
|
342
|
|
|
""" |
|
343
|
|
|
if isinstance(data, str): |
|
344
|
|
|
date = datetime.strptime(data, "%Y-%m-%dT%H:%M:%S") |
|
345
|
|
|
elif isinstance(data, dict): |
|
346
|
|
|
date = datetime(**data) |
|
347
|
|
|
else: |
|
348
|
|
|
return None |
|
349
|
|
|
return date.replace(tzinfo=timezone.utc) |
|
350
|
|
|
|
|
351
|
|
|
|
|
352
|
|
|
def _read_from_filename(yml_file_path: Path) -> dict: |
|
353
|
|
|
"""Read from yml filename.""" |
|
354
|
|
|
spec_dict, _ = read_from_filename(yml_file_path) |
|
355
|
|
|
return spec_dict |
|
356
|
|
|
|
|
357
|
|
|
|
|
358
|
|
|
def load_spec(yml_file_path: Path): |
|
359
|
|
|
"""Load and validate spec object given a yml file path.""" |
|
360
|
|
|
spec = _read_from_filename(yml_file_path) |
|
361
|
|
|
validate_spec(spec) |
|
362
|
|
|
return Spec.from_dict(spec) |
|
363
|
|
|
|
|
364
|
|
|
|
|
365
|
|
|
def _request_validation_result_or_400(errors: OpenAPIError) -> None: |
|
366
|
|
|
"""Raise HTTP 400.""" |
|
367
|
|
|
error_response = "The request body contains invalid API data." |
|
368
|
|
|
if not errors.__cause__: |
|
369
|
|
|
error_response = str(errors) |
|
370
|
|
|
elif (hasattr(errors.__cause__, "schema_errors") and |
|
371
|
|
|
errors.__cause__.schema_errors): |
|
372
|
|
|
schema_errors = errors.__cause__.schema_errors |
|
373
|
|
|
for error in schema_errors: |
|
374
|
|
|
error_response += f", {error.message} for field" |
|
375
|
|
|
error_response += f" {'/'.join(map(str,error.path))}." |
|
376
|
|
|
raise HTTPException(400, detail=error_response) |
|
377
|
|
|
|
|
378
|
|
|
|
|
379
|
|
|
def validate_openapi_request( |
|
380
|
|
|
spec: Spec, request: Request, loop: AbstractEventLoop |
|
381
|
|
|
) -> bytes: |
|
382
|
|
|
"""Validate a Request given an OpenAPI spec. |
|
383
|
|
|
|
|
384
|
|
|
This function is meant to be called from a synchronous context |
|
385
|
|
|
since StarletteOpenAPIRequest internally uses `asgiref.sync.AsyncToSync` |
|
386
|
|
|
and its forcing not to use the current running event loop. |
|
387
|
|
|
""" |
|
388
|
|
|
body = get_body(request, loop) |
|
389
|
|
|
if body: |
|
390
|
|
|
content_type_json_or_415(request) |
|
391
|
|
|
openapi_request = StarletteOpenAPIRequest(request, body) |
|
392
|
|
|
try: |
|
393
|
|
|
unmarshal_request(openapi_request, spec) |
|
394
|
|
|
except OpenAPIError as err: |
|
395
|
|
|
_request_validation_result_or_400(err) |
|
396
|
|
|
return body |
|
397
|
|
|
|
|
398
|
|
|
|
|
399
|
|
|
async def avalidate_openapi_request( |
|
400
|
|
|
spec: Spec, |
|
401
|
|
|
request: Request, |
|
402
|
|
|
) -> bytes: |
|
403
|
|
|
"""Async validate_openapi_request. |
|
404
|
|
|
|
|
405
|
|
|
This function is for async routes. It also returns the request body bytes. |
|
406
|
|
|
It does not try to assume that it'll have a loadable json body to work |
|
407
|
|
|
seamlessly with as many type of endpoints with minimal friction. |
|
408
|
|
|
You can use await aget_json_or_400(request) to get the request body. |
|
409
|
|
|
|
|
410
|
|
|
Example: |
|
411
|
|
|
|
|
412
|
|
|
await avalidate_openapi_request(self.spec, request) |
|
413
|
|
|
body = await aget_json_or_400(request) |
|
414
|
|
|
""" |
|
415
|
|
|
body = await request.body() |
|
416
|
|
|
if body: |
|
417
|
|
|
content_type_json_or_415(request) |
|
418
|
|
|
openapi_request = AStarletteOpenAPIRequest(request, body) |
|
419
|
|
|
try: |
|
420
|
|
|
unmarshal_request(openapi_request, spec) |
|
421
|
|
|
except OpenAPIError as err: |
|
422
|
|
|
_request_validation_result_or_400(err) |
|
423
|
|
|
return body |
|
424
|
|
|
|
|
425
|
|
|
|
|
426
|
|
|
def validate_openapi(spec): |
|
427
|
|
|
"""Decorator to validate a REST endpoint input. |
|
428
|
|
|
|
|
429
|
|
|
Uses the schema defined in the openapi.yml file |
|
430
|
|
|
to validate. |
|
431
|
|
|
""" |
|
432
|
|
|
def validate_decorator(func): |
|
433
|
|
|
@functools.wraps(func) |
|
434
|
|
|
def wrapper_validate(*args, **kwargs): |
|
435
|
|
|
request: Request = None |
|
436
|
|
|
napp = None |
|
437
|
|
|
for arg in args: |
|
438
|
|
|
if isinstance(arg, Request): |
|
439
|
|
|
request = arg |
|
440
|
|
|
if hasattr(arg, "controller"): |
|
441
|
|
|
napp = arg |
|
442
|
|
|
if request and napp: |
|
443
|
|
|
break |
|
444
|
|
|
if not request: |
|
445
|
|
|
err = f"{func.__name__} args doesn't have a Request argument" |
|
446
|
|
|
raise RuntimeError(err) |
|
447
|
|
|
if not napp: |
|
448
|
|
|
err = f"{func.__name__} should be a NApp method to get ev_loop" |
|
449
|
|
|
raise RuntimeError(err) |
|
450
|
|
|
validate_openapi_request(spec, request, napp.controller.loop) |
|
451
|
|
|
return func(*args, **kwargs) |
|
452
|
|
|
return wrapper_validate |
|
453
|
|
|
return validate_decorator |
|
454
|
|
|
|