1
|
1 |
|
import copy |
2
|
1 |
|
from re import compile |
3
|
1 |
|
import time |
4
|
1 |
|
import requests |
5
|
|
|
|
6
|
1 |
|
from .rest import RequestsNetworkWrapper, ServiceClient |
7
|
|
|
|
8
|
1 |
|
from .rest_client import RESTServiceClient, Endpoint |
9
|
|
|
|
10
|
1 |
|
from .custom_query_object import CustomQueryObject |
11
|
1 |
|
import os |
12
|
1 |
|
import logging |
13
|
|
|
|
14
|
1 |
|
logger = logging.getLogger(__name__) |
15
|
|
|
|
16
|
1 |
|
_name_checker = compile(r"^[\w -]+$") |
17
|
|
|
|
18
|
|
|
|
19
|
1 |
|
def _check_endpoint_type(name): |
20
|
1 |
|
if not isinstance(name, str): |
21
|
1 |
|
raise TypeError("Endpoint name must be a string") |
22
|
|
|
|
23
|
1 |
|
if name == "": |
24
|
1 |
|
raise ValueError("Endpoint name cannot be empty") |
25
|
|
|
|
26
|
|
|
|
27
|
1 |
|
def _check_hostname(name): |
28
|
1 |
|
_check_endpoint_type(name) |
29
|
1 |
|
hostname_checker = compile(r"^^http(s)?://[\w.-]+(/)?(:\d+)?(/)?$") |
30
|
|
|
|
31
|
1 |
|
if not hostname_checker.match(name): |
32
|
1 |
|
raise ValueError( |
33
|
|
|
f"endpoint name {name} should be in http(s)://<hostname>" |
34
|
|
|
"[:<port>] and hostname may consist only of: " |
35
|
|
|
"a-z, A-Z, 0-9, underscore and hyphens." |
36
|
|
|
) |
37
|
|
|
|
38
|
|
|
|
39
|
1 |
|
def _check_endpoint_name(name): |
40
|
|
|
"""Checks that the endpoint name is valid by comparing it with an RE and |
41
|
|
|
checking that it is not reserved.""" |
42
|
1 |
|
_check_endpoint_type(name) |
43
|
|
|
|
44
|
1 |
|
if not _name_checker.match(name): |
45
|
1 |
|
raise ValueError( |
46
|
|
|
f"endpoint name {name} can only contain: a-z, A-Z, 0-9," |
47
|
|
|
" underscore, hyphens and spaces." |
48
|
|
|
) |
49
|
|
|
|
50
|
|
|
|
51
|
1 |
|
class Client: |
52
|
1 |
|
def __init__(self, endpoint, query_timeout=1000): |
53
|
|
|
""" |
54
|
|
|
Connects to a running server. |
55
|
|
|
|
56
|
|
|
The class constructor takes a server address which is then used to |
57
|
|
|
connect for all subsequent member APIs. |
58
|
|
|
|
59
|
|
|
Parameters |
60
|
|
|
---------- |
61
|
|
|
endpoint : str, optional |
62
|
|
|
The server URL. |
63
|
|
|
|
64
|
|
|
query_timeout : float, optional |
65
|
|
|
The timeout for query operations. |
66
|
|
|
""" |
67
|
1 |
|
_check_hostname(endpoint) |
68
|
|
|
|
69
|
1 |
|
self._endpoint = endpoint |
70
|
|
|
|
71
|
1 |
|
session = requests.session() |
72
|
1 |
|
session.verify = False |
73
|
1 |
|
requests.packages.urllib3.disable_warnings() |
74
|
|
|
|
75
|
|
|
# Setup the communications layer. |
76
|
1 |
|
network_wrapper = RequestsNetworkWrapper(session) |
77
|
1 |
|
service_client = ServiceClient(self._endpoint, network_wrapper) |
78
|
|
|
|
79
|
1 |
|
self._service = RESTServiceClient(service_client) |
80
|
1 |
|
if not type(query_timeout) in (int, float) or query_timeout <= 0: |
81
|
1 |
|
query_timeout = 0.0 |
82
|
1 |
|
self._service.query_timeout = query_timeout |
83
|
|
|
|
84
|
1 |
|
def __repr__(self): |
85
|
|
|
return ( |
86
|
|
|
"<" |
87
|
|
|
+ self.__class__.__name__ |
88
|
|
|
+ " object at " |
89
|
|
|
+ hex(id(self)) |
90
|
|
|
+ " connected to " |
91
|
|
|
+ repr(self._endpoint) |
92
|
|
|
+ ">" |
93
|
|
|
) |
94
|
|
|
|
95
|
1 |
|
def get_status(self): |
96
|
|
|
""" |
97
|
|
|
Gets the status of the deployed endpoints. |
98
|
|
|
|
99
|
|
|
Returns |
100
|
|
|
------- |
101
|
|
|
dict |
102
|
|
|
Keys are endpoints and values are dicts describing the state of |
103
|
|
|
the endpoint. |
104
|
|
|
|
105
|
|
|
Examples |
106
|
|
|
-------- |
107
|
|
|
.. sourcecode:: python |
108
|
|
|
{ |
109
|
|
|
u'foo': { |
110
|
|
|
u'status': u'LoadFailed', |
111
|
|
|
u'last_error': u'error mesasge', |
112
|
|
|
u'version': 1, |
113
|
|
|
u'type': u'model', |
114
|
|
|
}, |
115
|
|
|
} |
116
|
|
|
""" |
117
|
1 |
|
return self._service.get_status() |
118
|
|
|
|
119
|
|
|
# |
120
|
|
|
# Query |
121
|
|
|
# |
122
|
|
|
|
123
|
1 |
|
@property |
124
|
1 |
|
def query_timeout(self): |
125
|
|
|
"""The timeout for queries in milliseconds.""" |
126
|
1 |
|
return self._service.query_timeout |
127
|
|
|
|
128
|
1 |
|
@query_timeout.setter |
129
|
1 |
|
def query_timeout(self, value): |
130
|
1 |
|
if type(value) in (int, float) and value > 0: |
131
|
1 |
|
self._service.query_timeout = value |
132
|
|
|
|
133
|
1 |
|
def query(self, name, *args, **kwargs): |
134
|
|
|
"""Query an endpoint. |
135
|
|
|
|
136
|
|
|
Parameters |
137
|
|
|
---------- |
138
|
|
|
name : str |
139
|
|
|
The name of the endpoint. |
140
|
|
|
|
141
|
|
|
*args : list of anything |
142
|
|
|
Ordered parameters to the endpoint. |
143
|
|
|
|
144
|
|
|
**kwargs : dict of anything |
145
|
|
|
Named parameters to the endpoint. |
146
|
|
|
|
147
|
|
|
Returns |
148
|
|
|
------- |
149
|
|
|
dict |
150
|
|
|
Keys are: |
151
|
|
|
model: the name of the endpoint |
152
|
|
|
version: the version used. |
153
|
|
|
response: the response to the query. |
154
|
|
|
uuid : a unique id for the request. |
155
|
|
|
""" |
156
|
1 |
|
return self._service.query(name, *args, **kwargs) |
157
|
|
|
|
158
|
|
|
# |
159
|
|
|
# Endpoints |
160
|
|
|
# |
161
|
|
|
|
162
|
1 |
|
def get_endpoints(self, type=None): |
163
|
|
|
"""Returns all deployed endpoints. |
164
|
|
|
|
165
|
|
|
Examples |
166
|
|
|
-------- |
167
|
|
|
.. sourcecode:: python |
168
|
|
|
{"clustering": |
169
|
|
|
{"description": "", |
170
|
|
|
"docstring": "-- no docstring found in query function --", |
171
|
|
|
"creation_time": 1469511182, |
172
|
|
|
"version": 1, |
173
|
|
|
"dependencies": [], |
174
|
|
|
"last_modified_time": 1469511182, |
175
|
|
|
"type": "model", |
176
|
|
|
"target": null, |
177
|
|
|
"is_public": True} |
178
|
|
|
"add": { |
179
|
|
|
"description": "", |
180
|
|
|
"docstring": "-- no docstring found in query function --", |
181
|
|
|
"creation_time": 1469505967, |
182
|
|
|
"version": 1, |
183
|
|
|
"dependencies": [], |
184
|
|
|
"last_modified_time": 1469505967, |
185
|
|
|
"type": "model", |
186
|
|
|
"target": null, |
187
|
|
|
"is_public": False} |
188
|
|
|
} |
189
|
|
|
""" |
190
|
1 |
|
return self._service.get_endpoints(type) |
191
|
|
|
|
192
|
1 |
|
def _get_endpoint_upload_destination(self): |
193
|
|
|
"""Returns the endpoint upload destination.""" |
194
|
1 |
|
return self._service.get_endpoint_upload_destination()["path"] |
195
|
|
|
|
196
|
1 |
|
def deploy(self, name, obj, description="", schema=None, override=False, is_public=False): |
197
|
|
|
"""Deploys a Python function as an endpoint in the server. |
198
|
|
|
|
199
|
|
|
Parameters |
200
|
|
|
---------- |
201
|
|
|
name : str |
202
|
|
|
A unique identifier for the endpoint. |
203
|
|
|
|
204
|
|
|
obj : function |
205
|
|
|
Refers to a user-defined function with any signature. However both |
206
|
|
|
input and output of the function need to be JSON serializable. |
207
|
|
|
|
208
|
|
|
description : str, optional |
209
|
|
|
The description for the endpoint. This string will be returned by |
210
|
|
|
the ``endpoints`` API. |
211
|
|
|
|
212
|
|
|
schema : dict, optional |
213
|
|
|
The schema of the function, containing information about input and |
214
|
|
|
output parameters, and respective examples. Providing a schema for |
215
|
|
|
a deployed function lets other users of the service discover how to |
216
|
|
|
use it. Refer to schema.generate_schema for more information on |
217
|
|
|
how to generate the schema. |
218
|
|
|
|
219
|
|
|
override : bool |
220
|
|
|
Whether to override (update) an existing endpoint. If False and |
221
|
|
|
there is already an endpoint with that name, it will raise a |
222
|
|
|
RuntimeError. If True and there is already an endpoint with that |
223
|
|
|
name, it will deploy a new version on top of it. |
224
|
|
|
|
225
|
|
|
is_public : bool, optional |
226
|
|
|
Whether a function should be public for viewing from within tableau. If |
227
|
|
|
False, function will not appear in the custom functions explorer within |
228
|
|
|
Tableau. If True, function will be visible ta anyone on a site with this |
229
|
|
|
analytics extension configured |
230
|
|
|
|
231
|
|
|
See Also |
232
|
|
|
-------- |
233
|
|
|
remove, get_endpoints |
234
|
|
|
""" |
235
|
1 |
|
endpoint = self.get_endpoints().get(name) |
236
|
1 |
|
version = 1 |
237
|
1 |
|
if endpoint: |
238
|
1 |
|
if not override: |
239
|
|
|
raise RuntimeError( |
240
|
|
|
f"An endpoint with that name ({name}) already" |
241
|
|
|
' exists. Use "override = True" to force update ' |
242
|
|
|
"an existing endpoint." |
243
|
|
|
) |
244
|
|
|
|
245
|
1 |
|
version = endpoint.version + 1 |
246
|
|
|
|
247
|
1 |
|
obj = self._gen_endpoint(name, obj, description, version, schema, is_public) |
248
|
|
|
|
249
|
1 |
|
self._upload_endpoint(obj) |
250
|
|
|
|
251
|
1 |
|
if version == 1: |
252
|
1 |
|
self._service.add_endpoint(Endpoint(**obj)) |
253
|
|
|
else: |
254
|
1 |
|
self._service.set_endpoint(Endpoint(**obj)) |
255
|
|
|
|
256
|
1 |
|
self._wait_for_endpoint_deployment(obj["name"], obj["version"]) |
257
|
|
|
|
258
|
1 |
|
def remove(self, name): |
259
|
|
|
'''Removes an endpoint dict. |
260
|
|
|
|
261
|
|
|
Parameters |
262
|
|
|
---------- |
263
|
|
|
name : str |
264
|
|
|
Endpoint name to remove''' |
265
|
|
|
self._service.remove_endpoint(name) |
266
|
|
|
|
267
|
1 |
|
def update_endpoint_info(self, name, description=None, schema=None, is_public=None): |
268
|
|
|
'''Updates description, schema, or is public for an existing endpoint |
269
|
|
|
|
270
|
|
|
Parameters |
271
|
|
|
---------- |
272
|
|
|
name : str |
273
|
|
|
Name of the endpoint that to be updated. If endpoint does not exist |
274
|
|
|
runtime error will be thrown |
275
|
|
|
|
276
|
|
|
description : str, optional |
277
|
|
|
The description for the endpoint. This string will be returned by |
278
|
|
|
the ``endpoints`` API. |
279
|
|
|
|
280
|
|
|
schema : dict, optional |
281
|
|
|
The schema of the function, containing information about input and |
282
|
|
|
output parameters, and respective examples. Providing a schema for |
283
|
|
|
a deployed function lets other users of the service discover how to |
284
|
|
|
use it. Refer to schema.generate_schema for more information on |
285
|
|
|
how to generate the schema. |
286
|
|
|
|
287
|
|
|
is_public : bool, optional |
288
|
|
|
Whether a function should be public for viewing from within tableau. If |
289
|
|
|
False, function will not appear in the custom functions explorer within |
290
|
|
|
Tableau. If True, function will be visible to anyone on a site with this |
291
|
|
|
analytics extension configured |
292
|
|
|
''' |
293
|
|
|
|
294
|
|
|
endpoint = self.get_endpoints().get(name) |
295
|
|
|
|
296
|
|
|
if not endpoint: |
297
|
|
|
raise RuntimeError( |
298
|
|
|
f"No endpoint with that name ({name}) exists" |
299
|
|
|
" Please select an existing endpoint to update" |
300
|
|
|
) |
301
|
|
|
|
302
|
|
|
if description is not None: |
303
|
|
|
if type(description) is not str: |
304
|
|
|
raise RuntimeError( |
305
|
|
|
f"Type of description must be string" |
306
|
|
|
) |
307
|
|
|
endpoint.description = description |
308
|
|
|
if schema is not None: |
309
|
|
|
if type(schema) is not dict: |
310
|
|
|
raise RuntimeError( |
311
|
|
|
f"Type of schema must be dictionary" |
312
|
|
|
) |
313
|
|
|
endpoint.schema = schema |
314
|
|
|
if is_public is not None: |
315
|
|
|
if type(is_public) is not bool: |
316
|
|
|
raise RuntimeError( |
317
|
|
|
f"Type of is_public must be bool" |
318
|
|
|
) |
319
|
|
|
endpoint.is_public = is_public |
320
|
|
|
|
321
|
|
|
dest_path = self._get_endpoint_upload_destination() |
322
|
|
|
|
323
|
|
|
endpoint.src_path = os.path.join( |
324
|
|
|
dest_path, "endpoints", endpoint.name, str(endpoint.version) |
325
|
|
|
) |
326
|
|
|
|
327
|
|
|
self._service.set_endpoint(endpoint) |
328
|
|
|
|
329
|
1 |
|
def _gen_endpoint(self, name, obj, description, version=1, schema=None, is_public=False): |
330
|
|
|
"""Generates an endpoint dict. |
331
|
|
|
|
332
|
|
|
Parameters |
333
|
|
|
---------- |
334
|
|
|
name : str |
335
|
|
|
Endpoint name to add or update |
336
|
|
|
|
337
|
|
|
obj : func |
338
|
|
|
Object that backs the endpoint. See add() for a complete |
339
|
|
|
description. |
340
|
|
|
|
341
|
|
|
description : str |
342
|
|
|
Description of the endpoint |
343
|
|
|
|
344
|
|
|
version : int |
345
|
|
|
The version. Defaults to 1. |
346
|
|
|
|
347
|
|
|
is_public : bool |
348
|
|
|
True if function should be visible in the custom functions explorer |
349
|
|
|
within Tableau |
350
|
|
|
|
351
|
|
|
Returns |
352
|
|
|
------- |
353
|
|
|
dict |
354
|
|
|
Keys: |
355
|
|
|
name : str |
356
|
|
|
The name provided. |
357
|
|
|
|
358
|
|
|
version : int |
359
|
|
|
The version provided. |
360
|
|
|
|
361
|
|
|
description : str |
362
|
|
|
The provided description. |
363
|
|
|
|
364
|
|
|
type : str |
365
|
|
|
The type of the endpoint. |
366
|
|
|
|
367
|
|
|
endpoint_obj : object |
368
|
|
|
The wrapper around the obj provided that can be used to |
369
|
|
|
generate the code and dependencies for the endpoint. |
370
|
|
|
|
371
|
|
|
Raises |
372
|
|
|
------ |
373
|
|
|
TypeError |
374
|
|
|
When obj is not one of the expected types. |
375
|
|
|
""" |
376
|
|
|
# check for invalid PO names |
377
|
1 |
|
_check_endpoint_name(name) |
378
|
|
|
|
379
|
1 |
|
if description is None: |
380
|
|
|
description = obj.__doc__.strip() or "" if isinstance(obj.__doc__, str) else "" |
381
|
|
|
|
382
|
1 |
|
endpoint_object = CustomQueryObject(query=obj, description=description,) |
383
|
|
|
|
384
|
1 |
|
return { |
385
|
|
|
"name": name, |
386
|
|
|
"version": version, |
387
|
|
|
"description": description, |
388
|
|
|
"type": "model", |
389
|
|
|
"endpoint_obj": endpoint_object, |
390
|
|
|
"dependencies": endpoint_object.get_dependencies(), |
391
|
|
|
"methods": endpoint_object.get_methods(), |
392
|
|
|
"required_files": [], |
393
|
|
|
"required_packages": [], |
394
|
|
|
"schema": copy.copy(schema), |
395
|
|
|
"is_public": is_public, |
396
|
|
|
} |
397
|
|
|
|
398
|
1 |
|
def _upload_endpoint(self, obj): |
399
|
|
|
"""Sends the endpoint across the wire.""" |
400
|
1 |
|
endpoint_obj = obj["endpoint_obj"] |
401
|
|
|
|
402
|
1 |
|
dest_path = self._get_endpoint_upload_destination() |
403
|
|
|
|
404
|
|
|
# Upload the endpoint |
405
|
1 |
|
obj["src_path"] = os.path.join( |
406
|
|
|
dest_path, "endpoints", obj["name"], str(obj["version"]) |
407
|
|
|
) |
408
|
|
|
|
409
|
1 |
|
endpoint_obj.save(obj["src_path"]) |
410
|
|
|
|
411
|
1 |
|
def _wait_for_endpoint_deployment( |
412
|
|
|
self, endpoint_name, version=1, interval=1.0, |
413
|
|
|
): |
414
|
|
|
""" |
415
|
|
|
Waits for the endpoint to be deployed by calling get_status() and |
416
|
|
|
checking the versions deployed of the endpoint against the expected |
417
|
|
|
version. If all the versions are equal to or greater than the version |
418
|
|
|
expected, then it will return. Uses time.sleep(). |
419
|
|
|
""" |
420
|
1 |
|
logger.info( |
421
|
|
|
f"Waiting for endpoint {endpoint_name} to deploy to " f"version {version}" |
422
|
|
|
) |
423
|
1 |
|
start = time.time() |
424
|
1 |
|
while True: |
425
|
1 |
|
ep_status = self.get_status() |
426
|
1 |
|
try: |
427
|
1 |
|
ep = ep_status[endpoint_name] |
428
|
|
|
except KeyError: |
429
|
|
|
logger.info( |
430
|
|
|
f"Endpoint {endpoint_name} doesn't " "exist in endpoints yet" |
431
|
|
|
) |
432
|
|
|
else: |
433
|
1 |
|
logger.info(f"ep={ep}") |
434
|
|
|
|
435
|
1 |
|
if ep["status"] == "LoadFailed": |
436
|
|
|
raise RuntimeError(f'LoadFailed: {ep["last_error"]}') |
437
|
|
|
|
438
|
1 |
|
elif ep["status"] == "LoadSuccessful": |
439
|
1 |
|
if ep["version"] >= version: |
440
|
1 |
|
logger.info("LoadSuccessful") |
441
|
1 |
|
break |
442
|
|
|
else: |
443
|
|
|
logger.info("LoadSuccessful but wrong version") |
444
|
|
|
|
445
|
|
|
if time.time() - start > 10: |
446
|
|
|
raise RuntimeError("Waited more then 10s for deployment") |
447
|
|
|
|
448
|
|
|
logger.info(f"Sleeping {interval}...") |
449
|
|
|
time.sleep(interval) |
450
|
|
|
|
451
|
1 |
|
def set_credentials(self, username, password): |
452
|
|
|
""" |
453
|
|
|
Set credentials for all the TabPy client-server communication |
454
|
|
|
where client is tabpy-tools and server is tabpy-server. |
455
|
|
|
|
456
|
|
|
Parameters |
457
|
|
|
---------- |
458
|
|
|
username : str |
459
|
|
|
User name (login). Username is case insensitive. |
460
|
|
|
|
461
|
|
|
password : str |
462
|
|
|
Password in plain text. |
463
|
|
|
""" |
464
|
|
|
self._service.set_credentials(username, password) |
465
|
|
|
|