1
|
|
|
from argparse import ArgumentParser |
2
|
|
|
import concurrent.futures |
3
|
|
|
import configparser |
4
|
|
|
import logging |
5
|
|
|
from logging import config |
6
|
|
|
import multiprocessing |
7
|
|
|
import os |
8
|
|
|
import shutil |
9
|
|
|
import tabpy.tabpy_server |
10
|
|
|
from tabpy.tabpy import __version__ |
11
|
|
|
from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters |
12
|
|
|
from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters |
13
|
|
|
from tabpy.tabpy_server.app.util import parse_pwd_file |
14
|
|
|
from tabpy.tabpy_server.management.state import TabPyState |
15
|
|
|
from tabpy.tabpy_server.management.util import _get_state_from_file |
16
|
|
|
from tabpy.tabpy_server.psws.callbacks\ |
17
|
|
|
import (init_model_evaluator, init_ps_server) |
18
|
|
|
from tabpy.tabpy_server.psws.python_service\ |
19
|
|
|
import (PythonService, PythonServiceHandler) |
20
|
|
|
from tabpy.tabpy_server.handlers\ |
21
|
|
|
import (EndpointHandler, EndpointsHandler, |
22
|
|
|
EvaluationPlaneHandler, QueryPlaneHandler, |
23
|
|
|
ServiceInfoHandler, StatusHandler, |
24
|
|
|
UploadDestinationHandler) |
25
|
|
|
import tornado |
26
|
|
|
|
27
|
|
|
|
28
|
|
|
logger = logging.getLogger(__name__) |
29
|
|
|
|
30
|
|
|
|
31
|
|
|
class TabPyApp: |
32
|
|
|
''' |
33
|
|
|
TabPy application class for keeping context like settings, state, etc. |
34
|
|
|
''' |
35
|
|
|
|
36
|
|
|
settings = {} |
37
|
|
|
subdirectory = "" |
38
|
|
|
tabpy_state = None |
39
|
|
|
python_service = None |
40
|
|
|
credentials = {} |
41
|
|
|
|
42
|
|
|
def __init__(self, config_file=None): |
43
|
|
|
if config_file is None: |
44
|
|
|
cli_args = self._parse_cli_arguments() |
45
|
|
|
if cli_args.config is not None: |
46
|
|
|
config_file = cli_args.config |
47
|
|
|
else: |
48
|
|
|
config_file = os.path.join(os.path.dirname(__file__), |
49
|
|
|
os.path.pardir, 'common', |
50
|
|
|
'default.conf') |
51
|
|
|
|
52
|
|
|
if os.path.isfile(config_file): |
53
|
|
|
try: |
54
|
|
|
logging.config.fileConfig( |
55
|
|
|
config_file, disable_existing_loggers=False) |
56
|
|
|
except KeyError: |
57
|
|
|
logging.basicConfig(level=logging.DEBUG) |
58
|
|
|
|
59
|
|
|
self._parse_config(config_file) |
60
|
|
|
|
61
|
|
|
def run(self): |
62
|
|
|
application = self._create_tornado_web_app() |
63
|
|
|
|
64
|
|
|
init_model_evaluator( |
65
|
|
|
self.settings, |
66
|
|
|
self.tabpy_state, |
67
|
|
|
self.python_service) |
68
|
|
|
|
69
|
|
|
protocol = self.settings[SettingsParameters.TransferProtocol] |
70
|
|
|
if protocol == 'http': |
71
|
|
|
application.listen(self.settings[SettingsParameters.Port]) |
72
|
|
|
elif protocol == 'https': |
73
|
|
|
application.listen(self.settings[SettingsParameters.Port], |
74
|
|
|
ssl_options={ |
75
|
|
|
'certfile': self.settings[SettingsParameters.CertificateFile], |
76
|
|
|
'keyfile': self.settings[SettingsParameters.KeyFile] |
77
|
|
|
}) |
78
|
|
|
else: |
79
|
|
|
msg = f'Unsupported transfer protocol {protocol}.' |
80
|
|
|
logger.critical(msg) |
81
|
|
|
raise RuntimeError(msg) |
82
|
|
|
|
83
|
|
|
logger.info( |
84
|
|
|
'Web service listening on port ' |
85
|
|
|
f'{str(self.settings[SettingsParameters.Port])}') |
86
|
|
|
tornado.ioloop.IOLoop.instance().start() |
87
|
|
|
|
88
|
|
|
def _create_tornado_web_app(self): |
89
|
|
|
logger.info('Initializing TabPy...') |
90
|
|
|
tornado.ioloop.IOLoop.instance().run_sync( |
91
|
|
|
lambda: init_ps_server(self.settings, self.tabpy_state)) |
92
|
|
|
logger.info('Done initializing TabPy.') |
93
|
|
|
|
94
|
|
|
executor = concurrent.futures.ThreadPoolExecutor( |
95
|
|
|
max_workers=multiprocessing.cpu_count()) |
96
|
|
|
|
97
|
|
|
# initialize Tornado application |
98
|
|
|
application = tornado.web.Application([ |
99
|
|
|
# skip MainHandler to use StaticFileHandler .* page requests and |
100
|
|
|
# default to index.html |
101
|
|
|
# (r"/", MainHandler), |
102
|
|
|
(self.subdirectory + r'/query/([^/]+)', QueryPlaneHandler, |
103
|
|
|
dict(app=self)), |
104
|
|
|
(self.subdirectory + r'/status', StatusHandler, |
105
|
|
|
dict(app=self)), |
106
|
|
|
(self.subdirectory + r'/info', ServiceInfoHandler, |
107
|
|
|
dict(app=self)), |
108
|
|
|
(self.subdirectory + r'/endpoints', EndpointsHandler, |
109
|
|
|
dict(app=self)), |
110
|
|
|
(self.subdirectory + r'/endpoints/([^/]+)?', EndpointHandler, |
111
|
|
|
dict(app=self)), |
112
|
|
|
(self.subdirectory + r'/evaluate', EvaluationPlaneHandler, |
113
|
|
|
dict(executor=executor, |
114
|
|
|
app=self)), |
115
|
|
|
(self.subdirectory + |
116
|
|
|
r'/configurations/endpoint_upload_destination', |
117
|
|
|
UploadDestinationHandler, |
118
|
|
|
dict(app=self)), |
119
|
|
|
(self.subdirectory + r'/(.*)', tornado.web.StaticFileHandler, |
120
|
|
|
dict(path=self.settings[SettingsParameters.StaticPath], |
121
|
|
|
default_filename="index.html")), |
122
|
|
|
], debug=False, **self.settings) |
123
|
|
|
|
124
|
|
|
return application |
125
|
|
|
|
126
|
|
|
@staticmethod |
127
|
|
|
def _parse_cli_arguments(): |
128
|
|
|
''' |
129
|
|
|
Parse command line arguments. Expected arguments: |
130
|
|
|
* --config: string |
131
|
|
|
''' |
132
|
|
|
parser = ArgumentParser(description='Run TabPy Server.') |
133
|
|
|
parser.add_argument('--config', help='Path to a config file.') |
134
|
|
|
return parser.parse_args() |
135
|
|
|
|
136
|
|
|
def _parse_config(self, config_file): |
137
|
|
|
"""Provide consistent mechanism for pulling in configuration. |
138
|
|
|
|
139
|
|
|
Attempt to retain backward compatibility for |
140
|
|
|
existing implementations by grabbing port |
141
|
|
|
setting from CLI first. |
142
|
|
|
|
143
|
|
|
Take settings in the following order: |
144
|
|
|
|
145
|
|
|
1. CLI arguments if present |
146
|
|
|
2. config file |
147
|
|
|
3. OS environment variables (for ease of |
148
|
|
|
setting defaults if not present) |
149
|
|
|
4. current defaults if a setting is not present in any location |
150
|
|
|
|
151
|
|
|
Additionally provide similar configuration capabilities in between |
152
|
|
|
config file and environment variables. |
153
|
|
|
For consistency use the same variable name in the config file as |
154
|
|
|
in the os environment. |
155
|
|
|
For naming standards use all capitals and start with 'TABPY_' |
156
|
|
|
""" |
157
|
|
|
self.settings = {} |
158
|
|
|
self.subdirectory = "" |
159
|
|
|
self.tabpy_state = None |
160
|
|
|
self.python_service = None |
161
|
|
|
self.credentials = {} |
162
|
|
|
|
163
|
|
|
parser = configparser.ConfigParser() |
164
|
|
|
|
165
|
|
|
if os.path.isfile(config_file): |
166
|
|
|
with open(config_file) as f: |
167
|
|
|
parser.read_string(f.read()) |
168
|
|
|
else: |
169
|
|
|
logger.warning( |
170
|
|
|
f'Unable to find config file at {config_file}, ' |
171
|
|
|
'using default settings.') |
172
|
|
|
|
173
|
|
|
def set_parameter(settings_key, |
174
|
|
|
config_key, |
175
|
|
|
default_val=None, |
176
|
|
|
check_env_var=False): |
177
|
|
|
key_is_set = False |
178
|
|
|
|
179
|
|
|
if config_key is not None and\ |
180
|
|
|
parser.has_section('TabPy') and\ |
181
|
|
|
parser.has_option('TabPy', config_key): |
182
|
|
|
self.settings[settings_key] = parser.get('TabPy', config_key) |
183
|
|
|
key_is_set = True |
184
|
|
|
logger.debug( |
185
|
|
|
f'Parameter {settings_key} set to ' |
186
|
|
|
f'"{self.settings[settings_key]}" ' |
187
|
|
|
'from config file') |
188
|
|
|
|
189
|
|
|
if not key_is_set and check_env_var: |
190
|
|
|
val = os.getenv(config_key) |
191
|
|
|
if val is not None: |
192
|
|
|
self.settings[settings_key] = val |
193
|
|
|
key_is_set = True |
194
|
|
|
logger.debug( |
195
|
|
|
f'Parameter {settings_key} set to ' |
196
|
|
|
f'"{self.settings[settings_key]}" ' |
197
|
|
|
'from environment variable') |
198
|
|
|
|
199
|
|
|
if not key_is_set and default_val is not None: |
200
|
|
|
self.settings[settings_key] = default_val |
201
|
|
|
key_is_set = True |
202
|
|
|
logger.debug( |
203
|
|
|
f'Parameter {settings_key} set to ' |
204
|
|
|
f'"{self.settings[settings_key]}" ' |
205
|
|
|
'from default value') |
206
|
|
|
|
207
|
|
|
if not key_is_set: |
208
|
|
|
logger.debug( |
209
|
|
|
f'Parameter {settings_key} is not set') |
210
|
|
|
|
211
|
|
|
set_parameter(SettingsParameters.Port, ConfigParameters.TABPY_PORT, |
212
|
|
|
default_val=9004, check_env_var=True) |
213
|
|
|
set_parameter(SettingsParameters.ServerVersion, None, |
214
|
|
|
default_val=__version__) |
215
|
|
|
|
216
|
|
|
set_parameter(SettingsParameters.EvaluateTimeout, |
217
|
|
|
ConfigParameters.TABPY_EVALUATE_TIMEOUT, |
218
|
|
|
default_val=30) |
219
|
|
|
try: |
220
|
|
|
self.settings[SettingsParameters.EvaluateTimeout] = float( |
221
|
|
|
self.settings[SettingsParameters.EvaluateTimeout]) |
222
|
|
|
except ValueError: |
223
|
|
|
logger.warning( |
224
|
|
|
'Evaluate timeout must be a float type. Defaulting ' |
225
|
|
|
'to evaluate timeout of 30 seconds.') |
226
|
|
|
self.settings[SettingsParameters.EvaluateTimeout] = 30 |
227
|
|
|
|
228
|
|
|
pkg_path = os.path.dirname(tabpy.__file__) |
229
|
|
|
set_parameter(SettingsParameters.UploadDir, |
230
|
|
|
ConfigParameters.TABPY_QUERY_OBJECT_PATH, |
231
|
|
|
default_val=os.path.join(pkg_path, |
232
|
|
|
'tmp', 'query_objects'), |
233
|
|
|
check_env_var=True) |
234
|
|
|
if not os.path.exists(self.settings[SettingsParameters.UploadDir]): |
235
|
|
|
os.makedirs(self.settings[SettingsParameters.UploadDir]) |
236
|
|
|
|
237
|
|
|
# set and validate transfer protocol |
238
|
|
|
set_parameter(SettingsParameters.TransferProtocol, |
239
|
|
|
ConfigParameters.TABPY_TRANSFER_PROTOCOL, |
240
|
|
|
default_val='http') |
241
|
|
|
self.settings[SettingsParameters.TransferProtocol] =\ |
242
|
|
|
self.settings[SettingsParameters.TransferProtocol].lower() |
243
|
|
|
|
244
|
|
|
set_parameter(SettingsParameters.CertificateFile, |
245
|
|
|
ConfigParameters.TABPY_CERTIFICATE_FILE) |
246
|
|
|
set_parameter(SettingsParameters.KeyFile, |
247
|
|
|
ConfigParameters.TABPY_KEY_FILE) |
248
|
|
|
self._validate_transfer_protocol_settings() |
249
|
|
|
|
250
|
|
|
# if state.ini does not exist try and create it - remove |
251
|
|
|
# last dependence on batch/shell script |
252
|
|
|
set_parameter(SettingsParameters.StateFilePath, |
253
|
|
|
ConfigParameters.TABPY_STATE_PATH, |
254
|
|
|
default_val=os.path.join(pkg_path, 'tabpy_server'), |
255
|
|
|
check_env_var=True) |
256
|
|
|
self.settings[SettingsParameters.StateFilePath] = os.path.realpath( |
257
|
|
|
os.path.normpath( |
258
|
|
|
os.path.expanduser( |
259
|
|
|
self.settings[SettingsParameters.StateFilePath]))) |
260
|
|
|
state_file_dir = self.settings[SettingsParameters.StateFilePath] |
261
|
|
|
state_file_path = os.path.join(state_file_dir, 'state.ini') |
262
|
|
|
if not os.path.isfile(state_file_path): |
263
|
|
|
state_file_template_path = os.path.join( |
264
|
|
|
pkg_path, 'tabpy_server', 'state.ini.template') |
265
|
|
|
logger.debug(f'File {state_file_path} not found, creating from ' |
266
|
|
|
f'template {state_file_template_path}...') |
267
|
|
|
shutil.copy(state_file_template_path, state_file_path) |
268
|
|
|
|
269
|
|
|
logger.info(f'Loading state from state file {state_file_path}') |
270
|
|
|
tabpy_state = _get_state_from_file(state_file_dir) |
271
|
|
|
self.tabpy_state = TabPyState( |
272
|
|
|
config=tabpy_state, settings=self.settings) |
273
|
|
|
|
274
|
|
|
self.python_service = PythonServiceHandler(PythonService()) |
275
|
|
|
self.settings['compress_response'] = True |
276
|
|
|
set_parameter(SettingsParameters.StaticPath, |
277
|
|
|
ConfigParameters.TABPY_STATIC_PATH, |
278
|
|
|
default_val='./') |
279
|
|
|
self.settings[SettingsParameters.StaticPath] =\ |
280
|
|
|
os.path.abspath(self.settings[SettingsParameters.StaticPath]) |
281
|
|
|
logger.debug(f'Static pages folder set to ' |
282
|
|
|
f'"{self.settings[SettingsParameters.StaticPath]}"') |
283
|
|
|
|
284
|
|
|
# Set subdirectory from config if applicable |
285
|
|
|
if tabpy_state.has_option("Service Info", "Subdirectory"): |
286
|
|
|
self.subdirectory = "/" + \ |
287
|
|
|
tabpy_state.get("Service Info", "Subdirectory") |
288
|
|
|
|
289
|
|
|
# If passwords file specified load credentials |
290
|
|
|
set_parameter(ConfigParameters.TABPY_PWD_FILE, |
291
|
|
|
ConfigParameters.TABPY_PWD_FILE) |
292
|
|
|
if ConfigParameters.TABPY_PWD_FILE in self.settings: |
293
|
|
|
if not self._parse_pwd_file(): |
294
|
|
|
msg = ('Failed to read passwords file ' |
295
|
|
|
f'{self.settings[ConfigParameters.TABPY_PWD_FILE]}') |
296
|
|
|
logger.critical(msg) |
297
|
|
|
raise RuntimeError(msg) |
298
|
|
|
else: |
299
|
|
|
logger.info( |
300
|
|
|
"Password file is not specified: " |
301
|
|
|
"Authentication is not enabled") |
302
|
|
|
|
303
|
|
|
features = self._get_features() |
304
|
|
|
self.settings[SettingsParameters.ApiVersions] =\ |
305
|
|
|
{'v1': {'features': features}} |
306
|
|
|
|
307
|
|
|
set_parameter(SettingsParameters.LogRequestContext, |
308
|
|
|
ConfigParameters.TABPY_LOG_DETAILS, |
309
|
|
|
default_val='false') |
310
|
|
|
self.settings[SettingsParameters.LogRequestContext] = ( |
311
|
|
|
self.settings[SettingsParameters.LogRequestContext].lower() != |
312
|
|
|
'false') |
313
|
|
|
call_context_state =\ |
314
|
|
|
'enabled' if self.settings[SettingsParameters.LogRequestContext]\ |
315
|
|
|
else 'disabled' |
316
|
|
|
logger.info(f'Call context logging is {call_context_state}') |
317
|
|
|
|
318
|
|
|
def _validate_transfer_protocol_settings(self): |
319
|
|
|
if SettingsParameters.TransferProtocol not in self.settings: |
320
|
|
|
msg = 'Missing transfer protocol information.' |
321
|
|
|
logger.critical(msg) |
322
|
|
|
raise RuntimeError(msg) |
323
|
|
|
|
324
|
|
|
protocol = self.settings[SettingsParameters.TransferProtocol] |
325
|
|
|
|
326
|
|
|
if protocol == 'http': |
327
|
|
|
return |
328
|
|
|
|
329
|
|
|
if protocol != 'https': |
330
|
|
|
msg = f'Unsupported transfer protocol: {protocol}' |
331
|
|
|
logger.critical(msg) |
332
|
|
|
raise RuntimeError(msg) |
333
|
|
|
|
334
|
|
|
self._validate_cert_key_state( |
335
|
|
|
'The parameter(s) {} must be set.', |
336
|
|
|
SettingsParameters.CertificateFile in self.settings, |
337
|
|
|
SettingsParameters.KeyFile in self.settings) |
338
|
|
|
cert = self.settings[SettingsParameters.CertificateFile] |
339
|
|
|
|
340
|
|
|
self._validate_cert_key_state( |
341
|
|
|
'The parameter(s) {} must point to ' |
342
|
|
|
'an existing file.', |
343
|
|
|
os.path.isfile(cert), |
344
|
|
|
os.path.isfile(self.settings[SettingsParameters.KeyFile])) |
345
|
|
|
tabpy.tabpy_server.app.util.validate_cert(cert) |
346
|
|
|
|
347
|
|
|
@staticmethod |
348
|
|
|
def _validate_cert_key_state(msg, cert_valid, key_valid): |
349
|
|
|
cert_and_key_param = ( |
350
|
|
|
f'{ConfigParameters.TABPY_CERTIFICATE_FILE} and ' |
351
|
|
|
f'{ConfigParameters.TABPY_KEY_FILE}') |
352
|
|
|
https_error = 'Error using HTTPS: ' |
353
|
|
|
err = None |
354
|
|
|
if not cert_valid and not key_valid: |
355
|
|
|
err = https_error + msg.format(cert_and_key_param) |
356
|
|
|
elif not cert_valid: |
357
|
|
|
err = https_error + \ |
358
|
|
|
msg.format(ConfigParameters.TABPY_CERTIFICATE_FILE) |
359
|
|
|
elif not key_valid: |
360
|
|
|
err = https_error + msg.format(ConfigParameters.TABPY_KEY_FILE) |
361
|
|
|
|
362
|
|
|
if err is not None: |
363
|
|
|
logger.critical(err) |
364
|
|
|
raise RuntimeError(err) |
365
|
|
|
|
366
|
|
|
def _parse_pwd_file(self): |
367
|
|
|
succeeded, self.credentials = parse_pwd_file( |
368
|
|
|
self.settings[ConfigParameters.TABPY_PWD_FILE]) |
369
|
|
|
|
370
|
|
|
if succeeded and len(self.credentials) == 0: |
371
|
|
|
logger.error('No credentials found') |
372
|
|
|
succeeded = False |
373
|
|
|
|
374
|
|
|
return succeeded |
375
|
|
|
|
376
|
|
|
def _get_features(self): |
377
|
|
|
features = {} |
378
|
|
|
|
379
|
|
|
# Check for auth |
380
|
|
|
if ConfigParameters.TABPY_PWD_FILE in self.settings: |
381
|
|
|
features['authentication'] = { |
382
|
|
|
'required': True, 'methods': { |
383
|
|
|
'basic-auth': {}}} |
384
|
|
|
|
385
|
|
|
return features |
386
|
|
|
|