Passed
Push — master ( 227024...1d0698 )
by Oleksandr
02:44
created

TabPyApp._parse_config()   F

Complexity

Conditions 19

Size

Total Lines 181
Code Lines 123

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
eloc 123
dl 0
loc 181
rs 0.4199
c 0
b 0
f 0
cc 19
nop 2

How to fix   Long Method    Complexity   

Long Method

Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.

For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.

Commonly applied refactorings include:

Complexity

Complex classes like tabpy.tabpy_server.app.app.TabPyApp._parse_config() often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.

Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.

1
from argparse import ArgumentParser
2
import concurrent.futures
3
import configparser
4
import logging
5
from logging import config
6
import multiprocessing
7
import os
8
import shutil
9
import tabpy.tabpy_server
10
from tabpy.tabpy import __version__
11
from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters
12
from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters
13
from tabpy.tabpy_server.app.util import parse_pwd_file
14
from tabpy.tabpy_server.management.state import TabPyState
15
from tabpy.tabpy_server.management.util import _get_state_from_file
16
from tabpy.tabpy_server.psws.callbacks\
17
    import (init_model_evaluator, init_ps_server)
18
from tabpy.tabpy_server.psws.python_service\
19
    import (PythonService, PythonServiceHandler)
20
from tabpy.tabpy_server.handlers\
21
    import (EndpointHandler, EndpointsHandler,
22
            EvaluationPlaneHandler, QueryPlaneHandler,
23
            ServiceInfoHandler, StatusHandler,
24
            UploadDestinationHandler)
25
import tornado
26
27
28
logger = logging.getLogger(__name__)
29
30
31
class TabPyApp:
32
    '''
33
    TabPy application class for keeping context like settings, state, etc.
34
    '''
35
36
    settings = {}
37
    subdirectory = ""
38
    tabpy_state = None
39
    python_service = None
40
    credentials = {}
41
42
    def __init__(self, config_file=None):
43
        if config_file is None:
44
            cli_args = self._parse_cli_arguments()
45
            if cli_args.config is not None:
46
                config_file = cli_args.config
47
            else:
48
                config_file = os.path.join(os.path.dirname(__file__),
49
                                           os.path.pardir, 'common',
50
                                           'default.conf')
51
52
        if os.path.isfile(config_file):
53
            try:
54
                logging.config.fileConfig(
55
                    config_file, disable_existing_loggers=False)
56
            except KeyError:
57
                logging.basicConfig(level=logging.DEBUG)
58
59
        self._parse_config(config_file)
60
61
    def run(self):
62
        application = self._create_tornado_web_app()
63
64
        init_model_evaluator(
65
            self.settings,
66
            self.tabpy_state,
67
            self.python_service)
68
69
        protocol = self.settings[SettingsParameters.TransferProtocol]
70
        if protocol == 'http':
71
            application.listen(self.settings[SettingsParameters.Port])
72
        elif protocol == 'https':
73
            application.listen(self.settings[SettingsParameters.Port],
74
                               ssl_options={
75
                'certfile': self.settings[SettingsParameters.CertificateFile],
76
                'keyfile': self.settings[SettingsParameters.KeyFile]
77
            })
78
        else:
79
            msg = f'Unsupported transfer protocol {protocol}.'
80
            logger.critical(msg)
81
            raise RuntimeError(msg)
82
83
        logger.info(
84
            'Web service listening on port '
85
            f'{str(self.settings[SettingsParameters.Port])}')
86
        tornado.ioloop.IOLoop.instance().start()
87
88
    def _create_tornado_web_app(self):
89
        logger.info('Initializing TabPy...')
90
        tornado.ioloop.IOLoop.instance().run_sync(
91
            lambda: init_ps_server(self.settings, self.tabpy_state))
92
        logger.info('Done initializing TabPy.')
93
94
        executor = concurrent.futures.ThreadPoolExecutor(
95
            max_workers=multiprocessing.cpu_count())
96
97
        # initialize Tornado application
98
        application = tornado.web.Application([
99
            # skip MainHandler to use StaticFileHandler .* page requests and
100
            # default to index.html
101
            # (r"/", MainHandler),
102
            (self.subdirectory + r'/query/([^/]+)', QueryPlaneHandler,
103
             dict(app=self)),
104
            (self.subdirectory + r'/status', StatusHandler,
105
             dict(app=self)),
106
            (self.subdirectory + r'/info', ServiceInfoHandler,
107
             dict(app=self)),
108
            (self.subdirectory + r'/endpoints', EndpointsHandler,
109
             dict(app=self)),
110
            (self.subdirectory + r'/endpoints/([^/]+)?', EndpointHandler,
111
             dict(app=self)),
112
            (self.subdirectory + r'/evaluate', EvaluationPlaneHandler,
113
             dict(executor=executor,
114
                  app=self)),
115
            (self.subdirectory +
116
             r'/configurations/endpoint_upload_destination',
117
             UploadDestinationHandler,
118
             dict(app=self)),
119
            (self.subdirectory + r'/(.*)', tornado.web.StaticFileHandler,
120
             dict(path=self.settings[SettingsParameters.StaticPath],
121
                  default_filename="index.html")),
122
        ], debug=False, **self.settings)
123
124
        return application
125
126
    @staticmethod
127
    def _parse_cli_arguments():
128
        '''
129
        Parse command line arguments. Expected arguments:
130
        * --config: string
131
        '''
132
        parser = ArgumentParser(description='Run TabPy Server.')
133
        parser.add_argument('--config', help='Path to a config file.')
134
        return parser.parse_args()
135
136
    def _parse_config(self, config_file):
137
        """Provide consistent mechanism for pulling in configuration.
138
139
        Attempt to retain backward compatibility for
140
        existing implementations by grabbing port
141
        setting from CLI first.
142
143
        Take settings in the following order:
144
145
        1. CLI arguments if present
146
        2. config file
147
        3. OS environment variables (for ease of
148
           setting defaults if not present)
149
        4. current defaults if a setting is not present in any location
150
151
        Additionally provide similar configuration capabilities in between
152
        config file and environment variables.
153
        For consistency use the same variable name in the config file as
154
        in the os environment.
155
        For naming standards use all capitals and start with 'TABPY_'
156
        """
157
        self.settings = {}
158
        self.subdirectory = ""
159
        self.tabpy_state = None
160
        self.python_service = None
161
        self.credentials = {}
162
163
        parser = configparser.ConfigParser()
164
165
        if os.path.isfile(config_file):
166
            with open(config_file) as f:
167
                parser.read_string(f.read())
168
        else:
169
            logger.warning(
170
                f'Unable to find config file at {config_file}, '
171
                'using default settings.')
172
173
        def set_parameter(settings_key,
174
                          config_key,
175
                          default_val=None,
176
                          check_env_var=False):
177
            key_is_set = False
178
179
            if config_key is not None and\
180
               parser.has_section('TabPy') and\
181
               parser.has_option('TabPy', config_key):
182
                self.settings[settings_key] = parser.get('TabPy', config_key)
183
                key_is_set = True
184
                logger.debug(
185
                    f'Parameter {settings_key} set to '
186
                    f'"{self.settings[settings_key]}" '
187
                    'from config file')
188
189
            if not key_is_set and check_env_var:
190
                val = os.getenv(config_key)
191
                if val is not None:
192
                    self.settings[settings_key] = val
193
                    key_is_set = True
194
                    logger.debug(
195
                        f'Parameter {settings_key} set to '
196
                        f'"{self.settings[settings_key]}" '
197
                        'from environment variable')
198
199
            if not key_is_set and default_val is not None:
200
                self.settings[settings_key] = default_val
201
                key_is_set = True
202
                logger.debug(
203
                    f'Parameter {settings_key} set to '
204
                    f'"{self.settings[settings_key]}" '
205
                    'from default value')
206
207
            if not key_is_set:
208
                logger.debug(
209
                    f'Parameter {settings_key} is not set')
210
211
        set_parameter(SettingsParameters.Port, ConfigParameters.TABPY_PORT,
212
                      default_val=9004, check_env_var=True)
213
        set_parameter(SettingsParameters.ServerVersion, None,
214
                      default_val=__version__)
215
216
        set_parameter(SettingsParameters.EvaluateTimeout,
217
                      ConfigParameters.TABPY_EVALUATE_TIMEOUT,
218
                      default_val=30)
219
        try:
220
            self.settings[SettingsParameters.EvaluateTimeout] = float(
221
                self.settings[SettingsParameters.EvaluateTimeout])
222
        except ValueError:
223
            logger.warning(
224
                'Evaluate timeout must be a float type. Defaulting '
225
                'to evaluate timeout of 30 seconds.')
226
            self.settings[SettingsParameters.EvaluateTimeout] = 30
227
228
        pkg_path = os.path.dirname(tabpy.__file__)
229
        set_parameter(SettingsParameters.UploadDir,
230
                      ConfigParameters.TABPY_QUERY_OBJECT_PATH,
231
                      default_val=os.path.join(pkg_path,
232
                                               'tmp', 'query_objects'),
233
                      check_env_var=True)
234
        if not os.path.exists(self.settings[SettingsParameters.UploadDir]):
235
            os.makedirs(self.settings[SettingsParameters.UploadDir])
236
237
        # set and validate transfer protocol
238
        set_parameter(SettingsParameters.TransferProtocol,
239
                      ConfigParameters.TABPY_TRANSFER_PROTOCOL,
240
                      default_val='http')
241
        self.settings[SettingsParameters.TransferProtocol] =\
242
            self.settings[SettingsParameters.TransferProtocol].lower()
243
244
        set_parameter(SettingsParameters.CertificateFile,
245
                      ConfigParameters.TABPY_CERTIFICATE_FILE)
246
        set_parameter(SettingsParameters.KeyFile,
247
                      ConfigParameters.TABPY_KEY_FILE)
248
        self._validate_transfer_protocol_settings()
249
250
        # if state.ini does not exist try and create it - remove
251
        # last dependence on batch/shell script
252
        set_parameter(SettingsParameters.StateFilePath,
253
                      ConfigParameters.TABPY_STATE_PATH,
254
                      default_val=os.path.join(pkg_path, 'tabpy_server'),
255
                      check_env_var=True)
256
        self.settings[SettingsParameters.StateFilePath] = os.path.realpath(
257
            os.path.normpath(
258
                os.path.expanduser(
259
                    self.settings[SettingsParameters.StateFilePath])))
260
        state_file_dir = self.settings[SettingsParameters.StateFilePath]
261
        state_file_path = os.path.join(state_file_dir, 'state.ini')
262
        if not os.path.isfile(state_file_path):
263
            state_file_template_path = os.path.join(
264
                pkg_path, 'tabpy_server', 'state.ini.template')
265
            logger.debug(f'File {state_file_path} not found, creating from '
266
                         f'template {state_file_template_path}...')
267
            shutil.copy(state_file_template_path, state_file_path)
268
269
        logger.info(f'Loading state from state file {state_file_path}')
270
        tabpy_state = _get_state_from_file(state_file_dir)
271
        self.tabpy_state = TabPyState(
272
            config=tabpy_state, settings=self.settings)
273
274
        self.python_service = PythonServiceHandler(PythonService())
275
        self.settings['compress_response'] = True
276
        set_parameter(SettingsParameters.StaticPath,
277
                      ConfigParameters.TABPY_STATIC_PATH,
278
                      default_val='./')
279
        self.settings[SettingsParameters.StaticPath] =\
280
            os.path.abspath(self.settings[SettingsParameters.StaticPath])
281
        logger.debug(f'Static pages folder set to '
282
                     f'"{self.settings[SettingsParameters.StaticPath]}"')
283
284
        # Set subdirectory from config if applicable
285
        if tabpy_state.has_option("Service Info", "Subdirectory"):
286
            self.subdirectory = "/" + \
287
                tabpy_state.get("Service Info", "Subdirectory")
288
289
        # If passwords file specified load credentials
290
        set_parameter(ConfigParameters.TABPY_PWD_FILE,
291
                      ConfigParameters.TABPY_PWD_FILE)
292
        if ConfigParameters.TABPY_PWD_FILE in self.settings:
293
            if not self._parse_pwd_file():
294
                msg = ('Failed to read passwords file '
295
                       f'{self.settings[ConfigParameters.TABPY_PWD_FILE]}')
296
                logger.critical(msg)
297
                raise RuntimeError(msg)
298
        else:
299
            logger.info(
300
                "Password file is not specified: "
301
                "Authentication is not enabled")
302
303
        features = self._get_features()
304
        self.settings[SettingsParameters.ApiVersions] =\
305
            {'v1': {'features': features}}
306
307
        set_parameter(SettingsParameters.LogRequestContext,
308
                      ConfigParameters.TABPY_LOG_DETAILS,
309
                      default_val='false')
310
        self.settings[SettingsParameters.LogRequestContext] = (
311
            self.settings[SettingsParameters.LogRequestContext].lower() !=
312
            'false')
313
        call_context_state =\
314
            'enabled' if self.settings[SettingsParameters.LogRequestContext]\
315
            else 'disabled'
316
        logger.info(f'Call context logging is {call_context_state}')
317
318
    def _validate_transfer_protocol_settings(self):
319
        if SettingsParameters.TransferProtocol not in self.settings:
320
            msg = 'Missing transfer protocol information.'
321
            logger.critical(msg)
322
            raise RuntimeError(msg)
323
324
        protocol = self.settings[SettingsParameters.TransferProtocol]
325
326
        if protocol == 'http':
327
            return
328
329
        if protocol != 'https':
330
            msg = f'Unsupported transfer protocol: {protocol}'
331
            logger.critical(msg)
332
            raise RuntimeError(msg)
333
334
        self._validate_cert_key_state(
335
            'The parameter(s) {} must be set.',
336
            SettingsParameters.CertificateFile in self.settings,
337
            SettingsParameters.KeyFile in self.settings)
338
        cert = self.settings[SettingsParameters.CertificateFile]
339
340
        self._validate_cert_key_state(
341
            'The parameter(s) {} must point to '
342
            'an existing file.',
343
            os.path.isfile(cert),
344
            os.path.isfile(self.settings[SettingsParameters.KeyFile]))
345
        tabpy.tabpy_server.app.util.validate_cert(cert)
346
347
    @staticmethod
348
    def _validate_cert_key_state(msg, cert_valid, key_valid):
349
        cert_and_key_param = (
350
            f'{ConfigParameters.TABPY_CERTIFICATE_FILE} and '
351
            f'{ConfigParameters.TABPY_KEY_FILE}')
352
        https_error = 'Error using HTTPS: '
353
        err = None
354
        if not cert_valid and not key_valid:
355
            err = https_error + msg.format(cert_and_key_param)
356
        elif not cert_valid:
357
            err = https_error + \
358
                msg.format(ConfigParameters.TABPY_CERTIFICATE_FILE)
359
        elif not key_valid:
360
            err = https_error + msg.format(ConfigParameters.TABPY_KEY_FILE)
361
362
        if err is not None:
363
            logger.critical(err)
364
            raise RuntimeError(err)
365
366
    def _parse_pwd_file(self):
367
        succeeded, self.credentials = parse_pwd_file(
368
            self.settings[ConfigParameters.TABPY_PWD_FILE])
369
370
        if succeeded and len(self.credentials) == 0:
371
            logger.error('No credentials found')
372
            succeeded = False
373
374
        return succeeded
375
376
    def _get_features(self):
377
        features = {}
378
379
        # Check for auth
380
        if ConfigParameters.TABPY_PWD_FILE in self.settings:
381
            features['authentication'] = {
382
                'required': True, 'methods': {
383
                    'basic-auth': {}}}
384
385
        return features
386