Test Failed
Push — master ( e8b070...b526e1 )
by Oleksandr
33:34 queued 26:33
created

tabpy.tabpy_server.app.app   B

Complexity

Total Complexity 52

Size/Duplication

Total Lines 438
Duplicated Lines 0 %

Test Coverage

Coverage 82.16%

Importance

Changes 0
Metric Value
wmc 52
eloc 291
dl 0
loc 438
ccs 152
cts 185
cp 0.8216
rs 7.44
c 0
b 0
f 0

10 Methods

Rating   Name   Duplication   Size   Complexity  
B TabPyApp._create_tornado_web_app() 0 73 3
A TabPyApp.run() 0 33 3
B TabPyApp._set_parameter() 0 29 8
A TabPyApp._parse_pwd_file() 0 10 3
B TabPyApp._validate_cert_key_state() 0 18 6
A TabPyApp.__init__() 0 14 4
A TabPyApp._validate_transfer_protocol_settings() 0 29 4
D TabPyApp._parse_config() 0 132 11
A TabPyApp._get_features() 0 11 2
A TabPyApp._build_tabpy_state() 0 17 2

1 Function

Rating   Name   Duplication   Size   Complexity  
B _init_asyncio_patch() 0 17 6

How to fix   Complexity   

Complexity

Complex classes like tabpy.tabpy_server.app.app often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.

Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.

1 1
import concurrent.futures
2 1
import configparser
3 1
import logging
4 1
import multiprocessing
5 1
import os
6 1
import shutil
7 1
import signal
8 1
import sys
9 1
import tabpy
10 1
from tabpy.tabpy import __version__
11 1
from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters
12 1
from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters
13 1
from tabpy.tabpy_server.app.util import parse_pwd_file
14 1
from tabpy.tabpy_server.management.state import TabPyState
15 1
from tabpy.tabpy_server.management.util import _get_state_from_file
16 1
from tabpy.tabpy_server.psws.callbacks import init_model_evaluator, init_ps_server
17 1
from tabpy.tabpy_server.psws.python_service import PythonService, PythonServiceHandler
18 1
from tabpy.tabpy_server.handlers import (
19 1
    EndpointHandler,
20
    EndpointsHandler,
21
    EvaluationPlaneHandler,
22
    QueryPlaneHandler,
23
    ServiceInfoHandler,
24
    StatusHandler,
25
    UploadDestinationHandler,
26
)
27
import tornado
28 1
29
30
logger = logging.getLogger(__name__)
31 1
32
33
def _init_asyncio_patch():
34 1
    """
35
    Select compatible event loop for Tornado 5+.
36
    As of Python 3.8, the default event loop on Windows is `proactor`,
37
    however Tornado requires the old default "selector" event loop.
38
    As Tornado has decided to leave this to users to set, MkDocs needs
39
    to set it. See https://github.com/tornadoweb/tornado/issues/2608.
40
    """
41
    if sys.platform.startswith("win") and sys.version_info >= (3, 8):
42 1
        import asyncio
43
        try:
44
            from asyncio import WindowsSelectorEventLoopPolicy
45
        except ImportError:
46
            pass  # Can't assign a policy which doesn't exist.
47
        else:
48
            if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy):
49
                asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy())
50
51
52
class TabPyApp:
53 1
    """
54
    TabPy application class for keeping context like settings, state, etc.
55
    """
56
57
    settings = {}
58 1
    subdirectory = ""
59 1
    tabpy_state = None
60 1
    python_service = None
61 1
    credentials = {}
62 1
63
    def __init__(self, config_file):
64 1
        if config_file is None:
65 1
            config_file = os.path.join(
66 1
                os.path.dirname(__file__), os.path.pardir, "common", "default.conf"
67
            )
68
69
        if os.path.isfile(config_file):
70 1
            try:
71 1
                from logging import config
72 1
                config.fileConfig(config_file, disable_existing_loggers=False)
73 1
            except KeyError:
74 1
                logging.basicConfig(level=logging.DEBUG)
75
76 1
        self._parse_config(config_file)
77
78 1
    def run(self):
79
        application = self._create_tornado_web_app()
80
        max_request_size = (
81
            int(self.settings[SettingsParameters.MaxRequestSizeInMb]) * 1024 * 1024
82
        )
83
        logger.info(f"Setting max request size to {max_request_size} bytes")
84
85
        init_model_evaluator(self.settings, self.tabpy_state, self.python_service)
86
87
        protocol = self.settings[SettingsParameters.TransferProtocol]
88
        ssl_options = None
89
        if protocol == "https":
90
            ssl_options = {
91
                "certfile": self.settings[SettingsParameters.CertificateFile],
92
                "keyfile": self.settings[SettingsParameters.KeyFile],
93
            }
94
        elif protocol != "http":
95
            msg = f"Unsupported transfer protocol {protocol}."
96
            logger.critical(msg)
97
            raise RuntimeError(msg)
98
99
        application.listen(
100
            self.settings[SettingsParameters.Port],
101
            ssl_options=ssl_options,
102
            max_buffer_size=max_request_size,
103
            max_body_size=max_request_size,
104
        )
105
106
        logger.info(
107
            "Web service listening on port "
108
            f"{str(self.settings[SettingsParameters.Port])}"
109
        )
110
        tornado.ioloop.IOLoop.instance().start()
111
112 1
    def _create_tornado_web_app(self):
113 1
        class TabPyTornadoApp(tornado.web.Application):
114 1
            is_closing = False
115
116 1
            def signal_handler(self, signal, _):
117
                logger.critical(f"Exiting on signal {signal}...")
118
                self.is_closing = True
119
120 1
            def try_exit(self):
121
                if self.is_closing:
122
                    tornado.ioloop.IOLoop.instance().stop()
123
                    logger.info("Shutting down TabPy...")
124
125 1
        logger.info("Initializing TabPy...")
126 1
        tornado.ioloop.IOLoop.instance().run_sync(
127
            lambda: init_ps_server(self.settings, self.tabpy_state)
128
        )
129 1
        logger.info("Done initializing TabPy.")
130
131 1
        executor = concurrent.futures.ThreadPoolExecutor(
132
            max_workers=multiprocessing.cpu_count()
133
        )
134
135
        # initialize Tornado application
136 1
        _init_asyncio_patch()
137 1
        application = TabPyTornadoApp(
138
            [
139
                # skip MainHandler to use StaticFileHandler .* page requests and
140
                # default to index.html
141
                # (r"/", MainHandler),
142
                (
143
                    self.subdirectory + r"/query/([^/]+)",
144
                    QueryPlaneHandler,
145
                    dict(app=self),
146
                ),
147
                (self.subdirectory + r"/status", StatusHandler, dict(app=self)),
148
                (self.subdirectory + r"/info", ServiceInfoHandler, dict(app=self)),
149
                (self.subdirectory + r"/endpoints", EndpointsHandler, dict(app=self)),
150
                (
151
                    self.subdirectory + r"/endpoints/([^/]+)?",
152
                    EndpointHandler,
153
                    dict(app=self),
154
                ),
155
                (
156
                    self.subdirectory + r"/evaluate",
157
                    EvaluationPlaneHandler,
158
                    dict(executor=executor, app=self),
159
                ),
160
                (
161
                    self.subdirectory + r"/configurations/endpoint_upload_destination",
162
                    UploadDestinationHandler,
163
                    dict(app=self),
164
                ),
165
                (
166
                    self.subdirectory + r"/(.*)",
167
                    tornado.web.StaticFileHandler,
168
                    dict(
169
                        path=self.settings[SettingsParameters.StaticPath],
170
                        default_filename="index.html",
171
                    ),
172
                ),
173
            ],
174
            debug=False,
175
            **self.settings,
176
        )
177
178 1
        signal.signal(signal.SIGINT, application.signal_handler)
179 1
        tornado.ioloop.PeriodicCallback(application.try_exit, 500).start()
180
181 1
        signal.signal(signal.SIGINT, application.signal_handler)
182 1
        tornado.ioloop.PeriodicCallback(application.try_exit, 500).start()
183
184 1
        return application
185
186 1
    def _set_parameter(self, parser, settings_key, config_key, default_val, parse_function):
187 1
        key_is_set = False
188
189 1
        if (
190
            config_key is not None
191
            and parser.has_section("TabPy")
192
            and parser.has_option("TabPy", config_key)
193
        ):
194 1
            if parse_function is None:
195 1
                parse_function = parser.get
196 1
            self.settings[settings_key] = parse_function("TabPy", config_key)
197 1
            key_is_set = True
198 1
            logger.debug(
199
                f"Parameter {settings_key} set to "
200
                f'"{self.settings[settings_key]}" '
201
                "from config file or environment variable"
202
            )
203
204 1
        if not key_is_set and default_val is not None:
205 1
            self.settings[settings_key] = default_val
206 1
            key_is_set = True
207 1
            logger.debug(
208
                f"Parameter {settings_key} set to "
209
                f'"{self.settings[settings_key]}" '
210
                "from default value"
211
            )
212
213 1
        if not key_is_set:
214 1
            logger.debug(f"Parameter {settings_key} is not set")
215
216 1
    def _parse_config(self, config_file):
217
        """Provide consistent mechanism for pulling in configuration.
218
219
        Attempt to retain backward compatibility for
220
        existing implementations by grabbing port
221
        setting from CLI first.
222
223
        Take settings in the following order:
224
225
        1. CLI arguments if present
226
        2. config file
227
        3. OS environment variables (for ease of
228
           setting defaults if not present)
229
        4. current defaults if a setting is not present in any location
230
231
        Additionally provide similar configuration capabilities in between
232
        config file and environment variables.
233
        For consistency use the same variable name in the config file as
234
        in the os environment.
235
        For naming standards use all capitals and start with 'TABPY_'
236
        """
237 1
        self.settings = {}
238 1
        self.subdirectory = ""
239 1
        self.tabpy_state = None
240 1
        self.python_service = None
241 1
        self.credentials = {}
242
243 1
        pkg_path = os.path.dirname(tabpy.__file__)
244
245 1
        parser = configparser.ConfigParser(os.environ)
246
        logger.info(f"Parsing config file {config_file}")
247 1
248 1
        file_exists = False
249 1
        if os.path.isfile(config_file):
250
            try:
251 1
                with open(config_file, 'r') as f:
252
                    parser.read_string(f.read())
253
                    file_exists = True
254
            except Exception:
255
                pass
256 1
257
        if not file_exists:
258
            logger.warning(
259
                f"Unable to open config file {config_file}, "
260
                "using default settings."
261
            )
262
263
        settings_parameters = [
264
            (SettingsParameters.Port, ConfigParameters.TABPY_PORT, 9004, None),
265
            (SettingsParameters.ServerVersion, None, __version__, None),
266
            (SettingsParameters.EvaluateTimeout, ConfigParameters.TABPY_EVALUATE_TIMEOUT,
267
             30, parser.getfloat),
268
            (SettingsParameters.UploadDir, ConfigParameters.TABPY_QUERY_OBJECT_PATH,
269
             os.path.join(pkg_path, "tmp", "query_objects"), None),
270
            (SettingsParameters.TransferProtocol, ConfigParameters.TABPY_TRANSFER_PROTOCOL,
271
             "http", None),
272
            (SettingsParameters.CertificateFile, ConfigParameters.TABPY_CERTIFICATE_FILE,
273
             None, None),
274
            (SettingsParameters.KeyFile, ConfigParameters.TABPY_KEY_FILE, None, None),
275
            (SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH,
276
             os.path.join(pkg_path, "tabpy_server"), None),
277
            (SettingsParameters.StaticPath, ConfigParameters.TABPY_STATIC_PATH,
278
             os.path.join(pkg_path, "tabpy_server", "static"), None),
279 1
            (ConfigParameters.TABPY_PWD_FILE, ConfigParameters.TABPY_PWD_FILE, None, None),
280 1
            (SettingsParameters.LogRequestContext, ConfigParameters.TABPY_LOG_DETAILS,
281
             "false", None),
282 1
            (SettingsParameters.MaxRequestSizeInMb, ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB,
283 1
             100, None),
284
        ]
285
286 1
        for setting, parameter, default_val, parse_function in settings_parameters:
287
            self._set_parameter(parser, setting, parameter, default_val, parse_function)
288
289
        if not os.path.exists(self.settings[SettingsParameters.UploadDir]):
290 1
            os.makedirs(self.settings[SettingsParameters.UploadDir])
291
292
        # set and validate transfer protocol
293
        self.settings[SettingsParameters.TransferProtocol] = self.settings[
294 1
            SettingsParameters.TransferProtocol
295
        ].lower()
296
297
        self._validate_transfer_protocol_settings()
298
299 1
        # if state.ini does not exist try and create it - remove
300
        # last dependence on batch/shell script
301 1
        self.settings[SettingsParameters.StateFilePath] = os.path.realpath(
302 1
            os.path.normpath(
303 1
                os.path.expanduser(self.settings[SettingsParameters.StateFilePath])
304
            )
305
        )
306 1
        state_config, self.tabpy_state = self._build_tabpy_state()
307
308
        self.python_service = PythonServiceHandler(PythonService())
309
        self.settings["compress_response"] = True
310
        self.settings[SettingsParameters.StaticPath] = os.path.abspath(
311
            self.settings[SettingsParameters.StaticPath]
312 1
        )
313 1
        logger.debug(
314
            f"Static pages folder set to "
315
            f'"{self.settings[SettingsParameters.StaticPath]}"'
316 1
        )
317 1
318 1
        # Set subdirectory from config if applicable
319
        if state_config.has_option("Service Info", "Subdirectory"):
320
            self.subdirectory = "/" + state_config.get("Service Info", "Subdirectory")
321
322 1
        # If passwords file specified load credentials
323 1
        if ConfigParameters.TABPY_PWD_FILE in self.settings:
324
            if not self._parse_pwd_file():
325 1
                msg = (
326
                    "Failed to read passwords file "
327
                    f"{self.settings[ConfigParameters.TABPY_PWD_FILE]}"
328
                )
329 1
                logger.critical(msg)
330 1
                raise RuntimeError(msg)
331
        else:
332 1
            logger.info(
333
                "Password file is not specified: " "Authentication is not enabled"
334
            )
335 1
336
        features = self._get_features()
337
        self.settings[SettingsParameters.ApiVersions] = {"v1": {"features": features}}
338
339
        self.settings[SettingsParameters.LogRequestContext] = (
340 1
            self.settings[SettingsParameters.LogRequestContext].lower() != "false"
341
        )
342 1
        call_context_state = (
343 1
            "enabled"
344
            if self.settings[SettingsParameters.LogRequestContext]
345
            else "disabled"
346
        )
347
        logger.info(f"Call context logging is {call_context_state}")
348 1
349
    def _validate_transfer_protocol_settings(self):
350 1
        if SettingsParameters.TransferProtocol not in self.settings:
351 1
            msg = "Missing transfer protocol information."
352
            logger.critical(msg)
353 1
            raise RuntimeError(msg)
354 1
355 1
        protocol = self.settings[SettingsParameters.TransferProtocol]
356 1
357
        if protocol == "http":
358 1
            return
359
360
        if protocol != "https":
361
            msg = f"Unsupported transfer protocol: {protocol}"
362
            logger.critical(msg)
363 1
            raise RuntimeError(msg)
364
365 1
        self._validate_cert_key_state(
366
            "The parameter(s) {} must be set.",
367
            SettingsParameters.CertificateFile in self.settings,
368
            SettingsParameters.KeyFile in self.settings,
369
        )
370 1
        cert = self.settings[SettingsParameters.CertificateFile]
371
372 1
        self._validate_cert_key_state(
373
            "The parameter(s) {} must point to " "an existing file.",
374 1
            os.path.isfile(cert),
375
            os.path.isfile(self.settings[SettingsParameters.KeyFile]),
376
        )
377
        tabpy.tabpy_server.app.util.validate_cert(cert)
378 1
379 1
    @staticmethod
380 1
    def _validate_cert_key_state(msg, cert_valid, key_valid):
381 1
        cert_and_key_param = (
382 1
            f"{ConfigParameters.TABPY_CERTIFICATE_FILE} and "
383 1
            f"{ConfigParameters.TABPY_KEY_FILE}"
384 1
        )
385 1
        https_error = "Error using HTTPS: "
386
        err = None
387 1
        if not cert_valid and not key_valid:
388 1
            err = https_error + msg.format(cert_and_key_param)
389 1
        elif not cert_valid:
390
            err = https_error + msg.format(ConfigParameters.TABPY_CERTIFICATE_FILE)
391 1
        elif not key_valid:
392 1
            err = https_error + msg.format(ConfigParameters.TABPY_KEY_FILE)
393
394
        if err is not None:
395
            logger.critical(err)
396 1
            raise RuntimeError(err)
397 1
398 1
    def _parse_pwd_file(self):
399
        succeeded, self.credentials = parse_pwd_file(
400 1
            self.settings[ConfigParameters.TABPY_PWD_FILE]
401
        )
402 1
403 1
        if succeeded and len(self.credentials) == 0:
404
            logger.error("No credentials found")
405
            succeeded = False
406 1
407 1
        return succeeded
408
409
    def _get_features(self):
410
        features = {}
411
412 1
        # Check for auth
413
        if ConfigParameters.TABPY_PWD_FILE in self.settings:
414 1
            features["authentication"] = {
415 1
                "required": True,
416 1
                "methods": {"basic-auth": {}},
417 1
            }
418 1
419
        return features
420
421
    def _build_tabpy_state(self):
422
        pkg_path = os.path.dirname(tabpy.__file__)
423
        state_file_dir = self.settings[SettingsParameters.StateFilePath]
424
        state_file_path = os.path.join(state_file_dir, "state.ini")
425
        if not os.path.isfile(state_file_path):
426
            state_file_template_path = os.path.join(
427
                pkg_path, "tabpy_server", "state.ini.template"
428 1
            )
429 1
            logger.debug(
430 1
                f"File {state_file_path} not found, creating from "
431
                f"template {state_file_template_path}..."
432
            )
433
            shutil.copy(state_file_template_path, state_file_path)
434
435
        logger.info(f"Loading state from state file {state_file_path}")
436
        tabpy_state = _get_state_from_file(state_file_dir)
437
        return tabpy_state, TabPyState(config=tabpy_state, settings=self.settings)
438