binux /
pyspider
| Conditions | 40 |
| Total Lines | 124 |
| Lines | 0 |
| Ratio | 0 % |
| Changes | 0 | ||
Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.
For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.
Commonly applied refactorings include:
If many parameters/temporary variables are present:
Complex classes like _connect_database() often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
| 1 | #!/usr/bin/env python |
||
| 49 | def _connect_database(url): # NOQA |
||
| 50 | parsed = urlparse(url) |
||
| 51 | |||
| 52 | scheme = parsed.scheme.split('+') |
||
| 53 | if len(scheme) == 1: |
||
| 54 | raise Exception('wrong scheme format: %s' % parsed.scheme) |
||
| 55 | else: |
||
| 56 | engine, dbtype = scheme[0], scheme[-1] |
||
| 57 | other_scheme = "+".join(scheme[1:-1]) |
||
| 58 | |||
| 59 | if dbtype not in ('taskdb', 'projectdb', 'resultdb'): |
||
| 60 | raise LookupError('unknown database type: %s, ' |
||
| 61 | 'type should be one of ["taskdb", "projectdb", "resultdb"]', dbtype) |
||
| 62 | |||
| 63 | if engine == 'mysql': |
||
| 64 | parames = {} |
||
| 65 | if parsed.username: |
||
| 66 | parames['user'] = parsed.username |
||
| 67 | if parsed.password: |
||
| 68 | parames['passwd'] = parsed.password |
||
| 69 | if parsed.hostname: |
||
| 70 | parames['host'] = parsed.hostname |
||
| 71 | if parsed.port: |
||
| 72 | parames['port'] = parsed.port |
||
| 73 | if parsed.path.strip('/'): |
||
| 74 | parames['database'] = parsed.path.strip('/') |
||
| 75 | |||
| 76 | if dbtype == 'taskdb': |
||
| 77 | from .mysql.taskdb import TaskDB |
||
| 78 | return TaskDB(**parames) |
||
| 79 | elif dbtype == 'projectdb': |
||
| 80 | from .mysql.projectdb import ProjectDB |
||
| 81 | return ProjectDB(**parames) |
||
| 82 | elif dbtype == 'resultdb': |
||
| 83 | from .mysql.resultdb import ResultDB |
||
| 84 | return ResultDB(**parames) |
||
| 85 | else: |
||
| 86 | raise LookupError |
||
| 87 | elif engine == 'sqlite': |
||
| 88 | if parsed.path.startswith('//'): |
||
| 89 | path = '/' + parsed.path.strip('/') |
||
| 90 | elif parsed.path.startswith('/'): |
||
| 91 | path = './' + parsed.path.strip('/') |
||
| 92 | elif not parsed.path: |
||
| 93 | path = ':memory:' |
||
| 94 | else: |
||
| 95 | raise Exception('error path: %s' % parsed.path) |
||
| 96 | |||
| 97 | if dbtype == 'taskdb': |
||
| 98 | from .sqlite.taskdb import TaskDB |
||
| 99 | return TaskDB(path) |
||
| 100 | elif dbtype == 'projectdb': |
||
| 101 | from .sqlite.projectdb import ProjectDB |
||
| 102 | return ProjectDB(path) |
||
| 103 | elif dbtype == 'resultdb': |
||
| 104 | from .sqlite.resultdb import ResultDB |
||
| 105 | return ResultDB(path) |
||
| 106 | else: |
||
| 107 | raise LookupError |
||
| 108 | elif engine == 'mongodb': |
||
| 109 | url = url.replace(parsed.scheme, 'mongodb') |
||
| 110 | parames = {} |
||
| 111 | if parsed.path.strip('/'): |
||
| 112 | parames['database'] = parsed.path.strip('/') |
||
| 113 | |||
| 114 | if dbtype == 'taskdb': |
||
| 115 | from .mongodb.taskdb import TaskDB |
||
| 116 | return TaskDB(url, **parames) |
||
| 117 | elif dbtype == 'projectdb': |
||
| 118 | from .mongodb.projectdb import ProjectDB |
||
| 119 | return ProjectDB(url, **parames) |
||
| 120 | elif dbtype == 'resultdb': |
||
| 121 | from .mongodb.resultdb import ResultDB |
||
| 122 | return ResultDB(url, **parames) |
||
| 123 | else: |
||
| 124 | raise LookupError |
||
| 125 | elif engine == 'sqlalchemy': |
||
| 126 | if not other_scheme: |
||
| 127 | raise Exception('wrong scheme format: %s' % parsed.scheme) |
||
| 128 | url = url.replace(parsed.scheme, other_scheme) |
||
| 129 | |||
| 130 | if dbtype == 'taskdb': |
||
| 131 | from .sqlalchemy.taskdb import TaskDB |
||
| 132 | return TaskDB(url) |
||
| 133 | elif dbtype == 'projectdb': |
||
| 134 | from .sqlalchemy.projectdb import ProjectDB |
||
| 135 | return ProjectDB(url) |
||
| 136 | elif dbtype == 'resultdb': |
||
| 137 | from .sqlalchemy.resultdb import ResultDB |
||
| 138 | return ResultDB(url) |
||
| 139 | else: |
||
| 140 | raise LookupError |
||
| 141 | elif engine == 'redis': |
||
| 142 | if dbtype == 'taskdb': |
||
| 143 | from .redis.taskdb import TaskDB |
||
| 144 | return TaskDB(parsed.hostname, parsed.port, |
||
| 145 | int(parsed.path.strip('/') or 0)) |
||
| 146 | else: |
||
| 147 | raise LookupError('not supported dbtype: %s', dbtype) |
||
| 148 | elif engine == 'local': |
||
| 149 | scripts = url.split('//', 1)[1].split(',') |
||
| 150 | if dbtype == 'projectdb': |
||
| 151 | from .local.projectdb import ProjectDB |
||
| 152 | return ProjectDB(scripts) |
||
| 153 | else: |
||
| 154 | raise LookupError('not supported dbtype: %s', dbtype) |
||
| 155 | elif engine == 'elasticsearch' or engine == 'es': |
||
| 156 | index = parse_qs(parsed.query) |
||
| 157 | if 'index' in index and index['index']: |
||
| 158 | index = index['index'][0] |
||
| 159 | else: |
||
| 160 | index = 'pyspider' |
||
| 161 | |||
| 162 | if dbtype == 'projectdb': |
||
| 163 | from .elasticsearch.projectdb import ProjectDB |
||
| 164 | return ProjectDB([parsed.netloc], index=index) |
||
| 165 | elif dbtype == 'resultdb': |
||
| 166 | from .elasticsearch.resultdb import ResultDB |
||
| 167 | return ResultDB([parsed.netloc], index=index) |
||
| 168 | elif dbtype == 'taskdb': |
||
| 169 | from .elasticsearch.taskdb import TaskDB |
||
| 170 | return TaskDB([parsed.netloc], index=index) |
||
| 171 | else: |
||
| 172 | raise Exception('unknown engine: %s' % engine) |
||
| 173 |
