binux /
pyspider
| Conditions | 45 |
| Total Lines | 158 |
| Lines | 0 |
| Ratio | 0 % |
Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.
For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.
Commonly applied refactorings include:
If many parameters/temporary variables are present:
Complex classes like pyspider.fetcher.Fetcher.http_fetch() often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
| 1 | #!/usr/bin/env python |
||
| 181 | def http_fetch(self, url, task, callback): |
||
| 182 | '''HTTP fetcher''' |
||
| 183 | start_time = time.time() |
||
| 184 | |||
| 185 | self.on_fetch('http', task) |
||
| 186 | fetch = copy.deepcopy(self.default_options) |
||
| 187 | fetch['url'] = url |
||
| 188 | fetch['headers'] = tornado.httputil.HTTPHeaders(fetch['headers']) |
||
| 189 | fetch['headers']['User-Agent'] = self.user_agent |
||
| 190 | task_fetch = task.get('fetch', {}) |
||
| 191 | for each in self.allowed_options: |
||
| 192 | if each in task_fetch: |
||
| 193 | fetch[each] = task_fetch[each] |
||
| 194 | fetch['headers'].update(task_fetch.get('headers', {})) |
||
| 195 | |||
| 196 | if task.get('track'): |
||
| 197 | track_headers = tornado.httputil.HTTPHeaders( |
||
| 198 | task.get('track', {}).get('fetch', {}).get('headers') or {}) |
||
| 199 | track_ok = task.get('track', {}).get('process', {}).get('ok', False) |
||
| 200 | else: |
||
| 201 | track_headers = {} |
||
| 202 | track_ok = False |
||
| 203 | # proxy |
||
| 204 | proxy_string = None |
||
| 205 | if isinstance(task_fetch.get('proxy'), six.string_types): |
||
| 206 | proxy_string = task_fetch['proxy'] |
||
| 207 | elif self.proxy and task_fetch.get('proxy', True): |
||
| 208 | proxy_string = self.proxy |
||
| 209 | if proxy_string: |
||
| 210 | if '://' not in proxy_string: |
||
| 211 | proxy_string = 'http://' + proxy_string |
||
| 212 | proxy_splited = urlsplit(proxy_string) |
||
| 213 | if proxy_splited.username: |
||
| 214 | fetch['proxy_username'] = proxy_splited.username |
||
| 215 | if six.PY2: |
||
| 216 | fetch['proxy_username'] = fetch['proxy_username'].encode('utf8') |
||
| 217 | if proxy_splited.password: |
||
| 218 | fetch['proxy_password'] = proxy_splited.password |
||
| 219 | if six.PY2: |
||
| 220 | fetch['proxy_password'] = fetch['proxy_password'].encode('utf8') |
||
| 221 | fetch['proxy_host'] = proxy_splited.hostname.encode('utf8') |
||
| 222 | if six.PY2: |
||
| 223 | fetch['proxy_host'] = fetch['proxy_host'].encode('utf8') |
||
| 224 | fetch['proxy_port'] = proxy_splited.port or 8080 |
||
| 225 | |||
| 226 | # etag |
||
| 227 | if task_fetch.get('etag', True): |
||
| 228 | _t = None |
||
| 229 | if isinstance(task_fetch.get('etag'), six.string_types): |
||
| 230 | _t = task_fetch.get('etag') |
||
| 231 | elif track_ok: |
||
| 232 | _t = track_headers.get('etag') |
||
| 233 | if _t: |
||
| 234 | fetch['headers'].setdefault('If-None-Match', _t) |
||
| 235 | # last modifed |
||
| 236 | if task_fetch.get('last_modified', True): |
||
| 237 | _t = None |
||
| 238 | if isinstance(task_fetch.get('last_modifed'), six.string_types): |
||
| 239 | _t = task_fetch.get('last_modifed') |
||
| 240 | elif track_ok: |
||
| 241 | _t = track_headers.get('last-modified') |
||
| 242 | if _t: |
||
| 243 | fetch['headers'].setdefault('If-Modified-Since', _t) |
||
| 244 | |||
| 245 | session = cookies.RequestsCookieJar() |
||
| 246 | |||
| 247 | # fix for tornado request obj |
||
| 248 | if 'Cookie' in fetch['headers']: |
||
| 249 | c = http_cookies.SimpleCookie() |
||
| 250 | try: |
||
| 251 | c.load(fetch['headers']['Cookie']) |
||
| 252 | except AttributeError: |
||
| 253 | c.load(utils.utf8(fetch['headers']['Cookie'])) |
||
| 254 | for key in c: |
||
| 255 | session.set(key, c[key]) |
||
| 256 | del fetch['headers']['Cookie'] |
||
| 257 | fetch['follow_redirects'] = False |
||
| 258 | if 'timeout' in fetch: |
||
| 259 | fetch['connect_timeout'] = fetch['request_timeout'] = fetch['timeout'] |
||
| 260 | del fetch['timeout'] |
||
| 261 | if 'data' in fetch: |
||
| 262 | fetch['body'] = fetch['data'] |
||
| 263 | del fetch['data'] |
||
| 264 | if 'cookies' in fetch: |
||
| 265 | session.update(fetch['cookies']) |
||
| 266 | del fetch['cookies'] |
||
| 267 | |||
| 268 | store = {} |
||
| 269 | store['max_redirects'] = task_fetch.get('max_redirects', 5) |
||
| 270 | |||
| 271 | def handle_response(response): |
||
| 272 | extract_cookies_to_jar(session, response.request, response.headers) |
||
| 273 | if (response.code in (301, 302, 303, 307) |
||
| 274 | and response.headers.get('Location') |
||
| 275 | and task_fetch.get('allow_redirects', True)): |
||
| 276 | if store['max_redirects'] <= 0: |
||
| 277 | error = tornado.httpclient.HTTPError( |
||
| 278 | 599, 'Maximum (%d) redirects followed' % task_fetch.get('max_redirects', 5), |
||
| 279 | response) |
||
| 280 | return handle_error(error) |
||
| 281 | if response.code in (302, 303): |
||
| 282 | fetch['method'] = 'GET' |
||
| 283 | if 'body' in fetch: |
||
| 284 | del fetch['body'] |
||
| 285 | fetch['url'] = urljoin(fetch['url'], response.headers['Location']) |
||
| 286 | fetch['request_timeout'] -= time.time() - start_time |
||
| 287 | if fetch['request_timeout'] < 0: |
||
| 288 | fetch['request_timeout'] = 0.1 |
||
| 289 | fetch['connect_timeout'] = fetch['request_timeout'] |
||
| 290 | store['max_redirects'] -= 1 |
||
| 291 | return make_request(fetch) |
||
| 292 | |||
| 293 | result = {} |
||
| 294 | result['orig_url'] = url |
||
| 295 | result['content'] = response.body or '' |
||
| 296 | result['headers'] = dict(response.headers) |
||
| 297 | result['status_code'] = response.code |
||
| 298 | result['url'] = response.effective_url or url |
||
| 299 | result['cookies'] = session.get_dict() |
||
| 300 | result['time'] = time.time() - start_time |
||
| 301 | result['save'] = task_fetch.get('save') |
||
| 302 | if response.error: |
||
| 303 | result['error'] = utils.text(response.error) |
||
| 304 | if 200 <= response.code < 300: |
||
| 305 | logger.info("[%d] %s:%s %s %.2fs", response.code, |
||
| 306 | task.get('project'), task.get('taskid'), |
||
| 307 | url, result['time']) |
||
| 308 | else: |
||
| 309 | logger.warning("[%d] %s:%s %s %.2fs", response.code, |
||
| 310 | task.get('project'), task.get('taskid'), |
||
| 311 | url, result['time']) |
||
| 312 | callback('http', task, result) |
||
| 313 | self.on_result('http', task, result) |
||
| 314 | return task, result |
||
| 315 | |||
| 316 | handle_error = lambda x: self.handle_error('http', |
||
| 317 | url, task, start_time, callback, x) |
||
| 318 | |||
| 319 | def make_request(fetch): |
||
| 320 | try: |
||
| 321 | request = tornado.httpclient.HTTPRequest(**fetch) |
||
| 322 | cookie_header = cookies.get_cookie_header(session, request) |
||
| 323 | if cookie_header: |
||
| 324 | request.headers['Cookie'] = cookie_header |
||
| 325 | if self.async: |
||
| 326 | self.http_client.fetch(request, handle_response) |
||
| 327 | else: |
||
| 328 | return handle_response(self.http_client.fetch(request)) |
||
| 329 | except tornado.httpclient.HTTPError as e: |
||
| 330 | if e.response: |
||
| 331 | return handle_response(e.response) |
||
| 332 | else: |
||
| 333 | return handle_error(e) |
||
| 334 | except Exception as e: |
||
| 335 | logger.exception(fetch) |
||
| 336 | return handle_error(e) |
||
| 337 | |||
| 338 | return make_request(fetch) |
||
| 339 | |||
| 526 |
