binux /
pyspider
| @@ 66-79 (lines=14) @@ | ||
| 63 | self.table.create(self.engine, checkfirst=True) |
|
| 64 | self.table.indexes.clear() |
|
| 65 | ||
| 66 | @staticmethod |
|
| 67 | def _parse(data): |
|
| 68 | for key, value in list(six.iteritems(data)): |
|
| 69 | if isinstance(value, six.binary_type): |
|
| 70 | data[key] = utils.text(value) |
|
| 71 | for each in ('schedule', 'fetch', 'process', 'track'): |
|
| 72 | if each in data: |
|
| 73 | if data[each]: |
|
| 74 | if isinstance(data[each], bytearray): |
|
| 75 | data[each] = str(data[each]) |
|
| 76 | data[each] = json.loads(data[each]) |
|
| 77 | else: |
|
| 78 | data[each] = {} |
|
| 79 | return data |
|
| 80 | ||
| 81 | @staticmethod |
|
| 82 | def _stringify(data): |
|
| @@ 31-42 (lines=12) @@ | ||
| 28 | self.database[collection_name].ensure_index('status') |
|
| 29 | self.database[collection_name].ensure_index('taskid') |
|
| 30 | ||
| 31 | def _parse(self, data): |
|
| 32 | if '_id' in data: |
|
| 33 | del data['_id'] |
|
| 34 | for each in ('schedule', 'fetch', 'process', 'track'): |
|
| 35 | if each in data: |
|
| 36 | if data[each]: |
|
| 37 | if isinstance(data[each], bytearray): |
|
| 38 | data[each] = str(data[each]) |
|
| 39 | data[each] = json.loads(data[each], 'utf8') |
|
| 40 | else: |
|
| 41 | data[each] = {} |
|
| 42 | return data |
|
| 43 | ||
| 44 | def _stringify(self, data): |
|
| 45 | for each in ('schedule', 'fetch', 'process', 'track'): |
|
