binux /
pyspider
| @@ 167-177 (lines=11) @@ | ||
| 164 | self.priority_queue.put(task) |
|
| 165 | self.mutex.release() |
|
| 166 | ||
| 167 | def _check_processing(self): |
|
| 168 | now = time.time() |
|
| 169 | self.mutex.acquire() |
|
| 170 | while self.processing.qsize() and self.processing.top and self.processing.top.exetime < now: |
|
| 171 | task = self.processing.get_nowait() |
|
| 172 | if task.taskid is None: |
|
| 173 | continue |
|
| 174 | task.exetime = 0 |
|
| 175 | self.priority_queue.put(task) |
|
| 176 | logger.info("processing: retry %s", task.taskid) |
|
| 177 | self.mutex.release() |
|
| 178 | ||
| 179 | def put(self, taskid, priority=0, exetime=0): |
|
| 180 | """ |
|
| @@ 158-165 (lines=8) @@ | ||
| 155 | self._check_time_queue() |
|
| 156 | self._check_processing() |
|
| 157 | ||
| 158 | def _check_time_queue(self): |
|
| 159 | now = time.time() |
|
| 160 | self.mutex.acquire() |
|
| 161 | while self.time_queue.qsize() and self.time_queue.top and self.time_queue.top.exetime < now: |
|
| 162 | task = self.time_queue.get_nowait() |
|
| 163 | task.exetime = 0 |
|
| 164 | self.priority_queue.put(task) |
|
| 165 | self.mutex.release() |
|
| 166 | ||
| 167 | def _check_processing(self): |
|
| 168 | now = time.time() |
|
