Complex classes like Crawler often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes. You can also have a look at the cohesion graph to spot any un-connected, or weakly-connected components.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use Crawler, and based on these observations, apply Extract Interface, too.
| 1 | <?php |
||
| 24 | class Crawler |
||
| 25 | { |
||
| 26 | /** @var \GuzzleHttp\Client */ |
||
| 27 | protected $client; |
||
| 28 | |||
| 29 | /** @var \Psr\Http\Message\UriInterface */ |
||
| 30 | protected $baseUrl; |
||
| 31 | |||
| 32 | /** @var array[\Spatie\Crawler\CrawlObserver] */ |
||
| 33 | protected $crawlObservers; |
||
| 34 | |||
| 35 | /** @var \Spatie\Crawler\CrawlProfile */ |
||
| 36 | protected $crawlProfile; |
||
| 37 | |||
| 38 | /** @var int */ |
||
| 39 | protected $concurrency; |
||
| 40 | |||
| 41 | /** @var \Spatie\Crawler\CrawlQueue\CrawlQueue */ |
||
| 42 | protected $crawlQueue; |
||
| 43 | |||
| 44 | /** @var int */ |
||
| 45 | protected $crawledUrlCount = 0; |
||
| 46 | |||
| 47 | /** @var int|null */ |
||
| 48 | protected $maximumCrawlCount = null; |
||
| 49 | |||
| 50 | /** @var int */ |
||
| 51 | protected $maximumResponseSize = 1024 * 1024 * 2; |
||
| 52 | |||
| 53 | /** @var int|null */ |
||
| 54 | protected $maximumDepth = null; |
||
| 55 | |||
| 56 | /** @var \Tree\Node\Node */ |
||
| 57 | protected $depthTree; |
||
| 58 | |||
| 59 | /** @var bool */ |
||
| 60 | protected $executeJavaScript = false; |
||
| 61 | |||
| 62 | /** @var Browsershot */ |
||
| 63 | protected $browsershot = null; |
||
| 64 | |||
| 65 | protected static $defaultClientOptions = [ |
||
| 66 | RequestOptions::COOKIES => true, |
||
| 67 | RequestOptions::CONNECT_TIMEOUT => 10, |
||
| 68 | RequestOptions::TIMEOUT => 10, |
||
| 69 | RequestOptions::ALLOW_REDIRECTS => false, |
||
| 70 | ]; |
||
| 71 | |||
| 72 | /** |
||
| 73 | * @param array $clientOptions |
||
| 74 | * |
||
| 75 | * @return static |
||
| 76 | */ |
||
| 77 | public static function create(array $clientOptions = []) |
||
| 87 | |||
| 88 | public function __construct(Client $client, int $concurrency = 10) |
||
| 98 | |||
| 99 | /** |
||
| 100 | * @param int $concurrency |
||
| 101 | * |
||
| 102 | * @return $this |
||
| 103 | */ |
||
| 104 | public function setConcurrency(int $concurrency) |
||
| 110 | |||
| 111 | /** |
||
| 112 | * Responses that are larger that then specified value will be ignored. |
||
| 113 | * |
||
| 114 | * @param int $maximumResponseSizeInBytes |
||
| 115 | * |
||
| 116 | * @return $this |
||
| 117 | */ |
||
| 118 | public function setMaximumResponseSize(int $maximumResponseSizeInBytes) |
||
| 124 | |||
| 125 | /** |
||
| 126 | * @param int $maximumCrawlCount |
||
| 127 | * |
||
| 128 | * @return $this |
||
| 129 | */ |
||
| 130 | public function setMaximumCrawlCount(int $maximumCrawlCount) |
||
| 136 | |||
| 137 | /** |
||
| 138 | * @param int $maximumDepth |
||
| 139 | * |
||
| 140 | * @return $this |
||
| 141 | */ |
||
| 142 | public function setMaximumDepth(int $maximumDepth) |
||
| 148 | |||
| 149 | /** |
||
| 150 | * @param CrawlQueue $crawlQueue |
||
| 151 | * @return $this |
||
| 152 | */ |
||
| 153 | public function setCrawlQueue(CrawlQueue $crawlQueue) |
||
| 159 | |||
| 160 | /** |
||
| 161 | * @return $this |
||
| 162 | */ |
||
| 163 | public function executeJavaScript() |
||
| 169 | |||
| 170 | /** |
||
| 171 | * @return $this |
||
| 172 | */ |
||
| 173 | public function doNotExecuteJavaScript() |
||
| 179 | |||
| 180 | /** |
||
| 181 | * @param \Spatie\Crawler\CrawlObserver|array[\Spatie\Crawler\CrawlObserver] $crawlObservers |
||
|
|
|||
| 182 | * |
||
| 183 | * @return $this |
||
| 184 | */ |
||
| 185 | public function setCrawlObserver($crawlObservers) |
||
| 193 | |||
| 194 | public function setCrawlObservers(array $crawlObservers) |
||
| 200 | |||
| 201 | public function addCrawlObserver(CrawlObserver $crawlObserver) |
||
| 207 | |||
| 208 | /** |
||
| 209 | * @param \Spatie\Crawler\CrawlProfile $crawlProfile |
||
| 210 | * |
||
| 211 | * @return $this |
||
| 212 | */ |
||
| 213 | public function setCrawlProfile(CrawlProfile $crawlProfile) |
||
| 219 | |||
| 220 | /** |
||
| 221 | * @param \Psr\Http\Message\UriInterface|string $baseUrl |
||
| 222 | */ |
||
| 223 | public function startCrawling($baseUrl) |
||
| 251 | |||
| 252 | protected function startCrawlingQueue() |
||
| 288 | |||
| 289 | public function endsWith($haystack, $needle) |
||
| 294 | |||
| 295 | protected function convertBodyToString(StreamInterface $bodyStream, $readMaximumBytes = 1024 * 1024 * 2): string |
||
| 303 | |||
| 304 | /** |
||
| 305 | * @param ResponseInterface|null $response |
||
| 306 | * @param CrawlUrl $crawlUrl |
||
| 307 | */ |
||
| 308 | protected function handleCrawled(ResponseInterface $response, CrawlUrl $crawlUrl) |
||
| 318 | |||
| 319 | /** |
||
| 320 | * @param RequestException $exception |
||
| 321 | * @param CrawlUrl $crawlUrl |
||
| 322 | */ |
||
| 323 | protected function handleCrawlFailed(RequestException $exception, CrawlUrl $crawlUrl) |
||
| 333 | |||
| 334 | protected function getCrawlRequests(): Generator |
||
| 355 | |||
| 356 | protected function addAllLinksToCrawlQueue(string $html, UriInterface $foundOnUrl) |
||
| 393 | |||
| 394 | protected function shouldCrawl(Node $node): bool |
||
| 402 | |||
| 403 | protected function extractAllLinks(string $html, UriInterface $foundOnUrl): Collection |
||
| 421 | |||
| 422 | protected function normalizeUrl(UriInterface $url): UriInterface |
||
| 426 | |||
| 427 | protected function hasCrawlableScheme(UriInterface $uri): bool |
||
| 431 | |||
| 432 | protected function addtoDepthTree(Node $node, UriInterface $url, UriInterface $parentUrl) |
||
| 454 | |||
| 455 | protected function getBodyAfterExecutingJavaScript(UriInterface $foundOnUrl): string |
||
| 463 | |||
| 464 | protected function getBrowsershot(): Browsershot |
||
| 474 | |||
| 475 | public function setBrowsershot(Browsershot $browsershot) |
||
| 481 | |||
| 482 | protected function addToCrawlQueue(CrawlUrl $crawlUrl) |
||
| 490 | |||
| 491 | protected function maximumCrawlCountReached(): bool |
||
| 499 | } |
||
| 500 |
This check marks PHPDoc comments that could not be parsed by our parser. To see which comment annotations we can parse, please refer to our documentation on supported doc-types.