Crawler::getCrawlRequests() - Code Metrics - Inspection of "Add change crawl queue support" - spatie/crawler - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#87)

by Peter

created 2017-12-10 17:23 UTC

Crawler::getCrawlRequests() A

↳ Parent: Crawler

Complexity

Conditions	4
Paths	4

Size

Total Lines	18
Code Lines	9

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
dl	0
loc	18
rs	9.2
c	0
b	0
f	0
cc	4
eloc	9
nc	4
nop	0

<?php

namespace Spatie\Crawler;

use Generator;
use Tree\Node\Node;
use GuzzleHttp\Pool;
use GuzzleHttp\Client;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\RequestOptions;
use Illuminate\Support\Collection;
use Spatie\Browsershot\Browsershot;
use Symfony\Component\DomCrawler\Link;
use Psr\Http\Message\ResponseInterface;
use GuzzleHttp\Exception\RequestException;
use Symfony\Component\DomCrawler\Crawler as DomCrawler;

class Crawler
{
    /** @var \GuzzleHttp\Client */
    protected $client;

    /** @var \Spatie\Crawler\Url */
    protected $baseUrl;

    /** @var \Spatie\Crawler\CrawlObserver */
    protected $crawlObserver;

    /** @var \Spatie\Crawler\CrawlProfile */
    protected $crawlProfile;

    /** @var int */
    protected $concurrency;

    /** @var \Spatie\Crawler\CrawlQueue */
    protected $crawlQueue;

    /** @var int */
    protected $crawledUrlCount = 0;

    /** @var int|null */
    protected $maximumCrawlCount = null;

    /** @var int|null */
    protected $maximumDepth = null;

    /** @var \Tree\Node\Node */
    protected $depthTree;

    /** @var false */
    protected $executeJavaScript = false;

    /** @var string|null */
    protected $pathToChromeBinary = null;

    protected static $defaultClientOptions = [
        RequestOptions::COOKIES => true,
        RequestOptions::CONNECT_TIMEOUT => 10,
        RequestOptions::TIMEOUT => 10,
        RequestOptions::ALLOW_REDIRECTS => false,
    ];

    /**
     * @param array $clientOptions
     *
     * @return static
     */
    public static function create(array $clientOptions = [])
    {
        $clientOptions = (count($clientOptions))
            ? $clientOptions
            : self::$defaultClientOptions;

        $client = new Client($clientOptions);

        return new static($client);
    }

    public function __construct(Client $client, int $concurrency = 10)
    {
        $this->client = $client;

        $this->concurrency = $concurrency;

        $this->crawlProfile = new CrawlAllUrls();

        $this->crawlQueue = new CollectionCrawlQueue();
    }

    /**
     * @param int $concurrency
     *
     * @return $this
     */
    public function setConcurrency(int $concurrency)
    {
        $this->concurrency = $concurrency;

        return $this;
    }

    /**
     * @param int $maximumCrawlCount
     *
     * @return $this
     */
    public function setMaximumCrawlCount(int $maximumCrawlCount)
    {
        $this->maximumCrawlCount = $maximumCrawlCount;

        return $this;
    }

    /**
     * @param int $maximumDepth
     *
     * @return $this
     */
    public function setMaximumDepth(int $maximumDepth)
    {
        $this->maximumDepth = $maximumDepth;

        return $this;
    }

    /**
     * @param CrawlQueue $crawlQueue
     * @return $this
     */
    public function setCrawlQueue(CrawlQueue $crawlQueue)
    {
        $this->crawlQueue = $crawlQueue;

        return $this;
    }

    /**
     * @return $this
     */
    public function executeJavaScript($pathToChromeBinary = null)
    {
        $this->executeJavaScript = true;
$answer = 42;

$correct = false;

$correct = (bool) $answer;

        $this->pathToChromeBinary = $pathToChromeBinary;

        return $this;
    }

    /**
     * @return $this
     */
    public function doNotExecuteJavaScript()
    {
        $this->executeJavaScript = false;

        return $this;
    }

    /**
     * @param \Spatie\Crawler\CrawlObserver $crawlObserver
     *
     * @return $this
     */
    public function setCrawlObserver(CrawlObserver $crawlObserver)
    {
        $this->crawlObserver = $crawlObserver;

        return $this;
    }

    /**
     * @param \Spatie\Crawler\CrawlProfile $crawlProfile
     *
     * @return $this
     */
    public function setCrawlProfile(CrawlProfile $crawlProfile)
    {
        $this->crawlProfile = $crawlProfile;

        return $this;
    }

    /**
     * @param \Spatie\Crawler\Url|string $baseUrl
     */
    public function startCrawling($baseUrl)
    {
        if (! $baseUrl instanceof Url) {
            $baseUrl = Url::create($baseUrl);
        }

        $this->baseUrl = $baseUrl;

        $crawlUrl = CrawlUrl::create($baseUrl);

        $this->addToCrawlQueue($crawlUrl);

        $this->depthTree = new Node((string) $this->baseUrl);

        $this->startCrawlingQueue();

        $this->crawlObserver->finishedCrawling();
    }

    protected function startCrawlingQueue()
    {
        while ($this->crawlQueue->hasPendingUrls()) {
            $pool = new Pool($this->client, $this->getCrawlRequests(), [
                'concurrency' => $this->concurrency,
                'options' => $this->client->getConfig(),
                'fulfilled' => function (ResponseInterface $response, int $index) {
                    $crawlUrl = $this->crawlQueue->getUrlById($index);
                    $this->handleResponse($response, $crawlUrl);

                    if (! $this->crawlProfile instanceof CrawlSubdomains) {
                        if ($crawlUrl->url->host !== $this->baseUrl->host) {
                            return;
                        }
                    }

                    $this->addAllLinksToCrawlQueue(
                        (string) $response->getBody(),
                        $crawlUrl->url
                    );
                },
                'rejected' => function (RequestException $exception, int $index) {
                    $this->handleResponse(
                        $exception->getResponse(),
                        $this->crawlQueue->getUrlById($index)
                    );
                },
            ]);

            $promise = $pool->promise();
            $promise->wait();
        }
    }

    /**
     * @param ResponseInterface|null $response
     * @param CrawlUrl $crawlUrl
     */
    protected function handleResponse($response, CrawlUrl $crawlUrl)
    {
        $this->crawlObserver->hasBeenCrawled($crawlUrl->url, $response, $crawlUrl->foundOnUrl);
    }

    protected function getCrawlRequests(): Generator
    {
        while ($crawlUrl = $this->crawlQueue->getFirstPendingUrl()) {
            if (! $this->crawlProfile->shouldCrawl($crawlUrl->url)) {
                continue;
            }

            if ($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)) {
                continue;
            }

            $this->crawlObserver->willCrawl($crawlUrl->url);

            $this->crawlQueue->markAsProcessed($crawlUrl);

            yield $crawlUrl->getId() => new Request('GET', (string) $crawlUrl->url);
        }
    }

    protected function addAllLinksToCrawlQueue(string $html, Url $foundOnUrl)
    {
        $allLinks = $this->extractAllLinks($html, $foundOnUrl);

        collect($allLinks)
            ->filter(function (Url $url) {
                return $url->hasCrawlableScheme();
            })
            ->map(function (Url $url) {
                return $this->normalizeUrl($url);
            })
            ->filter(function (Url $url) {
                return $this->crawlProfile->shouldCrawl($url);
            })
            ->reject(function ($url) {
                return $this->crawlQueue->has($url);
            })
            ->each(function (Url $url) use ($foundOnUrl) {
                $node = $this->addtoDepthTree($this->depthTree, (string) $url, $foundOnUrl);

                if (! $this->shouldCrawl($node)) {
                    return;
                }

                if ($this->maximumCrawlCountReached()) {
                    return;
                }

                $crawlUrl = CrawlUrl::create($url, $foundOnUrl);

                $this->addToCrawlQueue($crawlUrl);
            });
    }

    protected function shouldCrawl(Node $node): bool
    {
        if (is_null($this->maximumDepth)) {
            return true;
        }

        return $node->getDepth() <= $this->maximumDepth;
    }

    protected function extractAllLinks(string $html, Url $foundOnUrl): Collection
    {
        if ($this->executeJavaScript) {
            $html = $this->getBodyAfterExecutingJavaScript($foundOnUrl);
        }

        $domCrawler = new DomCrawler($html, $foundOnUrl);

        return collect($domCrawler->filterXpath('//a')->links())
            ->map(function (Link $link) {
                return Url::create($link->getUri());
            });
    }

    protected function normalizeUrl(Url $url): Url
    {
        return $url->removeFragment();
    }

    protected function addtoDepthTree(Node $node, string $url, string $parentUrl)
    {
        $returnNode = null;

        if ($node->getValue() === $parentUrl) {
            $newNode = new Node($url);

            $node->addChild($newNode);

            return $newNode;
        }

        foreach ($node->getChildren() as $currentNode) {
            $returnNode = $this->addtoDepthTree($currentNode, $url, $parentUrl);

            if (! is_null($returnNode)) {
                break;
            }
        }

        return $returnNode;
    }

    protected function getBodyAfterExecutingJavaScript(Url $foundOnUrl): string
    {
        $browsershot = Browsershot::url((string) $foundOnUrl);

        if ($this->pathToChromeBinary) {
''   == false // true
''   == null  // true
'ab' == false // false
'ab' == null  // false

// It is often better to use strict comparison
'' === false // false
'' === null  // false
            $browsershot->setChromePath($this->pathToChromeBinary);
        }

        $html = $browsershot->bodyHtml();

        return html_entity_decode($html);
    }

    protected function addToCrawlQueue(CrawlUrl $crawlUrl)
    {
        $this->crawledUrlCount++;

        $this->crawlQueue->add($crawlUrl);

        return $this;
    }

    protected function maximumCrawlCountReached(): bool
    {
        if (is_null($this->maximumCrawlCount)) {
            return false;
        }

        return $this->crawledUrlCount >= $this->maximumCrawlCount;
    }
}


1			<?php
2
3			namespace Spatie\Crawler;
4
5			use Generator;
6			use Tree\Node\Node;
7			use GuzzleHttp\Pool;
8			use GuzzleHttp\Client;
9			use GuzzleHttp\Psr7\Request;
10			use GuzzleHttp\RequestOptions;
11			use Illuminate\Support\Collection;
12			use Spatie\Browsershot\Browsershot;
13			use Symfony\Component\DomCrawler\Link;
14			use Psr\Http\Message\ResponseInterface;
15			use GuzzleHttp\Exception\RequestException;
16			use Symfony\Component\DomCrawler\Crawler as DomCrawler;
17
18			class Crawler
19			{
20			/** @var \GuzzleHttp\Client */
21			protected $client;
22
23			/** @var \Spatie\Crawler\Url */
24			protected $baseUrl;
25
26			/** @var \Spatie\Crawler\CrawlObserver */
27			protected $crawlObserver;
28
29			/** @var \Spatie\Crawler\CrawlProfile */
30			protected $crawlProfile;
31
32			/** @var int */
33			protected $concurrency;
34
35			/** @var \Spatie\Crawler\CrawlQueue */
36			protected $crawlQueue;
37
38			/** @var int */
39			protected $crawledUrlCount = 0;
40
41			/** @var int\|null */
42			protected $maximumCrawlCount = null;
43
44			/** @var int\|null */
45			protected $maximumDepth = null;
46
47			/** @var \Tree\Node\Node */
48			protected $depthTree;
49
50			/** @var false */
51			protected $executeJavaScript = false;
52
53			/** @var string\|null */
54			protected $pathToChromeBinary = null;
55
56			protected static $defaultClientOptions = [
57			RequestOptions::COOKIES => true,
58			RequestOptions::CONNECT_TIMEOUT => 10,
59			RequestOptions::TIMEOUT => 10,
60			RequestOptions::ALLOW_REDIRECTS => false,
61			];
62
63			/**
64			* @param array $clientOptions
65			*
66			* @return static
67			*/
68			public static function create(array $clientOptions = [])
69			{
70			$clientOptions = (count($clientOptions))
71			? $clientOptions
72			: self::$defaultClientOptions;
73
74			$client = new Client($clientOptions);
75
76			return new static($client);
77			}
78
79			public function __construct(Client $client, int $concurrency = 10)
80			{
81			$this->client = $client;
82
83			$this->concurrency = $concurrency;
84
85			$this->crawlProfile = new CrawlAllUrls();
86
87			$this->crawlQueue = new CollectionCrawlQueue();
88			}
89
90			/**
91			* @param int $concurrency
92			*
93			* @return $this
94			*/
95			public function setConcurrency(int $concurrency)
96			{
97			$this->concurrency = $concurrency;
98
99			return $this;
100			}
101
102			/**
103			* @param int $maximumCrawlCount
104			*
105			* @return $this
106			*/
107			public function setMaximumCrawlCount(int $maximumCrawlCount)
108			{
109			$this->maximumCrawlCount = $maximumCrawlCount;
110
111			return $this;
112			}
113
114			/**
115			* @param int $maximumDepth
116			*
117			* @return $this
118			*/
119			public function setMaximumDepth(int $maximumDepth)
120			{
121			$this->maximumDepth = $maximumDepth;
122
123			return $this;
124			}
125
126			/**
127			* @param CrawlQueue $crawlQueue
128			* @return $this
129			*/
130			public function setCrawlQueue(CrawlQueue $crawlQueue)
131			{
132			$this->crawlQueue = $crawlQueue;
133
134			return $this;
135			}
136
137			/**
138			* @return $this
139			*/
140			public function executeJavaScript($pathToChromeBinary = null)
141			{
142			$this->executeJavaScript = true;
			0 ignored issues – show Documentation Bug introduced 2017-09-21 20:55 UTC by Report Bug Copy Issue Report The property `$executeJavaScript` was declared of type `false`, but `true` is of type `boolean`. Maybe add a type cast? This check looks for assignments to scalar types that may be of the wrong type. To ensure the code behaves as expected, it may be a good idea to add an explicit type cast. $answer = 42; $correct = false; $correct = (bool) $answer; Loading history...
143
144			$this->pathToChromeBinary = $pathToChromeBinary;
145
146			return $this;
147			}
148
149			/**
150			* @return $this
151			*/
152			public function doNotExecuteJavaScript()
153			{
154			$this->executeJavaScript = false;
155
156			return $this;
157			}
158
159			/**
160			* @param \Spatie\Crawler\CrawlObserver $crawlObserver
161			*
162			* @return $this
163			*/
164			public function setCrawlObserver(CrawlObserver $crawlObserver)
165			{
166			$this->crawlObserver = $crawlObserver;
167
168			return $this;
169			}
170
171			/**
172			* @param \Spatie\Crawler\CrawlProfile $crawlProfile
173			*
174			* @return $this
175			*/
176			public function setCrawlProfile(CrawlProfile $crawlProfile)
177			{
178			$this->crawlProfile = $crawlProfile;
179
180			return $this;
181			}
182
183			/**
184			* @param \Spatie\Crawler\Url\|string $baseUrl
185			*/
186			public function startCrawling($baseUrl)
187			{
188			if (! $baseUrl instanceof Url) {
189			$baseUrl = Url::create($baseUrl);
190			}
191
192			$this->baseUrl = $baseUrl;
193
194			$crawlUrl = CrawlUrl::create($baseUrl);
195
196			$this->addToCrawlQueue($crawlUrl);
197
198			$this->depthTree = new Node((string) $this->baseUrl);
199
200			$this->startCrawlingQueue();
201
202			$this->crawlObserver->finishedCrawling();
203			}
204
205			protected function startCrawlingQueue()
206			{
207			while ($this->crawlQueue->hasPendingUrls()) {
208			$pool = new Pool($this->client, $this->getCrawlRequests(), [
209			'concurrency' => $this->concurrency,
210			'options' => $this->client->getConfig(),
211			'fulfilled' => function (ResponseInterface $response, int $index) {
212			$crawlUrl = $this->crawlQueue->getUrlById($index);
213			$this->handleResponse($response, $crawlUrl);
214
215			if (! $this->crawlProfile instanceof CrawlSubdomains) {
216			if ($crawlUrl->url->host !== $this->baseUrl->host) {
217			return;
218			}
219			}
220
221			$this->addAllLinksToCrawlQueue(
222			(string) $response->getBody(),
223			$crawlUrl->url
224			);
225			},
226			'rejected' => function (RequestException $exception, int $index) {
227			$this->handleResponse(
228			$exception->getResponse(),
229			$this->crawlQueue->getUrlById($index)
230			);
231			},
232			]);
233
234			$promise = $pool->promise();
235			$promise->wait();
236			}
237			}
238
239			/**
240			* @param ResponseInterface\|null $response
241			* @param CrawlUrl $crawlUrl
242			*/
243			protected function handleResponse($response, CrawlUrl $crawlUrl)
244			{
245			$this->crawlObserver->hasBeenCrawled($crawlUrl->url, $response, $crawlUrl->foundOnUrl);
246			}
247
248			protected function getCrawlRequests(): Generator
249			{
250			while ($crawlUrl = $this->crawlQueue->getFirstPendingUrl()) {
251			if (! $this->crawlProfile->shouldCrawl($crawlUrl->url)) {
252			continue;
253			}
254
255			if ($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)) {
256			continue;
257			}
258
259			$this->crawlObserver->willCrawl($crawlUrl->url);
260
261			$this->crawlQueue->markAsProcessed($crawlUrl);
262
263			yield $crawlUrl->getId() => new Request('GET', (string) $crawlUrl->url);
264			}
265			}
266
267			protected function addAllLinksToCrawlQueue(string $html, Url $foundOnUrl)
268			{
269			$allLinks = $this->extractAllLinks($html, $foundOnUrl);
270
271			collect($allLinks)
272			->filter(function (Url $url) {
273			return $url->hasCrawlableScheme();
274			})
275			->map(function (Url $url) {
276			return $this->normalizeUrl($url);
277			})
278			->filter(function (Url $url) {
279			return $this->crawlProfile->shouldCrawl($url);
280			})
281			->reject(function ($url) {
282			return $this->crawlQueue->has($url);
283			})
284			->each(function (Url $url) use ($foundOnUrl) {
285			$node = $this->addtoDepthTree($this->depthTree, (string) $url, $foundOnUrl);
286
287			if (! $this->shouldCrawl($node)) {
288			return;
289			}
290
291			if ($this->maximumCrawlCountReached()) {
292			return;
293			}
294
295			$crawlUrl = CrawlUrl::create($url, $foundOnUrl);
296
297			$this->addToCrawlQueue($crawlUrl);
298			});
299			}
300
301			protected function shouldCrawl(Node $node): bool
302			{
303			if (is_null($this->maximumDepth)) {
304			return true;
305			}
306
307			return $node->getDepth() <= $this->maximumDepth;
308			}
309
310			protected function extractAllLinks(string $html, Url $foundOnUrl): Collection
311			{
312			if ($this->executeJavaScript) {
313			$html = $this->getBodyAfterExecutingJavaScript($foundOnUrl);
314			}
315
316			$domCrawler = new DomCrawler($html, $foundOnUrl);
317
318			return collect($domCrawler->filterXpath('//a')->links())
319			->map(function (Link $link) {
320			return Url::create($link->getUri());
321			});
322			}
323
324			protected function normalizeUrl(Url $url): Url
325			{
326			return $url->removeFragment();
327			}
328
329			protected function addtoDepthTree(Node $node, string $url, string $parentUrl)
330			{
331			$returnNode = null;
332
333			if ($node->getValue() === $parentUrl) {
334			$newNode = new Node($url);
335
336			$node->addChild($newNode);
337
338			return $newNode;
339			}
340
341			foreach ($node->getChildren() as $currentNode) {
342			$returnNode = $this->addtoDepthTree($currentNode, $url, $parentUrl);
343
344			if (! is_null($returnNode)) {
345			break;
346			}
347			}
348
349			return $returnNode;
350			}
351
352			protected function getBodyAfterExecutingJavaScript(Url $foundOnUrl): string
353			{
354			$browsershot = Browsershot::url((string) $foundOnUrl);
355
356			if ($this->pathToChromeBinary) {
			0 ignored issues – show Bug Best Practice introduced 2017-09-21 21:04 UTC by Report Bug Copy Issue Report The expression `$this->pathToChromeBinary` of type `string\|null` is loosely compared to `true`; this is ambiguous if the string can be empty. You might want to explicitly use `!== null` instead. In PHP, under loose comparison (like `==`, or `!=`, or `switch` conditions), values of different types might be equal. For `string` values, the empty string `''` is a special case, in particular the following results might be unexpected: '' == false // true '' == null // true 'ab' == false // false 'ab' == null // false // It is often better to use strict comparison '' === false // false '' === null // false Loading history...
357			$browsershot->setChromePath($this->pathToChromeBinary);
358			}
359
360			$html = $browsershot->bodyHtml();
361
362			return html_entity_decode($html);
363			}
364
365			protected function addToCrawlQueue(CrawlUrl $crawlUrl)
366			{
367			$this->crawledUrlCount++;
368
369			$this->crawlQueue->add($crawlUrl);
370
371			return $this;
372			}
373
374			protected function maximumCrawlCountReached(): bool
375			{
376			if (is_null($this->maximumCrawlCount)) {
377			return false;
378			}
379
380			return $this->crawledUrlCount >= $this->maximumCrawlCount;
381			}
382			}
383

spatie / crawler

Pull Request — master (#87)

Crawler::getCrawlRequests() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like