Crawler::updateResolvedUrl() - Code Metrics - Inspection of "Switch to https for prerender.io client, correct u..." - mediamonks/crawler - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( e21061...77fa8f )

unknown

created 2017-10-17 12:38 UTC

Crawler::updateResolvedUrl() A

↳ Parent: Crawler

Complexity

Conditions	2
Paths	2

Size

Total Lines	9
Code Lines	5

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	6
CRAP Score	2

Importance

Changes

Metric	Value
dl	0
loc	9
ccs	6
cts	6
cp	1
rs	9.6666
c	0
b	0
f	0
cc	2
eloc	5
nc	2
nop	1
crap	2

<?php

namespace MediaMonks\Crawler;

use MediaMonks\Crawler\Exception\RequestException;
use MediaMonks\Crawler\Exception\UnsupportedUrlException;
use MediaMonks\Crawler\Url\Matcher\UrlMatcherInterface;
use MediaMonks\Crawler\Url\Normalizer\UrlNormalizerInterface;
use Symfony\Component\BrowserKit\Client;
use Symfony\Component\DomCrawler\Crawler as DomCrawler;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;

class Crawler implements LoggerAwareInterface
{
    /**
     * @var Client
     */
    private $client;

    /**
     * @var int
     */
    private $limit = 0;

    /**
     * @var bool
     */
    private $stopOnError = false;

    /**
     * @var bool
     */
    private $exceptionOnError = false;

    /**
     * @var UrlMatcherInterface[]
     */
    private $whitelistUrlMatchers = [];

    /**
     * @var UrlMatcherInterface[]
     */
    private $blacklistUrlMatchers = [];

    /**
     * @var UrlNormalizerInterface[]
     */
    private $urlNormalizers = [];

    /**
     * @var Url
     */
    private $baseUrl;

    /**
     * @var array
     */
    private $urlsCrawled = [];

    /**
     * @var array
     */
    private $urlsQueued = [];

    /**
     * @var array
     */
    private $urlsRejected = [];

    /**
     * @var array
     */
    private $urlsReturned = [];

    /**
     * @var LoggerInterface
     */
    private $logger = null;

    /**
     * @param Client $client
     */
    public function __construct(Client $client = null)
    {
        if (empty($client)) {
            $client = new \Goutte\Client();
        }

        $this->setClient($client);

        return $this;
    }

    /**
     * @param Client $client
     */
    public function setClient(Client $client)
    {
        $this->client = $client;
    }

    /**
     * @return Client
     */
    public function getClient()
    {
        return $this->client;
    }

    /**
     * @return int
     */
    public function getLimit()
    {
        return $this->limit;
    }

    /**
     * @param int $limit
     * @return $this
     */
    public function setLimit($limit)
    {
        $this->limit = $limit;

        return $this;
    }

    /**
     * @return boolean
     */
    public function getStopOnError()
    {
        return $this->stopOnError;
    }

    /**
     * @param boolean $stopOnError
     * @return $this
     */
    public function setStopOnError($stopOnError)
    {
        $this->stopOnError = $stopOnError;

        return $this;
    }

    /**
     * @return boolean
     */
    public function getExceptionOnError()
    {
        return $this->exceptionOnError;
    }

    /**
     * @param boolean $exceptionOnError
     * @return $this
     */
    public function setExceptionOnError($exceptionOnError)
    {
        $this->exceptionOnError = $exceptionOnError;

        return $this;
    }

    /**
     * @return array
     */
    public function getUrlsCrawled()
    {
        return $this->urlsCrawled;
    }

    /**
     * @return array
     */
    public function getUrlsQueued()
    {
        return $this->urlsQueued;
    }

    /**
     * @return array
     */
    public function getUrlsRejected()
    {
        return $this->urlsRejected;
    }

    /**
     * @return array
     */
    public function getUrlsReturned()
    {
        return $this->urlsReturned;
    }

    /**
     * @param $urlMatchers
     * @return $this
     */
    public function setWhitelistUrlMatchers(array $urlMatchers)
    {
        $this->clearWhitelistUrlMatchers();
        foreach ($urlMatchers as $matcher) {
            $this->addWhitelistUrlMatcher($matcher);
        }

        return $this;
    }

    /**
     * @return Url\Matcher\UrlMatcherInterface[]
     */
    public function getWhitelistUrlMatchers()
    {
        return $this->whitelistUrlMatchers;
    }

    /**
     * @param UrlMatcherInterface $urlMatcher
     * @return $this
     */
    public function addWhitelistUrlMatcher(UrlMatcherInterface $urlMatcher)
    {
        $this->whitelistUrlMatchers[] = $urlMatcher;

        return $this;
    }

    /**
     * @return $this
     */
    public function clearWhitelistUrlMatchers()
    {
        $this->whitelistUrlMatchers = [];

        return $this;
    }

    /**
     * @param array $urlMatchers
     * @return $this
     */
    public function setBlacklistUrlMatchers(array $urlMatchers)
    {
        $this->clearBlacklistUrlMatchers();
        foreach ($urlMatchers as $matcher) {
            $this->addBlacklistUrlMatcher($matcher);
        }

        return $this;
    }

    /**
     * @return Url\Matcher\UrlMatcherInterface[]
     */
    public function getBlacklistUrlMatchers()
    {
        return $this->blacklistUrlMatchers;
    }

    /**
     * @param UrlMatcherInterface $urlMatcher
     * @return $this
     */
    public function addBlacklistUrlMatcher(UrlMatcherInterface $urlMatcher)
    {
        $this->blacklistUrlMatchers[] = $urlMatcher;

        return $this;
    }

    /**
     * @return $this
     */
    public function clearBlacklistUrlMatchers()
    {
        $this->blacklistUrlMatchers = [];

        return $this;
    }

    /**
     * @param array $normalizers
     * @return $this
     */
    public function setUrlNormalizers(array $normalizers)
    {
        $this->clearUrlNormalizers();

        foreach ($normalizers as $normalizer) {
            $this->addUrlNormalizer($normalizer);
        }

        return $this;
    }

    /**
     * @return UrlNormalizerInterface[]
     */
    public function getUrlNormalizers()
    {
        return $this->urlNormalizers;
    }

    /**
     * @param UrlNormalizerInterface $normalizer
     * @return $this
     */
    public function addUrlNormalizer(UrlNormalizerInterface $normalizer)
    {
        $this->urlNormalizers[] = $normalizer;

        return $this;
    }

    /**
     * @return $this
     */
    public function clearUrlNormalizers()
    {
        $this->urlNormalizers = [];

        return $this;
    }

    /**
     * @return LoggerInterface
     */
    public function getLogger()
    {
        if (is_null($this->logger)) {
            $this->logger = new NullLogger();
        }

        return $this->logger;
    }

    /**
     * @param LoggerInterface $logger
     * @return $this
     */
    public function setLogger(LoggerInterface $logger)
    {
        $this->logger = $logger;

        return $this;
    }

    /**
     * @param Url $url
     */
    protected function addUrlToQueue(Url $url)
    {
        $this->urlsQueued[(string)$url] = $url;
    }

    /**
     * @param $url
     * @return Url
     * @throws \Exception
     */
    protected function createHttpUrlString($url)
    {
        try {
            return Url::createFromString($url);
        }
        catch (\Exception $e) {
            $this->getLogger()->warning(
                sprintf('Url %s could not be converted to an object: %s', $url, $e->getMessage())
            );

            throw new UnsupportedUrlException($url);
        }
    }

    /**
     * @param Url $url
     */
    protected function reset(Url $url)
    {
        $this->baseUrl = $url;
        $this->urlsCrawled = [];
        $this->urlsQueued = [];

        $this->addUrlToQueue($url);
    }

    /**
     * @param string $url
     * @return \Generator|Page[]
     * @throws RequestException
     */
    public function crawl($url)
    {
        $this->reset($this->createHttpUrlString($url));

        while (count($this->urlsQueued) > 0) {

            $url = array_shift($this->urlsQueued);

            try {
                $crawler = $this->requestPage((string)$url);
                $url = $this->updateResolvedUrl($url);
            } catch (\Exception $e) {
                $this->getLogger()->error(sprintf('Error requesting page %s: %s', $url, $e->getMessage()));

                if ($this->getStopOnError()) {
                    return;
                }
                if ($this->getExceptionOnError()) {
                    throw new RequestException($e->getMessage(), $e->getCode(), $e);
                }

                continue;
            }

            $this->urlsCrawled[] = (string)$url;
            $this->updateQueue($crawler);

            if ($this->shouldReturnUrl($url)) {
                $this->getLogger()->debug(sprintf('Return url "%s"', $url));

                $this->urlsReturned[] = (string)$url;

                yield new Page($url, $crawler, $this->client->getResponse());
/** @return stdClass|null */
function mayReturnNull() { }

function doesNotAcceptNull(stdClass $x) { }

// With potential error.
function withoutCheck() {
    $x = mayReturnNull();
    doesNotAcceptNull($x); // Potential error here.
}

// Safe - Alternative 1
function withCheck1() {
    $x = mayReturnNull();
    if ( ! $x instanceof stdClass) {
        throw new \LogicException('$x must be defined.');
    }
    doesNotAcceptNull($x);
}

// Safe - Alternative 2
function withCheck2() {
    $x = mayReturnNull();
    if ($x instanceof stdClass) {
        doesNotAcceptNull($x);
    }
}
            }

            if ($this->isLimitReached()) {
                $this->getLogger()->info(sprintf('Crawl limit of %d was reach', $this->limit));

                return;
            }
        }
    }

    /**
     * @param Url $url
     * @return Url
     */
    protected function updateResolvedUrl(Url $url)
    {
        $request = $this->client->getRequest();
        if (!empty($request)) {
            $url = $this->createHttpUrlString($request->getUri());
        }

        return $url;
    }

    /**
     * @param DomCrawler $crawler
     */
    protected function updateQueue(DomCrawler $crawler)
    {
        foreach ($this->extractUrlsFromCrawler($crawler) as $url) {
            $this->getLogger()->debug(sprintf('Found url %s in page', $url));
            try {
                $url = $this->normalizeUrl($this->createHttpUrlString($url));

                if ($this->shouldCrawlUrl($url)) {
                    $this->addUrlToQueue($url);
                }
            } catch (\Exception $e) {
                $this->urlsRejected[] = $url;
            }
        }
    }

    /**
     * @param Url $url
     * @return Url
     */
    protected function normalizeUrl(Url $url)
    {
        foreach ($this->urlNormalizers as $normalizer) {
            $url = $normalizer->normalize($url);
        }

        return $url;
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function shouldReturnUrl(Url $url)
    {
        if (!empty($this->whitelistUrlMatchers)) {
            if (!$this->isUrlWhitelisted($url)) {
                $this->getLogger()->info(sprintf('Skipped "%s" because it is not whitelisted', $url));

                return false;
            }
        }

        if ($this->isUrlBlacklisted($url)) {
            $this->getLogger()->info(sprintf('Skipped "%s" because it is blacklisted', $url));

            return false;
        }

        return true;
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function isUrlWhitelisted(Url $url)
    {
        foreach ($this->whitelistUrlMatchers as $matcher) {
            if ($matcher->matches($url)) {
                return true;
            }
        }

        return false;
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function isUrlBlacklisted(Url $url)
    {
        foreach ($this->blacklistUrlMatchers as $matcher) {
            if ($matcher->matches($url)) {
                return true;
            }
        }

        return false;
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function shouldCrawlUrl(Url $url)
    {
        if ($this->isUrlRejected($url)
            || $this->isUrlCrawled($url)
            || $this->isUrlQueued($url)
        ) {
            return false;
        }

        if (!$this->isUrlPartOfBaseUrl($url)) {
            $this->urlsRejected[] = (string)$url;

            return false;
        }

        return true;
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function isUrlRejected(Url $url)
    {
        return in_array((string)$url, $this->urlsRejected);
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function isUrlCrawled(Url $url)
    {
        return in_array((string)$url, $this->urlsCrawled);
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function isUrlQueued(Url $url)
    {
        return isset($this->urlsQueued[(string)$url]);
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function isUrlPartOfBaseUrl(Url $url)
    {
        $baseUrlString = (string)$this->baseUrl;
        $this->getLogger()->debug($baseUrlString.' - '.$url);
        if (strpos((string)$url, $baseUrlString) === false) {
            return false;
        }

        return true;
    }

    /**
     * @return bool
     */
    protected function isLimitReached()
    {
        return (!empty($this->limit) && count($this->urlsReturned) === $this->limit);
    }

    /**
     * @param DomCrawler $crawler
     * @return array
     */
    protected function extractUrlsFromCrawler(DomCrawler $crawler)
    {
        return $crawler->filter('a')->each(
            function (DomCrawler $node) {
                return $node->link()->getUri();
            }
        );
    }

    /**
     * @param string $url
     * @return DomCrawler
     */
    protected function requestPage($url)
    {
        $this->getLogger()->info(sprintf('Crawling page %s', $url));
        $crawler = $this->client->request('GET', $url);
        $this->getLogger()->info(sprintf('Crawled page %s', $url));

        return $crawler;
    }
}


1		<?php
2
3		namespace MediaMonks\Crawler;
4
5		use MediaMonks\Crawler\Exception\RequestException;
6		use MediaMonks\Crawler\Exception\UnsupportedUrlException;
7		use MediaMonks\Crawler\Url\Matcher\UrlMatcherInterface;
8		use MediaMonks\Crawler\Url\Normalizer\UrlNormalizerInterface;
9		use Symfony\Component\BrowserKit\Client;
10		use Symfony\Component\DomCrawler\Crawler as DomCrawler;
11		use Psr\Log\LoggerAwareInterface;
12		use Psr\Log\LoggerInterface;
13		use Psr\Log\NullLogger;
14
15		class Crawler implements LoggerAwareInterface
16		{
17		/**
18		* @var Client
19		*/
20		private $client;
21
22		/**
23		* @var int
24		*/
25		private $limit = 0;
26
27		/**
28		* @var bool
29		*/
30		private $stopOnError = false;
31
32		/**
33		* @var bool
34		*/
35		private $exceptionOnError = false;
36
37		/**
38		* @var UrlMatcherInterface[]
39		*/
40		private $whitelistUrlMatchers = [];
41
42		/**
43		* @var UrlMatcherInterface[]
44		*/
45		private $blacklistUrlMatchers = [];
46
47		/**
48		* @var UrlNormalizerInterface[]
49		*/
50		private $urlNormalizers = [];
51
52		/**
53		* @var Url
54		*/
55		private $baseUrl;
56
57		/**
58		* @var array
59		*/
60		private $urlsCrawled = [];
61
62		/**
63		* @var array
64		*/
65		private $urlsQueued = [];
66
67		/**
68		* @var array
69		*/
70		private $urlsRejected = [];
71
72		/**
73		* @var array
74		*/
75		private $urlsReturned = [];
76
77		/**
78		* @var LoggerInterface
79		*/
80		private $logger = null;
81
82		/**
83		* @param Client $client
84		*/
85	15	public function __construct(Client $client = null)
86		{
87	15	if (empty($client)) {
88	6	$client = new \Goutte\Client();
89	5	}
90
91	15	$this->setClient($client);
92
93	15	return $this;
94		}
95
96		/**
97		* @param Client $client
98		*/
99	15	public function setClient(Client $client)
100		{
101	15	$this->client = $client;
102	15	}
103
104		/**
105		* @return Client
106		*/
107	2	public function getClient()
108		{
109	2	return $this->client;
110		}
111
112		/**
113		* @return int
114		*/
115	2	public function getLimit()
116		{
117	2	return $this->limit;
118		}
119
120		/**
121		* @param int $limit
122		* @return $this
123		*/
124	2	public function setLimit($limit)
125		{
126	2	$this->limit = $limit;
127
128	2	return $this;
129		}
130
131		/**
132		* @return boolean
133		*/
134	5	public function getStopOnError()
135		{
136	5	return $this->stopOnError;
137		}
138
139		/**
140		* @param boolean $stopOnError
141		* @return $this
142		*/
143	2	public function setStopOnError($stopOnError)
144		{
145	2	$this->stopOnError = $stopOnError;
146
147	2	return $this;
148		}
149
150		/**
151		* @return boolean
152		*/
153	2	public function getExceptionOnError()
154		{
155	2	return $this->exceptionOnError;
156		}
157
158		/**
159		* @param boolean $exceptionOnError
160		* @return $this
161		*/
162	1	public function setExceptionOnError($exceptionOnError)
163		{
164	1	$this->exceptionOnError = $exceptionOnError;
165
166	1	return $this;
167		}
168
169		/**
170		* @return array
171		*/
172	9	public function getUrlsCrawled()
173		{
174	9	return $this->urlsCrawled;
175		}
176
177		/**
178		* @return array
179		*/
180	2	public function getUrlsQueued()
181		{
182	2	return $this->urlsQueued;
183		}
184
185		/**
186		* @return array
187		*/
188	2	public function getUrlsRejected()
189		{
190	2	return $this->urlsRejected;
191		}
192
193		/**
194		* @return array
195		*/
196	4	public function getUrlsReturned()
197		{
198	4	return $this->urlsReturned;
199		}
200
201		/**
202		* @param $urlMatchers
203		* @return $this
204		*/
205	1	public function setWhitelistUrlMatchers(array $urlMatchers)
206		{
207	1	$this->clearWhitelistUrlMatchers();
208	1	foreach ($urlMatchers as $matcher) {
209	1	$this->addWhitelistUrlMatcher($matcher);
210	1	}
211
212	1	return $this;
213		}
214
215		/**
216		* @return Url\Matcher\UrlMatcherInterface[]
217		*/
218	2	public function getWhitelistUrlMatchers()
219		{
220	2	return $this->whitelistUrlMatchers;
221		}
222
223		/**
224		* @param UrlMatcherInterface $urlMatcher
225		* @return $this
226		*/
227	2	public function addWhitelistUrlMatcher(UrlMatcherInterface $urlMatcher)
228		{
229	2	$this->whitelistUrlMatchers[] = $urlMatcher;
230
231	2	return $this;
232		}
233
234		/**
235		* @return $this
236		*/
237	1	public function clearWhitelistUrlMatchers()
238		{
239	1	$this->whitelistUrlMatchers = [];
240
241	1	return $this;
242		}
243
244		/**
245		* @param array $urlMatchers
246		* @return $this
247		*/
248	1	public function setBlacklistUrlMatchers(array $urlMatchers)
249		{
250	1	$this->clearBlacklistUrlMatchers();
251	1	foreach ($urlMatchers as $matcher) {
252	1	$this->addBlacklistUrlMatcher($matcher);
253	1	}
254
255	1	return $this;
256		}
257
258		/**
259		* @return Url\Matcher\UrlMatcherInterface[]
260		*/
261	2	public function getBlacklistUrlMatchers()
262		{
263	2	return $this->blacklistUrlMatchers;
264		}
265
266		/**
267		* @param UrlMatcherInterface $urlMatcher
268		* @return $this
269		*/
270	2	public function addBlacklistUrlMatcher(UrlMatcherInterface $urlMatcher)
271		{
272	2	$this->blacklistUrlMatchers[] = $urlMatcher;
273
274	2	return $this;
275		}
276
277		/**
278		* @return $this
279		*/
280	1	public function clearBlacklistUrlMatchers()
281		{
282	1	$this->blacklistUrlMatchers = [];
283
284	1	return $this;
285		}
286
287		/**
288		* @param array $normalizers
289		* @return $this
290		*/
291	1	public function setUrlNormalizers(array $normalizers)
292		{
293	1	$this->clearUrlNormalizers();
294
295	1	foreach ($normalizers as $normalizer) {
296	1	$this->addUrlNormalizer($normalizer);
297	1	}
298
299	1	return $this;
300		}
301
302		/**
303		* @return UrlNormalizerInterface[]
304		*/
305	1	public function getUrlNormalizers()
306		{
307	1	return $this->urlNormalizers;
308		}
309
310		/**
311		* @param UrlNormalizerInterface $normalizer
312		* @return $this
313		*/
314	2	public function addUrlNormalizer(UrlNormalizerInterface $normalizer)
315		{
316	2	$this->urlNormalizers[] = $normalizer;
317
318	2	return $this;
319		}
320
321		/**
322		* @return $this
323		*/
324	1	public function clearUrlNormalizers()
325		{
326	1	$this->urlNormalizers = [];
327
328	1	return $this;
329		}
330
331		/**
332		* @return LoggerInterface
333		*/
334	13	public function getLogger()
335		{
336	13	if (is_null($this->logger)) {
337	12	$this->logger = new NullLogger();
338	12	}
339
340	13	return $this->logger;
341		}
342
343		/**
344		* @param LoggerInterface $logger
345		* @return $this
346		*/
347	1	public function setLogger(LoggerInterface $logger)
348		{
349	1	$this->logger = $logger;
350
351	1	return $this;
352		}
353
354		/**
355		* @param Url $url
356		*/
357	11	protected function addUrlToQueue(Url $url)
358		{
359	11	$this->urlsQueued[(string)$url] = $url;
360	11	}
361
362		/**
363		* @param $url
364		* @return Url
365		* @throws \Exception
366		*/
367	10	protected function createHttpUrlString($url)
368		{
369		try {
370	10	return Url::createFromString($url);
371		}
372	6	catch (\Exception $e) {
373	6	$this->getLogger()->warning(
374	6	sprintf('Url %s could not be converted to an object: %s', $url, $e->getMessage())
375	6	);
376
377	6	throw new UnsupportedUrlException($url);
378		}
379		}
380
381		/**
382		* @param Url $url
383		*/
384	11	protected function reset(Url $url)
385		{
386	11	$this->baseUrl = $url;
387	11	$this->urlsCrawled = [];
388	11	$this->urlsQueued = [];
389
390	11	$this->addUrlToQueue($url);
391	11	}
392
393		/**
394		* @param string $url
395		* @return \Generator\|Page[]
396		* @throws RequestException
397		*/
398	10	public function crawl($url)
399		{
400	10	$this->reset($this->createHttpUrlString($url));
401
402	10	while (count($this->urlsQueued) > 0) {
403
404	10	$url = array_shift($this->urlsQueued);
405
406		try {
407	10	$crawler = $this->requestPage((string)$url);
408	10	$url = $this->updateResolvedUrl($url);
409	10	} catch (\Exception $e) {
410	3	$this->getLogger()->error(sprintf('Error requesting page %s: %s', $url, $e->getMessage()));
411
412	3	if ($this->getStopOnError()) {
413	1	return;
414		}
415	2	if ($this->getExceptionOnError()) {
416	1	throw new RequestException($e->getMessage(), $e->getCode(), $e);
417		}
418
419	1	continue;
420		}
421
422	9	$this->urlsCrawled[] = (string)$url;
423	9	$this->updateQueue($crawler);
424
425	9	if ($this->shouldReturnUrl($url)) {
426	9	$this->getLogger()->debug(sprintf('Return url "%s"', $url));
427
428	9	$this->urlsReturned[] = (string)$url;
429
430	9	yield new Page($url, $crawler, $this->client->getResponse());
		0 ignored issues – show Bug introduced 2017-08-11 12:18 UTC by Report Bug Copy Issue Report It seems like `$crawler` defined by `$this->requestPage((string) $url)` on line `407` can be `null`; however, `MediaMonks\Crawler\Page::__construct()` does not accept `null`, maybe add an additional type check? Unless you are absolutely sure that the expression can never be null because of other conditions, we strongly recommend to add an additional type check to your code: /** @return stdClass\|null */ function mayReturnNull() { } function doesNotAcceptNull(stdClass $x) { } // With potential error. function withoutCheck() { $x = mayReturnNull(); doesNotAcceptNull($x); // Potential error here. } // Safe - Alternative 1 function withCheck1() { $x = mayReturnNull(); if ( ! $x instanceof stdClass) { throw new \LogicException('$x must be defined.'); } doesNotAcceptNull($x); } // Safe - Alternative 2 function withCheck2() { $x = mayReturnNull(); if ($x instanceof stdClass) { doesNotAcceptNull($x); } } Loading history...
431	9	}
432
433	9	if ($this->isLimitReached()) {
434	1	$this->getLogger()->info(sprintf('Crawl limit of %d was reach', $this->limit));
435
436	1	return;
437		}
438	9	}
439	7	}
440
441		/**
442		* @param Url $url
443		* @return Url
444		*/
445	10	protected function updateResolvedUrl(Url $url)
446		{
447	10	$request = $this->client->getRequest();
448	9	if (!empty($request)) {
449	1	$url = $this->createHttpUrlString($request->getUri());
450	1	}
451
452	9	return $url;
453		}
454
455		/**
456		* @param DomCrawler $crawler
457		*/
458	9	protected function updateQueue(DomCrawler $crawler)
459		{
460	9	foreach ($this->extractUrlsFromCrawler($crawler) as $url) {
461	7	$this->getLogger()->debug(sprintf('Found url %s in page', $url));
462		try {
463	7	$url = $this->normalizeUrl($this->createHttpUrlString($url));
464
465	7	if ($this->shouldCrawlUrl($url)) {
466	7	$this->addUrlToQueue($url);
467	7	}
468	7	} catch (\Exception $e) {
469	6	$this->urlsRejected[] = $url;
470		}
471	9	}
472	9	}
473
474		/**
475		* @param Url $url
476		* @return Url
477		*/
478	7	protected function normalizeUrl(Url $url)
479		{
480	7	foreach ($this->urlNormalizers as $normalizer) {
481	1	$url = $normalizer->normalize($url);
482	7	}
483
484	7	return $url;
485		}
486
487		/**
488		* @param Url $url
489		* @return bool
490		*/
491	9	protected function shouldReturnUrl(Url $url)
492		{
493	9	if (!empty($this->whitelistUrlMatchers)) {
494	1	if (!$this->isUrlWhitelisted($url)) {
495	1	$this->getLogger()->info(sprintf('Skipped "%s" because it is not whitelisted', $url));
496
497	1	return false;
498		}
499	1	}
500
501	9	if ($this->isUrlBlacklisted($url)) {
502	1	$this->getLogger()->info(sprintf('Skipped "%s" because it is blacklisted', $url));
503
504	1	return false;
505		}
506
507	9	return true;
508		}
509
510		/**
511		* @param Url $url
512		* @return bool
513		*/
514	1	protected function isUrlWhitelisted(Url $url)
515		{
516	1	foreach ($this->whitelistUrlMatchers as $matcher) {
517	1	if ($matcher->matches($url)) {
518	1	return true;
519		}
520	1	}
521
522	1	return false;
523		}
524
525		/**
526		* @param Url $url
527		* @return bool
528		*/
529	9	protected function isUrlBlacklisted(Url $url)
530		{
531	9	foreach ($this->blacklistUrlMatchers as $matcher) {
532	1	if ($matcher->matches($url)) {
533	1	return true;
534		}
535	9	}
536
537	9	return false;
538		}
539
540		/**
541		* @param Url $url
542		* @return bool
543		*/
544	8	protected function shouldCrawlUrl(Url $url)
545		{
546	8	if ($this->isUrlRejected($url)
547	8	\|\| $this->isUrlCrawled($url)
548	8	\|\| $this->isUrlQueued($url)
549	8	) {
550	6	return false;
551		}
552
553	8	if (!$this->isUrlPartOfBaseUrl($url)) {
554	7	$this->urlsRejected[] = (string)$url;
555
556	7	return false;
557		}
558
559	8	return true;
560		}
561
562		/**
563		* @param Url $url
564		* @return bool
565		*/
566	8	protected function isUrlRejected(Url $url)
567		{
568	8	return in_array((string)$url, $this->urlsRejected);
569		}
570
571		/**
572		* @param Url $url
573		* @return bool
574		*/
575	8	protected function isUrlCrawled(Url $url)
576		{
577	8	return in_array((string)$url, $this->urlsCrawled);
578		}
579
580		/**
581		* @param Url $url
582		* @return bool
583		*/
584	8	protected function isUrlQueued(Url $url)
585		{
586	8	return isset($this->urlsQueued[(string)$url]);
587		}
588
589		/**
590		* @param Url $url
591		* @return bool
592		*/
593	8	protected function isUrlPartOfBaseUrl(Url $url)
594		{
595	8	$baseUrlString = (string)$this->baseUrl;
596	8	$this->getLogger()->debug($baseUrlString.' - '.$url);
597	8	if (strpos((string)$url, $baseUrlString) === false) {
598	7	return false;
599		}
600
601	8	return true;
602		}
603
604		/**
605		* @return bool
606		*/
607	9	protected function isLimitReached()
608		{
609	9	return (!empty($this->limit) && count($this->urlsReturned) === $this->limit);
610		}
611
612		/**
613		* @param DomCrawler $crawler
614		* @return array
615		*/
616	9	protected function extractUrlsFromCrawler(DomCrawler $crawler)
617		{
618	9	return $crawler->filter('a')->each(
619	7	function (DomCrawler $node) {
620	7	return $node->link()->getUri();
621		}
622	9	);
623		}
624
625		/**
626		* @param string $url
627		* @return DomCrawler
628		*/
629	10	protected function requestPage($url)
630		{
631	10	$this->getLogger()->info(sprintf('Crawling page %s', $url));
632	10	$crawler = $this->client->request('GET', $url);
633	10	$this->getLogger()->info(sprintf('Crawled page %s', $url));
634
635	10	return $crawler;
636		}
637		}
638

mediamonks / crawler

Push — master ( e21061...77fa8f )

Crawler::updateResolvedUrl() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like