Crawler::updateQueue() - Code Metrics - Inspection of "- More tests added" - mediamonks/crawler - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 1380cb...844cd9 )

unknown

created 2016-11-25 13:28 UTC

Crawler::updateQueue() A

↳ Parent: Crawler

Complexity

Conditions	4
Paths	6

Size

Total Lines	18
Code Lines	11

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	14
CRAP Score	4

Importance

Changes

Metric	Value
dl	0
loc	18
ccs	14
cts	14
cp	1
rs	9.2
c	0
b	0
f	0
cc	4
eloc	11
nc	6
nop	1
crap	4

<?php

namespace MediaMonks\Crawler;

use MediaMonks\Crawler\Url\Matcher\UrlMatcherInterface;
use MediaMonks\Crawler\Url\Normalizer\UrlNormalizerInterface;
use Symfony\Component\BrowserKit\Client;
use Symfony\Component\DomCrawler\Crawler as DomCrawler;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;

class Crawler implements LoggerAwareInterface
{
    /**
     * @var Client
     */
    private $client;

    /**
     * @var int
     */
    private $limit = 0;

    /**
     * @var bool
     */
    private $stopOnError = false;

    /**
     * @var UrlMatcherInterface[]
     */
    private $whitelistUrlMatchers = [];

    /**
     * @var UrlMatcherInterface[]
     */
    private $blacklistUrlMatchers = [];

    /**
     * @var UrlNormalizerInterface[]
     */
    private $urlNormalizers = [];

    /**
     * @var Url
     */
    private $baseUrl;

    /**
     * @var array
     */
    private $urlsCrawled = [];

    /**
     * @var array
     */
    private $urlsQueued = [];

    /**
     * @var array
     */
    private $urlsRejected = [];

    /**
     * @var array
     */
    private $urlsReturned = [];

    /**
     * @var LoggerInterface
     */
    private $logger = null;

    /**
     * @param Client $client
     * @param array $options
     */
    public function __construct(Client $client = null, array $options = [])
    {
        if (empty($client)) {
            $client = new \Goutte\Client();
        }

        $this->setClient($client);
        $this->setOptions($options);

        return $this;
    }

    /**
     * @param Client $client
     */
    public function setClient(Client $client)
    {
        $this->client = $client;
    }

    /**
     * @return Client
     */
    public function getClient()
    {
        return $this->client;
    }

    /**
     * @param array $options
     */
    public function setOptions(array $options)
    {
        if (isset($options['limit'])) {
            $this->setLimit($options['limit']);
        }
        if (isset($options['stop_on_error'])) {
            $this->setStopOnError($options['stop_on_error']);
        }
        if (isset($options['logger'])) {
            $this->setLogger($options['logger']);
        }
        if (isset($options['whitelist_url_matchers'])) {
            $this->setWhitelistUrlMatchers($options['whitelist_url_matchers']);
        }
        if (isset($options['blacklist_url_matchers'])) {
            $this->setBlacklistUrlMatchers($options['blacklist_url_matchers']);
        }
        if (isset($options['blacklist_url_matchers'])) {
            $this->setBlacklistUrlMatchers($options['blacklist_url_matchers']);
        }
    }

    /**
     * @return int
     */
    public function getLimit()
    {
        return $this->limit;
    }

    /**
     * @param int $limit
     * @return $this
     */
    public function setLimit($limit)
    {
        $this->limit = $limit;

        return $this;
    }

    /**
     * @return boolean
     */
    public function getStopOnError()
    {
        return $this->stopOnError;
    }

    /**
     * @param boolean $stopOnError
     * @return Crawler
     */
    public function setStopOnError($stopOnError)
    {
        $this->stopOnError = $stopOnError;

        return $this;
    }

    /**
     * @return array
     */
    public function getUrlsCrawled()
    {
        return $this->urlsCrawled;
    }

    /**
     * @return array
     */
    public function getUrlsQueued()
    {
        return $this->urlsQueued;
    }

    /**
     * @return array
     */
    public function getUrlsRejected()
    {
        return $this->urlsRejected;
    }

    /**
     * @return array
     */
    public function getUrlsReturned()
    {
        return $this->urlsReturned;
    }

    /**
     * @param $urlMatchers
     * @return $this
     */
    public function setWhitelistUrlMatchers(array $urlMatchers)
    {
        $this->clearWhitelistUrlMatchers();
        foreach ($urlMatchers as $matcher) {
            $this->addWhitelistUrlMatcher($matcher);
        }

        return $this;
    }

    /**
     * @return Url\Matcher\UrlMatcherInterface[]
     */
    public function getWhitelistUrlMatchers()
    {
        return $this->whitelistUrlMatchers;
    }

    /**
     * @param UrlMatcherInterface $urlMatcher
     * @return $this
     */
    public function addWhitelistUrlMatcher(UrlMatcherInterface $urlMatcher)
    {
        $this->whitelistUrlMatchers[] = $urlMatcher;

        return $this;
    }

    /**
     * @return $this
     */
    public function clearWhitelistUrlMatchers()
    {
        $this->whitelistUrlMatchers = [];

        return $this;
    }

    /**
     * @param array $urlMatchers
     * @return $this
     */
    public function setBlacklistUrlMatchers(array $urlMatchers)
    {
        $this->clearBlacklistUrlMatchers();
        foreach ($urlMatchers as $matcher) {
            $this->addBlacklistUrlMatcher($matcher);
        }

        return $this;
    }

    /**
     * @return Url\Matcher\UrlMatcherInterface[]
     */
    public function getBlacklistUrlMatchers()
    {
        return $this->blacklistUrlMatchers;
    }

    /**
     * @param UrlMatcherInterface $urlMatcher
     * @return $this
     */
    public function addBlacklistUrlMatcher(UrlMatcherInterface $urlMatcher)
    {
        $this->blacklistUrlMatchers[] = $urlMatcher;

        return $this;
    }

    /**
     * @return $this
     */
    public function clearBlacklistUrlMatchers()
    {
        $this->blacklistUrlMatchers = [];

        return $this;
    }

    /**
     * @param array $normalizers
     * @return $this
     */
    public function setUrlNormalizers(array $normalizers)
    {
        $this->clearUrlNormalizers();

        foreach ($normalizers as $normalizer) {
            $this->addUrlNormalizer($normalizer);
        }

        return $this;
    }

    /**
     * @return UrlNormalizerInterface[]
     */
    public function getUrlNormalizers()
    {
        return $this->urlNormalizers;
    }

    /**
     * @param UrlNormalizerInterface $normalizer
     * @return $this
     */
    public function addUrlNormalizer(UrlNormalizerInterface $normalizer)
    {
        $this->urlNormalizers[] = $normalizer;

        return $this;
    }

    /**
     * @return $this
     */
    public function clearUrlNormalizers()
    {
        $this->urlNormalizers = [];

        return $this;
    }

    /**
     * @return LoggerInterface
     */
    public function getLogger()
    {
        if (is_null($this->logger)) {
            $this->logger = new NullLogger();
        }

        return $this->logger;
    }

    /**
     * @param LoggerInterface $logger
     * @return $this
     */
    public function setLogger(LoggerInterface $logger)
    {
        $this->logger = $logger;

        return $this;
    }

    /**
     * @param Url $url
     */
    protected function addUrlToQueue(Url $url)
    {
        $this->urlsQueued[(string)$url] = $url;
    }

    /**
     * @param $url
     * @return Url
     */
    protected function createHttpUrlString($url)
    {
        return Url::createFromString($url);
    }

    /**
     * @param Url $url
     */
    protected function reset(Url $url)
    {
        $this->baseUrl = $url;
        $this->urlsCrawled = [];
        $this->urlsQueued = [];

        $this->addUrlToQueue($url);
    }

    /**
     * @param string $url
     * @return \Generator|void
     */
    public function crawl($url)
    {
        $url = $this->createHttpUrlString($url);
        $this->reset($url);

        while (count($this->urlsQueued) > 0) {

            $url = array_shift($this->urlsQueued);

            try {
                $crawler = $this->requestPage((string)$url);
            } catch (\Exception $e) {
                $this->getLogger()->error(sprintf('Error requesting page %s: %s', $url, $e->getMessage()));

                if ($this->getStopOnError()) {
                    return;
                }

                continue;
            }

            $this->urlsCrawled[] = (string)$url;
            $this->updateQueue($crawler);

            if ($this->shouldReturnUrl($url)) {
                $this->getLogger()->debug(sprintf('Return url "%s"', $url));

                $this->urlsReturned[] = (string)$url;

                yield new Page($url, $crawler);
            }

            if ($this->isLimitReached()) {
                $this->getLogger()->info(sprintf('Crawl limit of %d was reach', $this->limit));

                return;
            }
        }
    }

    /**
     * @param DomCrawler $crawler
     */
    protected function updateQueue(DomCrawler $crawler)
    {
        foreach ($this->extractUrlsFromCrawler($crawler) as $url) {
            $this->getLogger()->debug(sprintf('Found url %s in page', $url));
            try {
                $url = $this->normalizeUrl($this->createHttpUrlString($url));

                if ($this->shouldCrawlUrl($url)) {
                    $this->addUrlToQueue($url);
                }
            } catch (\Exception $e) {
                $this->getLogger()->warning(
                    sprintf('Url %s could not be converted to an object: %s', $url, $e->getMessage())
                );
                $this->urlsRejected[] = $url;
            }
        }
    }

    /**
     * @param Url $url
     * @return Url
     */
    protected function normalizeUrl(Url $url)
    {
        foreach ($this->urlNormalizers as $normalizer) {
            $url = $normalizer->normalize($url);
        }

        return $url;
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function shouldReturnUrl(Url $url)
    {
        if (!empty($this->whitelistUrlMatchers)) {
            if (!$this->isUrlWhitelisted($url)) {
                $this->getLogger()->info(sprintf('Skipped "%s" because it is not whitelisted', $url));

                return false;
            }
        }

        if ($this->isUrlBlacklisted($url)) {
            $this->getLogger()->info(sprintf('Skipped "%s" because it is blacklisted', $url));

            return false;
        }

        return true;
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function isUrlWhitelisted(Url $url)
    {
        foreach ($this->whitelistUrlMatchers as $matcher) {
            if ($matcher->matches($url)) {
                return true;
            }
        }

        return false;
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function isUrlBlacklisted(Url $url)
    {
        foreach ($this->blacklistUrlMatchers as $matcher) {
            if ($matcher->matches($url)) {
                return true;
            }
        }

        return false;
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function shouldCrawlUrl(Url $url)
    {
        $urlString = (string)$url;
        if (in_array($urlString, $this->urlsRejected)) {
            return false;
        }
        if (in_array($urlString, $this->urlsCrawled)) {
            return false;
        }
        if (isset($this->urlsQueued[$urlString])) {
            return false;
        }

        if (!$this->isUrlPartOfBaseUrl($url)) {
            $this->urlsRejected[] = (string)$url;

            return false;
        }

        return true;
    }

    /**
     * @param Url $url
     * @return bool
     */
    protected function isUrlPartOfBaseUrl(Url $url)
    {
        $baseUrlString = (string)$this->baseUrl;
        $this->getLogger()->debug($baseUrlString.' - '.$url);
        if (strpos((string)$url, $baseUrlString) === false) {
            return false;
        }

        return true;
    }

    /**
     * @return bool
     */
    private function isLimitReached()
    {
        return (!empty($this->limit) && count($this->urlsReturned) === $this->limit);
    }

    /**
     * @param DomCrawler $crawler
     * @return array
     */
    protected function extractUrlsFromCrawler(DomCrawler $crawler)
    {
        return $crawler->filter('a')->each(
            function (DomCrawler $node) {
                return $node->link()->getUri();
            }
        );
    }

    /**
     * @param $url
     * @return DomCrawler
     */
    protected function requestPage($url)
    {
        $this->getLogger()->info(sprintf('Crawling page %s', $url));
        $crawler = $this->client->request('GET', $url);
        $this->getLogger()->info(sprintf('Crawled page %s', $url));

        return $crawler;
    }
}


1		<?php
2
3		namespace MediaMonks\Crawler;
4
5		use MediaMonks\Crawler\Url\Matcher\UrlMatcherInterface;
6		use MediaMonks\Crawler\Url\Normalizer\UrlNormalizerInterface;
7		use Symfony\Component\BrowserKit\Client;
8		use Symfony\Component\DomCrawler\Crawler as DomCrawler;
9		use Psr\Log\LoggerAwareInterface;
10		use Psr\Log\LoggerInterface;
11		use Psr\Log\NullLogger;
12
13		class Crawler implements LoggerAwareInterface
14		{
15		/**
16		* @var Client
17		*/
18		private $client;
19
20		/**
21		* @var int
22		*/
23		private $limit = 0;
24
25		/**
26		* @var bool
27		*/
28		private $stopOnError = false;
29
30		/**
31		* @var UrlMatcherInterface[]
32		*/
33		private $whitelistUrlMatchers = [];
34
35		/**
36		* @var UrlMatcherInterface[]
37		*/
38		private $blacklistUrlMatchers = [];
39
40		/**
41		* @var UrlNormalizerInterface[]
42		*/
43		private $urlNormalizers = [];
44
45		/**
46		* @var Url
47		*/
48		private $baseUrl;
49
50		/**
51		* @var array
52		*/
53		private $urlsCrawled = [];
54
55		/**
56		* @var array
57		*/
58		private $urlsQueued = [];
59
60		/**
61		* @var array
62		*/
63		private $urlsRejected = [];
64
65		/**
66		* @var array
67		*/
68		private $urlsReturned = [];
69
70		/**
71		* @var LoggerInterface
72		*/
73		private $logger = null;
74
75		/**
76		* @param Client $client
77		* @param array $options
78		*/
79	11	public function __construct(Client $client = null, array $options = [])
80		{
81	11	if (empty($client)) {
82	5	$client = new \Goutte\Client();
83	5	}
84
85	11	$this->setClient($client);
86	11	$this->setOptions($options);
87
88	11	return $this;
89		}
90
91		/**
92		* @param Client $client
93		*/
94	11	public function setClient(Client $client)
95		{
96	11	$this->client = $client;
97	11	}
98
99		/**
100		* @return Client
101		*/
102	2	public function getClient()
103		{
104	2	return $this->client;
105		}
106
107		/**
108		* @param array $options
109		*/
110	11	public function setOptions(array $options)
111		{
112	11	if (isset($options['limit'])) {
113	2	$this->setLimit($options['limit']);
114	1	}
115	11	if (isset($options['stop_on_error'])) {
116	1	$this->setStopOnError($options['stop_on_error']);
117	1	}
118	11	if (isset($options['logger'])) {
119	1	$this->setLogger($options['logger']);
120	1	}
121	11	if (isset($options['whitelist_url_matchers'])) {
122	1	$this->setWhitelistUrlMatchers($options['whitelist_url_matchers']);
123	1	}
124	11	if (isset($options['blacklist_url_matchers'])) {
125	1	$this->setBlacklistUrlMatchers($options['blacklist_url_matchers']);
126	1	}
127	11	if (isset($options['blacklist_url_matchers'])) {
128	1	$this->setBlacklistUrlMatchers($options['blacklist_url_matchers']);
129	1	}
130	11	}
131
132		/**
133		* @return int
134		*/
135	3	public function getLimit()
136		{
137	3	return $this->limit;
138		}
139
140		/**
141		* @param int $limit
142		* @return $this
143		*/
144	3	public function setLimit($limit)
145		{
146	3	$this->limit = $limit;
147
148	3	return $this;
149		}
150
151		/**
152		* @return boolean
153		*/
154	3	public function getStopOnError()
155		{
156	3	return $this->stopOnError;
157		}
158
159		/**
160		* @param boolean $stopOnError
161		* @return Crawler
162		*/
163	2	public function setStopOnError($stopOnError)
164		{
165	2	$this->stopOnError = $stopOnError;
166
167	2	return $this;
168		}
169
170		/**
171		* @return array
172		*/
173	7	public function getUrlsCrawled()
174		{
175	7	return $this->urlsCrawled;
176		}
177
178		/**
179		* @return array
180		*/
181	2	public function getUrlsQueued()
182		{
183	2	return $this->urlsQueued;
184		}
185
186		/**
187		* @return array
188		*/
189	2	public function getUrlsRejected()
190		{
191	2	return $this->urlsRejected;
192		}
193
194		/**
195		* @return array
196		*/
197	4	public function getUrlsReturned()
198		{
199	4	return $this->urlsReturned;
200		}
201
202		/**
203		* @param $urlMatchers
204		* @return $this
205		*/
206	2	public function setWhitelistUrlMatchers(array $urlMatchers)
207		{
208	2	$this->clearWhitelistUrlMatchers();
209	2	foreach ($urlMatchers as $matcher) {
210	2	$this->addWhitelistUrlMatcher($matcher);
211	2	}
212
213	2	return $this;
214		}
215
216		/**
217		* @return Url\Matcher\UrlMatcherInterface[]
218		*/
219	3	public function getWhitelistUrlMatchers()
220		{
221	3	return $this->whitelistUrlMatchers;
222		}
223
224		/**
225		* @param UrlMatcherInterface $urlMatcher
226		* @return $this
227		*/
228	3	public function addWhitelistUrlMatcher(UrlMatcherInterface $urlMatcher)
229		{
230	3	$this->whitelistUrlMatchers[] = $urlMatcher;
231
232	3	return $this;
233		}
234
235		/**
236		* @return $this
237		*/
238	2	public function clearWhitelistUrlMatchers()
239		{
240	2	$this->whitelistUrlMatchers = [];
241
242	2	return $this;
243		}
244
245		/**
246		* @param array $urlMatchers
247		* @return $this
248		*/
249	2	public function setBlacklistUrlMatchers(array $urlMatchers)
250		{
251	2	$this->clearBlacklistUrlMatchers();
252	2	foreach ($urlMatchers as $matcher) {
253	2	$this->addBlacklistUrlMatcher($matcher);
254	2	}
255
256	2	return $this;
257		}
258
259		/**
260		* @return Url\Matcher\UrlMatcherInterface[]
261		*/
262	3	public function getBlacklistUrlMatchers()
263		{
264	3	return $this->blacklistUrlMatchers;
265		}
266
267		/**
268		* @param UrlMatcherInterface $urlMatcher
269		* @return $this
270		*/
271	3	public function addBlacklistUrlMatcher(UrlMatcherInterface $urlMatcher)
272		{
273	3	$this->blacklistUrlMatchers[] = $urlMatcher;
274
275	3	return $this;
276		}
277
278		/**
279		* @return $this
280		*/
281	2	public function clearBlacklistUrlMatchers()
282		{
283	2	$this->blacklistUrlMatchers = [];
284
285	2	return $this;
286		}
287
288		/**
289		* @param array $normalizers
290		* @return $this
291		*/
292	1	public function setUrlNormalizers(array $normalizers)
293		{
294	1	$this->clearUrlNormalizers();
295
296	1	foreach ($normalizers as $normalizer) {
297	1	$this->addUrlNormalizer($normalizer);
298	1	}
299
300	1	return $this;
301		}
302
303		/**
304		* @return UrlNormalizerInterface[]
305		*/
306	1	public function getUrlNormalizers()
307		{
308	1	return $this->urlNormalizers;
309		}
310
311		/**
312		* @param UrlNormalizerInterface $normalizer
313		* @return $this
314		*/
315	2	public function addUrlNormalizer(UrlNormalizerInterface $normalizer)
316		{
317	2	$this->urlNormalizers[] = $normalizer;
318
319	2	return $this;
320		}
321
322		/**
323		* @return $this
324		*/
325	1	public function clearUrlNormalizers()
326		{
327	1	$this->urlNormalizers = [];
328
329	1	return $this;
330		}
331
332		/**
333		* @return LoggerInterface
334		*/
335	9	public function getLogger()
336		{
337	9	if (is_null($this->logger)) {
338	7	$this->logger = new NullLogger();
339	7	}
340
341	9	return $this->logger;
342		}
343
344		/**
345		* @param LoggerInterface $logger
346		* @return $this
347		*/
348	2	public function setLogger(LoggerInterface $logger)
349		{
350	2	$this->logger = $logger;
351
352	2	return $this;
353		}
354
355		/**
356		* @param Url $url
357		*/
358	6	protected function addUrlToQueue(Url $url)
359		{
360	6	$this->urlsQueued[(string)$url] = $url;
361	6	}
362
363		/**
364		* @param $url
365		* @return Url
366		*/
367	6	protected function createHttpUrlString($url)
368		{
369	6	return Url::createFromString($url);
370		}
371
372		/**
373		* @param Url $url
374		*/
375	6	protected function reset(Url $url)
376		{
377	6	$this->baseUrl = $url;
378	6	$this->urlsCrawled = [];
379	6	$this->urlsQueued = [];
380
381	6	$this->addUrlToQueue($url);
382	6	}
383
384		/**
385		* @param string $url
386		* @return \Generator\|void
387		*/
388	6	public function crawl($url)
389		{
390	6	$url = $this->createHttpUrlString($url);
391	6	$this->reset($url);
392
393	6	while (count($this->urlsQueued) > 0) {
394
395	6	$url = array_shift($this->urlsQueued);
396
397		try {
398	6	$crawler = $this->requestPage((string)$url);
399	6	} catch (\Exception $e) {
400		$this->getLogger()->error(sprintf('Error requesting page %s: %s', $url, $e->getMessage()));
401
402		if ($this->getStopOnError()) {
403		return;
404		}
405
406		continue;
407		}
408
409	6	$this->urlsCrawled[] = (string)$url;
410	6	$this->updateQueue($crawler);
411
412	6	if ($this->shouldReturnUrl($url)) {
413	6	$this->getLogger()->debug(sprintf('Return url "%s"', $url));
414
415	6	$this->urlsReturned[] = (string)$url;
416
417	6	yield new Page($url, $crawler);
418	6	}
419
420	6	if ($this->isLimitReached()) {
421	1	$this->getLogger()->info(sprintf('Crawl limit of %d was reach', $this->limit));
422
423	1	return;
424		}
425	6	}
426	5	}
427
428		/**
429		* @param DomCrawler $crawler
430		*/
431	6	protected function updateQueue(DomCrawler $crawler)
432		{
433	6	foreach ($this->extractUrlsFromCrawler($crawler) as $url) {
434	5	$this->getLogger()->debug(sprintf('Found url %s in page', $url));
435		try {
436	5	$url = $this->normalizeUrl($this->createHttpUrlString($url));
437
438	5	if ($this->shouldCrawlUrl($url)) {
439	5	$this->addUrlToQueue($url);
440	5	}
441	5	} catch (\Exception $e) {
442	5	$this->getLogger()->warning(
443	5	sprintf('Url %s could not be converted to an object: %s', $url, $e->getMessage())
444	5	);
445	5	$this->urlsRejected[] = $url;
446		}
447	6	}
448	6	}
449
450		/**
451		* @param Url $url
452		* @return Url
453		*/
454	5	protected function normalizeUrl(Url $url)
455		{
456	5	foreach ($this->urlNormalizers as $normalizer) {
457	1	$url = $normalizer->normalize($url);
458	5	}
459
460	5	return $url;
461		}
462
463		/**
464		* @param Url $url
465		* @return bool
466		*/
467	6	protected function shouldReturnUrl(Url $url)
468		{
469	6	if (!empty($this->whitelistUrlMatchers)) {
470	1	if (!$this->isUrlWhitelisted($url)) {
471	1	$this->getLogger()->info(sprintf('Skipped "%s" because it is not whitelisted', $url));
472
473	1	return false;
474		}
475	1	}
476
477	6	if ($this->isUrlBlacklisted($url)) {
478	1	$this->getLogger()->info(sprintf('Skipped "%s" because it is blacklisted', $url));
479
480	1	return false;
481		}
482
483	6	return true;
484		}
485
486		/**
487		* @param Url $url
488		* @return bool
489		*/
490	1	protected function isUrlWhitelisted(Url $url)
491		{
492	1	foreach ($this->whitelistUrlMatchers as $matcher) {
493	1	if ($matcher->matches($url)) {
494	1	return true;
495		}
496	1	}
497
498	1	return false;
499		}
500
501		/**
502		* @param Url $url
503		* @return bool
504		*/
505	6	protected function isUrlBlacklisted(Url $url)
506		{
507	6	foreach ($this->blacklistUrlMatchers as $matcher) {
508	1	if ($matcher->matches($url)) {
509	1	return true;
510		}
511	6	}
512
513	6	return false;
514		}
515
516		/**
517		* @param Url $url
518		* @return bool
519		*/
520	5	protected function shouldCrawlUrl(Url $url)
521		{
522	5	$urlString = (string)$url;
523	5	if (in_array($urlString, $this->urlsRejected)) {
524		return false;
525		}
526	5	if (in_array($urlString, $this->urlsCrawled)) {
527	4	return false;
528		}
529	5	if (isset($this->urlsQueued[$urlString])) {
530	1	return false;
531		}
532
533	5	if (!$this->isUrlPartOfBaseUrl($url)) {
534	5	$this->urlsRejected[] = (string)$url;
535
536	5	return false;
537		}
538
539	5	return true;
540		}
541
542		/**
543		* @param Url $url
544		* @return bool
545		*/
546	5	protected function isUrlPartOfBaseUrl(Url $url)
547		{
548	5	$baseUrlString = (string)$this->baseUrl;
549	5	$this->getLogger()->debug($baseUrlString.' - '.$url);
550	5	if (strpos((string)$url, $baseUrlString) === false) {
551	5	return false;
552		}
553
554	5	return true;
555		}
556
557		/**
558		* @return bool
559		*/
560	6	private function isLimitReached()
561		{
562	6	return (!empty($this->limit) && count($this->urlsReturned) === $this->limit);
563		}
564
565		/**
566		* @param DomCrawler $crawler
567		* @return array
568		*/
569	6	protected function extractUrlsFromCrawler(DomCrawler $crawler)
570		{
571	6	return $crawler->filter('a')->each(
572	5	function (DomCrawler $node) {
573	5	return $node->link()->getUri();
574		}
575	6	);
576		}
577
578		/**
579		* @param $url
580		* @return DomCrawler
581		*/
582	6	protected function requestPage($url)
583		{
584	6	$this->getLogger()->info(sprintf('Crawling page %s', $url));
585	6	$crawler = $this->client->request('GET', $url);
586	6	$this->getLogger()->info(sprintf('Crawled page %s', $url));
587
588	6	return $crawler;
589		}
590		}
591

mediamonks / crawler

Push — master ( 1380cb...844cd9 )

Crawler::updateQueue() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like