SpiderTest::testCrawlFailedRequest() - Code Metrics - Inspection of "Merge pull request #27 from mvdbos/feature/increas..." - mvdbos/php-spider - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 6f4e11...e12aed )

by Matthijs

created 2016-01-29 15:18 UTC

SpiderTest::testCrawlFailedRequest() A

↳ Parent: SpiderTest

Complexity

Conditions	1
Paths	1

Size

Total Lines	13
Code Lines	8

Duplication

Lines	0
Ratio	0 %

Importance

Changes	5
Bugs	0	Features	1

Metric	Value
c	5
b	0
f	1
dl	0
loc	13
rs	9.4285
cc	1
eloc	8
nc	1
nop	0

<?php
namespace VDB\Spider;

use Exception;
use GuzzleHttp\Psr7\Response;
use PHPUnit_Framework_MockObject_MockObject;
use VDB\Spider\Discoverer\XPathExpressionDiscoverer;
use VDB\Spider\Tests\TestCase;
use VDB\Spider\QueueManager\InMemoryQueueManager;
use VDB\Spider\Uri\DiscoveredUri;
use VDB\Uri\Uri;

/**
 */
class SpiderTest extends TestCase
{
    /**
     * @var Spider
     */
    protected $spider;

    /**
     * @var PHPUnit_Framework_MockObject_MockObject
     */
    protected $requestHandler;

    /** @var DiscoveredUri */
    protected $linkA;
    /** @var DiscoveredUri */
    protected $linkB;
    /** @var DiscoveredUri */
    protected $linkC;
    /** @var DiscoveredUri */
    protected $linkD;
    /** @var DiscoveredUri */
    protected $linkE;
    /** @var DiscoveredUri */
    protected $linkF;
    /** @var DiscoveredUri */
    protected $linkG;

    /** @var Response */
    protected $responseA;
    /** @var Response */
    protected $responseB;
    /** @var Response */
    protected $responseC;
    /** @var Response */
    protected $responseD;
    /** @var Response */
    protected $responseE;
    /** @var Response */
    protected $responseF;
    /** @var Response */
    protected $responseG;

    /** @var string */
    protected $hrefA;
    protected $hrefB;
    protected $hrefC;
    protected $hrefD;
    protected $hrefE;
    protected $hrefF;
    protected $hrefG;

    /**
     * @var array An associative array, containing a map of $this->linkX to $this->responseX.
     */
    protected $linkToResponseMap = [];

    /**
     * Sets up the fixture, for example, opens a network connection.
     * This method is called before a test is executed.
     *
     * Setting up the following structure:
     *
     * 0:        A
     *          /|\
     * 1:      B C E
     *        /| | |
     * 2:    D F G |
     *         | _ |
     *
     * Note: E links to F.
     */
    protected function setUp()
    {
        $this->spider = new Spider('http://php-spider.org/A');

        $this->requestHandler = $this->getMock('VDB\Spider\RequestHandler\RequestHandlerInterface');

        $this->hrefA = 'http://php-spider.org/A';
        $this->hrefB = 'http://php-spider.org/B';
        $this->hrefC = 'http://php-spider.org/C';
        $this->hrefD = 'http://php-spider.org/D';
        $this->hrefE = 'http://php-spider.org/E';
        $this->hrefF = 'http://php-spider.org/F';
        $this->hrefG = 'http://php-spider.org/G';

        $this->linkA = new DiscoveredUri(new Uri($this->hrefA));
        $this->linkB = new DiscoveredUri(new Uri($this->hrefB));
        $this->linkC = new DiscoveredUri(new Uri($this->hrefC));
        $this->linkD = new DiscoveredUri(new Uri($this->hrefD));
        $this->linkE = new DiscoveredUri(new Uri($this->hrefE));
        $this->linkF = new DiscoveredUri(new Uri($this->hrefF));
        $this->linkG = new DiscoveredUri(new Uri($this->hrefG));

        $this->linkA->setDepthFound(0);
        $this->linkB->setDepthFound(1);
        $this->linkC->setDepthFound(1);
        $this->linkD->setDepthFound(2);
        $this->linkE->setDepthFound(1);
        $this->linkF->setDepthFound(2);
        $this->linkG->setDepthFound(2);

        $htmlA = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceA.html');
        $this->responseA = new Response(200, [], $htmlA);

        $htmlB = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceB.html');
        $this->responseB = new Response(200, [], $htmlB);

        $htmlC = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceC.html');
        $this->responseC = new Response(200, [], $htmlC);

        $htmlD = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceD.html');
        $this->responseD = new Response(200, [], $htmlD);

        $htmlE = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceE.html');
        $this->responseE = new Response(200, [], $htmlE);

        $htmlF = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceF.html');
        $this->responseF = new Response(200, [], $htmlF);

        $htmlG = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceG.html');
        $this->responseG = new Response(200, [], $htmlG);

        $this->linkToResponseMap[$this->linkA->toString()] = $this->responseA;
        $this->linkToResponseMap[$this->linkB->toString()] = $this->responseB;
        $this->linkToResponseMap[$this->linkC->toString()] = $this->responseC;
        $this->linkToResponseMap[$this->linkD->toString()] = $this->responseD;
        $this->linkToResponseMap[$this->linkE->toString()] = $this->responseE;
        $this->linkToResponseMap[$this->linkF->toString()] = $this->responseF;
        $this->linkToResponseMap[$this->linkG->toString()] = $this->responseG;

        $this->requestHandler
            ->expects($this->any())
            ->method('request')
            ->will($this->returnCallback(array($this, 'doTestRequest')));

        $this->spider->getDownloader()->setRequestHandler($this->requestHandler);

        $this->spider->getDiscovererSet()->set(new XPathExpressionDiscoverer('//a'));
    }

    /**
     * @return Resource
     * @throws \ErrorException
     */
    public function doTestRequest()
    {
        $link = func_get_arg(0);

        if (array_key_exists($link->toString(), $this->linkToResponseMap)) {
            return $this->getResource($link, $this->linkToResponseMap[$link->toString()]);
        }

        throw new \ErrorException('The requested URI was not stubbed: ' . $link->toString());
    }

    /**
     * @covers VDB\Spider\Spider
     *
     * Behaviour as explained here: https://en.wikipedia.org/wiki/Depth-first_search#Example
     */
    public function testCrawlDFSDefaultBehaviour()
    {
        $this->spider->getDiscovererSet()->maxDepth = 10;

        $this->spider->crawl();

        $expected = array(
            $this->linkA,
            $this->linkE,
            $this->linkF,
            $this->linkC,
            $this->linkG,
            $this->linkB,
            $this->linkD
        );

        $this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
    }

    /**
     * @covers VDB\Spider\Spider
     */
    public function testCrawlBFSDefaultBehaviour()
    {
        $this->spider->getQueueManager()->setTraversalAlgorithm(InMemoryQueueManager::ALGORITHM_BREADTH_FIRST);
        $this->spider->getDiscovererSet()->maxDepth = 1000;

        $this->spider->crawl();

        $expected = array(
            $this->linkA,
            $this->linkB,
            $this->linkC,
            $this->linkE,
            $this->linkD,
            $this->linkF,
            $this->linkG
        );

        $this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
    }

    private function compareUriArray($expected, $actual)
    {
        foreach ($actual as $index => $resource) {
            $this->assertEquals($resource->getUri(), $expected[$index]);
        }
    }

    /**
     * @covers VDB\Spider\Spider
     *
     * Behaviour as explained here: https://en.wikipedia.org/wiki/Depth-first_search#Example
     *
     * Given the following structure:
     *
     * 0:        A
     *          /|\
     * 1:      B C E
     *        /| | |
     * 2:    D F G |
     *         | _ |
     *
     * We expect the following result: A, E, C, B
     *
     */
    public function testCrawlDFSMaxDepthOne()
    {
        $this->spider->getDiscovererSet()->maxDepth = 1;

        $this->spider->crawl();

        $expected = array(
            $this->linkA,
            $this->linkE,
            $this->linkC,
            $this->linkB,
        );

        $this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
    }

    /**
     * @covers VDB\Spider\Spider
     */
    public function testCrawlBFSMaxDepthOne()
    {
        $this->spider->getQueueManager()->setTraversalAlgorithm(InMemoryQueueManager::ALGORITHM_BREADTH_FIRST);
        $this->spider->getDiscovererSet()->maxDepth = 1;

        $this->spider->crawl();

        $expected = array(
            $this->linkA,
            $this->linkB,
            $this->linkC,
            $this->linkE,
        );

        $this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
    }

    /**
     * @covers VDB\Spider\Spider
     */
    public function testCrawlDFSMaxQueueSize()
    {
        $this->spider->getDiscovererSet()->maxDepth = 1000;
        $this->spider->getDownloader()->setDownloadLimit(3);

        $this->spider->crawl();

        $expected = array(
            $this->linkA,
            $this->linkE,
            $this->linkF,
        );

        $this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
    }

    /**
     * @covers VDB\Spider\Spider
     */
    public function testCrawlBFSMaxQueueSize()
    {
        $this->spider->getQueueManager()->setTraversalAlgorithm(InMemoryQueueManager::ALGORITHM_BREADTH_FIRST);
        $this->spider->getDiscovererSet()->maxDepth = 1000;
        $this->spider->getDownloader()->setDownloadLimit(3);

        $this->spider->crawl();

        $expected = array(
            $this->linkA,
            $this->linkB,
            $this->linkC,
        );

        $this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
    }

    /**
     * @covers VDB\Spider\Spider
     */
    public function testCrawlFailedRequest()
    {
        $this->requestHandler
            ->expects($this->any())
            ->method('request')
            ->will(
                $this->throwException(new Exception('Failed mock request!'))
            );

        $this->spider->crawl();

        $this->assertCount(0, $this->spider->getDownloader()->getPersistenceHandler(), 'Persisted count');
    }
}


1		<?php
2		namespace VDB\Spider;
3
4		use Exception;
5		use GuzzleHttp\Psr7\Response;
6		use PHPUnit_Framework_MockObject_MockObject;
7		use VDB\Spider\Discoverer\XPathExpressionDiscoverer;
8		use VDB\Spider\Tests\TestCase;
9		use VDB\Spider\QueueManager\InMemoryQueueManager;
10		use VDB\Spider\Uri\DiscoveredUri;
11		use VDB\Uri\Uri;
12
13		/**
14		*/
15		class SpiderTest extends TestCase
16		{
17		/**
18		* @var Spider
19		*/
20		protected $spider;
21
22		/**
23		* @var PHPUnit_Framework_MockObject_MockObject
24		*/
25		protected $requestHandler;
26
27		/** @var DiscoveredUri */
28		protected $linkA;
29		/** @var DiscoveredUri */
30		protected $linkB;
31		/** @var DiscoveredUri */
32		protected $linkC;
33		/** @var DiscoveredUri */
34		protected $linkD;
35		/** @var DiscoveredUri */
36		protected $linkE;
37		/** @var DiscoveredUri */
38		protected $linkF;
39		/** @var DiscoveredUri */
40		protected $linkG;
41
42		/** @var Response */
43		protected $responseA;
44		/** @var Response */
45		protected $responseB;
46		/** @var Response */
47		protected $responseC;
48		/** @var Response */
49		protected $responseD;
50		/** @var Response */
51		protected $responseE;
52		/** @var Response */
53		protected $responseF;
54		/** @var Response */
55		protected $responseG;
56
57		/** @var string */
58		protected $hrefA;
59		protected $hrefB;
60		protected $hrefC;
61		protected $hrefD;
62		protected $hrefE;
63		protected $hrefF;
64		protected $hrefG;
65
66		/**
67		* @var array An associative array, containing a map of $this->linkX to $this->responseX.
68		*/
69		protected $linkToResponseMap = [];
70
71		/**
72		* Sets up the fixture, for example, opens a network connection.
73		* This method is called before a test is executed.
74		*
75		* Setting up the following structure:
76		*
77		* 0: A
78		* /\|\
79		* 1: B C E
80		* /\| \| \|
81		* 2: D F G \|
82		* \| _ \|
83		*
84		* Note: E links to F.
85		*/
86		protected function setUp()
87		{
88		$this->spider = new Spider('http://php-spider.org/A');
89
90		$this->requestHandler = $this->getMock('VDB\Spider\RequestHandler\RequestHandlerInterface');
91
92		$this->hrefA = 'http://php-spider.org/A';
93		$this->hrefB = 'http://php-spider.org/B';
94		$this->hrefC = 'http://php-spider.org/C';
95		$this->hrefD = 'http://php-spider.org/D';
96		$this->hrefE = 'http://php-spider.org/E';
97		$this->hrefF = 'http://php-spider.org/F';
98		$this->hrefG = 'http://php-spider.org/G';
99
100		$this->linkA = new DiscoveredUri(new Uri($this->hrefA));
101		$this->linkB = new DiscoveredUri(new Uri($this->hrefB));
102		$this->linkC = new DiscoveredUri(new Uri($this->hrefC));
103		$this->linkD = new DiscoveredUri(new Uri($this->hrefD));
104		$this->linkE = new DiscoveredUri(new Uri($this->hrefE));
105		$this->linkF = new DiscoveredUri(new Uri($this->hrefF));
106		$this->linkG = new DiscoveredUri(new Uri($this->hrefG));
107
108		$this->linkA->setDepthFound(0);
109		$this->linkB->setDepthFound(1);
110		$this->linkC->setDepthFound(1);
111		$this->linkD->setDepthFound(2);
112		$this->linkE->setDepthFound(1);
113		$this->linkF->setDepthFound(2);
114		$this->linkG->setDepthFound(2);
115
116		$htmlA = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceA.html');
117		$this->responseA = new Response(200, [], $htmlA);
118
119		$htmlB = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceB.html');
120		$this->responseB = new Response(200, [], $htmlB);
121
122		$htmlC = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceC.html');
123		$this->responseC = new Response(200, [], $htmlC);
124
125		$htmlD = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceD.html');
126		$this->responseD = new Response(200, [], $htmlD);
127
128		$htmlE = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceE.html');
129		$this->responseE = new Response(200, [], $htmlE);
130
131		$htmlF = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceF.html');
132		$this->responseF = new Response(200, [], $htmlF);
133
134		$htmlG = file_get_contents(__DIR__ . '/Fixtures/SpiderTestHTMLResourceG.html');
135		$this->responseG = new Response(200, [], $htmlG);
136
137		$this->linkToResponseMap[$this->linkA->toString()] = $this->responseA;
138		$this->linkToResponseMap[$this->linkB->toString()] = $this->responseB;
139		$this->linkToResponseMap[$this->linkC->toString()] = $this->responseC;
140		$this->linkToResponseMap[$this->linkD->toString()] = $this->responseD;
141		$this->linkToResponseMap[$this->linkE->toString()] = $this->responseE;
142		$this->linkToResponseMap[$this->linkF->toString()] = $this->responseF;
143		$this->linkToResponseMap[$this->linkG->toString()] = $this->responseG;
144
145		$this->requestHandler
146		->expects($this->any())
147		->method('request')
148		->will($this->returnCallback(array($this, 'doTestRequest')));
149
150		$this->spider->getDownloader()->setRequestHandler($this->requestHandler);
151
152		$this->spider->getDiscovererSet()->set(new XPathExpressionDiscoverer('//a'));
153		}
154
155		/**
156		* @return Resource
157		* @throws \ErrorException
158		*/
159		public function doTestRequest()
160		{
161		$link = func_get_arg(0);
162
163		if (array_key_exists($link->toString(), $this->linkToResponseMap)) {
164		return $this->getResource($link, $this->linkToResponseMap[$link->toString()]);
165		}
166
167		throw new \ErrorException('The requested URI was not stubbed: ' . $link->toString());
168		}
169
170		/**
171		* @covers VDB\Spider\Spider
172		*
173		* Behaviour as explained here: https://en.wikipedia.org/wiki/Depth-first_search#Example
174		*/
175	View Code Duplication	public function testCrawlDFSDefaultBehaviour()
176		{
177		$this->spider->getDiscovererSet()->maxDepth = 10;
178
179		$this->spider->crawl();
180
181		$expected = array(
182		$this->linkA,
183		$this->linkE,
184		$this->linkF,
185		$this->linkC,
186		$this->linkG,
187		$this->linkB,
188		$this->linkD
189		);
190
191		$this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
192		}
193
194		/**
195		* @covers VDB\Spider\Spider
196		*/
197	View Code Duplication	public function testCrawlBFSDefaultBehaviour()
198		{
199		$this->spider->getQueueManager()->setTraversalAlgorithm(InMemoryQueueManager::ALGORITHM_BREADTH_FIRST);
200		$this->spider->getDiscovererSet()->maxDepth = 1000;
201
202		$this->spider->crawl();
203
204		$expected = array(
205		$this->linkA,
206		$this->linkB,
207		$this->linkC,
208		$this->linkE,
209		$this->linkD,
210		$this->linkF,
211		$this->linkG
212		);
213
214		$this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
215		}
216
217		private function compareUriArray($expected, $actual)
218		{
219		foreach ($actual as $index => $resource) {
220		$this->assertEquals($resource->getUri(), $expected[$index]);
221		}
222		}
223
224		/**
225		* @covers VDB\Spider\Spider
226		*
227		* Behaviour as explained here: https://en.wikipedia.org/wiki/Depth-first_search#Example
228		*
229		* Given the following structure:
230		*
231		* 0: A
232		* /\|\
233		* 1: B C E
234		* /\| \| \|
235		* 2: D F G \|
236		* \| _ \|
237		*
238		* We expect the following result: A, E, C, B
239		*
240		*/
241		public function testCrawlDFSMaxDepthOne()
242		{
243		$this->spider->getDiscovererSet()->maxDepth = 1;
244
245		$this->spider->crawl();
246
247		$expected = array(
248		$this->linkA,
249		$this->linkE,
250		$this->linkC,
251		$this->linkB,
252		);
253
254		$this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
255		}
256
257		/**
258		* @covers VDB\Spider\Spider
259		*/
260	View Code Duplication	public function testCrawlBFSMaxDepthOne()
261		{
262		$this->spider->getQueueManager()->setTraversalAlgorithm(InMemoryQueueManager::ALGORITHM_BREADTH_FIRST);
263		$this->spider->getDiscovererSet()->maxDepth = 1;
264
265		$this->spider->crawl();
266
267		$expected = array(
268		$this->linkA,
269		$this->linkB,
270		$this->linkC,
271		$this->linkE,
272		);
273
274		$this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
275		}
276
277		/**
278		* @covers VDB\Spider\Spider
279		*/
280	View Code Duplication	public function testCrawlDFSMaxQueueSize()
281		{
282		$this->spider->getDiscovererSet()->maxDepth = 1000;
283		$this->spider->getDownloader()->setDownloadLimit(3);
284
285		$this->spider->crawl();
286
287		$expected = array(
288		$this->linkA,
289		$this->linkE,
290		$this->linkF,
291		);
292
293		$this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
294		}
295
296		/**
297		* @covers VDB\Spider\Spider
298		*/
299	View Code Duplication	public function testCrawlBFSMaxQueueSize()
300		{
301		$this->spider->getQueueManager()->setTraversalAlgorithm(InMemoryQueueManager::ALGORITHM_BREADTH_FIRST);
302		$this->spider->getDiscovererSet()->maxDepth = 1000;
303		$this->spider->getDownloader()->setDownloadLimit(3);
304
305		$this->spider->crawl();
306
307		$expected = array(
308		$this->linkA,
309		$this->linkB,
310		$this->linkC,
311		);
312
313		$this->compareUriArray($expected, $this->spider->getDownloader()->getPersistenceHandler());
314		}
315
316		/**
317		* @covers VDB\Spider\Spider
318		*/
319		public function testCrawlFailedRequest()
320		{
321		$this->requestHandler
322		->expects($this->any())
323		->method('request')
324		->will(
325		$this->throwException(new Exception('Failed mock request!'))
326		);
327
328		$this->spider->crawl();
329
330		$this->assertCount(0, $this->spider->getDownloader()->getPersistenceHandler(), 'Persisted count');
331		}
332		}
333

mvdbos / php-spider

Push — master ( 6f4e11...e12aed )

SpiderTest::testCrawlFailedRequest() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like