|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
/* |
|
4
|
|
|
* This file is part of the Spider package. |
|
5
|
|
|
* |
|
6
|
|
|
* (c) Matthijs van den Bos <[email protected]> |
|
7
|
|
|
* |
|
8
|
|
|
* For the full copyright and license information, please view the LICENSE |
|
9
|
|
|
* file that was distributed with this source code. |
|
10
|
|
|
*/ |
|
11
|
|
|
|
|
12
|
|
|
namespace VDB\Spider\Tests\Downloader; |
|
13
|
|
|
|
|
14
|
|
|
use GuzzleHttp\Psr7\Response; |
|
15
|
|
|
use VDB\Spider\Tests\TestCase; |
|
16
|
|
|
use VDB\Spider\Downloader\DownloaderInterface; |
|
17
|
|
|
use VDB\Spider\Downloader\Downloader; |
|
18
|
|
|
use VDB\Spider\Uri\DiscoveredUri; |
|
19
|
|
|
use VDB\Spider\Resource; |
|
20
|
|
|
use VDB\Uri\Uri; |
|
21
|
|
|
|
|
22
|
|
|
/** |
|
23
|
|
|
* |
|
24
|
|
|
*/ |
|
25
|
|
|
class DownloaderTest extends TestCase |
|
26
|
|
|
{ |
|
27
|
|
|
/** |
|
28
|
|
|
* @var Downloader |
|
29
|
|
|
*/ |
|
30
|
|
|
private $downloader; |
|
31
|
|
|
|
|
32
|
|
|
/** |
|
33
|
|
|
* @var Resource |
|
34
|
|
|
*/ |
|
35
|
|
|
protected $resource; |
|
36
|
|
|
|
|
37
|
|
|
/** |
|
38
|
|
|
* @var string |
|
39
|
|
|
*/ |
|
40
|
|
|
protected $html; |
|
41
|
|
|
|
|
42
|
|
|
public function setUp() |
|
43
|
|
|
{ |
|
44
|
|
|
$this->html = file_get_contents(__DIR__ . '/../Fixtures/DownloaderTestHTMLResource.html'); |
|
45
|
|
|
$this->resource = new Resource( |
|
46
|
|
|
new DiscoveredUri(new Uri('/domains/special', 'http://example.org')), |
|
47
|
|
|
new Response(200, [], $this->html) |
|
48
|
|
|
); |
|
49
|
|
|
|
|
50
|
|
|
$this->downloader = new Downloader(); |
|
51
|
|
|
|
|
52
|
|
|
$requestHandler = $this->getMock('VDB\Spider\RequestHandler\RequestHandlerInterface'); |
|
53
|
|
|
$requestHandler |
|
54
|
|
|
->expects($this->any()) |
|
55
|
|
|
->method('request') |
|
56
|
|
|
->will($this->returnValue($this->resource)); |
|
57
|
|
|
|
|
58
|
|
|
$this->downloader->setRequestHandler($requestHandler); |
|
59
|
|
|
} |
|
60
|
|
|
|
|
61
|
|
|
/** |
|
62
|
|
|
* @covers VDB\Spider\Downloader\Downloader::download() |
|
63
|
|
|
* @covers VDB\Spider\Downloader\Downloader::getPersistenceHandler() |
|
64
|
|
|
* @covers VDB\Spider\Downloader\Downloader::fetchResource() |
|
65
|
|
|
* @covers VDB\Spider\Downloader\Downloader::getRequestHandler() |
|
66
|
|
|
* @covers VDB\Spider\Downloader\Downloader::getRequestHandler() |
|
67
|
|
|
*/ |
|
68
|
|
|
public function testDownload() |
|
69
|
|
|
{ |
|
70
|
|
|
$resource = $this->downloader->download(new DiscoveredUri(new Uri('http://foobar.org'))); |
|
71
|
|
|
$this->assertInstanceOf('VDB\\Spider\\Resource', $resource); |
|
72
|
|
|
} |
|
73
|
|
|
|
|
74
|
|
|
/** |
|
75
|
|
|
* @covers VDB\Spider\Downloader\Downloader::download() |
|
76
|
|
|
* @covers VDB\Spider\Downloader\Downloader::fetchResource() |
|
77
|
|
|
* @covers VDB\Spider\Downloader\Downloader::getPersistenceHandler() |
|
78
|
|
|
* @covers VDB\Spider\Downloader\Downloader::getRequestHandler() |
|
79
|
|
|
* @covers VDB\Spider\Downloader\Downloader::setRequestHandler() |
|
80
|
|
|
*/ |
|
81
|
|
View Code Duplication |
public function testDownloadFailed() |
|
|
|
|
|
|
82
|
|
|
{ |
|
83
|
|
|
$requestHandler = $this->getMock('VDB\Spider\RequestHandler\RequestHandlerInterface'); |
|
84
|
|
|
$requestHandler |
|
85
|
|
|
->expects($this->any()) |
|
86
|
|
|
->method('request') |
|
87
|
|
|
->will($this->throwException(new \Exception)); |
|
88
|
|
|
$this->downloader->setRequestHandler($requestHandler); |
|
89
|
|
|
|
|
90
|
|
|
$resource = $this->downloader->download(new DiscoveredUri(new Uri('http://foobar.org'))); |
|
91
|
|
|
|
|
92
|
|
|
$this->assertFalse($resource); |
|
93
|
|
|
} |
|
94
|
|
|
|
|
95
|
|
|
/** |
|
96
|
|
|
* @covers VDB\Spider\Downloader\Downloader::download() |
|
97
|
|
|
* @covers VDB\Spider\Downloader\Downloader::getRequestHandler() |
|
98
|
|
|
* @covers VDB\Spider\Downloader\Downloader::getPersistenceHandler() |
|
99
|
|
|
* @covers VDB\Spider\Downloader\Downloader::matchesPostfetchFilter() |
|
100
|
|
|
* @covers VDB\Spider\Downloader\Downloader::fetchResource() |
|
101
|
|
|
* @covers VDB\Spider\Downloader\Downloader::addPostFetchFilter() |
|
102
|
|
|
*/ |
|
103
|
|
View Code Duplication |
public function testFilterNotMatches() |
|
|
|
|
|
|
104
|
|
|
{ |
|
105
|
|
|
$filterNeverMatch = $this->getMock('VDB\Spider\Filter\PostFetchFilterInterface'); |
|
106
|
|
|
$filterNeverMatch |
|
107
|
|
|
->expects($this->any()) |
|
108
|
|
|
->method('match') |
|
109
|
|
|
->will($this->returnValue(false)); |
|
110
|
|
|
$this->downloader->addPostFetchFilter($filterNeverMatch); |
|
111
|
|
|
|
|
112
|
|
|
$resource = $this->downloader->download(new DiscoveredUri(new Uri('http://foobar.org'))); |
|
113
|
|
|
|
|
114
|
|
|
$this->assertInstanceOf('VDB\\Spider\\Resource', $resource); |
|
115
|
|
|
} |
|
116
|
|
|
|
|
117
|
|
|
/** |
|
118
|
|
|
* @covers VDB\Spider\Downloader\Downloader::download() |
|
119
|
|
|
* @covers VDB\Spider\Downloader\Downloader::getPersistenceHandler() |
|
120
|
|
|
* @covers VDB\Spider\Downloader\Downloader::getRequestHandler() |
|
121
|
|
|
* @covers VDB\Spider\Downloader\Downloader::matchesPostfetchFilter() |
|
122
|
|
|
* @covers VDB\Spider\Downloader\Downloader::fetchResource() |
|
123
|
|
|
* @covers VDB\Spider\Downloader\Downloader::addPostFetchFilter() |
|
124
|
|
|
*/ |
|
125
|
|
View Code Duplication |
public function testFilterMatches() |
|
|
|
|
|
|
126
|
|
|
{ |
|
127
|
|
|
$filterAlwaysMatch = $this->getMock('VDB\Spider\Filter\PostFetchFilterInterface'); |
|
128
|
|
|
$filterAlwaysMatch |
|
129
|
|
|
->expects($this->any()) |
|
130
|
|
|
->method('match') |
|
131
|
|
|
->will($this->returnValue(true)); |
|
132
|
|
|
$this->downloader->addPostFetchFilter($filterAlwaysMatch); |
|
133
|
|
|
|
|
134
|
|
|
$resource = $this->downloader->download(new DiscoveredUri(new Uri('http://foobar.org'))); |
|
135
|
|
|
|
|
136
|
|
|
$this->assertFalse($resource); |
|
137
|
|
|
} |
|
138
|
|
|
} |
|
139
|
|
|
|
mvdbos /
php-spider
Loading history...
Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation.
You can also find more detailed suggestions in the “Code” section of your repository.