|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
namespace Spatie\Crawler\Handlers; |
|
4
|
|
|
|
|
5
|
|
|
use Psr\Http\Message\ResponseInterface; |
|
6
|
|
|
use Psr\Http\Message\StreamInterface; |
|
7
|
|
|
use Spatie\Crawler\Crawler; |
|
8
|
|
|
use Spatie\Crawler\CrawlSubdomains; |
|
9
|
|
|
use Spatie\Crawler\CrawlUrl; |
|
10
|
|
|
use Spatie\Robots\RobotsHeaders; |
|
11
|
|
|
use Spatie\Robots\RobotsMeta; |
|
12
|
|
|
|
|
13
|
|
|
class CrawlRequestFulfilled |
|
14
|
|
|
{ |
|
15
|
|
|
/** @var \Spatie\Crawler\Crawler */ |
|
16
|
|
|
protected $crawler; |
|
17
|
|
|
|
|
18
|
|
|
public function __construct(Crawler $crawler) { |
|
19
|
|
|
$this->crawler = $crawler; |
|
20
|
|
|
} |
|
21
|
|
|
|
|
22
|
|
|
public function __invoke(ResponseInterface $response, $index) |
|
23
|
|
|
{ |
|
24
|
|
|
$crawlUrl = $this->crawler->getCrawlQueue()->getUrlById($index); |
|
25
|
|
|
|
|
26
|
|
|
$body = $this->convertBodyToString($response->getBody(), $this->crawler->getMaximumResponseSize()); |
|
27
|
|
|
|
|
28
|
|
|
$robotsHeaders = RobotsHeaders::create($response->getHeaders()); |
|
29
|
|
|
|
|
30
|
|
|
$robotsMeta = RobotsMeta::create($body); |
|
31
|
|
|
|
|
32
|
|
|
if (! $this->mayIndex($robotsHeaders, $robotsMeta)) { |
|
33
|
|
|
return; |
|
34
|
|
|
} |
|
35
|
|
|
|
|
36
|
|
|
$this->handleCrawled($response, $crawlUrl); |
|
37
|
|
|
|
|
38
|
|
|
if (! $this->crawler->getCrawlProfile() instanceof CrawlSubdomains) { |
|
39
|
|
|
if ($crawlUrl->url->getHost() !== $this->crawler->getBaseUrl()->getHost()) { |
|
40
|
|
|
return; |
|
41
|
|
|
} |
|
42
|
|
|
} |
|
43
|
|
|
|
|
44
|
|
|
if (! $this->mayFollow($robotsHeaders, $robotsMeta)) { |
|
45
|
|
|
return; |
|
46
|
|
|
} |
|
47
|
|
|
|
|
48
|
|
|
$this->crawler->addAllLinksToCrawlQueue( |
|
49
|
|
|
$body, |
|
50
|
|
|
$crawlUrl->url |
|
51
|
|
|
); |
|
52
|
|
|
} |
|
53
|
|
|
|
|
54
|
|
|
protected function convertBodyToString(StreamInterface $bodyStream, $readMaximumBytes = 1024 * 1024 * 2): string |
|
55
|
|
|
{ |
|
56
|
|
|
$bodyStream->rewind(); |
|
57
|
|
|
|
|
58
|
|
|
$body = $bodyStream->read($readMaximumBytes); |
|
59
|
|
|
|
|
60
|
|
|
return $body; |
|
61
|
|
|
} |
|
62
|
|
|
|
|
63
|
|
|
protected function handleCrawled(ResponseInterface $response, CrawlUrl $crawlUrl) |
|
64
|
|
|
{ |
|
65
|
|
|
foreach ($this->crawler->getCrawlObservers() as $crawlObserver) { |
|
66
|
|
|
$crawlObserver->crawled( |
|
67
|
|
|
$crawlUrl->url, |
|
68
|
|
|
$response, |
|
69
|
|
|
$crawlUrl->foundOnUrl |
|
70
|
|
|
); |
|
71
|
|
|
} |
|
72
|
|
|
} |
|
73
|
|
|
|
|
74
|
|
View Code Duplication |
protected function mayIndex(RobotsHeaders $robotsHeaders, RobotsMeta $robotsMeta): bool |
|
|
|
|
|
|
75
|
|
|
{ |
|
76
|
|
|
if (! $this->crawler->mustRespectRobots()) { |
|
77
|
|
|
return true; |
|
78
|
|
|
} |
|
79
|
|
|
|
|
80
|
|
|
if (! $robotsHeaders->mayIndex()) { |
|
81
|
|
|
return false; |
|
82
|
|
|
} |
|
83
|
|
|
|
|
84
|
|
|
if (! $robotsMeta->mayIndex()) { |
|
85
|
|
|
return false; |
|
86
|
|
|
} |
|
87
|
|
|
|
|
88
|
|
|
return true; |
|
89
|
|
|
} |
|
90
|
|
|
|
|
91
|
|
View Code Duplication |
protected function mayFollow(RobotsHeaders $robotsHeaders, RobotsMeta $robotsMeta): bool |
|
|
|
|
|
|
92
|
|
|
{ |
|
93
|
|
|
if (! $this->crawler->mustRespectRobots()) { |
|
94
|
|
|
return true; |
|
95
|
|
|
} |
|
96
|
|
|
|
|
97
|
|
|
if (! $robotsHeaders->mayFollow()) { |
|
98
|
|
|
return false; |
|
99
|
|
|
} |
|
100
|
|
|
|
|
101
|
|
|
if (! $robotsMeta->mayFollow()) { |
|
102
|
|
|
return false; |
|
103
|
|
|
} |
|
104
|
|
|
|
|
105
|
|
|
return true; |
|
106
|
|
|
} |
|
107
|
|
|
} |
|
108
|
|
|
|
spatie /
crawler
Loading history...
Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation.
You can also find more detailed suggestions in the “Code” section of your repository.