|
1
|
|
|
<?php |
|
2
|
|
|
declare(strict_types=1); |
|
3
|
|
|
namespace AOE\Crawler\CrawlStrategy; |
|
4
|
|
|
|
|
5
|
|
|
/* |
|
6
|
|
|
* This file is part of the TYPO3 Crawler Extension. |
|
7
|
|
|
* |
|
8
|
|
|
* It is free software; you can redistribute it and/or modify it under |
|
9
|
|
|
* the terms of the GNU General Public License, either version 2 |
|
10
|
|
|
* of the License, or any later version. |
|
11
|
|
|
* |
|
12
|
|
|
* For the full copyright and license information, please read the |
|
13
|
|
|
* LICENSE.txt file that was distributed with this source code. |
|
14
|
|
|
* |
|
15
|
|
|
* The TYPO3 project - inspiring people to share! |
|
16
|
|
|
*/ |
|
17
|
|
|
|
|
18
|
|
|
use GuzzleHttp\Exception\ServerException; |
|
19
|
|
|
use Psr\Http\Message\UriInterface; |
|
20
|
|
|
use Psr\Log\LoggerAwareInterface; |
|
21
|
|
|
use Psr\Log\LoggerAwareTrait; |
|
22
|
|
|
use TYPO3\CMS\Core\Http\RequestFactory; |
|
23
|
|
|
use TYPO3\CMS\Core\Utility\GeneralUtility; |
|
24
|
|
|
|
|
25
|
|
|
/** |
|
26
|
|
|
* Calls Guzzle / CURL (based on TYPO3 settings) for fetching a URL. |
|
27
|
|
|
*/ |
|
28
|
|
|
class GuzzleExecutionStrategy implements LoggerAwareInterface |
|
29
|
|
|
{ |
|
30
|
|
|
use LoggerAwareTrait; |
|
31
|
|
|
|
|
32
|
|
|
/** |
|
33
|
|
|
* Sets up a CURL / Guzzle Request for fetching the request. |
|
34
|
|
|
* |
|
35
|
|
|
* @param UriInterface $url |
|
36
|
|
|
* @param string $crawlerId |
|
37
|
|
|
* @return bool|mixed |
|
38
|
|
|
*/ |
|
39
|
|
|
public function fetchUrlContents(UriInterface $url, string $crawlerId) |
|
40
|
|
|
{ |
|
41
|
|
|
$reqHeaders = $this->buildRequestHeaders($crawlerId); |
|
42
|
|
|
|
|
43
|
|
|
$options = ['headers' => $reqHeaders]; |
|
44
|
|
|
if ($url->getUserInfo()) { |
|
45
|
|
|
$options['auth'] = explode(':', $url->getUserInfo()); |
|
46
|
|
|
} |
|
47
|
|
|
try { |
|
48
|
|
|
$url = (string)$url; |
|
49
|
|
|
$response = GeneralUtility::makeInstance(RequestFactory::class) |
|
50
|
|
|
->request( |
|
51
|
|
|
$url, |
|
52
|
|
|
'GET', |
|
53
|
|
|
$options |
|
54
|
|
|
); |
|
55
|
|
|
$contents = $response->getBody()->getContents(); |
|
56
|
|
|
return unserialize($contents); |
|
57
|
|
|
} catch (ServerException $e) { |
|
|
|
|
|
|
58
|
|
|
$this->logger->debug( |
|
59
|
|
|
sprintf('Error while opening "%s"', $url), |
|
60
|
|
|
['crawlerId' => $crawlerId] |
|
61
|
|
|
); |
|
62
|
|
|
return false; |
|
63
|
|
|
} |
|
64
|
|
|
} |
|
65
|
|
|
|
|
66
|
|
|
/** |
|
67
|
|
|
* Builds HTTP request headers. |
|
68
|
|
|
* |
|
69
|
|
|
* @param string $crawlerId |
|
70
|
|
|
* @return array |
|
71
|
|
|
*/ |
|
72
|
|
|
protected function buildRequestHeaders(string $crawlerId): array |
|
73
|
|
|
{ |
|
74
|
|
|
return [ |
|
75
|
|
|
'Connection' => 'close', |
|
76
|
|
|
'X-T3Crawler' => $crawlerId, |
|
77
|
|
|
'User-Agent' => 'TYPO3 crawler' |
|
78
|
|
|
]; |
|
79
|
|
|
} |
|
80
|
|
|
|
|
81
|
|
|
} |
|
82
|
|
|
|
AOEpeople /
crawler
Scrutinizer analyzes your
composer.json/composer.lockfile if available to determine the classes, and functions that are defined by your dependencies.It seems like the listed class was neither found in your dependencies, nor was it found in the analyzed files in your repository. If you are using some other form of dependency management, you might want to disable this analysis.