|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
namespace PiedWeb\UrlHarvester; |
|
4
|
|
|
|
|
5
|
|
|
use PiedWeb\Curl\Request as CurlRequest; |
|
6
|
|
|
use PiedWeb\Curl\Response; |
|
7
|
|
|
|
|
8
|
|
|
/** |
|
9
|
|
|
* Request a page and get it only if it's an html page. |
|
10
|
|
|
*/ |
|
11
|
|
|
class Request |
|
12
|
|
|
{ |
|
13
|
|
|
/** |
|
14
|
|
|
* @var string |
|
15
|
|
|
*/ |
|
16
|
|
|
private $url; |
|
17
|
|
|
|
|
18
|
|
|
/** |
|
19
|
|
|
* @var string |
|
20
|
|
|
*/ |
|
21
|
|
|
private $userAgent; |
|
22
|
|
|
|
|
23
|
|
|
/** |
|
24
|
|
|
* @var string |
|
25
|
|
|
*/ |
|
26
|
|
|
private $language; |
|
27
|
|
|
|
|
28
|
|
|
/** |
|
29
|
|
|
* @var string |
|
30
|
|
|
*/ |
|
31
|
|
|
private $proxy; |
|
32
|
|
|
|
|
33
|
|
|
/** |
|
34
|
|
|
* @var string |
|
35
|
|
|
*/ |
|
36
|
|
|
private $downloadOnly; |
|
37
|
|
|
|
|
38
|
|
|
/** |
|
39
|
|
|
* @var CurlRequest |
|
40
|
|
|
*/ |
|
41
|
|
|
private $request; |
|
42
|
|
|
|
|
43
|
|
|
/** |
|
44
|
|
|
* @var Response |
|
45
|
|
|
*/ |
|
46
|
|
|
private $response; |
|
47
|
|
|
|
|
48
|
|
|
/** |
|
49
|
|
|
* @param string $url |
|
50
|
|
|
* @param string $userAgent |
|
51
|
|
|
* @param string $language |
|
52
|
|
|
* @param bool $tryHttps |
|
53
|
|
|
* |
|
54
|
|
|
* @return self |
|
55
|
|
|
*/ |
|
56
|
|
|
public static function make( |
|
57
|
|
|
string $url, |
|
58
|
|
|
string $userAgent, |
|
59
|
|
|
$downloadOnly = '200;html', |
|
60
|
|
|
string $language = 'en,en-US;q=0.5', |
|
61
|
30 |
|
?string $proxy = null |
|
62
|
|
|
) { |
|
63
|
|
|
$request = new Request($url); |
|
64
|
|
|
|
|
65
|
|
|
$request->userAgent = $userAgent; |
|
66
|
|
|
$request->downloadOnly = $downloadOnly; |
|
67
|
|
|
$request->language = $language; |
|
68
|
|
|
$request->proxy = $proxy; |
|
69
|
30 |
|
|
|
70
|
|
|
return $request->request(); |
|
71
|
30 |
|
} |
|
72
|
30 |
|
|
|
73
|
30 |
|
private function __construct($url) |
|
74
|
30 |
|
{ |
|
75
|
30 |
|
/* |
|
76
|
|
|
if (!filter_var($string, FILTER_VALIDATE_URL)) { |
|
77
|
30 |
|
throw new \Exception('URL invalid: '.$string); |
|
78
|
|
|
}**/ |
|
79
|
30 |
|
$this->url = $url; |
|
80
|
|
|
} |
|
81
|
|
|
|
|
82
|
30 |
|
/** |
|
83
|
|
|
* Prepare headers as a normal browser (same order, same content). |
|
84
|
|
|
* |
|
85
|
|
|
* @return array |
|
86
|
|
|
*/ |
|
87
|
|
|
private function prepareHeadersForRequest() |
|
88
|
30 |
|
{ |
|
89
|
30 |
|
$host = parse_url($this->url, PHP_URL_HOST); |
|
90
|
|
|
|
|
91
|
|
|
$headers = []; |
|
92
|
|
|
$headers[] = 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'; |
|
93
|
|
|
$headers[] = 'Accept-Encoding: gzip, deflate'; |
|
94
|
|
|
$headers[] = 'Accept-Language: '.$this->language; |
|
95
|
|
|
$headers[] = 'Connection: keep-alive'; |
|
96
|
30 |
|
|
|
97
|
|
|
if ($host) { |
|
98
|
30 |
|
//$headers[] = 'Host: '.$host; |
|
99
|
|
|
} |
|
100
|
30 |
|
// Referer |
|
101
|
30 |
|
|
|
102
|
30 |
|
$headers[] = 'Upgrade-Insecure-Requests: 1'; |
|
103
|
30 |
|
$headers[] = 'User-Agent: '.$this->userAgent; |
|
104
|
30 |
|
|
|
105
|
|
|
return $headers; |
|
106
|
30 |
|
} |
|
107
|
|
|
|
|
108
|
|
|
/** |
|
109
|
|
|
*/ |
|
110
|
|
|
private function request() |
|
111
|
30 |
|
{ |
|
112
|
30 |
|
$this->request = new CurlRequest($this->url); |
|
113
|
|
|
$this->request |
|
114
|
30 |
|
->setReturnHeader() |
|
115
|
|
|
->setEncodingGzip() |
|
116
|
|
|
->setUserAgent($this->userAgent) |
|
117
|
|
|
->setDefaultSpeedOptions() |
|
118
|
|
|
->setOpt(CURLOPT_SSL_VERIFYHOST, 0) |
|
119
|
|
|
->setOpt(CURLOPT_SSL_VERIFYPEER, 0) |
|
120
|
30 |
|
->setOpt(CURLOPT_MAXREDIRS, 1) |
|
121
|
|
|
->setOpt(CURLOPT_COOKIE, false) |
|
122
|
30 |
|
->setOpt(CURLOPT_CONNECTTIMEOUT, 20) |
|
123
|
30 |
|
->setOpt(CURLOPT_TIMEOUT, 80); |
|
124
|
30 |
|
|
|
125
|
30 |
|
$this->setDownloadOnly(); |
|
126
|
30 |
|
|
|
127
|
30 |
|
if ($this->proxy) { |
|
128
|
30 |
|
$this->request->setProxy($this->proxy); |
|
129
|
30 |
|
} |
|
130
|
30 |
|
|
|
131
|
30 |
|
$this->request->setOpt(CURLOPT_HTTPHEADER, $this->prepareHeadersForRequest()); |
|
132
|
30 |
|
|
|
133
|
30 |
|
$this->response = $this->request->exec(); |
|
|
|
|
|
|
134
|
|
|
|
|
135
|
30 |
|
return $this->response; |
|
136
|
|
|
} |
|
137
|
30 |
|
|
|
138
|
|
|
protected function setDownloadOnly() |
|
139
|
|
|
{ |
|
140
|
|
|
if ($this->downloadOnly) { |
|
141
|
30 |
|
if ('200;html' == $this->downloadOnly) { |
|
142
|
|
|
$download = new \PiedWeb\Curl\MultipleCheckInHeaders(); |
|
143
|
30 |
|
$this->request->setDownloadOnlyIf([$download, 'check']); |
|
144
|
|
|
} elseif (is_callable($this->downloadOnly)) { |
|
145
|
|
|
$this->request->setDownloadOnlyIf($this->downloadOnly); |
|
146
|
30 |
|
} |
|
147
|
|
|
} |
|
148
|
|
|
} |
|
149
|
|
|
} |
|
150
|
|
|
|
PiedWeb /
UrlHarvester
Our type inference engine has found a suspicous assignment of a value to a property. This check raises an issue when a value that can be of a mixed type is assigned to a property that is type hinted more strictly.
For example, imagine you have a variable
$accountIdthat can either hold an Id object or false (if there is no account id yet). Your code now assigns that value to theidproperty of an instance of theAccountclass. This class holds a proper account, so the id value must no longer be false.Either this assignment is in error or a type check should be added for that assignment.