1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* vipnytt/RobotsTxtParser |
4
|
|
|
* |
5
|
|
|
* @link https://github.com/VIPnytt/RobotsTxtParser |
6
|
|
|
* @license https://github.com/VIPnytt/RobotsTxtParser/blob/master/LICENSE The MIT License (MIT) |
7
|
|
|
*/ |
8
|
|
|
|
9
|
|
|
namespace vipnytt\RobotsTxtParser\Parser\Directives; |
10
|
|
|
|
11
|
|
|
use vipnytt\RobotsTxtParser\Client\Directives\UserAgentClient; |
12
|
|
|
use vipnytt\RobotsTxtParser\Handler\Directives\SubDirectiveHandler; |
13
|
|
|
use vipnytt\RobotsTxtParser\Handler\RenderHandler; |
14
|
|
|
use vipnytt\RobotsTxtParser\RobotsTxtInterface; |
15
|
|
|
use vipnytt\UserAgentParser as UserAgentStringParser; |
16
|
|
|
|
17
|
|
|
/** |
18
|
|
|
* Class UserAgentParser |
19
|
|
|
* |
20
|
|
|
* @package vipnytt\RobotsTxtParser\Parser\Directives |
21
|
|
|
*/ |
22
|
|
|
class UserAgentParser implements ParserInterface, RobotsTxtInterface |
23
|
|
|
{ |
24
|
|
|
use DirectiveParserTrait; |
25
|
|
|
|
26
|
|
|
/** |
27
|
|
|
* Sub directives white list |
28
|
|
|
*/ |
29
|
|
|
const SUB_DIRECTIVES = [ |
30
|
|
|
self::DIRECTIVE_ALLOW => 'allow', |
31
|
|
|
self::DIRECTIVE_CACHE_DELAY => 'cacheDelay', |
32
|
|
|
self::DIRECTIVE_COMMENT => 'comment', |
33
|
|
|
self::DIRECTIVE_CRAWL_DELAY => 'crawlDelay', |
34
|
|
|
self::DIRECTIVE_DISALLOW => 'disallow', |
35
|
|
|
self::DIRECTIVE_NO_INDEX => 'noindex', |
36
|
|
|
self::DIRECTIVE_REQUEST_RATE => 'requestRate', |
37
|
|
|
self::DIRECTIVE_ROBOT_VERSION => 'robotVersion', |
38
|
|
|
self::DIRECTIVE_VISIT_TIME => 'visitTime', |
39
|
|
|
]; |
40
|
|
|
|
41
|
|
|
/** |
42
|
|
|
* Base uri |
43
|
|
|
* @var string |
44
|
|
|
*/ |
45
|
|
|
private $base; |
46
|
|
|
|
47
|
|
|
/** |
48
|
|
|
* Effective uri |
49
|
|
|
* @var string |
50
|
|
|
*/ |
51
|
|
|
private $effective; |
52
|
|
|
|
53
|
|
|
/** |
54
|
|
|
* User-agent handler |
55
|
|
|
* @var SubDirectiveHandler[] |
56
|
|
|
*/ |
57
|
|
|
private $handler = []; |
58
|
|
|
|
59
|
|
|
/** |
60
|
|
|
* Current User-agent(s) |
61
|
|
|
* @var string[] |
62
|
|
|
*/ |
63
|
|
|
private $current = []; |
64
|
|
|
|
65
|
|
|
/** |
66
|
|
|
* Append User-agent |
67
|
|
|
* @var bool |
68
|
|
|
*/ |
69
|
|
|
private $append = false; |
70
|
|
|
|
71
|
|
|
/** |
72
|
|
|
* User-agent directive count |
73
|
|
|
* @var int[] |
74
|
|
|
*/ |
75
|
|
|
private $count = []; |
76
|
|
|
|
77
|
|
|
/** |
78
|
|
|
* User-agent client cache |
79
|
|
|
* @var UserAgentClient[] |
80
|
|
|
*/ |
81
|
|
|
private $client = []; |
82
|
|
|
|
83
|
|
|
/** |
84
|
|
|
* UserAgent constructor. |
85
|
|
|
* |
86
|
|
|
* @param string $base |
87
|
|
|
* @param string $effective |
88
|
|
|
*/ |
89
|
|
|
public function __construct($base, $effective) |
90
|
|
|
{ |
91
|
|
|
$this->base = $base; |
92
|
|
|
$this->effective = $effective; |
93
|
|
|
$this->set(self::USER_AGENT); |
94
|
|
|
$this->append = false; |
95
|
|
|
} |
96
|
|
|
|
97
|
|
|
/** |
98
|
|
|
* Set new User-agent |
99
|
|
|
* |
100
|
|
|
* @param string $userAgent |
101
|
|
|
* @return bool |
102
|
|
|
*/ |
103
|
|
|
private function set($userAgent) |
104
|
|
|
{ |
105
|
|
|
if (!$this->append) { |
106
|
|
|
$this->current = []; |
107
|
|
|
} |
108
|
|
|
$userAgent = mb_strtolower($userAgent); |
109
|
|
|
if (in_array(self::USER_AGENT, array_merge($this->current, [$userAgent]))) { |
110
|
|
|
$this->current = []; |
111
|
|
|
$userAgent = self::USER_AGENT; |
112
|
|
|
} |
113
|
|
|
$this->current[] = $userAgent; |
114
|
|
|
if (!in_array($userAgent, array_keys($this->handler))) { |
115
|
|
|
$this->handler[$userAgent] = new SubDirectiveHandler($this->base, $this->effective, $userAgent); |
116
|
|
|
$this->count[$userAgent] = 0; |
117
|
|
|
} |
118
|
|
|
$this->append = true; |
119
|
|
|
return true; |
120
|
|
|
} |
121
|
|
|
|
122
|
|
|
/** |
123
|
|
|
* Add |
124
|
|
|
* |
125
|
|
|
* @param string $line |
126
|
|
|
* @return bool |
127
|
|
|
*/ |
128
|
|
|
public function add($line) |
129
|
|
|
{ |
130
|
|
|
if (($pair = $this->generateRulePair($line, array_merge([self::DIRECTIVE_USER_AGENT], array_keys(self::SUB_DIRECTIVES)))) === false) { |
131
|
|
|
$this->append = false; |
132
|
|
|
return false; |
133
|
|
|
} |
134
|
|
|
if ($pair['directive'] === self::DIRECTIVE_USER_AGENT) { |
135
|
|
|
return $this->set($pair['value']); |
136
|
|
|
} |
137
|
|
|
$this->append = false; |
138
|
|
|
$result = []; |
139
|
|
|
foreach ($this->current as $userAgent) { |
140
|
|
|
$result[] = $this->handler[$userAgent]->{self::SUB_DIRECTIVES[$pair['directive']]}()->add($pair['value']); |
141
|
|
|
$this->count[$userAgent]++; |
142
|
|
|
} |
143
|
|
|
return in_array(true, $result, true); |
144
|
|
|
} |
145
|
|
|
|
146
|
|
|
/** |
147
|
|
|
* Render |
148
|
|
|
* |
149
|
|
|
* @param RenderHandler $handler |
150
|
|
|
* @return bool |
151
|
|
|
*/ |
152
|
|
|
public function render(RenderHandler $handler) |
153
|
|
|
{ |
154
|
|
|
return $handler->getLevel() >= 3 ? $this->renderExtensive($handler) : $this->renderCompressed($handler); |
155
|
|
|
} |
156
|
|
|
|
157
|
|
|
/** |
158
|
|
|
* Render extensive |
159
|
|
|
* |
160
|
|
|
* @param RenderHandler $handler |
161
|
|
|
* @return bool |
162
|
|
|
*/ |
163
|
|
|
private function renderExtensive(RenderHandler $handler) |
164
|
|
|
{ |
165
|
|
|
$userAgents = $this->getUserAgents(); |
166
|
|
|
rsort($userAgents); |
167
|
|
|
foreach ($userAgents as $userAgent) { |
168
|
|
|
$handler->add(self::DIRECTIVE_USER_AGENT, $userAgent); |
169
|
|
|
$this->renderAdd($userAgent, $handler); |
170
|
|
|
} |
171
|
|
|
return true; |
172
|
|
|
} |
173
|
|
|
|
174
|
|
|
/** |
175
|
|
|
* User-agent list |
176
|
|
|
* |
177
|
|
|
* @return string[] |
178
|
|
|
*/ |
179
|
|
|
public function getUserAgents() |
180
|
|
|
{ |
181
|
|
|
$list = array_keys(array_filter($this->count)); |
182
|
|
|
sort($list); |
183
|
|
|
return $list; |
184
|
|
|
} |
185
|
|
|
|
186
|
|
|
private function renderAdd($userAgent, RenderHandler $handler) |
187
|
|
|
{ |
188
|
|
|
$this->handler[$userAgent]->robotVersion()->render($handler); |
189
|
|
|
$this->handler[$userAgent]->visitTime()->render($handler); |
190
|
|
|
$this->handler[$userAgent]->noIndex()->render($handler); |
191
|
|
|
$this->handler[$userAgent]->disallow()->render($handler); |
192
|
|
|
$this->handler[$userAgent]->allow()->render($handler); |
193
|
|
|
$this->handler[$userAgent]->crawlDelay()->render($handler); |
194
|
|
|
$this->handler[$userAgent]->cacheDelay()->render($handler); |
195
|
|
|
$this->handler[$userAgent]->requestRate()->render($handler); |
196
|
|
|
$this->handler[$userAgent]->comment()->render($handler); |
197
|
|
|
} |
198
|
|
|
|
199
|
|
|
/** |
200
|
|
|
* Render compressed |
201
|
|
|
* |
202
|
|
|
* @param RenderHandler $handler |
203
|
|
|
* @return bool |
204
|
|
|
*/ |
205
|
|
|
private function renderCompressed(RenderHandler $handler) |
206
|
|
|
{ |
207
|
|
|
$pair = $this->export(); |
208
|
|
|
while (!empty($pair)) { |
209
|
|
|
$groupMembers = current($pair); |
210
|
|
|
foreach (array_keys($pair, $groupMembers) as $userAgent) { |
211
|
|
|
$handler->add(self::DIRECTIVE_USER_AGENT, $userAgent); |
212
|
|
|
unset($pair[$userAgent]); |
213
|
|
|
} |
214
|
|
|
if (isset($userAgent)) { |
215
|
|
|
$this->renderAdd($userAgent, $handler); |
216
|
|
|
unset($userAgent); |
217
|
|
|
} |
218
|
|
|
} |
219
|
|
|
return true; |
220
|
|
|
} |
221
|
|
|
|
222
|
|
|
/** |
223
|
|
|
* Export |
224
|
|
|
* |
225
|
|
|
* @return array |
226
|
|
|
*/ |
227
|
|
|
public function export() |
228
|
|
|
{ |
229
|
|
|
$array = []; |
230
|
|
|
foreach ($this->getUserAgents() as $userAgent) { |
231
|
|
|
$array[$userAgent] = $this->client($userAgent)->export(); |
232
|
|
|
} |
233
|
|
|
return $array; |
234
|
|
|
} |
235
|
|
|
|
236
|
|
|
/** |
237
|
|
|
* Client |
238
|
|
|
* |
239
|
|
|
* @param string $product |
240
|
|
|
* @param float|int|string|null $version |
241
|
|
|
* @param int|null $statusCode |
242
|
|
|
* @return UserAgentClient |
243
|
|
|
*/ |
244
|
|
|
public function client($product = self::USER_AGENT, $version = null, $statusCode = null) |
245
|
|
|
{ |
246
|
|
|
$userAgentString = $version === null ? $product : $product . '/' . $version; |
247
|
|
|
if (isset($this->client[$userAgentString])) { |
248
|
|
|
// Already cached |
249
|
|
|
return $this->client[$userAgentString]; |
250
|
|
|
} elseif (isset($this->handler[$userAgentString])) { |
251
|
|
|
// 100% match |
252
|
|
|
return $this->client[$userAgentString] = new UserAgentClient($this->handler[$userAgentString], $this->base, $statusCode, $product); |
253
|
|
|
} |
254
|
|
|
$userAgentParser = new UserAgentStringParser($product, $version); |
255
|
|
|
if (($match = $userAgentParser->getMostSpecific($this->getUserAgents())) === false) { |
256
|
|
|
$match = self::USER_AGENT; |
257
|
|
|
} |
258
|
|
|
return $this->client[$userAgentString] = new UserAgentClient($this->handler[$match], $this->base, $statusCode, $product); |
259
|
|
|
} |
260
|
|
|
} |
261
|
|
|
|