1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* vipnytt/RobotsTxtParser |
4
|
|
|
* |
5
|
|
|
* @link https://github.com/VIPnytt/RobotsTxtParser |
6
|
|
|
* @license https://github.com/VIPnytt/RobotsTxtParser/blob/master/LICENSE The MIT License (MIT) |
7
|
|
|
*/ |
8
|
|
|
|
9
|
|
|
namespace vipnytt\RobotsTxtParser\Parser\Directives; |
10
|
|
|
|
11
|
|
|
use vipnytt\RobotsTxtParser\Client\Directives\UserAgentClient; |
12
|
|
|
use vipnytt\RobotsTxtParser\Client\Directives\UserAgentTools; |
13
|
|
|
use vipnytt\RobotsTxtParser\Handler\Directives\SubDirectiveHandler; |
14
|
|
|
use vipnytt\RobotsTxtParser\Handler\RenderHandler; |
15
|
|
|
use vipnytt\RobotsTxtParser\RobotsTxtInterface; |
16
|
|
|
use vipnytt\UserAgentParser as UserAgentStringParser; |
17
|
|
|
|
18
|
|
|
/** |
19
|
|
|
* Class UserAgentParser |
20
|
|
|
* |
21
|
|
|
* @package vipnytt\RobotsTxtParser\Parser\Directives |
22
|
|
|
*/ |
23
|
|
|
class UserAgentParser implements ParserInterface, RobotsTxtInterface |
24
|
|
|
{ |
25
|
|
|
use DirectiveParserTrait; |
26
|
|
|
|
27
|
|
|
/** |
28
|
|
|
* Sub directives white list |
29
|
|
|
*/ |
30
|
|
|
const SUB_DIRECTIVES = [ |
31
|
|
|
self::DIRECTIVE_ALLOW => 'allow', |
32
|
|
|
self::DIRECTIVE_CACHE_DELAY => 'cacheDelay', |
33
|
|
|
self::DIRECTIVE_COMMENT => 'comment', |
34
|
|
|
self::DIRECTIVE_CRAWL_DELAY => 'crawlDelay', |
35
|
|
|
self::DIRECTIVE_DISALLOW => 'disallow', |
36
|
|
|
self::DIRECTIVE_NO_INDEX => 'noIndex', |
37
|
|
|
self::DIRECTIVE_REQUEST_RATE => 'requestRate', |
38
|
|
|
self::DIRECTIVE_ROBOT_VERSION => 'robotVersion', |
39
|
|
|
self::DIRECTIVE_VISIT_TIME => 'visitTime', |
40
|
|
|
]; |
41
|
|
|
|
42
|
|
|
/** |
43
|
|
|
* Base uri |
44
|
|
|
* @var string |
45
|
|
|
*/ |
46
|
|
|
private $base; |
47
|
|
|
|
48
|
|
|
/** |
49
|
|
|
* User-agent handler |
50
|
|
|
* @var SubDirectiveHandler[] |
51
|
|
|
*/ |
52
|
|
|
private $handler = []; |
53
|
|
|
|
54
|
|
|
/** |
55
|
|
|
* Current User-agent(s) |
56
|
|
|
* @var string[] |
57
|
|
|
*/ |
58
|
|
|
private $current = []; |
59
|
|
|
|
60
|
|
|
/** |
61
|
|
|
* Append User-agent |
62
|
|
|
* @var bool |
63
|
|
|
*/ |
64
|
|
|
private $append = false; |
65
|
|
|
|
66
|
|
|
/** |
67
|
|
|
* UserAgent constructor. |
68
|
|
|
* |
69
|
|
|
* @param string $base |
70
|
|
|
*/ |
71
|
|
|
public function __construct($base) |
72
|
|
|
{ |
73
|
|
|
$this->base = $base; |
74
|
|
|
$this->handlerAdd(self::USER_AGENT); |
75
|
|
|
} |
76
|
|
|
|
77
|
|
|
/** |
78
|
|
|
* Add sub-directive handler |
79
|
|
|
* |
80
|
|
|
* @param string $group |
81
|
|
|
* @return bool |
82
|
|
|
*/ |
83
|
|
|
private function handlerAdd($group) |
84
|
|
|
{ |
85
|
|
|
if (!in_array($group, array_keys($this->handler))) { |
86
|
|
|
$this->handler[$group] = new SubDirectiveHandler($this->base, $group); |
87
|
|
|
return true; |
88
|
|
|
} |
89
|
|
|
return false; |
90
|
|
|
} |
91
|
|
|
|
92
|
|
|
/** |
93
|
|
|
* Add line |
94
|
|
|
* |
95
|
|
|
* @param string $line |
96
|
|
|
* @return bool |
97
|
|
|
*/ |
98
|
|
|
public function add($line) |
99
|
|
|
{ |
100
|
|
|
if ($line == '' || |
101
|
|
|
($pair = $this->generateRulePair($line, [-1 => self::DIRECTIVE_USER_AGENT] + array_keys(self::SUB_DIRECTIVES))) === false) { |
102
|
|
|
return $this->append = false; |
103
|
|
|
} |
104
|
|
|
if ($pair[0] === self::DIRECTIVE_USER_AGENT) { |
105
|
|
|
return $this->set($pair[1]); |
106
|
|
|
} |
107
|
|
|
$this->append = false; |
108
|
|
|
$result = []; |
109
|
|
|
foreach ($this->current as $group) { |
110
|
|
|
$result[] = $this->handler[$group]->{self::SUB_DIRECTIVES[$pair[0]]}->add($pair[1]); |
111
|
|
|
$this->handler[$group]->count++; |
112
|
|
|
} |
113
|
|
|
return in_array(true, $result, true); |
114
|
|
|
} |
115
|
|
|
|
116
|
|
|
/** |
117
|
|
|
* Set new User-agent |
118
|
|
|
* |
119
|
|
|
* @param string $group |
120
|
|
|
* @return bool |
121
|
|
|
*/ |
122
|
|
|
private function set($group) |
123
|
|
|
{ |
124
|
|
|
if (!$this->append) { |
125
|
|
|
$this->current = []; |
126
|
|
|
} |
127
|
|
|
$group = mb_strtolower($group); |
128
|
|
|
$this->current[] = $group; |
129
|
|
|
$this->handlerAdd($group); |
130
|
|
|
$this->append = true; |
131
|
|
|
return true; |
132
|
|
|
} |
133
|
|
|
|
134
|
|
|
/** |
135
|
|
|
* Render |
136
|
|
|
* |
137
|
|
|
* @param RenderHandler $handler |
138
|
|
|
* @return bool |
139
|
|
|
*/ |
140
|
|
|
public function render(RenderHandler $handler) |
141
|
|
|
{ |
142
|
|
|
return $handler->getLevel() == 2 ? $this->renderExtensive($handler) : $this->renderCompressed($handler); |
143
|
|
|
} |
144
|
|
|
|
145
|
|
|
/** |
146
|
|
|
* Render extensive |
147
|
|
|
* |
148
|
|
|
* @param RenderHandler $handler |
149
|
|
|
* @return bool |
150
|
|
|
*/ |
151
|
|
|
private function renderExtensive(RenderHandler $handler) |
152
|
|
|
{ |
153
|
|
|
$userAgents = $this->getUserAgents(); |
154
|
|
|
rsort($userAgents); |
155
|
|
|
foreach ($userAgents as $userAgent) { |
156
|
|
|
$handler->add(self::DIRECTIVE_USER_AGENT, $userAgent); |
157
|
|
|
$this->renderAdd($userAgent, $handler); |
158
|
|
|
} |
159
|
|
|
return true; |
160
|
|
|
} |
161
|
|
|
|
162
|
|
|
/** |
163
|
|
|
* User-agent list |
164
|
|
|
* |
165
|
|
|
* @return string[] |
166
|
|
|
*/ |
167
|
|
|
public function getUserAgents() |
168
|
|
|
{ |
169
|
|
|
$list = array_keys($this->handler); |
170
|
|
|
sort($list); |
171
|
|
|
return $list; |
172
|
|
|
} |
173
|
|
|
|
174
|
|
|
/** |
175
|
|
|
* Add sub-directives to the RenderHandler |
176
|
|
|
* |
177
|
|
|
* @param string $userAgent |
178
|
|
|
* @param RenderHandler $handler |
179
|
|
|
*/ |
180
|
|
|
private function renderAdd($userAgent, RenderHandler $handler) |
181
|
|
|
{ |
182
|
|
|
if ($userAgent !== self::USER_AGENT && |
183
|
|
|
$this->handler[$userAgent]->count === 0 |
184
|
|
|
) { |
185
|
|
|
$handler->add(self::DIRECTIVE_DISALLOW, ''); |
186
|
|
|
return; |
187
|
|
|
} |
188
|
|
|
$this->handler[$userAgent]->robotVersion->render($handler); |
189
|
|
|
$this->handler[$userAgent]->visitTime->render($handler); |
190
|
|
|
$this->handler[$userAgent]->noIndex->render($handler); |
191
|
|
|
$this->handler[$userAgent]->disallow->render($handler); |
192
|
|
|
$this->handler[$userAgent]->allow->render($handler); |
193
|
|
|
$this->handler[$userAgent]->crawlDelay->render($handler); |
194
|
|
|
$this->handler[$userAgent]->cacheDelay->render($handler); |
195
|
|
|
$this->handler[$userAgent]->requestRate->render($handler); |
196
|
|
|
$this->handler[$userAgent]->comment->render($handler); |
197
|
|
|
} |
198
|
|
|
|
199
|
|
|
/** |
200
|
|
|
* Render compressed |
201
|
|
|
* |
202
|
|
|
* @param RenderHandler $handler |
203
|
|
|
* @return bool |
204
|
|
|
*/ |
205
|
|
|
private function renderCompressed(RenderHandler $handler) |
206
|
|
|
{ |
207
|
|
|
$pair = $this->export(); |
208
|
|
|
while (!empty($pair)) { |
209
|
|
|
$groupMembers = current($pair); |
210
|
|
|
foreach (array_keys($pair, $groupMembers) as $userAgent) { |
211
|
|
|
$handler->add(self::DIRECTIVE_USER_AGENT, $userAgent); |
212
|
|
|
unset($pair[$userAgent]); |
213
|
|
|
} |
214
|
|
|
if (isset($userAgent)) { |
215
|
|
|
$this->renderAdd($userAgent, $handler); |
216
|
|
|
unset($userAgent); |
217
|
|
|
} |
218
|
|
|
} |
219
|
|
|
return true; |
220
|
|
|
} |
221
|
|
|
|
222
|
|
|
/** |
223
|
|
|
* Export |
224
|
|
|
* |
225
|
|
|
* @return array |
226
|
|
|
*/ |
227
|
|
|
public function export() |
228
|
|
|
{ |
229
|
|
|
$array = []; |
230
|
|
|
foreach ($this->getUserAgents() as $group) { |
231
|
|
|
if ($group == self::USER_AGENT && |
232
|
|
|
$this->handler[$group]->count === 0 |
233
|
|
|
) { |
234
|
|
|
continue; |
235
|
|
|
} |
236
|
|
|
$array[$group] = (new UserAgentTools($this->handler[$group], $this->base))->export(); |
237
|
|
|
} |
238
|
|
|
return $array; |
239
|
|
|
} |
240
|
|
|
|
241
|
|
|
/** |
242
|
|
|
* Client |
243
|
|
|
* |
244
|
|
|
* @param string $product |
245
|
|
|
* @param float|int|string|null $version |
246
|
|
|
* @param int|null $statusCode |
247
|
|
|
* @return UserAgentClient |
248
|
|
|
*/ |
249
|
|
|
public function client($product = self::USER_AGENT, $version = null, $statusCode = null) |
250
|
|
|
{ |
251
|
|
|
$parser = new UserAgentStringParser($product, $version); |
252
|
|
|
$match = $parser->getUserAgent(); |
253
|
|
|
if (!isset($this->handler[$match])) { |
254
|
|
|
// User-agent does not match any rule sets |
255
|
|
|
if (($match = $parser->getMostSpecific($this->getUserAgents())) === false) { |
256
|
|
|
$match = self::USER_AGENT; |
257
|
|
|
} |
258
|
|
|
} |
259
|
|
|
return new UserAgentClient($this->handler[$match], $this->base, $statusCode, $parser->getProduct()); |
260
|
|
|
} |
261
|
|
|
} |
262
|
|
|
|