|
1
|
|
|
<?php |
|
2
|
|
|
/** |
|
3
|
|
|
* vipnytt/RobotsTxtParser |
|
4
|
|
|
* |
|
5
|
|
|
* @link https://github.com/VIPnytt/RobotsTxtParser |
|
6
|
|
|
* @license https://github.com/VIPnytt/RobotsTxtParser/blob/master/LICENSE The MIT License (MIT) |
|
7
|
|
|
*/ |
|
8
|
|
|
|
|
9
|
|
|
namespace vipnytt\RobotsTxtParser\Client\Cache\MySQL; |
|
10
|
|
|
|
|
11
|
|
|
use vipnytt\RobotsTxtParser\Client\Cache\BaseCore; |
|
12
|
|
|
use vipnytt\RobotsTxtParser\Exceptions; |
|
13
|
|
|
use vipnytt\RobotsTxtParser\RobotsTxtInterface; |
|
14
|
|
|
use vipnytt\RobotsTxtParser\TxtClient; |
|
15
|
|
|
use vipnytt\RobotsTxtParser\UriClient; |
|
16
|
|
|
|
|
17
|
|
|
/** |
|
18
|
|
|
* Class Base |
|
19
|
|
|
* |
|
20
|
|
|
* @see https://vipnytt.github.io/RobotsTxtParser/methods/Cache.html for documentation |
|
21
|
|
|
* @package vipnytt\RobotsTxtParser\Handler\Cache\MySQL |
|
22
|
|
|
*/ |
|
23
|
|
|
class Base extends BaseCore implements RobotsTxtInterface |
|
24
|
|
|
{ |
|
25
|
|
|
/** |
|
26
|
|
|
* Debug - Get raw data |
|
27
|
|
|
* |
|
28
|
|
|
* @return array |
|
29
|
|
|
*/ |
|
30
|
|
View Code Duplication |
public function debug() |
|
|
|
|
|
|
31
|
|
|
{ |
|
32
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
33
|
|
|
SELECT * |
|
34
|
|
|
FROM robotstxt__cache1 |
|
35
|
|
|
WHERE base = :base; |
|
36
|
|
|
SQL |
|
37
|
|
|
); |
|
38
|
|
|
$query->bindValue('base', $this->base, \PDO::PARAM_STR); |
|
39
|
|
|
$query->execute(); |
|
40
|
|
|
return $query->rowCount() > 0 ? $query->fetch(\PDO::FETCH_ASSOC) : []; |
|
41
|
|
|
} |
|
42
|
|
|
|
|
43
|
|
|
/** |
|
44
|
|
|
* Parser client |
|
45
|
|
|
* |
|
46
|
|
|
* @return TxtClient |
|
47
|
|
|
* @throws Exceptions\OutOfSyncException |
|
48
|
|
|
* @throws Exceptions\DatabaseException |
|
49
|
|
|
*/ |
|
50
|
|
|
public function client() |
|
51
|
|
|
{ |
|
52
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
53
|
|
|
SELECT |
|
54
|
|
|
content, |
|
55
|
|
|
statusCode, |
|
56
|
|
|
nextUpdate, |
|
57
|
|
|
effective, |
|
58
|
|
|
worker, |
|
59
|
|
|
UNIX_TIMESTAMP() |
|
60
|
|
|
FROM robotstxt__cache1 |
|
61
|
|
|
WHERE base = :base; |
|
62
|
|
|
SQL |
|
63
|
|
|
); |
|
64
|
|
|
$query->bindValue('base', $this->base, \PDO::PARAM_STR); |
|
65
|
|
|
$query->execute(); |
|
66
|
|
|
if ($query->rowCount() > 0) { |
|
67
|
|
|
$row = $query->fetch(\PDO::FETCH_ASSOC); |
|
68
|
|
|
$this->clockSyncCheck($row['UNIX_TIMESTAMP()'], self::OUT_OF_SYNC_TIME_LIMIT); |
|
69
|
|
|
if ($row['nextUpdate'] >= $row['UNIX_TIMESTAMP()']) { |
|
70
|
|
|
$this->markAsActive($row['worker']); |
|
71
|
|
|
return new TxtClient($this->base, $row['statusCode'], $row['content'], self::ENCODING, $row['effective'], $this->byteLimit); |
|
72
|
|
|
} |
|
73
|
|
|
} |
|
74
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
75
|
|
|
UPDATE robotstxt__cache1 |
|
76
|
|
|
SET worker = 0 |
|
77
|
|
|
WHERE base = :base; |
|
78
|
|
|
SQL |
|
79
|
|
|
); |
|
80
|
|
|
$query->bindValue('base', $this->base, \PDO::PARAM_STR); |
|
81
|
|
|
$query->execute(); |
|
82
|
|
|
return $this->refresh(); |
|
83
|
|
|
} |
|
84
|
|
|
|
|
85
|
|
|
/** |
|
86
|
|
|
* Mark robots.txt as active |
|
87
|
|
|
* |
|
88
|
|
|
* @param int|null $workerID |
|
89
|
|
|
* @return bool |
|
90
|
|
|
*/ |
|
91
|
|
|
private function markAsActive($workerID = 0) |
|
92
|
|
|
{ |
|
93
|
|
|
if ($workerID == 0) { |
|
94
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
95
|
|
|
UPDATE robotstxt__cache1 |
|
96
|
|
|
SET worker = NULL |
|
97
|
|
|
WHERE base = :base AND worker = 0; |
|
98
|
|
|
SQL |
|
99
|
|
|
); |
|
100
|
|
|
$query->bindValue('base', $this->base, \PDO::PARAM_STR); |
|
101
|
|
|
return $query->execute(); |
|
102
|
|
|
} |
|
103
|
|
|
return true; |
|
104
|
|
|
} |
|
105
|
|
|
|
|
106
|
|
|
/** |
|
107
|
|
|
* Update the robots.txt |
|
108
|
|
|
* |
|
109
|
|
|
* @return UriClient |
|
110
|
|
|
* @throws Exceptions\DatabaseException |
|
111
|
|
|
*/ |
|
112
|
|
|
public function refresh() |
|
113
|
|
|
{ |
|
114
|
|
|
$client = new UriClient($this->base, $this->curlOptions, $this->byteLimit); |
|
115
|
|
|
$effective = $client->getEffectiveUri(); |
|
116
|
|
|
if ($effective == $this->base) { |
|
117
|
|
|
$effective = null; |
|
118
|
|
|
} |
|
119
|
|
|
$statusCode = $client->getStatusCode(); |
|
120
|
|
|
$nextUpdate = $client->nextUpdate(); |
|
121
|
|
|
if (strpos($this->base, 'http') === 0 && |
|
122
|
|
|
( |
|
123
|
|
|
$statusCode === null || |
|
124
|
|
|
( |
|
125
|
|
|
$statusCode >= 500 && |
|
126
|
|
|
$statusCode < 600 |
|
127
|
|
|
) |
|
128
|
|
|
) && |
|
129
|
|
|
$this->displacePush($nextUpdate) |
|
130
|
|
|
) { |
|
131
|
|
|
return $client; |
|
132
|
|
|
} |
|
133
|
|
|
$validUntil = $client->validUntil(); |
|
134
|
|
|
$content = $client->render()->compressed(self::RENDER_LINE_SEPARATOR); |
|
135
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
136
|
|
|
INSERT INTO robotstxt__cache1 (base, content, statusCode, validUntil, nextUpdate, effective) |
|
137
|
|
|
VALUES (:base, :content, :statusCode, :validUntil, :nextUpdate, :effective) |
|
138
|
|
|
ON DUPLICATE KEY UPDATE content = :content, statusCode = :statusCode, validUntil = :validUntil, |
|
139
|
|
|
nextUpdate = :nextUpdate, effective = :effective, worker = 0; |
|
140
|
|
|
SQL |
|
141
|
|
|
); |
|
142
|
|
|
$query->bindValue('base', $this->base, \PDO::PARAM_STR); |
|
143
|
|
|
$query->bindValue('content', $content, \PDO::PARAM_STR); |
|
144
|
|
|
$query->bindValue('statusCode', $statusCode, \PDO::PARAM_INT | \PDO::PARAM_NULL); |
|
145
|
|
|
$query->bindValue('validUntil', $validUntil, \PDO::PARAM_INT); |
|
146
|
|
|
$query->bindValue('nextUpdate', $nextUpdate, \PDO::PARAM_INT); |
|
147
|
|
|
$query->bindValue('effective', $effective, \PDO::PARAM_STR | \PDO::PARAM_NULL); |
|
148
|
|
|
$query->execute(); |
|
149
|
|
|
return $client; |
|
150
|
|
|
} |
|
151
|
|
|
|
|
152
|
|
|
/** |
|
153
|
|
|
* Displace push timestamp |
|
154
|
|
|
* |
|
155
|
|
|
* @param int $nextUpdate |
|
156
|
|
|
* @return bool |
|
157
|
|
|
* @throws Exceptions\OutOfSyncException |
|
158
|
|
|
*/ |
|
159
|
|
|
private function displacePush($nextUpdate) |
|
160
|
|
|
{ |
|
161
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
162
|
|
|
SELECT |
|
163
|
|
|
validUntil, |
|
164
|
|
|
UNIX_TIMESTAMP() |
|
165
|
|
|
FROM robotstxt__cache1 |
|
166
|
|
|
WHERE base = :base; |
|
167
|
|
|
SQL |
|
168
|
|
|
); |
|
169
|
|
|
$query->bindValue('base', $this->base, \PDO::PARAM_STR); |
|
170
|
|
|
$query->execute(); |
|
171
|
|
|
if ($query->rowCount() > 0) { |
|
172
|
|
|
$row = $query->fetch(\PDO::FETCH_ASSOC); |
|
173
|
|
|
$this->clockSyncCheck($row['UNIX_TIMESTAMP()'], self::OUT_OF_SYNC_TIME_LIMIT); |
|
174
|
|
|
if ($row['validUntil'] > $row['UNIX_TIMESTAMP()']) { |
|
175
|
|
|
$nextUpdate = min($row['validUntil'], $nextUpdate); |
|
176
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
177
|
|
|
UPDATE robotstxt__cache1 |
|
178
|
|
|
SET nextUpdate = :nextUpdate, worker = 0 |
|
179
|
|
|
WHERE base = :base; |
|
180
|
|
|
SQL |
|
181
|
|
|
); |
|
182
|
|
|
$query->bindValue('base', $this->base, \PDO::PARAM_STR); |
|
183
|
|
|
$query->bindValue('nextUpdate', $nextUpdate, \PDO::PARAM_INT); |
|
184
|
|
|
return $query->execute(); |
|
185
|
|
|
} |
|
186
|
|
|
} |
|
187
|
|
|
return false; |
|
188
|
|
|
} |
|
189
|
|
|
|
|
190
|
|
|
/** |
|
191
|
|
|
* Invalidate cache |
|
192
|
|
|
* |
|
193
|
|
|
* @return bool |
|
194
|
|
|
*/ |
|
195
|
|
|
public function invalidate() |
|
196
|
|
|
{ |
|
197
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
198
|
|
|
DELETE FROM robotstxt__cache1 |
|
199
|
|
|
WHERE base = :base; |
|
200
|
|
|
SQL |
|
201
|
|
|
); |
|
202
|
|
|
$query->bindValue('base', $this->base, \PDO::PARAM_STR); |
|
203
|
|
|
return $query->execute(); |
|
204
|
|
|
} |
|
205
|
|
|
} |
|
206
|
|
|
|
VIPnytt /
RobotsTxtParser
Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation.
You can also find more detailed suggestions in the “Code” section of your repository.