|
1
|
|
|
<?php |
|
2
|
|
|
/** |
|
3
|
|
|
* vipnytt/RobotsTxtParser |
|
4
|
|
|
* |
|
5
|
|
|
* @link https://github.com/VIPnytt/RobotsTxtParser |
|
6
|
|
|
* @license https://github.com/VIPnytt/RobotsTxtParser/blob/master/LICENSE The MIT License (MIT) |
|
7
|
|
|
*/ |
|
8
|
|
|
|
|
9
|
|
|
namespace vipnytt\RobotsTxtParser\Client\Cache\MySQL; |
|
10
|
|
|
|
|
11
|
|
|
use vipnytt\RobotsTxtParser\Client\Cache\ManageCore; |
|
12
|
|
|
use vipnytt\RobotsTxtParser\Exceptions; |
|
13
|
|
|
|
|
14
|
|
|
/** |
|
15
|
|
|
* Class Manage |
|
16
|
|
|
* |
|
17
|
|
|
* @see https://vipnytt.github.io/RobotsTxtParser/methods/Cache.html for documentation |
|
18
|
|
|
* @package vipnytt\RobotsTxtParser\Handler\Cache\MySQL |
|
19
|
|
|
*/ |
|
20
|
|
|
class Manage extends ManageCore |
|
21
|
|
|
{ |
|
22
|
|
|
/** |
|
23
|
|
|
* Process the update queue |
|
24
|
|
|
* |
|
25
|
|
|
* @param float|int $timeLimit |
|
26
|
|
|
* @param int|null $workerID |
|
27
|
|
|
* @return string[] |
|
28
|
|
|
* @throws Exceptions\DatabaseException |
|
29
|
|
|
*/ |
|
30
|
|
|
public function cron($timeLimit = self::CRON_EXEC_TIME, $workerID = self::WORKER_ID) |
|
31
|
|
|
{ |
|
32
|
|
|
$start = microtime(true); |
|
33
|
|
|
$workerID = $this->setWorkerID($workerID); |
|
34
|
|
|
$log = []; |
|
35
|
|
|
$lastCount = -1; |
|
36
|
|
|
while (count($log) > $lastCount && |
|
37
|
|
|
( |
|
38
|
|
|
empty($timeLimit) || |
|
39
|
|
|
$timeLimit > (microtime(true) - $start) |
|
40
|
|
|
) |
|
41
|
|
|
) { |
|
42
|
|
|
$lastCount = count($log); |
|
43
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
44
|
|
|
UPDATE robotstxt__cache1 |
|
45
|
|
|
SET worker = :workerID |
|
46
|
|
|
WHERE worker IS NULL AND nextUpdate <= UNIX_TIMESTAMP() |
|
47
|
|
|
ORDER BY nextUpdate ASC |
|
48
|
|
|
LIMIT 1; |
|
49
|
|
|
SQL |
|
50
|
|
|
); |
|
51
|
|
|
$query->bindValue('workerID', $workerID, \PDO::PARAM_INT); |
|
52
|
|
|
$query->execute(); |
|
53
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
54
|
|
|
SELECT base |
|
55
|
|
|
FROM robotstxt__cache1 |
|
56
|
|
|
WHERE worker = :workerID |
|
57
|
|
|
ORDER BY nextUpdate DESC |
|
58
|
|
|
LIMIT 10; |
|
59
|
|
|
SQL |
|
60
|
|
|
); |
|
61
|
|
|
$query->bindValue('workerID', $workerID, \PDO::PARAM_INT); |
|
62
|
|
|
$query->execute(); |
|
63
|
|
|
while ($baseUri = $query->fetch(\PDO::FETCH_COLUMN)) { |
|
64
|
|
|
if ((new Base($this->pdo, $baseUri, $this->curlOptions, $this->byteLimit))->refresh()) { |
|
65
|
|
|
$log[(string)microtime(true)] = $baseUri; |
|
66
|
|
|
} |
|
67
|
|
|
} |
|
68
|
|
|
} |
|
69
|
|
|
return $log; |
|
70
|
|
|
} |
|
71
|
|
|
|
|
72
|
|
|
/** |
|
73
|
|
|
* Set WorkerID |
|
74
|
|
|
* |
|
75
|
|
|
* @param int|null $workerID |
|
76
|
|
|
* @return int |
|
77
|
|
|
* @throws \InvalidArgumentException |
|
78
|
|
|
*/ |
|
79
|
|
|
private function setWorkerID($workerID) |
|
80
|
|
|
{ |
|
81
|
|
|
if ($workerID >= 1 && |
|
82
|
|
|
$workerID <= 255 |
|
83
|
|
|
) { |
|
84
|
|
|
return (int)$workerID; |
|
85
|
|
|
} elseif ($workerID !== null) { |
|
86
|
|
|
throw new \InvalidArgumentException('WorkerID out of range (1-255)'); |
|
87
|
|
|
} |
|
88
|
|
|
return rand(1, 255); |
|
89
|
|
|
} |
|
90
|
|
|
|
|
91
|
|
|
/** |
|
92
|
|
|
* Base class |
|
93
|
|
|
* |
|
94
|
|
|
* @param string $baseUri |
|
95
|
|
|
* @return Base |
|
96
|
|
|
*/ |
|
97
|
|
|
public function base($baseUri) |
|
98
|
|
|
{ |
|
99
|
|
|
return new Base($this->pdo, $baseUri, $this->curlOptions, $this->byteLimit); |
|
100
|
|
|
} |
|
101
|
|
|
|
|
102
|
|
|
/** |
|
103
|
|
|
* Clean the cache table |
|
104
|
|
|
* |
|
105
|
|
|
* @param int $delay |
|
106
|
|
|
* @return bool |
|
107
|
|
|
*/ |
|
108
|
|
View Code Duplication |
public function clean($delay = self::CLEAN_DELAY) |
|
|
|
|
|
|
109
|
|
|
{ |
|
110
|
|
|
$delay += self::CACHE_TIME; |
|
111
|
|
|
$query = $this->pdo->prepare(<<<SQL |
|
112
|
|
|
DELETE FROM robotstxt__cache1 |
|
113
|
|
|
WHERE nextUpdate < (UNIX_TIMESTAMP() - :delay); |
|
114
|
|
|
SQL |
|
115
|
|
|
); |
|
116
|
|
|
$query->bindValue('delay', $delay, \PDO::PARAM_INT); |
|
117
|
|
|
return $query->execute(); |
|
118
|
|
|
} |
|
119
|
|
|
} |
|
120
|
|
|
|
VIPnytt /
RobotsTxtParser
Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation.
You can also find more detailed suggestions in the “Code” section of your repository.