|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
/** |
|
4
|
|
|
* Nexcess.net Turpentine Extension for Magento |
|
5
|
|
|
* Copyright (C) 2012 Nexcess.net L.L.C. |
|
6
|
|
|
* |
|
7
|
|
|
* This program is free software; you can redistribute it and/or modify |
|
8
|
|
|
* it under the terms of the GNU General Public License as published by |
|
9
|
|
|
* the Free Software Foundation; either version 2 of the License, or |
|
10
|
|
|
* (at your option) any later version. |
|
11
|
|
|
* |
|
12
|
|
|
* This program is distributed in the hope that it will be useful, |
|
13
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
14
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
15
|
|
|
* GNU General Public License for more details. |
|
16
|
|
|
* |
|
17
|
|
|
* You should have received a copy of the GNU General Public License along |
|
18
|
|
|
* with this program; if not, write to the Free Software Foundation, Inc., |
|
19
|
|
|
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
|
20
|
|
|
*/ |
|
21
|
|
|
|
|
22
|
|
|
class Nexcessnet_Turpentine_Model_Observer_Cron extends Varien_Event_Observer { |
|
23
|
|
|
|
|
24
|
|
|
/** |
|
25
|
|
|
* Max time to crawl URLs if max_execution_time is 0 (unlimited) in seconds |
|
26
|
|
|
* |
|
27
|
|
|
* @var int |
|
28
|
|
|
*/ |
|
29
|
|
|
const MAX_CRAWL_TIME = 300; |
|
30
|
|
|
|
|
31
|
|
|
/** |
|
32
|
|
|
* Amount of time of execution time to leave for other cron processes |
|
33
|
|
|
* |
|
34
|
|
|
* @var int |
|
35
|
|
|
*/ |
|
36
|
|
|
const EXEC_TIME_BUFFER = 15; |
|
37
|
|
|
|
|
38
|
|
|
/** |
|
39
|
|
|
* Crawl available URLs in the queue until we get close to max_execution_time |
|
40
|
|
|
* (up to MAX_CRAWL_TIME) |
|
41
|
|
|
* |
|
42
|
|
|
* @param Varien_Object $eventObject |
|
43
|
|
|
* @return null |
|
44
|
|
|
*/ |
|
45
|
|
|
public function crawlUrls($eventObject) { |
|
46
|
|
|
$helper = Mage::helper('turpentine/cron'); |
|
47
|
|
|
if ($helper->getCrawlerEnabled()) { |
|
48
|
|
|
$maxRunTime = $helper->getAllowedRunTime(); |
|
49
|
|
|
if ($maxRunTime === 0) { |
|
50
|
|
|
$maxRunTime = self::MAX_CRAWL_TIME; |
|
51
|
|
|
} |
|
52
|
|
|
|
|
53
|
|
|
$batchSize = $helper->getCrawlerBatchSize(); |
|
54
|
|
|
$timeout = $helper->getCrawlerWaitPeriod(); |
|
55
|
|
|
$crawlCount = 0; |
|
56
|
|
|
|
|
57
|
|
|
// just in case we have a silly short max_execution_time |
|
58
|
|
|
$maxRunTime = abs($maxRunTime - self::EXEC_TIME_BUFFER); |
|
59
|
|
|
while (($helper->getRunTime() < $maxRunTime) && |
|
60
|
|
|
$url = $helper->getNextUrl()) { |
|
61
|
|
|
if ( ! $this->_crawlUrl($url)) { |
|
62
|
|
|
Mage::helper('turpentine/debug')->logWarn( |
|
63
|
|
|
'Failed to crawl URL: %s', $url ); |
|
64
|
|
|
} |
|
65
|
|
|
|
|
66
|
|
|
if ($crawlCount > 0 |
|
67
|
|
|
&& $timeout > 0 |
|
68
|
|
|
&& $batchSize > 0 |
|
69
|
|
|
&& $crawlCount % $batchSize == 0 |
|
70
|
|
|
) { |
|
71
|
|
|
Mage::helper('turpentine/debug')->logDebug('Crawled '.$crawlCount.' urls, sleeping for '.$timeout.' seconds'); |
|
72
|
|
|
sleep($timeout); |
|
73
|
|
|
} |
|
74
|
|
|
$crawlCount++; |
|
75
|
|
|
} |
|
76
|
|
|
} |
|
77
|
|
|
} |
|
78
|
|
|
|
|
79
|
|
|
/** |
|
80
|
|
|
* Queue all URLs |
|
81
|
|
|
* |
|
82
|
|
|
* @param Varien_Object $eventObject |
|
83
|
|
|
* @return null |
|
84
|
|
|
*/ |
|
85
|
|
|
public function queueAllUrls($eventObject) { |
|
86
|
|
|
$helper = Mage::helper('turpentine/cron'); |
|
87
|
|
|
if ($helper->getCrawlerEnabled()) { |
|
88
|
|
|
$helper->addUrlsToCrawlerQueue($helper->getAllUrls()); |
|
89
|
|
|
} |
|
90
|
|
|
} |
|
91
|
|
|
|
|
92
|
|
|
/** |
|
93
|
|
|
* Request a single URL, returns whether the request was successful or not |
|
94
|
|
|
* |
|
95
|
|
|
* @param string $url |
|
96
|
|
|
* @return bool |
|
97
|
|
|
*/ |
|
98
|
|
|
protected function _crawlUrl($url) { |
|
99
|
|
|
$client = Mage::helper('turpentine/cron')->getCrawlerClient(); |
|
100
|
|
|
$client->setUri($url); |
|
101
|
|
|
Mage::helper('turpentine/debug')->logDebug('Crawling URL: %s', $url); |
|
102
|
|
|
try { |
|
103
|
|
|
$response = $client->request(); |
|
104
|
|
|
} catch (Exception $e) { |
|
105
|
|
|
Mage::helper('turpentine/debug')->logWarn( |
|
106
|
|
|
'Error crawling URL (%s): %s', |
|
107
|
|
|
$url, $e->getMessage() ); |
|
108
|
|
|
return false; |
|
109
|
|
|
} |
|
110
|
|
|
return $response->isSuccessful(); |
|
111
|
|
|
} |
|
112
|
|
|
} |
|
113
|
|
|
|
nexcess /
magento-turpentine
