Nexcessnet_Turpentine_Model_Observer_Cron::crawlUrls() - Code Metrics - nexcess/magento-turpentine - Measure and Improve Code Quality continuously with Scrutinizer

crawlUrls() B
last analyzed 2018-10-30 17:49 UTC

↳ Parent: Nexcessnet_Turpentine_Model_Observer_Cron

Complexity

Conditions	10
Paths	11

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
dl	0
loc	33
rs	7.6666
c	0
b	0
f	0
cc	10
nc	11
nop	1

How to fix Complexity

<?php

/**
 * Nexcess.net Turpentine Extension for Magento
 * Copyright (C) 2012  Nexcess.net L.L.C.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

class Nexcessnet_Turpentine_Model_Observer_Cron extends Varien_Event_Observer {

    /**
     * Max time to crawl URLs if max_execution_time is 0 (unlimited) in seconds
     *
     * @var int
     */
    const MAX_CRAWL_TIME = 300;

    /**
     * Amount of time of execution time to leave for other cron processes
     *
     * @var int
     */
    const EXEC_TIME_BUFFER = 15;

    /**
     * Crawl available URLs in the queue until we get close to max_execution_time
     * (up to MAX_CRAWL_TIME)
     *
     * @param  Varien_Object $eventObject
     * @return null
     */
    public function crawlUrls($eventObject) {
        $helper = Mage::helper('turpentine/cron');
        if ($helper->getCrawlerEnabled()) {
            $maxRunTime = $helper->getAllowedRunTime();
            if ($maxRunTime === 0) {
                $maxRunTime = self::MAX_CRAWL_TIME;
            }

            $batchSize = $helper->getCrawlerBatchSize();
            $timeout = $helper->getCrawlerWaitPeriod();
            $crawlCount = 0;

            // just in case we have a silly short max_execution_time
            $maxRunTime = abs($maxRunTime - self::EXEC_TIME_BUFFER);
            while (($helper->getRunTime() < $maxRunTime) &&
                    $url = $helper->getNextUrl()) {
                if ( ! $this->_crawlUrl($url)) {
                    Mage::helper('turpentine/debug')->logWarn(
                        'Failed to crawl URL: %s', $url );
                }

                if ($crawlCount > 0
                    && $timeout > 0
                    && $batchSize > 0
                    && $crawlCount % $batchSize == 0
                ) {
                    Mage::helper('turpentine/debug')->logDebug('Crawled '.$crawlCount.' urls, sleeping for '.$timeout.' seconds');
                    sleep($timeout);
                }
                $crawlCount++;
            }
        }
    }

    /**
     * Queue all URLs
     *
     * @param  Varien_Object $eventObject
     * @return null
     */
    public function queueAllUrls($eventObject) {
        $helper = Mage::helper('turpentine/cron');
        if ($helper->getCrawlerEnabled()) {
            $helper->addUrlsToCrawlerQueue($helper->getAllUrls());
        }
    }

    /**
     * Request a single URL, returns whether the request was successful or not
     *
     * @param  string $url
     * @return bool
     */
    protected function _crawlUrl($url) {
        $client = Mage::helper('turpentine/cron')->getCrawlerClient();
        $client->setUri($url);
        Mage::helper('turpentine/debug')->logDebug('Crawling URL: %s', $url);
        try {
            $response = $client->request();
        } catch (Exception $e) {
            Mage::helper('turpentine/debug')->logWarn(
                'Error crawling URL (%s): %s',
                $url, $e->getMessage() );
            return false;
        }
        return $response->isSuccessful();
    }
}


1			<?php
2
3			/**
4			* Nexcess.net Turpentine Extension for Magento
5			* Copyright (C) 2012 Nexcess.net L.L.C.
6			*
7			* This program is free software; you can redistribute it and/or modify
8			* it under the terms of the GNU General Public License as published by
9			* the Free Software Foundation; either version 2 of the License, or
10			* (at your option) any later version.
11			*
12			* This program is distributed in the hope that it will be useful,
13			* but WITHOUT ANY WARRANTY; without even the implied warranty of
14			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15			* GNU General Public License for more details.
16			*
17			* You should have received a copy of the GNU General Public License along
18			* with this program; if not, write to the Free Software Foundation, Inc.,
19			* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20			*/
21
22			class Nexcessnet_Turpentine_Model_Observer_Cron extends Varien_Event_Observer {
23
24			/**
25			* Max time to crawl URLs if max_execution_time is 0 (unlimited) in seconds
26			*
27			* @var int
28			*/
29			const MAX_CRAWL_TIME = 300;
30
31			/**
32			* Amount of time of execution time to leave for other cron processes
33			*
34			* @var int
35			*/
36			const EXEC_TIME_BUFFER = 15;
37
38			/**
39			* Crawl available URLs in the queue until we get close to max_execution_time
40			* (up to MAX_CRAWL_TIME)
41			*
42			* @param Varien_Object $eventObject
43			* @return null
44			*/
45			public function crawlUrls($eventObject) {
46			$helper = Mage::helper('turpentine/cron');
47			if ($helper->getCrawlerEnabled()) {
48			$maxRunTime = $helper->getAllowedRunTime();
49			if ($maxRunTime === 0) {
50			$maxRunTime = self::MAX_CRAWL_TIME;
51			}
52
53			$batchSize = $helper->getCrawlerBatchSize();
54			$timeout = $helper->getCrawlerWaitPeriod();
55			$crawlCount = 0;
56
57			// just in case we have a silly short max_execution_time
58			$maxRunTime = abs($maxRunTime - self::EXEC_TIME_BUFFER);
59			while (($helper->getRunTime() < $maxRunTime) &&
60			$url = $helper->getNextUrl()) {
61			if ( ! $this->_crawlUrl($url)) {
62			Mage::helper('turpentine/debug')->logWarn(
63			'Failed to crawl URL: %s', $url );
64			}
65
66			if ($crawlCount > 0
67			&& $timeout > 0
68			&& $batchSize > 0
69			&& $crawlCount % $batchSize == 0
70			) {
71			Mage::helper('turpentine/debug')->logDebug('Crawled '.$crawlCount.' urls, sleeping for '.$timeout.' seconds');
72			sleep($timeout);
73			}
74			$crawlCount++;
75			}
76			}
77			}
78
79			/**
80			* Queue all URLs
81			*
82			* @param Varien_Object $eventObject
83			* @return null
84			*/
85			public function queueAllUrls($eventObject) {
86			$helper = Mage::helper('turpentine/cron');
87			if ($helper->getCrawlerEnabled()) {
88			$helper->addUrlsToCrawlerQueue($helper->getAllUrls());
89			}
90			}
91
92			/**
93			* Request a single URL, returns whether the request was successful or not
94			*
95			* @param string $url
96			* @return bool
97			*/
98			protected function _crawlUrl($url) {
99			$client = Mage::helper('turpentine/cron')->getCrawlerClient();
100			$client->setUri($url);
101			Mage::helper('turpentine/debug')->logDebug('Crawling URL: %s', $url);
102			try {
103			$response = $client->request();
104			} catch (Exception $e) {
105			Mage::helper('turpentine/debug')->logWarn(
106			'Error crawling URL (%s): %s',
107			$url, $e->getMessage() );
108			return false;
109			}
110			return $response->isSuccessful();
111			}
112			}
113

nexcess / magento-turpentine

crawlUrls() B last analyzed 2018-10-30 17:49 UTC

Complexity

Size

Duplication

Importance

How to fix Complexity

Long Method

Duplication Side-by-Side

Filter issues like

crawlUrls() B
last analyzed 2018-10-30 17:49 UTC