StaticCacheFullBuildJob::processUrl() - Code Metrics - Inspection of "Configuration / extensibility improvements." - silverstripe/silverstripe-staticpublishqueue - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#104)

unknown

created 2020-02-25 00:13 UTC

StaticCacheFullBuildJob::processUrl() A

↳ Parent: StaticCacheFullBuildJob

Complexity

Conditions	4
Paths	4

Size

Total Lines	11
Code Lines	6

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	4
eloc	6
nc	4
nop	2
dl	0
loc	11
rs	10
c	0
b	0
f	0

<?php

namespace SilverStripe\StaticPublishQueue\Job;

use SilverStripe\CMS\Model\SiteTree;
use SilverStripe\StaticPublishQueue\Contract\StaticallyPublishable;
use SilverStripe\StaticPublishQueue\Extension\Publishable\PublishableSiteTree;
use SilverStripe\StaticPublishQueue\Job;
use SilverStripe\StaticPublishQueue\Publisher;
use SilverStripe\StaticPublishQueue\Service\URLSanitisationService;
use SilverStripe\Versioned\Versioned;

/**
 * Class StaticCacheFullBuildJob
 * Adds all live pages to the queue for caching. Best implemented on a cron via StaticCacheFullBuildTask.
 *
 * WARNING: this job is completely unsuited for large websites as it's collecting URLs from all live pages
 * this will either eat up all available memory and / or stall on timeout
 * if your site has thousands of pages you need to consider a different static cache refresh solution
 * ideally, the whole site re-cache would be segmented into smaller chunks and spread across different times of the day
 *
 * @package SilverStripe\StaticPublishQueue\Job
 */
class StaticCacheFullBuildJob extends Job
{
    /**
     * @return string
     */
    public function getTitle()
    {
        return 'Generate static pages for all URLs';
    }

    /**
     * @return string
     */
    public function getSignature()
    {
        return md5(static::class);
    }

    public function setup()
    {
        parent::setup();

        $urls = $this->getAllLivePageURLs();
        $urls = array_keys($urls);
        $urlService = URLSanitisationService::create();
        $urlService->addURL($urls);

        $this->URLsToProcess = $urlService->getURLs(true);

        $this->URLsToCleanUp = [];


        // update total steps as we changed the URLs data since the parent call
        $this->totalSteps = count($this->jobData->URLsToProcess);

        $this->addMessage('Building all URLs');
        $this->addMessage(var_export(array_keys($this->URLsToProcess), true));
    }

    public function process()
    {
        // Remove any URLs which have already been processed
        if ($this->jobData->ProcessedURLs) {
            $this->jobData->URLsToProcess = array_diff_key(
                $this->jobData->URLsToProcess,
                $this->jobData->ProcessedURLs
            );
        }

        $chunkSize = $this->getChunkSize();

        // generate static cache for all live pages
        $count = 0;
        foreach ($this->jobData->URLsToProcess as $url => $priority) {
            $count += 1;
            if ($chunkSize > 0 && $count > $chunkSize) {
                break;
            }

            $this->processUrl($url, $priority);
        }

        // cleanup unused static cache files
        if (count($this->jobData->URLsToProcess) === 0) {
            $trimSlashes = function ($value) {
                return trim($value, '/');
            };

            // list of all URLs which have a static cache file
            $this->jobData->publishedURLs = array_map($trimSlashes, Publisher::singleton()->getPublishedURLs());

            // list of all URLs which were published as a part of this job
            $this->jobData->ProcessedURLs = array_map($trimSlashes, $this->jobData->ProcessedURLs);

            // determine stale URLs - those which were not published as a part of this job
            //but still have a static cache file
            $this->jobData->URLsToCleanUp = array_diff($this->jobData->publishedURLs, $this->jobData->ProcessedURLs);

            foreach ($this->jobData->URLsToCleanUp as $staleURL) {
                $purgeMeta = Publisher::singleton()->purgeURL($staleURL);
                $purgeMeta = (is_array($purgeMeta)) ? $purgeMeta : [];

                if (array_key_exists('success', $purgeMeta) && $purgeMeta['success']) {
                    unset($this->jobData->URLsToCleanUp[$staleURL]);

                    continue;
                }

                $this->handleFailedUrl($staleURL, $purgeMeta);
            }
        };

        $this->updateCompletedState();
    }

    /**
     * @param string $url
     * @param int $priority
     */
    protected function processUrl($url, $priority)
    {
        $meta = Publisher::singleton()->publishURL($url, true);
        $meta = (is_array($meta)) ? $meta : [];
        if (array_key_exists('success', $meta) && $meta['success']) {
            $this->markUrlAsProcessed($url);

            return;
        }

        $this->handleFailedUrl($url, $meta);
    }

    protected function updateCompletedState()
    {
        if (count($this->jobData->URLsToProcess) > 0) {
            return;
        }

        if (count($this->jobData->URLsToCleanUp) > 0) {
            return;
        }

        $this->isComplete = true;
    }

    /**
     *
     * @return array
     */
    protected function getAllLivePageURLs()
    {
        $urls = [];
        $this->extend('beforeGetAllLivePageURLs', $urls);
        $livePages = Versioned::get_by_stage(SiteTree::class, Versioned::LIVE);
        foreach ($livePages as $page) {
            if ($page->hasExtension(PublishableSiteTree::class) || $page instanceof StaticallyPublishable) {
                $urls = array_merge($urls, $page->urlsToCache());
            }
        }

        $this->extend('afterGetAllLivePageURLs', $urls);
        // @TODO look here when integrating subsites
        // if (class_exists(Subsite::class)) {
        //     Subsite::disable_subsite_filter(true);
        // }
        return $urls;
    }
}


1			<?php
2
3			namespace SilverStripe\StaticPublishQueue\Job;
4
5			use SilverStripe\CMS\Model\SiteTree;
6			use SilverStripe\StaticPublishQueue\Contract\StaticallyPublishable;
7			use SilverStripe\StaticPublishQueue\Extension\Publishable\PublishableSiteTree;
8			use SilverStripe\StaticPublishQueue\Job;
9			use SilverStripe\StaticPublishQueue\Publisher;
10			use SilverStripe\StaticPublishQueue\Service\URLSanitisationService;
11			use SilverStripe\Versioned\Versioned;
12
13			/**
14			* Class StaticCacheFullBuildJob
15			* Adds all live pages to the queue for caching. Best implemented on a cron via StaticCacheFullBuildTask.
16			*
17			* WARNING: this job is completely unsuited for large websites as it's collecting URLs from all live pages
18			* this will either eat up all available memory and / or stall on timeout
19			* if your site has thousands of pages you need to consider a different static cache refresh solution
20			* ideally, the whole site re-cache would be segmented into smaller chunks and spread across different times of the day
21			*
22			* @package SilverStripe\StaticPublishQueue\Job
23			*/
24			class StaticCacheFullBuildJob extends Job
25			{
26			/**
27			* @return string
28			*/
29			public function getTitle()
30			{
31			return 'Generate static pages for all URLs';
32			}
33
34			/**
35			* @return string
36			*/
37			public function getSignature()
38			{
39			return md5(static::class);
40			}
41
42			public function setup()
43			{
44			parent::setup();
45
46			$urls = $this->getAllLivePageURLs();
47			$urls = array_keys($urls);
48			$urlService = URLSanitisationService::create();
49			$urlService->addURL($urls);
50
51			$this->URLsToProcess = $urlService->getURLs(true);
			0 ignored issues – show Bug Best Practice introduced 2019-07-21 19:31 UTC by Report Bug Copy Issue Report The property `URLsToProcess` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
52			$this->URLsToCleanUp = [];
			0 ignored issues – show Bug Best Practice introduced 2019-07-21 19:31 UTC by Report Bug Copy Issue Report The property `URLsToCleanUp` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
53
54			// update total steps as we changed the URLs data since the parent call
55			$this->totalSteps = count($this->jobData->URLsToProcess);
56
57			$this->addMessage('Building all URLs');
58			$this->addMessage(var_export(array_keys($this->URLsToProcess), true));
59			}
60
61			public function process()
62			{
63			// Remove any URLs which have already been processed
64			if ($this->jobData->ProcessedURLs) {
65			$this->jobData->URLsToProcess = array_diff_key(
66			$this->jobData->URLsToProcess,
67			$this->jobData->ProcessedURLs
68			);
69			}
70
71			$chunkSize = $this->getChunkSize();
72
73			// generate static cache for all live pages
74			$count = 0;
75			foreach ($this->jobData->URLsToProcess as $url => $priority) {
76			$count += 1;
77			if ($chunkSize > 0 && $count > $chunkSize) {
78			break;
79			}
80
81			$this->processUrl($url, $priority);
82			}
83
84			// cleanup unused static cache files
85			if (count($this->jobData->URLsToProcess) === 0) {
86			$trimSlashes = function ($value) {
87			return trim($value, '/');
88			};
89
90			// list of all URLs which have a static cache file
91			$this->jobData->publishedURLs = array_map($trimSlashes, Publisher::singleton()->getPublishedURLs());
92
93			// list of all URLs which were published as a part of this job
94			$this->jobData->ProcessedURLs = array_map($trimSlashes, $this->jobData->ProcessedURLs);
95
96			// determine stale URLs - those which were not published as a part of this job
97			//but still have a static cache file
98			$this->jobData->URLsToCleanUp = array_diff($this->jobData->publishedURLs, $this->jobData->ProcessedURLs);
99
100			foreach ($this->jobData->URLsToCleanUp as $staleURL) {
101			$purgeMeta = Publisher::singleton()->purgeURL($staleURL);
102			$purgeMeta = (is_array($purgeMeta)) ? $purgeMeta : [];
103
104			if (array_key_exists('success', $purgeMeta) && $purgeMeta['success']) {
105			unset($this->jobData->URLsToCleanUp[$staleURL]);
106
107			continue;
108			}
109
110			$this->handleFailedUrl($staleURL, $purgeMeta);
111			}
112			};
113
114			$this->updateCompletedState();
115			}
116
117			/**
118			* @param string $url
119			* @param int $priority
120			*/
121			protected function processUrl($url, $priority)
122			{
123			$meta = Publisher::singleton()->publishURL($url, true);
124			$meta = (is_array($meta)) ? $meta : [];
125			if (array_key_exists('success', $meta) && $meta['success']) {
126			$this->markUrlAsProcessed($url);
127
128			return;
129			}
130
131			$this->handleFailedUrl($url, $meta);
132			}
133
134			protected function updateCompletedState()
135			{
136			if (count($this->jobData->URLsToProcess) > 0) {
137			return;
138			}
139
140			if (count($this->jobData->URLsToCleanUp) > 0) {
141			return;
142			}
143
144			$this->isComplete = true;
145			}
146
147			/**
148			*
149			* @return array
150			*/
151			protected function getAllLivePageURLs()
152			{
153			$urls = [];
154			$this->extend('beforeGetAllLivePageURLs', $urls);
155			$livePages = Versioned::get_by_stage(SiteTree::class, Versioned::LIVE);
156			foreach ($livePages as $page) {
157			if ($page->hasExtension(PublishableSiteTree::class) \|\| $page instanceof StaticallyPublishable) {
158			$urls = array_merge($urls, $page->urlsToCache());
159			}
160			}
161
162			$this->extend('afterGetAllLivePageURLs', $urls);
163			// @TODO look here when integrating subsites
164			// if (class_exists(Subsite::class)) {
165			// Subsite::disable_subsite_filter(true);
166			// }
167			return $urls;
168			}
169			}
170

silverstripe / silverstripe-staticpublishqueue

Pull Request — master (#104)

StaticCacheFullBuildJob::processUrl() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like