BatchedProcessorTest::testBatching() - Code Metrics - Inspection of "API Remove bind_manipulation_capture RequestFilter..." - silverstripe/silverstripe-fulltextsearch - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#184)

by Robbie

created 2017-12-04 20:42 UTC

BatchedProcessorTest::testBatching() B

↳ Parent: BatchedProcessorTest

Complexity

Conditions	2
Paths	2

Size

Total Lines	35
Code Lines	24

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
c	0
b	0
f	0
dl	0
loc	35
rs	8.8571
cc	2
eloc	24
nc	2
nop	0

<?php

namespace SilverStripe\FullTextSearch\Tests;

use SilverStripe\CMS\Model\SiteTree;
use SilverStripe\Core\Config\Config;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\SapphireTest;
use SilverStripe\FullTextSearch\Search\FullTextSearch;
use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessor_QueuedJobService;
use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessorTest_Index;
use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessorTest_Object;
use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateCommitJobProcessor;
use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateQueuedJobProcessor;
use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateBatchedProcessor;
use SilverStripe\FullTextSearch\Search\Updaters\SearchUpdater;
use SilverStripe\FullTextSearch\Search\Variants\SearchVariantVersioned;
use SilverStripe\Subsites\Extensions\SiteTreeSubsites;
use SilverStripe\Subsites\Model\Subsite;
use SilverStripe\ORM\FieldType\DBDatetime;
use SilverStripe\Versioned\Versioned;
use Symbiote\QueuedJobs\Services\QueuedJobService;
use Symbiote\QueuedJobs\Services\QueuedJob;

/**
 * Tests {@see SearchUpdateQueuedJobProcessor}
 */
class BatchedProcessorTest extends SapphireTest
{
    protected $usesDatabase = true;

    protected $oldProcessor;

    protected static $extra_dataobjects = [
        BatchedProcessorTest_Object::class,
    ];

    protected static $illegal_extensions = [
        SiteTree::class => [
            SiteTreeSubsites::class,
        ]
    ];

    public static function setUpBeforeClass()
    {
        // Disable illegal extensions if skipping this test
        if (class_exists(Subsite::class) || !interface_exists(QueuedJob::class)) {
            static::$illegal_extensions = [];
        }
        parent::setUpBeforeClass();
    }

    protected function setUp()
    {
        Config::modify()->set(SearchUpdater::class, 'flush_on_shutdown', false);

        parent::setUp();

        if (!interface_exists('Symbiote\QueuedJobs\Services\QueuedJob')) {
            $this->skipTest = true;
class MyClass { }

$x = new MyClass();
$x->foo = true;
            $this->markTestSkipped("These tests need the QueuedJobs module installed to run");
        }

        Config::modify()->set(QueuedJobService::class, 'use_shutdown_function', false);

        if (class_exists('Subsite')) {
            $this->skipTest = true;
            $this->markTestSkipped(get_class() . ' skipped when running with subsites');
        }

        DBDatetime::set_mock_now('2015-05-07 06:00:00');

        Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_size', 5);
        Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 0);
        Config::modify()->set(SearchUpdateCommitJobProcessor::class, 'cooldown', 600);

        Versioned::set_stage("Stage");

        Injector::inst()->registerService(new BatchedProcessor_QueuedJobService(), QueuedJobService::class);

        FullTextSearch::force_index_list(BatchedProcessorTest_Index::class);

        SearchUpdateCommitJobProcessor::$dirty_indexes = array();
        SearchUpdateCommitJobProcessor::$has_run = false;

        $this->oldProcessor = SearchUpdater::$processor;
        SearchUpdater::$processor = new SearchUpdateQueuedJobProcessor();
    }

    protected function tearDown()
    {
        if ($this->oldProcessor) {
            SearchUpdater::$processor = $this->oldProcessor;
        }
        FullTextSearch::force_index_list();
        parent::tearDown();
    }

    /**
     * @return SearchUpdateQueuedJobProcessor
     */
    protected function generateDirtyIds()
    {
        $processor = SearchUpdater::$processor;
        for ($id = 1; $id <= 42; $id++) {
            // Save to db
            $object = new BatchedProcessorTest_Object();
            $object->TestText = 'Object ' . $id;
<?php

/**
 * @property int $x
 * @property int $y
 * @property string $text
 */
class MyLabel
{
    private $properties;

    private $allowedProperties = array('x', 'y', 'text');

    public function __get($name)
    {
        if (isset($properties[$name]) && in_array($name, $this->allowedProperties)) {
            return $properties[$name];
        } else {
            return null;
        }
    }

    public function __set($name, $value)
    {
        if (in_array($name, $this->allowedProperties)) {
            $properties[$name] = $value;
        } else {
            throw new \LogicException("Property $name is not defined.");
        }
    }

}
            $object->write();
            // Add to index manually
            $processor->addDirtyIDs(
                BatchedProcessorTest_Object::class,
                array(array(
                    'id' => $id,
                    'state' => array(SearchVariantVersioned::class => 'Stage')
                )),
                BatchedProcessorTest_Index::class
            );
        }
        $processor->batchData();
        return $processor;
    }

    /**
     * Tests that large jobs are broken up into a suitable number of batches
     */
    public function testBatching()
    {
        $index = singleton(BatchedProcessorTest_Index::class);
        $index->reset();
        $processor = $this->generateDirtyIds();

        // Check initial state
        $data = $processor->getJobData();
        $this->assertEquals(9, $data->totalSteps);
        $this->assertEquals(0, $data->currentStep);
        $this->assertEmpty($data->isComplete);
        $this->assertEquals(0, count($index->getAdded()));

        // Advance state
        for ($pass = 1; $pass <= 8; $pass++) {
            $processor->process();
            $data = $processor->getJobData();
            $this->assertEquals($pass, $data->currentStep);
            $this->assertEquals($pass * 5, count($index->getAdded()));
        }

        // Last run should have two hanging items
        $processor->process();
        $data = $processor->getJobData();
        $this->assertEquals(9, $data->currentStep);
        $this->assertEquals(42, count($index->getAdded()));
        $this->assertTrue($data->isComplete);

        // Check any additional queued jobs
        $processor->afterComplete();
        $service = singleton(QueuedJobService::class);
        $jobs = $service->getJobs();
        $this->assertEquals(1, count($jobs));
        $this->assertInstanceOf(SearchUpdateCommitJobProcessor::class, $jobs[0]['job']);
    }

    /**
     * Test creation of multiple commit jobs
     */
    public function testMultipleCommits()
    {
        $index = singleton(BatchedProcessorTest_Index::class);
        $index->reset();

        // Test that running a commit immediately after submitting to the indexes
        // correctly commits
        $first = SearchUpdateCommitJobProcessor::queue();
        $second = SearchUpdateCommitJobProcessor::queue();

        $this->assertFalse($index->getIsCommitted());

        // First process will cause the commit
        $this->assertFalse($first->jobFinished());

        $first->process();

        $allMessages = $first->getMessages();

        $this->assertTrue($index->getIsCommitted());
        $this->assertTrue($first->jobFinished());

        $this->assertStringEndsWith('All indexes committed', $allMessages[2]);

        // Executing the subsequent processor should not re-trigger a commit
        $index->reset();
        $this->assertFalse($second->jobFinished());

        $second->process();

        $allMessages = $second->getMessages();

        $this->assertFalse($index->getIsCommitted());
        $this->assertTrue($second->jobFinished());

        $this->assertStringEndsWith('Indexing already completed this request: Discarding this job', $allMessages[0]);

        // Given that a third job is created, and the indexes are dirtied, attempting to run this job
        // should result in a delay
        $index->reset();
        $third = SearchUpdateCommitJobProcessor::queue();
        $this->assertFalse($third->jobFinished());

        $third->process();

        $this->assertTrue($third->jobFinished());

        $allMessages = $third->getMessages();

        $this->assertStringEndsWith(
            'Indexing already run this request, but incomplete. Re-scheduling for 2015-05-07 06:10:00',
            $allMessages[0]
        );
    }

    /**
     * Tests that the batch_soft_cap setting is properly respected
     */
    public function testSoftCap()
    {
        $this->markTestSkipped(
            '@todo PostgreSQL: This test passes in isolation, but not in conjunction with the previous test'
        );

        $index = singleton(BatchedProcessorTest_Index::class);
        $index->reset();

        $processor = $this->generateDirtyIds();

        // Test that increasing the soft cap to 2 will reduce the number of batches
        Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 2);
        $processor->batchData();
        $data = $processor->getJobData();
        $this->assertEquals(8, $data->totalSteps);

        // A soft cap of 1 should not fit in the hanging two items
        Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 1);
        $processor->batchData();
        $data = $processor->getJobData();
        $this->assertEquals(9, $data->totalSteps);

        // Extra large soft cap should fit both items
        Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 4);
        $processor->batchData();
        $data = $processor->getJobData();
        $this->assertEquals(8, $data->totalSteps);

        // Process all data and ensure that all are processed adequately
        for ($pass = 1; $pass <= 8; $pass++) {
            $processor->process();
        }
        $data = $processor->getJobData();
        $this->assertEquals(8, $data->currentStep);
        $this->assertEquals(42, count($index->getAdded()));
        $this->assertTrue($data->isComplete);
    }
}


1			<?php
2
3			namespace SilverStripe\FullTextSearch\Tests;
4
5			use SilverStripe\CMS\Model\SiteTree;
6			use SilverStripe\Core\Config\Config;
7			use SilverStripe\Core\Injector\Injector;
8			use SilverStripe\Dev\SapphireTest;
9			use SilverStripe\FullTextSearch\Search\FullTextSearch;
10			use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessor_QueuedJobService;
11			use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessorTest_Index;
12			use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessorTest_Object;
13			use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateCommitJobProcessor;
14			use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateQueuedJobProcessor;
15			use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateBatchedProcessor;
16			use SilverStripe\FullTextSearch\Search\Updaters\SearchUpdater;
17			use SilverStripe\FullTextSearch\Search\Variants\SearchVariantVersioned;
18			use SilverStripe\Subsites\Extensions\SiteTreeSubsites;
19			use SilverStripe\Subsites\Model\Subsite;
20			use SilverStripe\ORM\FieldType\DBDatetime;
21			use SilverStripe\Versioned\Versioned;
22			use Symbiote\QueuedJobs\Services\QueuedJobService;
23			use Symbiote\QueuedJobs\Services\QueuedJob;
24
25			/**
26			* Tests {@see SearchUpdateQueuedJobProcessor}
27			*/
28			class BatchedProcessorTest extends SapphireTest
29			{
30			protected $usesDatabase = true;
31
32			protected $oldProcessor;
33
34			protected static $extra_dataobjects = [
35			BatchedProcessorTest_Object::class,
36			];
37
38			protected static $illegal_extensions = [
39			SiteTree::class => [
40			SiteTreeSubsites::class,
41			]
42			];
43
44			public static function setUpBeforeClass()
45			{
46			// Disable illegal extensions if skipping this test
47			if (class_exists(Subsite::class) \|\| !interface_exists(QueuedJob::class)) {
48			static::$illegal_extensions = [];
49			}
50			parent::setUpBeforeClass();
51			}
52
53			protected function setUp()
54			{
55			Config::modify()->set(SearchUpdater::class, 'flush_on_shutdown', false);
56
57			parent::setUp();
58
59			if (!interface_exists('Symbiote\QueuedJobs\Services\QueuedJob')) {
60			$this->skipTest = true;
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The property `skipTest` does not exist. Did you maybe forget to declare it? In PHP it is possible to write to properties without declaring them. For example, the following is perfectly valid PHP code: class MyClass { } $x = new MyClass(); $x->foo = true; Generally, it is a good practice to explictly declare properties to avoid accidental typos and provide IDE auto-completion: class MyClass { public $foo; } $x = new MyClass(); $x->foo = true; Loading history...
61			$this->markTestSkipped("These tests need the QueuedJobs module installed to run");
62			}
63
64			Config::modify()->set(QueuedJobService::class, 'use_shutdown_function', false);
65
66			if (class_exists('Subsite')) {
67			$this->skipTest = true;
68			$this->markTestSkipped(get_class() . ' skipped when running with subsites');
69			}
70
71			DBDatetime::set_mock_now('2015-05-07 06:00:00');
72
73			Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_size', 5);
74			Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 0);
75			Config::modify()->set(SearchUpdateCommitJobProcessor::class, 'cooldown', 600);
76
77			Versioned::set_stage("Stage");
78
79			Injector::inst()->registerService(new BatchedProcessor_QueuedJobService(), QueuedJobService::class);
80
81			FullTextSearch::force_index_list(BatchedProcessorTest_Index::class);
82
83			SearchUpdateCommitJobProcessor::$dirty_indexes = array();
84			SearchUpdateCommitJobProcessor::$has_run = false;
85
86			$this->oldProcessor = SearchUpdater::$processor;
87			SearchUpdater::$processor = new SearchUpdateQueuedJobProcessor();
88			}
89
90			protected function tearDown()
91			{
92			if ($this->oldProcessor) {
93			SearchUpdater::$processor = $this->oldProcessor;
94			}
95			FullTextSearch::force_index_list();
96			parent::tearDown();
97			}
98
99			/**
100			* @return SearchUpdateQueuedJobProcessor
101			*/
102			protected function generateDirtyIds()
103			{
104			$processor = SearchUpdater::$processor;
105			for ($id = 1; $id <= 42; $id++) {
106			// Save to db
107			$object = new BatchedProcessorTest_Object();
108			$object->TestText = 'Object ' . $id;
			0 ignored issues – show Documentation introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The property `TestText` does not exist on `object<SilverStripe\Full...edProcessorTest_Object>`. Since you implemented `__set`, maybe consider adding a @property annotation. Since your code implements the magic setter `_set`, this function will be called for any write access on an undefined variable. You can add the `@property` annotation to your class or interface to document the existence of this variable. <?php /** * @property int $x * @property int $y * @property string $text */ class MyLabel { private $properties; private $allowedProperties = array('x', 'y', 'text'); public function __get($name) { if (isset($properties[$name]) && in_array($name, $this->allowedProperties)) { return $properties[$name]; } else { return null; } } public function __set($name, $value) { if (in_array($name, $this->allowedProperties)) { $properties[$name] = $value; } else { throw new \LogicException("Property $name is not defined."); } } } Since the property has write access only, you can use the @property-write annotation instead. Of course, you may also just have mistyped another name, in which case you should fix the error. See also the PhpDoc documentation for @property. Loading history...
109			$object->write();
110			// Add to index manually
111			$processor->addDirtyIDs(
112			BatchedProcessorTest_Object::class,
113			array(array(
114			'id' => $id,
115			'state' => array(SearchVariantVersioned::class => 'Stage')
116			)),
117			BatchedProcessorTest_Index::class
118			);
119			}
120			$processor->batchData();
121			return $processor;
122			}
123
124			/**
125			* Tests that large jobs are broken up into a suitable number of batches
126			*/
127			public function testBatching()
128			{
129			$index = singleton(BatchedProcessorTest_Index::class);
130			$index->reset();
131			$processor = $this->generateDirtyIds();
132
133			// Check initial state
134			$data = $processor->getJobData();
135			$this->assertEquals(9, $data->totalSteps);
136			$this->assertEquals(0, $data->currentStep);
137			$this->assertEmpty($data->isComplete);
138			$this->assertEquals(0, count($index->getAdded()));
139
140			// Advance state
141			for ($pass = 1; $pass <= 8; $pass++) {
142			$processor->process();
143			$data = $processor->getJobData();
144			$this->assertEquals($pass, $data->currentStep);
145			$this->assertEquals($pass * 5, count($index->getAdded()));
146			}
147
148			// Last run should have two hanging items
149			$processor->process();
150			$data = $processor->getJobData();
151			$this->assertEquals(9, $data->currentStep);
152			$this->assertEquals(42, count($index->getAdded()));
153			$this->assertTrue($data->isComplete);
154
155			// Check any additional queued jobs
156			$processor->afterComplete();
157			$service = singleton(QueuedJobService::class);
158			$jobs = $service->getJobs();
159			$this->assertEquals(1, count($jobs));
160			$this->assertInstanceOf(SearchUpdateCommitJobProcessor::class, $jobs[0]['job']);
161			}
162
163			/**
164			* Test creation of multiple commit jobs
165			*/
166			public function testMultipleCommits()
167			{
168			$index = singleton(BatchedProcessorTest_Index::class);
169			$index->reset();
170
171			// Test that running a commit immediately after submitting to the indexes
172			// correctly commits
173			$first = SearchUpdateCommitJobProcessor::queue();
174			$second = SearchUpdateCommitJobProcessor::queue();
175
176			$this->assertFalse($index->getIsCommitted());
177
178			// First process will cause the commit
179			$this->assertFalse($first->jobFinished());
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `jobFinished` cannot be called on `$first` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
180			$first->process();
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `process` cannot be called on `$first` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
181			$allMessages = $first->getMessages();
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `getMessages` cannot be called on `$first` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
182			$this->assertTrue($index->getIsCommitted());
183			$this->assertTrue($first->jobFinished());
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `jobFinished` cannot be called on `$first` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
184			$this->assertStringEndsWith('All indexes committed', $allMessages[2]);
185
186			// Executing the subsequent processor should not re-trigger a commit
187			$index->reset();
188			$this->assertFalse($second->jobFinished());
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `jobFinished` cannot be called on `$second` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
189			$second->process();
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `process` cannot be called on `$second` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
190			$allMessages = $second->getMessages();
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `getMessages` cannot be called on `$second` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
191			$this->assertFalse($index->getIsCommitted());
192			$this->assertTrue($second->jobFinished());
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `jobFinished` cannot be called on `$second` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
193			$this->assertStringEndsWith('Indexing already completed this request: Discarding this job', $allMessages[0]);
194
195			// Given that a third job is created, and the indexes are dirtied, attempting to run this job
196			// should result in a delay
197			$index->reset();
198			$third = SearchUpdateCommitJobProcessor::queue();
199			$this->assertFalse($third->jobFinished());
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `jobFinished` cannot be called on `$third` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
200			$third->process();
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `process` cannot be called on `$third` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
201			$this->assertTrue($third->jobFinished());
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `jobFinished` cannot be called on `$third` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
202			$allMessages = $third->getMessages();
			0 ignored issues – show Bug introduced 2017-11-15 03:22 UTC by Report Bug Copy Issue Report The method `getMessages` cannot be called on `$third` (of type `integer`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
203			$this->assertStringEndsWith(
204			'Indexing already run this request, but incomplete. Re-scheduling for 2015-05-07 06:10:00',
205			$allMessages[0]
206			);
207			}
208
209			/**
210			* Tests that the batch_soft_cap setting is properly respected
211			*/
212			public function testSoftCap()
213			{
214			$this->markTestSkipped(
215			'@todo PostgreSQL: This test passes in isolation, but not in conjunction with the previous test'
216			);
217
218			$index = singleton(BatchedProcessorTest_Index::class);
219			$index->reset();
220
221			$processor = $this->generateDirtyIds();
222
223			// Test that increasing the soft cap to 2 will reduce the number of batches
224			Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 2);
225			$processor->batchData();
226			$data = $processor->getJobData();
227			$this->assertEquals(8, $data->totalSteps);
228
229			// A soft cap of 1 should not fit in the hanging two items
230			Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 1);
231			$processor->batchData();
232			$data = $processor->getJobData();
233			$this->assertEquals(9, $data->totalSteps);
234
235			// Extra large soft cap should fit both items
236			Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 4);
237			$processor->batchData();
238			$data = $processor->getJobData();
239			$this->assertEquals(8, $data->totalSteps);
240
241			// Process all data and ensure that all are processed adequately
242			for ($pass = 1; $pass <= 8; $pass++) {
243			$processor->process();
244			}
245			$data = $processor->getJobData();
246			$this->assertEquals(8, $data->currentStep);
247			$this->assertEquals(42, count($index->getAdded()));
248			$this->assertTrue($data->isComplete);
249			}
250			}
251

silverstripe / silverstripe-fulltextsearch

Pull Request — master (#184)

BatchedProcessorTest::testBatching() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like