1 | <?php |
||
2 | |||
3 | namespace SilverStripe\FullTextSearch\Tests; |
||
4 | |||
5 | use SilverStripe\CMS\Model\SiteTree; |
||
6 | use SilverStripe\Core\Config\Config; |
||
7 | use SilverStripe\Core\Injector\Injector; |
||
8 | use SilverStripe\Dev\SapphireTest; |
||
9 | use SilverStripe\FullTextSearch\Search\FullTextSearch; |
||
10 | use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessor_QueuedJobService; |
||
11 | use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessorTest_Index; |
||
12 | use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessorTest_Object; |
||
13 | use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateCommitJobProcessor; |
||
14 | use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateQueuedJobProcessor; |
||
15 | use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateBatchedProcessor; |
||
16 | use SilverStripe\FullTextSearch\Search\Updaters\SearchUpdater; |
||
17 | use SilverStripe\FullTextSearch\Search\Variants\SearchVariantVersioned; |
||
18 | use SilverStripe\Subsites\Extensions\SiteTreeSubsites; |
||
19 | use SilverStripe\Subsites\Model\Subsite; |
||
20 | use SilverStripe\ORM\FieldType\DBDatetime; |
||
21 | use SilverStripe\Versioned\Versioned; |
||
22 | use Symbiote\QueuedJobs\Services\QueuedJobService; |
||
0 ignored issues
–
show
|
|||
23 | use Symbiote\QueuedJobs\Services\QueuedJob; |
||
24 | |||
25 | /** |
||
26 | * Tests {@see SearchUpdateQueuedJobProcessor} |
||
27 | */ |
||
28 | class BatchedProcessorTest extends SapphireTest |
||
29 | { |
||
30 | protected $usesDatabase = true; |
||
31 | |||
32 | protected $oldProcessor; |
||
33 | |||
34 | protected static $extra_dataobjects = [ |
||
35 | BatchedProcessorTest_Object::class, |
||
36 | ]; |
||
37 | |||
38 | protected static $illegal_extensions = [ |
||
39 | SiteTree::class => [ |
||
40 | SiteTreeSubsites::class, |
||
41 | ], |
||
42 | ]; |
||
43 | |||
44 | public static function setUpBeforeClass() |
||
45 | { |
||
46 | // Disable illegal extensions if skipping this test |
||
47 | if (class_exists(Subsite::class) || !interface_exists(QueuedJob::class)) { |
||
48 | static::$illegal_extensions = []; |
||
49 | } |
||
50 | parent::setUpBeforeClass(); |
||
51 | } |
||
52 | |||
53 | protected function setUp() |
||
54 | { |
||
55 | parent::setUp(); |
||
56 | |||
57 | if (!interface_exists(QueuedJob::class)) { |
||
58 | $this->markTestSkipped("These tests need the QueuedJobs module installed to run"); |
||
59 | } |
||
60 | |||
61 | if (class_exists(Subsite::class)) { |
||
62 | $this->markTestSkipped(get_class() . ' skipped when running with subsites'); |
||
63 | } |
||
64 | |||
65 | DBDatetime::set_mock_now('2015-05-07 06:00:00'); |
||
66 | |||
67 | Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_size', 5); |
||
68 | Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 0); |
||
69 | Config::modify()->set(SearchUpdateCommitJobProcessor::class, 'cooldown', 600); |
||
70 | |||
71 | Versioned::set_stage(Versioned::DRAFT); |
||
72 | |||
73 | Injector::inst()->registerService(new BatchedProcessor_QueuedJobService(), QueuedJobService::class); |
||
74 | |||
75 | FullTextSearch::force_index_list(BatchedProcessorTest_Index::class); |
||
76 | |||
77 | SearchUpdateCommitJobProcessor::$dirty_indexes = array(); |
||
78 | SearchUpdateCommitJobProcessor::$has_run = false; |
||
79 | |||
80 | $this->oldProcessor = SearchUpdater::$processor; |
||
81 | SearchUpdater::$processor = new SearchUpdateQueuedJobProcessor(); |
||
82 | } |
||
83 | |||
84 | protected function tearDown() |
||
85 | { |
||
86 | if ($this->oldProcessor) { |
||
87 | SearchUpdater::$processor = $this->oldProcessor; |
||
88 | } |
||
89 | FullTextSearch::force_index_list(); |
||
90 | parent::tearDown(); |
||
91 | } |
||
92 | |||
93 | /** |
||
94 | * @return SearchUpdateQueuedJobProcessor |
||
95 | */ |
||
96 | protected function generateDirtyIds() |
||
97 | { |
||
98 | $processor = SearchUpdater::$processor; |
||
99 | for ($id = 1; $id <= 42; $id++) { |
||
100 | // Save to db |
||
101 | $object = new BatchedProcessorTest_Object(); |
||
102 | $object->TestText = 'Object ' . $id; |
||
103 | $object->write(); |
||
104 | // Add to index manually |
||
105 | $processor->addDirtyIDs( |
||
106 | BatchedProcessorTest_Object::class, |
||
107 | array(array( |
||
108 | 'id' => $id, |
||
109 | 'state' => array(SearchVariantVersioned::class => 'Stage') |
||
110 | )), |
||
111 | BatchedProcessorTest_Index::class |
||
112 | ); |
||
113 | } |
||
114 | $processor->batchData(); |
||
115 | return $processor; |
||
116 | } |
||
117 | |||
118 | /** |
||
119 | * Tests that large jobs are broken up into a suitable number of batches |
||
120 | */ |
||
121 | public function testBatching() |
||
122 | { |
||
123 | $index = singleton(BatchedProcessorTest_Index::class); |
||
124 | $index->reset(); |
||
125 | $processor = $this->generateDirtyIds(); |
||
126 | |||
127 | // Check initial state |
||
128 | $data = $processor->getJobData(); |
||
129 | $this->assertEquals(9, $data->totalSteps); |
||
130 | $this->assertEquals(0, $data->currentStep); |
||
131 | $this->assertEmpty($data->isComplete); |
||
132 | $this->assertEquals(0, count($index->getAdded())); |
||
133 | |||
134 | // Advance state |
||
135 | for ($pass = 1; $pass <= 8; $pass++) { |
||
136 | $processor->process(); |
||
137 | $data = $processor->getJobData(); |
||
138 | $this->assertEquals($pass, $data->currentStep); |
||
139 | $this->assertEquals($pass * 5, count($index->getAdded())); |
||
140 | } |
||
141 | |||
142 | // Last run should have two hanging items |
||
143 | $processor->process(); |
||
144 | $data = $processor->getJobData(); |
||
145 | $this->assertEquals(9, $data->currentStep); |
||
146 | $this->assertEquals(42, count($index->getAdded())); |
||
147 | $this->assertTrue($data->isComplete); |
||
148 | |||
149 | // Check any additional queued jobs |
||
150 | $processor->afterComplete(); |
||
151 | $service = singleton(QueuedJobService::class); |
||
152 | $jobs = $service->getJobs(); |
||
153 | $this->assertEquals(1, count($jobs)); |
||
154 | $this->assertInstanceOf(SearchUpdateCommitJobProcessor::class, $jobs[0]['job']); |
||
155 | } |
||
156 | |||
157 | /** |
||
158 | * Test creation of multiple commit jobs |
||
159 | */ |
||
160 | public function testMultipleCommits() |
||
161 | { |
||
162 | $index = singleton(BatchedProcessorTest_Index::class); |
||
163 | $index->reset(); |
||
164 | |||
165 | // Test that running a commit immediately after submitting to the indexes |
||
166 | // correctly commits |
||
167 | $first = SearchUpdateCommitJobProcessor::queue(); |
||
168 | $second = SearchUpdateCommitJobProcessor::queue(); |
||
169 | |||
170 | $this->assertFalse($index->getIsCommitted()); |
||
171 | |||
172 | // First process will cause the commit |
||
173 | $this->assertFalse($first->jobFinished()); |
||
174 | $first->process(); |
||
175 | $allMessages = $first->getMessages(); |
||
176 | $this->assertTrue($index->getIsCommitted()); |
||
177 | $this->assertTrue($first->jobFinished()); |
||
178 | $this->assertStringEndsWith('All indexes committed', $allMessages[2]); |
||
179 | |||
180 | // Executing the subsequent processor should not re-trigger a commit |
||
181 | $index->reset(); |
||
182 | $this->assertFalse($second->jobFinished()); |
||
183 | $second->process(); |
||
184 | $allMessages = $second->getMessages(); |
||
185 | $this->assertFalse($index->getIsCommitted()); |
||
186 | $this->assertTrue($second->jobFinished()); |
||
187 | $this->assertStringEndsWith('Indexing already completed this request: Discarding this job', $allMessages[0]); |
||
188 | |||
189 | // Given that a third job is created, and the indexes are dirtied, attempting to run this job |
||
190 | // should result in a delay |
||
191 | $index->reset(); |
||
192 | $third = SearchUpdateCommitJobProcessor::queue(); |
||
193 | $this->assertFalse($third->jobFinished()); |
||
194 | $third->process(); |
||
195 | $this->assertTrue($third->jobFinished()); |
||
196 | $allMessages = $third->getMessages(); |
||
197 | $this->assertStringEndsWith( |
||
198 | 'Indexing already run this request, but incomplete. Re-scheduling for 2015-05-07 06:10:00', |
||
199 | $allMessages[0] |
||
200 | ); |
||
201 | } |
||
202 | |||
203 | /** |
||
204 | * Tests that the batch_soft_cap setting is properly respected |
||
205 | */ |
||
206 | public function testSoftCap() |
||
207 | { |
||
208 | $this->markTestIncomplete( |
||
209 | '@todo PostgreSQL: This test passes in isolation, but not in conjunction with the previous test' |
||
210 | ); |
||
211 | |||
212 | $index = singleton(BatchedProcessorTest_Index::class); |
||
213 | $index->reset(); |
||
214 | |||
215 | $processor = $this->generateDirtyIds(); |
||
216 | |||
217 | // Test that increasing the soft cap to 2 will reduce the number of batches |
||
218 | Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 2); |
||
219 | $processor->batchData(); |
||
220 | $data = $processor->getJobData(); |
||
221 | $this->assertEquals(8, $data->totalSteps); |
||
222 | |||
223 | // A soft cap of 1 should not fit in the hanging two items |
||
224 | Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 1); |
||
225 | $processor->batchData(); |
||
226 | $data = $processor->getJobData(); |
||
227 | $this->assertEquals(9, $data->totalSteps); |
||
228 | |||
229 | // Extra large soft cap should fit both items |
||
230 | Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 4); |
||
231 | $processor->batchData(); |
||
232 | $data = $processor->getJobData(); |
||
233 | $this->assertEquals(8, $data->totalSteps); |
||
234 | |||
235 | // Process all data and ensure that all are processed adequately |
||
236 | for ($pass = 1; $pass <= 8; $pass++) { |
||
237 | $processor->process(); |
||
238 | } |
||
239 | $data = $processor->getJobData(); |
||
240 | $this->assertEquals(8, $data->currentStep); |
||
241 | $this->assertEquals(42, count($index->getAdded())); |
||
242 | $this->assertTrue($data->isComplete); |
||
243 | } |
||
244 | } |
||
245 |
The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g.
excluded_paths: ["lib/*"]
, you can move it to the dependency path list as follows:For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths