Passed
Push — master ( 8518cf...808471 )
by Nicolaas
10:09
created

CleanUp::run()   A

Complexity

Conditions 4
Paths 8

Size

Total Lines 32
Code Lines 23

Duplication

Lines 0
Ratio 0 %

Importance

Changes 9
Bugs 0 Features 0
Metric Value
eloc 23
c 9
b 0
f 0
dl 0
loc 32
rs 9.552
cc 4
nc 8
nop 1
1
<?php
2
3
namespace Sunnysideup\DatabaseShareCleanUp;
4
5
use SilverStripe\Control\Director;
6
use SilverStripe\Control\HTTPRequest;
7
use SilverStripe\Dev\BuildTask;
8
9
use Sunnysideup\DatabaseShareCleanUp\Api\Anonymiser;
10
use Sunnysideup\DatabaseShareCleanUp\Api\DatabaseActions;
11
use Sunnysideup\Flush\FlushNow;
12
13
class CleanUp extends BuildTask
14
{
15
    /**
16
     * @var bool If set to FALSE, keep it from showing in the list
17
     * and from being executable through URL or CLI.
18
     */
19
    protected $enabled = true;
20
21
    /**
22
     * @var string Shown in the overview on the {@link TaskRunner}
23
     * HTML or CLI interface. Should be short and concise, no HTML allowed.
24
     */
25
    protected $title = 'Cleanup and anonymise database - CAREFUL! Data will be deleted.';
26
27
    /**
28
     * @var string Describe the implications the task has,
29
     * and the changes it makes. Accepts HTML formatting.
30
     */
31
    protected $description = 'Goes through database and deletes data that may expose personal information and bloat database.';
32
33
    protected $forReal = false;
34
35
    protected $anonymise = false;
36
37
    protected $removeObsolete = false;
38
39
    protected $removeOldVersions = false;
40
41
    protected $debug = false;
42
43
    protected $emptyFields = false;
44
45
    protected $removeRows = false;
46
47
    protected $selectedTables = false;
48
49
    protected $selectedTableList = [];
50
51
    protected $data = [];
52
53
    private static $tables_to_delete_forever = [];
54
55
    private static $tables_to_be_cleaned = [
56
        'LoginAttempt',
57
    ];
58
59
    private static $fields_to_be_cleaned = [];
60
61
    private static $field_table_comboes_to_be_cleaned = [];
62
63
    private static $tables_to_keep = [];
64
65
    private static $fields_to_keep = [
66
        'ClassName',
67
        'Created',
68
        'LastEdited',
69
    ];
70
71
    private static $field_table_combos_to_keep = [];
72
73
    private static $max_table_size_in_mb = 20;
74
75
    private static $max_column_size_in_mb = 2;
76
77
    private static $dependencies = [
78
        'anonymiser' => '%$' . Anonymiser::class,
79
        'database' => '%$' . DatabaseActions::class,
80
    ];
81
82
    private $anonymiser;
83
84
    private $database;
85
86
    /**
87
     * Set a custom url segment (to follow dev/tasks/)
88
     *
89
     * @config
90
     * @var string
91
     */
92
    private static $segment = 'database-share-clean-up';
93
94
    public function setAnonymiser($a)
95
    {
96
        $this->anonymiser = $a;
97
    }
98
99
    public function setDatabase($b)
100
    {
101
        $this->database = $b;
102
    }
103
104
    /**
105
     * @return bool
106
     */
107
    public function isEnabled()
108
    {
109
        return Director::isDev();
110
    }
111
112
    /**
113
     * Implement this method in the task subclass to
114
     * execute via the TaskRunner
115
     *
116
     * @param HTTPRequest $request
117
     */
118
    public function run($request)
119
    {
120
        $this->anonymise = (bool) $request->getVar('anonymise');
121
        $this->removeObsolete = (bool) $request->getVar('removeobsolete');
122
        $this->removeOldVersions = (bool) $request->getVar('removeoldversions');
123
        $this->removeRows = (bool) $request->getVar('removerows');
124
        $this->emptyFields = (bool) $request->getVar('emptyfields');
125
126
        $this->selectedTables = (bool) $request->getVar('selectedtables');
127
        $this->debug = (bool) $request->getVar('debug');
128
        $this->forReal = (bool) $request->getVar('forreal');
129
        if ($this->forReal) {
130
            $this->debug = true;
131
        }
132
        $this->selectedTableList = $request->getVar('selectedtablelist') ?? [];
133
134
        $this->anonymiser->setDatabaseActions($this->database);
135
        $this->database->setForReal($this->forReal);
136
        $this->database->setDebug($this->debug);
137
138
        if ($this->forReal) {
139
            FlushNow::do_flush('<h3>Running in FOR REAL mode</h3>', 'bad');
140
        } else {
141
            FlushNow::do_flush('<h3>Not runing FOR REAL</h3>', 'good');
142
        }
143
        if ($this->anonymise) {
144
            $this->anonymiser->AnonymisePresets();
145
        }
146
147
        $this->createForm();
148
        $this->runInner();
149
        $this->createTable();
150
    }
151
152
    protected function runInner()
153
    {
154
        $maxTableSize = $this->Config()->get('max_table_size_in_mb');
155
        $maxColumnSize = $this->Config()->get('max_column_size_in_mb');
156
157
        $tablesToDeleteForever = $this->Config()->get('tables_to_delete_forever');
158
159
        $tablesToKeep = $this->Config()->get('tables_to_keep');
160
        $fieldsToKeep = $this->Config()->get('fields_to_keep');
161
        $fieldTableCombosToKeep = $this->Config()->get('field_table_combos_to_keep');
162
163
        $tablesToBeCleaned = $this->Config()->get('tables_to_be_cleaned');
164
        $fieldsToBeCleaned = $this->Config()->get('fields_to_be_cleaned');
165
        $tableFieldCombosToBeCleaned = $this->Config()->get('field_table_comboes_to_be_cleaned');
166
167
        $tables = $this->database->getAllTables();
168
        foreach ($tables as $tableName) {
169
            $this->data[$tableName] = [
170
                'TableName' => $tableName,
171
                'SizeAfter' => 0,
172
                'SizeBefore' => 0,
173
                'Actions' => [],
174
            ];
175
176
            if (in_array($tableName, $tablesToKeep, true)) {
177
                if ($this->debug) {
178
                    $this->data[$tableName]['Actions'][] = 'Skipped because it is in list of tables to keep.';
179
                }
180
                continue;
181
            }
182
            if ($this->database->isEmptyTable($tableName)) {
183
                if ($this->debug) {
184
                    $this->data[$tableName]['Actions'][] = 'Skipped because table is empty.';
185
                }
186
                continue;
187
            }
188
            $this->data[$tableName]['SizeBefore'] = $this->database->getTableSizeInMegaBytes($tableName);
189
            if ($this->selectedTables && ! in_array($tableName, $this->selectedTableList, true)) {
190
                $this->data[$tableName]['Actions'][] = 'Skipped because it is not a selected table.';
191
                continue;
192
            }
193
194
            if ($this->removeObsolete) {
195
                if (in_array($tableName, $tablesToDeleteForever, true)) {
196
                    $this->database->deleteTable($tableName);
197
                    $this->data[$tableName]['Actions'][] = 'DELETING FOREVER.';
198
                    continue;
199
                }
200
                $outcome = $this->database->deleteObsoleteTables($tableName);
201
                if ($outcome) {
202
                    $this->data[$tableName]['Actions'][] = 'Deleted because it is obsolete.';
203
                }
204
            }
205
206
            if ($this->removeOldVersions) {
207
                $outcome = $this->database->emptyVersionedTable($tableName);
208
                if ($outcome) {
209
                    $this->data[$tableName]['Actions'][] = 'Remove all and replace with one entry for each record.';
210
                }
211
            }
212
213
            if ($this->anonymise) {
214
                $outcome = $this->anonymiser->AnonymiseTable($tableName);
215
                if ($outcome) {
216
                    $this->data[$tableName]['Actions'][] = 'Anonymised Table.';
217
                }
218
            }
219
            //get fields
220
            $fields = $this->database->getAllFieldsForOneTable($tableName);
221
222
            foreach ($fields as $fieldName) {
223
                if (substr($fieldName, -2) === 'ID') {
224
                    if ($this->debug) {
225
                        $this->data[$tableName]['Actions'][] = ' ... ' . $fieldName . ': skipping!';
226
                    }
227
                    continue;
228
                }
229
                if (in_array($fieldName, $fieldsToKeep, true)) {
230
                    if ($this->debug) {
231
                        $this->data[$tableName]['Actions'][] = ' ... ' . $fieldName . ': skipping!';
232
                    }
233
                    continue;
234
                }
235
236
                $combo = $tableName . '.' . $fieldName;
237
                if (in_array($combo, $fieldTableCombosToKeep, true)) {
238
                    if ($this->debug) {
239
                        $this->data[$tableName]['Actions'][] = ' ... ' . $fieldName . ': skipping.';
240
                    }
241
                    continue;
242
                }
243
                if ($this->anonymise) {
244
                    $outcome = $this->anonymiser->AnonymiseTableField($tableName, $fieldName);
245
                    if ($outcome) {
246
                        $this->data[$tableName]['Actions'][] = ' ... ' . $fieldName . ': anonymised.';
247
                    }
248
                }
249
                if ($this->emptyFields) {
250
                    $columnSize = $this->database->getColumnSizeInMegabytes($tableName, $fieldName);
251
                    $test1 = $columnSize > $maxColumnSize;
252
                    $test2 = in_array($fieldName, $fieldsToBeCleaned, true);
253
                    $test3 = in_array($combo, $tableFieldCombosToBeCleaned, true);
254
                    if ($test1 || $test2 || $test3) {
255
                        $percentageToKeep = $test2 || $test3 ? 0 : $maxColumnSize / $columnSize;
256
                        $outcome = $this->database->removeOldColumnsFromTable($tableName, $fieldName, $percentageToKeep);
257
                        if ($outcome) {
258
                            $this->data[$tableName]['Actions'][] = ' ... ' . $fieldName . ': Removed most rows.';
259
                        }
260
                    }
261
                }
262
            }
263
264
            // clean table
265
            if ($this->removeRows) {
266
                $removeAllRows = in_array($tableName, $tablesToBeCleaned, true);
267
                if ($removeAllRows) {
268
                    $this->database->removeOldRowsFromTable($tableName, 0.01);
269
                    $this->data[$tableName]['Actions'][] = 'Removed most rows.';
270
                } else {
271
                    $tableSize = $this->database->getTableSizeInMegaBytes($tableName);
272
                    if ($tableSize > $maxTableSize) {
273
                        $percentageToKeep = $maxTableSize / $tableSize;
274
                        $this->database->removeOldRowsFromTable($tableName, $percentageToKeep);
275
                        $this->data[$tableName]['Actions'][] = 'Removed old rows.';
276
                    }
277
                }
278
            }
279
            $this->data[$tableName]['SizeAfter'] = $this->database->getTableSizeInMegaBytes($tableName);
280
        }
281
    }
282
283
    protected function createForm()
284
    {
285
        $anonymise = $this->anonymise ? 'checked="checked"' : '';
286
        $removeObsolete = $this->removeObsolete ? 'checked="checked"' : '';
287
        $removeOldVersions = $this->removeOldVersions ? 'checked="checked"' : '';
288
        $removeRows = $this->removeRows ? 'checked="checked"' : '';
289
        $emptyFields = $this->emptyFields ? 'checked="checked"' : '';
290
        $selectedTables = $this->selectedTables ? 'checked="checked"' : '';
291
        $forReal = $this->forReal ? 'checked="checked"' : '';
292
        $debug = $this->debug ? 'checked="checked"' : '';
293
        echo <<<html
294
        <h3>All sizes in Megabytes</h3>
295
        <form method="get">
296
            <div style="
297
                background-color: pink;
298
                width: 300px;
299
                padding: 1vw;
300
                position: fixed;
301
                right: 0;
302
                top: 0;
303
                bottom: 0;
304
                border: 1px solid red;
305
            ">
306
                <h4>What actions to take?</h4>
307
                <div class="field" style="padding: 10px;">
308
                    <input type="checkbox" name="anonymise" {$anonymise} />
309
                    <label>anonymise</label>
310
                </div>
311
                <div class="field" style="padding: 10px;">
312
                    <input type="checkbox" name="removeoldversions" {$removeOldVersions} />
313
                    <label>remove versioned table entries</label>
314
                </div>
315
                <div class="field" style="padding: 10px;">
316
                    <input type="checkbox" name="removeobsolete" {$removeObsolete} />
317
                    <label>remove obsolete tables</label>
318
                </div>
319
320
                <div class="field" style="padding: 10px;">
321
                    <input type="checkbox" name="emptyfields" {$emptyFields} />
322
                    <label>empty large fields</label>
323
                </div>
324
325
                <div class="field" style="padding: 10px;">
326
                    <input type="checkbox" name="removerows" {$removeRows} />
327
                    <label>remove old rows if there are too many (not recommended)</label>
328
                </div>
329
330
                <hr />
331
                <h4>How to apply?</h4>
332
                <div class="field" style="padding: 10px;">
333
                    <input type="checkbox" name="selectedtables" {$selectedTables} />
334
                    <label>apply to selected tables only?</label>
335
                </div>
336
                <div class="field" style="padding: 10px;">
337
                    <input type="checkbox" name="forreal" {$forReal} />
338
                    <label>do it for real?</label>
339
                </div>
340
                <hr />
341
                <h4>See more info?</h4>
342
                <div class="field" style="padding: 10px;">
343
                    <input type="checkbox" name="debug" {$debug} />
344
                    <label>debug</label>
345
                </div>
346
                <hr />
347
                <div class="field" style="padding: 10px;">
348
                    <input type="submit" value="let's do it!" />
349
                </div>
350
            </div>
351
352
353
html;
354
    }
355
356
    protected function createTable()
357
    {
358
        $tbody = '';
359
        $totalSizeBefore = 0;
360
        $totalSizeAfter = 0;
361
        usort(
362
            $this->data,
363
            function ($a, $b) {
364
                return $a['SizeAfter'] <=> $b['SizeAfter'];
365
            }
366
        );
367
        foreach ($this->data as $data) {
368
            $totalSizeBefore += $data['SizeBefore'];
369
            $totalSizeAfter += $data['SizeAfter'];
370
            $actions = '';
371
            if (count($data['Actions'])) {
372
                $actions = '
373
                        <ul>
374
                            <li>
375
                            ' . implode('</li><li>', $data['Actions']) . '
376
                            </li>
377
                        </ul>';
378
            }
379
            $tableList = empty($this->selectedTableList[$data['TableName']]) ? '' : 'checked="checked"';
380
            $tbody .= '
381
                <tr>
382
                    <td>
383
                        <input type="checkbox" name="selectedtablelist[]" value="' . $data['TableName'] . '" ' . $tableList . ' />
384
                    </td>
385
                    <td>
386
                        ' . $data['TableName'] . '
387
                        ' . $actions . '
388
                    </td>
389
                    <td style="text-align: center;">
390
                        ' . $data['SizeBefore'] . '
391
                    </td>
392
                    <td style="text-align: center;">
393
                        ' . $data['SizeAfter'] . '
394
                    </td>
395
                </tr>';
396
        }
397
        $tfoot = '
398
                <tr>
399
                    <th>
400
                        &nbsp;
401
                    </th>
402
                    <th>
403
                        TOTAL
404
                    </th>
405
                    <th>
406
                        ' . $totalSizeBefore . '
407
                    </th>
408
                    <th>
409
                        ' . $totalSizeAfter . '
410
                    </th>
411
                </tr>
412
        ';
413
        echo '
414
        <table border="1" style="width: calc(100% - 380px);">
415
            <thead>
416
                <tr>
417
                    <th width="5%">
418
                        Select
419
                    </th>
420
                    <th width="65%">
421
                        Table Name
422
                    </th>
423
                    <th style="text-align: center;" width="15%">
424
                        Before
425
                    </th>
426
                    <th style="text-align: center;" width="15%">
427
                        After
428
                    </th>
429
                </tr>
430
            </thead>
431
            <tfoot>' . $tfoot . '</tfoot>
432
            <tbody>' . $tbody . '</tbody>
433
        </table>
434
    </form>';
435
    }
436
}
437