Passed
Push — master ( fcece8...566a2b )
by Nicolaas
09:36
created

CleanUp::formatBytes()   A

Complexity

Conditions 1
Paths 1

Size

Total Lines 6
Code Lines 3

Duplication

Lines 0
Ratio 0 %

Importance

Changes 1
Bugs 0 Features 0
Metric Value
eloc 3
c 1
b 0
f 0
dl 0
loc 6
rs 10
cc 1
nc 1
nop 2
1
<?php
2
3
namespace Sunnysideup\DatabaseShareCleanUp;
4
5
use SilverStripe\Control\HTTPRequest;
6
use SilverStripe\Dev\BuildTask;
7
use Sunnysideup\DatabaseShareCleanUp\Api\Anonymiser;
8
use Sunnysideup\DatabaseShareCleanUp\Api\DatabaseActions;
9
use Sunnysideup\Flush\FlushNow;
10
use Sunnysideup\Flush\FlushNowImplementor;
11
12
class CleanUp extends BuildTask
13
{
14
    /**
15
     * @var bool if set to FALSE, keep it from showing in the list
16
     *           and from being executable through URL or CLI
17
     */
18
    protected $enabled = true;
19
20
    /**
21
     * @var string Shown in the overview on the {@link TaskRunner}
22
     *             HTML or CLI interface. Should be short and concise, no HTML allowed.
23
     */
24
    protected $title = 'Cleanup and anonymise database - CAREFUL! Data will be deleted.';
25
26
    /**
27
     * @var string Describe the implications the task has,
28
     *             and the changes it makes. Accepts HTML formatting.
29
     */
30
    protected $description = 'Goes through database and deletes data that may expose personal information and bloat database.';
31
32
    protected $forReal = false;
33
34
    protected $anonymise = false;
35
36
    protected $removeObsolete = false;
37
38
    protected $removeOldVersions = false;
39
40
    protected $debug = false;
41
42
    protected $emptyFields = false;
43
44
    protected $removeRows = false;
45
46
    protected $selectedTables = false;
47
48
    protected $selectedTableList = [];
49
50
    protected $data = [];
51
52
    private static $tables_to_delete_forever = [];
53
54
    private static $tables_to_be_cleaned = [
55
        'LoginAttempt',
56
    ];
57
58
    private static $fields_to_be_cleaned = [];
59
60
    private static $field_table_comboes_to_be_cleaned = [];
61
62
    private static $tables_to_keep = [];
63
64
    private static $fields_to_keep = [
65
        'ClassName',
66
        'Created',
67
        'LastEdited',
68
    ];
69
70
    private static $field_table_combos_to_keep = [];
71
72
    private static $max_table_size_in_mb = 20;
73
74
    private static $max_column_size_in_mb = 2;
75
76
    private static $dependencies = [
77
        'anonymiser' => '%$' . Anonymiser::class,
78
        'database' => '%$' . DatabaseActions::class,
79
    ];
80
81
    private $anonymiser;
82
83
    private $database;
84
85
    /**
86
     * Set a custom url segment (to follow dev/tasks/).
87
     *
88
     * @config
89
     *
90
     * @var string
91
     */
92
    private static $segment = 'database-share-clean-up';
93
94
    public function setAnonymiser($anonymiser)
95
    {
96
        $this->anonymiser = $anonymiser;
97
98
        return $this;
99
    }
100
101
    public function setDatabase($database)
102
    {
103
        $this->database = $database;
104
105
        return $this;
106
    }
107
108
    /**
109
     * Implement this method in the task subclass to
110
     * execute via the TaskRunner.
111
     *
112
     * @param HTTPRequest $request
113
     */
114
    public function run($request)
115
    {
116
        $this->anonymise = (bool) $request->getVar('anonymise');
117
        $this->removeObsolete = (bool) $request->getVar('removeobsolete');
118
        $this->removeOldVersions = (bool) $request->getVar('removeoldversions');
119
        $this->removeRows = (bool) $request->getVar('removerows');
120
        $this->emptyFields = (bool) $request->getVar('emptyfields');
121
122
        $this->selectedTables = (bool) $request->getVar('selectedtables');
123
        $this->debug = (bool) $request->getVar('debug');
124
        $this->forReal = (bool) $request->getVar('forreal');
125
        if ($this->forReal) {
126
            $this->debug = true;
127
        }
128
        $this->selectedTableList = $request->getVar('selectedtablelist') ?? [];
129
130
        $this->anonymiser->setDatabaseActions($this->database);
131
        $this->database->setForReal($this->forReal);
132
        $this->database->setDebug($this->debug);
133
134
        if ($this->forReal) {
135
            FlushNowImplementor::do_flush('<h3>Running in FOR REAL mode</h3>', 'bad');
136
        } else {
137
            FlushNowImplementor::do_flush('<h3>Not runing FOR REAL</h3>', 'good');
138
        }
139
        if ($this->anonymise) {
140
            $this->anonymiser->AnonymisePresets();
141
        }
142
143
        $this->createForm();
144
        $this->runInner();
145
        $this->createTable();
146
        echo "MEMORY USED:" . $this->formatBytes(memory_get_peak_usage());
147
    }
148
149
    protected function runInner()
150
    {
151
        $maxTableSize = $this->Config()->get('max_table_size_in_mb');
152
        $maxColumnSize = $this->Config()->get('max_column_size_in_mb');
153
154
        $tablesToDeleteForever = $this->Config()->get('tables_to_delete_forever');
155
156
        $tablesToKeep = $this->Config()->get('tables_to_keep');
157
        $fieldsToKeep = $this->Config()->get('fields_to_keep');
158
        $fieldTableCombosToKeep = $this->Config()->get('field_table_combos_to_keep');
159
160
        $tablesToBeCleaned = $this->Config()->get('tables_to_be_cleaned');
161
        $fieldsToBeCleaned = $this->Config()->get('fields_to_be_cleaned');
162
        $tableFieldCombosToBeCleaned = $this->Config()->get('field_table_comboes_to_be_cleaned');
163
164
        $tables = $this->database->getAllTables();
165
        foreach ($tables as $tableName) {
166
            if (!$this->database->tableExists($tableName)) {
167
                continue;
168
            }
169
            $this->data[$tableName] = [
170
                'TableName' => $tableName,
171
                'SizeAfter' => 0,
172
                'SizeBefore' => 0,
173
                'Actions' => [],
174
            ];
175
176
            if (in_array($tableName, $tablesToKeep, true)) {
177
                if ($this->debug) {
178
                    $this->data[$tableName]['Actions'][] = 'Skipped because it is in list of tables to keep.';
179
                }
180
181
                continue;
182
            }
183
184
            if ($this->removeObsolete) {
185
                if (in_array($tableName, $tablesToDeleteForever, true)) {
186
                    $this->database->deleteTable($tableName);
187
                    $this->data[$tableName]['Actions'][] = 'DELETING FOREVER.';
188
189
                    continue;
190
                }
191
                $outcome = $this->database->deleteObsoleteTables($tableName);
192
                if ($outcome) {
193
                    $this->data[$tableName]['Actions'][] = 'Deleted because it is obsolete.';
194
                }
195
            }
196
197
            if ($this->database->isEmptyTable($tableName)) {
198
                if ($this->debug) {
199
                    $this->data[$tableName]['Actions'][] = 'Skipped because table is empty.';
200
                }
201
202
                continue;
203
            }
204
            $this->data[$tableName]['SizeBefore'] = $this->database->getTableSizeInMegaBytes($tableName);
205
            if ($this->selectedTables && !in_array($tableName, $this->selectedTableList, true)) {
206
                $this->data[$tableName]['Actions'][] = 'Skipped because it is not a selected table.';
207
208
                continue;
209
            }
210
211
            if ($this->removeOldVersions) {
212
                $outcome = $this->database->emptyVersionedTable($tableName);
213
                if ($outcome) {
214
                    $this->data[$tableName]['Actions'][] = 'Remove all versions.';
215
                }
216
            }
217
218
            if ($this->anonymise) {
219
                $outcome = $this->anonymiser->AnonymiseTable($tableName);
220
                if ($outcome) {
221
                    $this->data[$tableName]['Actions'][] = 'Anonymised Table.';
222
                }
223
            }
224
            //get fields
225
            $fields = $this->database->getAllFieldsForOneTable($tableName);
226
227
            foreach ($fields as $fieldName) {
228
                if ('ID' === substr((string) $fieldName, -2)) {
229
                    if ($this->debug) {
230
                        $this->data[$tableName]['Actions'][] = ' ... ' . $fieldName . ': skipping!';
231
                    }
232
233
                    continue;
234
                }
235
                if (in_array($fieldName, $fieldsToKeep, true)) {
236
                    if ($this->debug) {
237
                        $this->data[$tableName]['Actions'][] = ' ... ' . $fieldName . ': skipping (field is marked as KEEP)!';
238
                    }
239
240
                    continue;
241
                }
242
243
                $combo = $tableName . '.' . $fieldName;
244
                if (in_array($combo, $fieldTableCombosToKeep, true)) {
245
                    if ($this->debug) {
246
                        $this->data[$tableName]['Actions'][] = ' ... ' . $fieldName . ': skipping (table.field is marked as KEEP).';
247
                    }
248
249
                    continue;
250
                }
251
                if ($this->anonymise) {
252
                    $outcome = $this->anonymiser->AnonymiseTableField($tableName, $fieldName);
253
                    if ($outcome) {
254
                        $this->data[$tableName]['Actions'][] = ' ... ' . $fieldName . ': anonymised.';
255
                    }
256
                }
257
                if ($this->emptyFields) {
258
                    $columnSize = $this->database->getColumnSizeInMegabytes($tableName, $fieldName);
259
                    $test1 = $columnSize > $maxColumnSize;
260
                    $test2 = in_array($fieldName, $fieldsToBeCleaned, true);
261
                    $test3 = in_array($combo, $tableFieldCombosToBeCleaned, true);
262
                    if ($test1 || $test2 || $test3) {
263
                        $percentageToKeep = $test2 || $test3 ? 0 : $maxColumnSize / $columnSize;
264
                        $outcome = $this->database->removeOldColumnsFromTable($tableName, $fieldName, $percentageToKeep);
265
                        if ($outcome) {
266
                            $this->data[$tableName]['Actions'][] = ' ... ' . $fieldName . ': Removed most rows.';
267
                        }
268
                    }
269
                }
270
            }
271
272
            // clean table
273
            if ($this->removeRows) {
274
                $removeAllRows = in_array($tableName, $tablesToBeCleaned, true);
275
                if ($removeAllRows) {
276
                    $this->database->removeOldRowsFromTable($tableName, 0.01);
277
                    $this->data[$tableName]['Actions'][] = 'Removed most rows.';
278
                } else {
279
                    $tableSize = $this->database->getTableSizeInMegaBytes($tableName);
280
                    if ($tableSize > $maxTableSize) {
281
                        $percentageToKeep = $maxTableSize / $tableSize;
282
                        $this->database->removeOldRowsFromTable($tableName, $percentageToKeep);
283
                        $this->data[$tableName]['Actions'][] = 'Removed old rows.';
284
                    }
285
                }
286
            }
287
            $this->data[$tableName]['SizeAfter'] = $this->database->getTableSizeInMegaBytes($tableName);
288
        }
289
    }
290
291
    protected function createForm()
292
    {
293
        $anonymise = $this->anonymise ? 'checked="checked"' : '';
294
        $removeObsolete = $this->removeObsolete ? 'checked="checked"' : '';
295
        $removeOldVersions = $this->removeOldVersions ? 'checked="checked"' : '';
296
        $removeRows = $this->removeRows ? 'checked="checked"' : '';
297
        $emptyFields = $this->emptyFields ? 'checked="checked"' : '';
298
        $selectedTables = $this->selectedTables ? 'checked="checked"' : '';
299
        $forReal = $this->forReal ? 'checked="checked"' : '';
300
        $debug = $this->debug ? 'checked="checked"' : '';
301
        echo <<<html
302
        <h3>All sizes in Megabytes</h3>
303
        <form method="get">
304
            <div style="
305
                background-color: pink;
306
                width: 300px;
307
                padding: 1vw;
308
                position: fixed;
309
                right: 0;
310
                top: 0;
311
                bottom: 0;
312
                border: 1px solid red;
313
            ">
314
                <h4>What actions to take?</h4>
315
                <div class="field" style="padding: 10px;">
316
                    <input type="checkbox" name="anonymise" {$anonymise} />
317
                    <label>anonymise</label>
318
                </div>
319
                <div class="field" style="padding: 10px;">
320
                    <input type="checkbox" name="removeoldversions" {$removeOldVersions} />
321
                    <label>remove versioned table entries</label>
322
                </div>
323
                <div class="field" style="padding: 10px;">
324
                    <input type="checkbox" name="removeobsolete" {$removeObsolete} />
325
                    <label>remove obsolete tables</label>
326
                </div>
327
328
                <div class="field" style="padding: 10px;">
329
                    <input type="checkbox" name="emptyfields" {$emptyFields} />
330
                    <label>empty large fields</label>
331
                </div>
332
333
                <div class="field" style="padding: 10px;">
334
                    <input type="checkbox" name="removerows" {$removeRows} />
335
                    <label>remove old rows if there are too many (not recommended)</label>
336
                </div>
337
338
                <hr />
339
                <h4>How to apply?</h4>
340
                <div class="field" style="padding: 10px;">
341
                    <input type="checkbox" name="selectedtables" {$selectedTables} />
342
                    <label>apply to selected tables only?</label>
343
                </div>
344
                <div class="field" style="padding: 10px;">
345
                    <input type="checkbox" name="forreal" {$forReal} />
346
                    <label>do it for real?</label>
347
                </div>
348
                <hr />
349
                <h4>See more info?</h4>
350
                <div class="field" style="padding: 10px;">
351
                    <input type="checkbox" name="debug" {$debug} />
352
                    <label>debug</label>
353
                </div>
354
                <hr />
355
                <div class="field" style="padding: 10px;">
356
                    <input type="submit" value="let's do it!" />
357
                </div>
358
            </div>
359
360
361
html;
362
    }
363
364
    protected function createTable()
365
    {
366
        $tbody = '';
367
        $totalSizeBefore = 0;
368
        $totalSizeAfter = 0;
369
        usort(
370
            $this->data,
371
            function ($a, $b) {
372
                return $b['SizeBefore'] <=> $a['SizeBefore'];
373
            }
374
        );
375
        foreach ($this->data as $data) {
376
            $totalSizeBefore += $data['SizeBefore'];
377
            $totalSizeAfter += $data['SizeAfter'];
378
            $actions = '';
379
            if (count($data['Actions'])) {
380
                $actions = '
381
                        <ul>
382
                            <li>
383
                            ' . implode('</li><li>', $data['Actions']) . '
384
                            </li>
385
                        </ul>';
386
            }
387
            $tableList = empty($this->selectedTableList[$data['TableName']]) ? '' : 'checked="checked"';
388
            $tbody .= '
389
                <tr>
390
                    <td>
391
                        <input type="checkbox" name="selectedtablelist[]" value="' . $data['TableName'] . '" ' . $tableList . ' />
392
                    </td>
393
                    <td>
394
                        ' . $data['TableName'] . '
395
                        ' . $actions . '
396
                    </td>
397
                    <td style="text-align: center;">
398
                        ' . $data['SizeBefore'] . '
399
                    </td>
400
                    <td style="text-align: center;">
401
                        ' . $data['SizeAfter'] . '
402
                    </td>
403
                </tr>';
404
        }
405
        $tfoot = '
406
                <tr>
407
                    <th>
408
                        &nbsp;
409
                    </th>
410
                    <th>
411
                        TOTAL
412
                    </th>
413
                    <th>
414
                        ' . $totalSizeBefore . '
415
                    </th>
416
                    <th>
417
                        ' . $totalSizeAfter . '
418
                    </th>
419
                </tr>
420
        ';
421
        echo '
422
        <table border="1" style="width: calc(100% - 380px);">
423
            <thead>
424
                <tr>
425
                    <th width="5%">
426
                        Select
427
                    </th>
428
                    <th width="65%">
429
                        Table Name
430
                    </th>
431
                    <th style="text-align: center;" width="15%">
432
                        Before
433
                    </th>
434
                    <th style="text-align: center;" width="15%">
435
                        After
436
                    </th>
437
                </tr>
438
            </thead>
439
            <tfoot>' . $tfoot . '</tfoot>
440
            <tbody>' . $tbody . '</tbody>
441
        </table>
442
    </form>';
443
    }
444
445
    private function formatBytes($size, $precision = 2)
446
    {
447
        $base = log($size, 1024);
448
        $suffixes = array('', 'K', 'M', 'G', 'T');
449
450
        return round(pow(1024, $base - floor($base)), $precision) . ' ' . $suffixes[floor($base)];
451
    }
452
}
453