PreserverTrait::flush()   C
last analyzed

Complexity

Conditions 8
Paths 22

Size

Total Lines 42
Code Lines 35

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 0
CRAP Score 72

Importance

Changes 0
Metric Value
dl 0
loc 42
ccs 0
cts 35
cp 0
rs 5.3846
c 0
b 0
f 0
cc 8
eloc 35
nc 22
nop 3
crap 72
1
<?php
2
declare(strict_types = 1);
3
/**
4
 * Contains PreserverTrait Trait.
5
 *
6
 * PHP version 7.0+
7
 *
8
 * LICENSE:
9
 * This file is part of Yet Another Php Eve Api Library also know as Yapeal
10
 * which can be used to access the Eve Online API data and place it into a
11
 * database.
12
 * Copyright (C) 2014-2017 Michael Cummings
13
 *
14
 * This program is free software: you can redistribute it and/or modify it
15
 * under the terms of the GNU Lesser General Public License as published by the
16
 * Free Software Foundation, either version 3 of the License, or (at your
17
 * option) any later version.
18
 *
19
 * This program is distributed in the hope that it will be useful, but WITHOUT
20
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
21
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
22
 * for more details.
23
 *
24
 * You should have received a copy of the GNU Lesser General Public License
25
 * along with this program. If not, see
26
 * <http://spdx.org/licenses/LGPL-3.0.html>.
27
 *
28
 * You should be able to find a copy of this license in the COPYING-LESSER.md
29
 * file. A copy of the GNU GPL should also be available in the COPYING.md file.
30
 *
31
 * @copyright 2014-2017 Michael Cummings
32
 * @license   LGPL-3.0+
33
 * @author    Michael Cummings <[email protected]>
34
 */
35
namespace Yapeal\Sql;
36
37
use Yapeal\Event\EveApiEventInterface;
38
use Yapeal\Event\MediatorInterface;
39
use Yapeal\Log\Logger;
40
41
/**
42
 * Trait PreserverTrait
43
 *
44
 * @method CommonSqlQueries getCsq()
45
 * @method \PDO getPdo()
46
 * @method MediatorInterface getYem()
47
 */
48
trait PreserverTrait
49
{
50
    /**
51
     * @return string[]
52
     * @throws \LogicException
53
     */
54
    public function getPreserveTos(): array
55
    {
56
        if (0 === count($this->preserveTos)) {
57
            $mess = 'Tried to access preserveTos before it was set';
58
            throw new \LogicException($mess);
59
        }
60
        return $this->preserveTos;
61
    }
62
    /**
63
     * @param EveApiEventInterface $event
64
     * @param string               $eventName
65
     * @param MediatorInterface    $yem
66
     *
67
     * @return EveApiEventInterface
68
     * @throws \DomainException
69
     * @throws \InvalidArgumentException
70
     * @throws \LogicException
71
     * @throws \UnexpectedValueException
72
     */
73
    public function preserveEveApi(
74
        EveApiEventInterface $event,
75
        string $eventName,
76
        MediatorInterface $yem
77
    ): EveApiEventInterface {
78
        if (!$this->shouldPreserve()) {
79
            return $event;
80
        }
81
        $this->setYem($yem);
0 ignored issues
show
Bug introduced by
It seems like setYem() must be provided by classes using this trait. How about adding it as abstract method to this trait?

This check looks for methods that are used by a trait but not required by it.

To illustrate, let’s look at the following code example

trait Idable {
    public function equalIds(Idable $other) {
        return $this->getId() === $other->getId();
    }
}

The trait Idable provides a method equalsId that in turn relies on the method getId(). If this method does not exist on a class mixing in this trait, the method will fail.

Adding the getId() as an abstract method to the trait will make sure it is available.

Loading history...
82
        $data = $event->getData();
83
        $yem->triggerLogEvent('Yapeal.Log.log',
84
                Logger::DEBUG,
85
                $this->getReceivedEventMessage($data, $eventName, __CLASS__));
86
        if ('' === $data->getEveApiXml()) {
87
            return $event;
88
        }
89
        $this->getPdo()
90
            ->beginTransaction();
91
        try {
92
            foreach ($this->getPreserveTos() as $preserveTo) {
93
                $this->$preserveTo($data);
94
            }
95
            $this->getPdo()
96
                ->commit();
97
        } catch (\PDOException $exc) {
98
            $mess = 'Failed to upsert data of';
99
            $yem->triggerLogEvent('Yapeal.Log.log',
100
                    Logger::WARNING,
101
                    $this->createEveApiMessage($mess, $data),
102
                    ['exception' => $exc]);
103
            $this->getPdo()
104
                ->rollBack();
105
            return $event;
106
        }
107
        $yem->triggerLogEvent('Yapeal.Log.log', Logger::DEBUG, $this->getFinishedEventMessage($data, $eventName));
108
        return $event->setHandledSufficiently();
109
    }
110
    /**
111
     * Turn on or off preserving of Eve API data by this preserver.
112
     *
113
     * Allows class to stay registered for events but be enabled or disabled during runtime.
114
     *
115
     * @param boolean $value
116
     *
117
     * @return $this Fluent interface
118
     */
119
    public function setPreserve(bool $value = true)
120
    {
121
        $this->preserve = $value;
122
        return $this;
123
    }
124
    /**
125
     * Used to process the most common attribute rowset style of API data.
126
     *
127
     * Most Eve APIs use a set of rowset tags containing row tags. Some of them nest additional rowsets inside of the
128
     * rows like with the AssetList APIs where contents of hangers, ships, and other containers are done this way. A few
129
     * of the APIs are made up of a collection of rowset elements instead. The top level rowset tags have columns, key,
130
     * and name attributes. Each row tag inside of the rowset will have attributes with the same names as listed in the
131
     * columns attribute from the rowset. Depending on the API some of the row attributes may be missing and have known
132
     * default values that are used instead or are considered optional in the database table and can be NULL.
133
     *
134
     * @param \SimpleXMLElement[] $rows
135
     * @param array               $columnDefaults
136
     * @param string              $tableName
137
     *
138
     * @return static Fluent interface.
139
     * @throws \DomainException
140
     * @throws \InvalidArgumentException
141
     * @throws \UnexpectedValueException
142
     */
143
    protected function attributePreserveData(array $rows, array $columnDefaults, string $tableName)
144
    {
145
        $this->lastColumnCount = 0;
146
        $this->lastRowCount = 0;
147
        unset($this->pdoStatement);
148
        if (0 === $rowCount = count($rows)) {
149
            return $this;
150
        }
151
        $columnNames = array_keys($columnDefaults);
152
        /**
153
         * Determines the maximum number of rows per SQL query.
154
         *
155
         * ## Background
156
         *
157
         * Coming up with a good chunk size is harder than it seems. First there a lot of Eve APIs with just few rows
158
         * like Account APIKeyInfo or Corp AccountBalance then there are others like Eve AllianceList, or Corp AssetList
159
         * which have 1000s or maybe even 10000s of rows for the last one in some larger corps with a lot of offices.
160
         *
161
         * On the SQL side of things larger queries are generally more efficient but also take up a lot more memory to
162
         * build. Plus very large queries tend to exceed limits built into the driver or database server itself to
163
         * protect against DOS attacks etc.
164
         *
165
         * After a lot of feedback from application developers and issues reports the upper limit seems to be around
166
         * 1000 rows at least with MySQL which has been the only test platform used in the past with Yapeal-ng. The
167
         * other factor is the OS the database is running on. The Windows drivers at least for MySQL seem to cause the
168
         * most issues but as stated 1000 rows seems to keep the problems from turn up. There are some php.ini settings
169
         * that can be changed to help with using larger queries but not everyone has access to them depending on where
170
         * they're host their site and other reasons.
171
         *
172
         * So to summarize for the really large Eve APIs results you want to use as few large queries as you can without
173
         * exceeding database platform or OS limits while also not needlessly breaking up smaller results which would
174
         * hurt performance and efficiency.
175
         *
176
         * ## Explaining the code
177
         *
178
         *   1. Take the row count and divide it by 4 throwing away any remainder to help keep memory use down without
179
         *   create tons of queries to process which is less efficient.
180
         *   2. Make sure for larger Eve APIs not to exceed 1000 rows chunks using min().
181
         *   3. Insure small and medium size Eve APIs aren't broken up needlessly by enforcing minimum of 100 rows
182
         *   chunks by using max().
183
         *
184
         * @var int $chunkSize
185
         */
186
        $chunkSize = max(100, min(1000, intdiv($rowCount, 4)));
187
        for ($pos = 0; $pos <= $rowCount; $pos += $chunkSize) {
188
            $this->flush($this->processXmlRows(array_slice($rows, $pos, $chunkSize, false), $columnDefaults),
189
                $columnNames,
190
                $tableName);
191
        }
192
        return $this;
193
    }
194
    /**
195
     * Used by all styles of Eve APIs to prepare and execute their SQL 'upsert' queries.
196
     *
197
     * 'Upsert' is a commonly used term for updating any existing rows in a table and inserting all the ones that don't
198
     * already exist together at one time.
199
     *
200
     * The method also tracks if the prepared query can be re-used or not to take fuller advantage of them in cases
201
     * where all queries have the same number of database rows as is common with some of the larger APIs and a few that
202
     * always have a fixed number of rows.
203
     *
204
     * @param string[] $columns
205
     * @param string[] $columnNames
206
     * @param string   $tableName
207
     *
208
     * @return static Fluent interface.
209
     * @throws \DomainException
210
     * @throws \InvalidArgumentException
211
     * @throws \UnexpectedValueException
212
     */
213
    protected function flush(array $columns, array $columnNames, string $tableName)
214
    {
215
        if (0 === count($columns)) {
216
            return $this;
217
        }
218
        $rowCount = intdiv(count($columns), count($columnNames));
219
        $mess = sprintf('Have %s row(s) to upsert into %s table', $rowCount, $tableName);
220
        $this->getYem()
221
            ->triggerLogEvent('Yapeal.Log.log', Logger::INFO, $mess);
222
        $isNotPrepared = $this->lastColumnCount !== count($columnNames)
223
            || $this->lastRowCount !== $rowCount
224
            || null === $this->pdoStatement;
225
        if ($isNotPrepared) {
226
            $sql = $this->getCsq()
227
                ->getUpsert($tableName, $columnNames, $rowCount);
228
            $mess = preg_replace('%(,\([?,]*\))+%', ',...', $sql);
229
            if (PREG_NO_ERROR !== $lastError = preg_last_error()) {
230
                $constants = array_flip(get_defined_constants(true)['pcre']);
231
                $lastError = $constants[$lastError];
232
                $mess = 'Received preg error ' . $lastError;
233
                throw new \DomainException($mess);
234
            }
235
            $this->getYem()
236
                ->triggerLogEvent('Yapeal.Log.log', Logger::INFO, $mess);
237
            $this->pdoStatement = $this->getPdo()
238
                ->prepare($sql);
239
            $this->lastColumnCount = count($columnNames);
240
            $this->lastRowCount = $rowCount;
241
        }
242
        $mess = '';
243
        foreach ($columns as $column) {
244
            $mess .= $column . ',';
245
            if (256 <= strlen($mess)) {
246
                break;
247
            }
248
        }
249
        $mess = substr($mess, 0, 256) . '...';
250
        $this->getYem()
251
            ->triggerLogEvent('Yapeal.Log.log', Logger::DEBUG, $mess);
252
        $this->pdoStatement->execute($columns);
253
        return $this;
254
    }
255
    /**
256
     * Combines the column defaults with a set of rows.
257
     *
258
     * @param \SimpleXMLElement[] $rows
259
     *
260
     * @param array               $columnDefaults
261
     *
262
     * @return array
263
     */
264 1
    protected function processXmlRows(array $rows, array $columnDefaults): array
265
    {
266
        $callback = function (array $carry, \SimpleXMLElement $row) use ($columnDefaults): array {
267 1
            foreach ($columnDefaults as $key => $value) {
268 1
                $attribute = (string)$row[$key];
269 1
                $carry[] = '' !== $attribute ? $attribute : (string)$value;
270
            }
271 1
            return $carry;
272 1
        };
273 1
        return array_reduce($rows, $callback, []);
274
    }
275
    /**
276
     * Used to process the second most common style of API data.
277
     *
278
     * Transforms a list of XML tags and their values into column names and values. $columnDefaults is used to both set
279
     * default values for required columns and to act as a set of known column names.
280
     *
281
     * @param \SimpleXMLElement[] $elements
282
     * @param array               $columnDefaults
283
     * @param string              $tableName
284
     *
285
     * @return static Fluent interface.
286
     * @throws \DomainException
287
     * @throws \InvalidArgumentException
288
     * @throws \UnexpectedValueException
289
     */
290
    protected function valuesPreserveData(array $elements, array $columnDefaults, string $tableName)
291
    {
292
        if (0 === count($elements)) {
293
            return $this;
294
        }
295
        $defaultNames = array_keys($columnDefaults);
296
        $callback = function (array $carry, \SimpleXMLElement $element) use ($defaultNames): array {
297
            if (in_array($name = $element->getName(), $defaultNames, true)) {
298
                $carry[$name] = (string)$element;
299
            }
300
            return $carry;
301
        };
302
        /*
303
         * The array reduce returns only elements with names in $columnDefaults. It also converts them from
304
         * SimpleXMLElements to a plain associative array.
305
         * Array replace is used to overwrite the column default values with any values given in the filtered and
306
         * converted elements. This also assures they are in the correct order.
307
         */
308
        $columns = array_replace($columnDefaults, array_reduce($elements, $callback, []));
309
        return $this->flush(array_values($columns), $defaultNames, $tableName);
310
    }
311
    /**
312
     * @var string[] preserveTos
313
     */
314
    protected $preserveTos = [];
315
    /**
316
     * @return bool
317
     */
318
    private function shouldPreserve(): bool
319
    {
320
        return $this->preserve;
321
    }
322
    /**
323
     * @var int $lastColumnCount
324
     */
325
    private $lastColumnCount;
326
    /**
327
     * @var int lastRowCount
328
     */
329
    private $lastRowCount;
330
    /**
331
     * @var \PDOStatement $pdoStatement
332
     */
333
    private $pdoStatement;
334
    /**
335
     * @var bool $preserve
336
     */
337
    private $preserve = true;
338
}
339