DoctrineInsertUpdateLoader::clearLogs() - Code Metrics - Inspection of "Optimization DoctrineInsertUpdateLoader" - smartbooster/etl-bundle - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#35)

by Louis

created 2021-09-13 13:28 UTC

DoctrineInsertUpdateLoader::clearLogs() A

↳ Parent: DoctrineInsertUpdateLoader

Complexity

Conditions	1
Paths	1

Size

Total Lines	3
Code Lines	1

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	0
CRAP Score	2

Importance

Changes

Metric	Value
cc	1
eloc	1
nc	1
nop	0
dl	0
loc	3
ccs	0
cts	0
cp	0
crap	2
rs	10
c	0
b	0
f	0

<?php

namespace Smart\EtlBundle\Loader;

use Doctrine\ORM\EntityManager;
use Smart\EtlBundle\Entity\ImportableInterface;
use Smart\EtlBundle\Exception\Loader\EntityTypeNotHandledException;
use Smart\EtlBundle\Exception\Loader\EntityAlreadyRegisteredException;
use Smart\EtlBundle\Exception\Loader\LoaderException;
use Smart\EtlBundle\Exception\Loader\LoadUnvalidObjectsException;
use Smart\EtlBundle\Utils\ArrayUtils;
use Symfony\Component\PropertyAccess\PropertyAccess;
use Symfony\Component\PropertyAccess\PropertyAccessor;
use Symfony\Component\Validator\Validator\ValidatorInterface;

/**
 * Nicolas Bastien <[email protected]>
 */
class DoctrineInsertUpdateLoader implements LoaderInterface
{
    const VALIDATION_GROUPS = 'smart_etl_loader';
    const BATCH_SIZE = 30;

    /**
     * @var EntityManager
     */
    protected $entityManager;

    /**
     * @var array
     */
    protected $references;

    /**
     * @var PropertyAccessor
     */
    protected $accessor;

    /**
     * List of entities to extract
     * [
     *      'class' => []
     * ]
     * @var array
     */
    protected $entitiesToProcess = [];

    /**
     * @var array
     */
    protected $loadLogs = [];

    /**
     * @var array keep validation errors for each process object with his own associative data index
     */
    protected $arrayValidationErrors = [];

    /**
     * @var mixed
     */
    protected $processKey = null;

    protected ValidatorInterface $validator;

    /**
     * @param ValidatorInterface $validator
     */
    public function __construct($entityManager, ValidatorInterface $validator)
    {
        $this->entityManager = $entityManager;
        $this->validator = $validator;
        $this->accessor = PropertyAccess::createPropertyAccessor();
    }

    /**
     * @param string $entityClass
     * @param string $identifierProperty : if null this entity will be always insert
     * @param array $entityProperties properties to synchronize
     * @return $this
     */
    public function addEntityToProcess($entityClass, $identifierProperty, array $entityProperties = [])
    {
        if (isset($this->entitiesToProcess[$entityClass])) {
            throw new EntityAlreadyRegisteredException($entityClass);
        }

        $this->entitiesToProcess[$entityClass] = [
            'class' => $entityClass,
            'identifier' => $identifierProperty,
            'properties' => $entityProperties
        ];

        return $this;
    }

    /**
     * @throws \Exception
     */
    public function load(array $data)
    {
        $this->entityManager->beginTransaction();
        try {
            $index = 1;
            $dbObjects = $this->getDbObjects($data);

            foreach ($data as $key => $object) {
                $this->processKey = $key;
                $this->processObject($object, $dbObjects);

                if (($index % self::BATCH_SIZE) === 0) {
                    $this->entityManager->flush();
                }

                $index++;
            }

            if (count($this->arrayValidationErrors) > 0) {
                throw new LoadUnvalidObjectsException($this->arrayValidationErrors);
            }

            $this->entityManager->flush();
            $this->entityManager->commit();
        } catch (\Exception $e) {
            $this->entityManager->rollback();
            if ($e instanceof LoaderException) {
                throw $e;
            }

            throw new \Exception('EXCEPTION LOADER : ' . $e->getMessage());
        }
    }

    /**
     * @param  ImportableInterface $object
     * @return ImportableInterface
     * @throws \Exception
     * @throws \TypeError
     */
    protected function processObject($object, array $dbObjects)
    {
        $objectClass = get_class($object);
        if (!isset($this->entitiesToProcess[$objectClass])) {
            throw new EntityTypeNotHandledException($objectClass);
        }

        $validationErrors = $this->validator->validate($object, null, self::VALIDATION_GROUPS);
        if ($validationErrors->count() > 0) {
            $this->arrayValidationErrors[$this->processKey] = $validationErrors;

            return null;
        }

        $identifier = $this->accessor->getValue($object, $this->entitiesToProcess[$objectClass]['identifier']);
        //Replace relations by their reference
        foreach ($this->entitiesToProcess[$objectClass]['properties'] as $property) {
            $propertyValue = $this->accessor->getValue($object, $property);
            if ($this->isEntityRelation($propertyValue)) {
                $relation = $propertyValue; //better understanding

                $relationIdentifier = $this->accessor->getValue($relation, $this->entitiesToProcess[get_class($relation)]['identifier']);
                if (!isset($this->references[$relationIdentifier])) {
                    //new relation should be processed before
                    $this->processObject($relation, $dbObjects);
                }
                $this->accessor->setValue(
                    $object,
                    $property,
                    $this->references[$relationIdentifier]
                );
            } elseif ($propertyValue instanceof \Traversable) {
                foreach ($propertyValue as $k => $v) {
                    if ($this->isEntityRelation($v)) {
                        $relationIdentifier = $this->accessor->getValue($v, $this->entitiesToProcess[get_class($v)]['identifier']);
                        if (!isset($this->references[$relationIdentifier])) {
                            //new relation should be processed before
                            $this->processObject($v, $dbObjects);
                        }
                        $propertyValue[$k] = $this->references[$relationIdentifier];
                    }
                }
                $this->accessor->setValue(
                    $object,
                    $property,
                    $propertyValue
                );
            }
        }

        $dbObject = null;

        if (isset($dbObjects[$objectClass]) && isset($dbObjects[$objectClass][$identifier])) {
            $dbObject = $dbObjects[$objectClass][$identifier];
        }
        if ($dbObject === null) {
            if (!$object->isImported()) {
                $object->setImportedAt(new \DateTime());
            }
            $this->entityManager->persist($object);
            if (!is_null($identifier)) {
                $this->references[$identifier] = $object;
            }

            if (isset($this->loadLogs[$objectClass])) {
                $this->loadLogs[$objectClass]['nb_created']++;
            } else {
                $this->loadLogs[$objectClass] = [
                    'nb_created' => 1,
                    'nb_updated' => 0,
                ];
            }
        } else {
            // todo validate if there is no change (if so do not increase the nb_updated)
            foreach ($this->entitiesToProcess[$objectClass]['properties'] as $property) {
                $this->accessor->setValue($dbObject, $property, $this->accessor->getValue($object, $property));
            }
            if (!$dbObject->isImported()) {
                $dbObject->setImportedAt(new \DateTime());
            }
            $this->references[$identifier] = $dbObject;

            if (isset($this->loadLogs[$objectClass])) {
                $this->loadLogs[$objectClass]['nb_updated']++;
            } else {
                $this->loadLogs[$objectClass] = [
                    'nb_created' => 0,
                    'nb_updated' => 1,
                ];
            }
        }

        return $object;
    }

    /**
     * Check if $propertyValue is an entity relation to process
     *
     * @param  mixed $propertyValue
     * @return bool
     */
    protected function isEntityRelation($propertyValue)
    {
        return (is_object($propertyValue) && !($propertyValue instanceof \DateTime) && !($propertyValue instanceof \Traversable));
    }

    public function getLogs(): array
    {
        return $this->loadLogs;
    }

    public function clearLogs(): void
    {
        $this->loadLogs = [];
    }

    private function getDbObjects(array $datas): array
    {
        $dbObjectsParam = [];
        $toReturn = [];

        // construct of array with all db object identifier needed
        foreach ($datas as $object) {
            $objectClass = get_class($object);

            $dbObjectsParam = ArrayUtils::addMultidimensionalArrayValue($dbObjectsParam, $objectClass, $this->accessor->getValue($object, $this->entitiesToProcess[$objectClass]['identifier']));
            $dbObjectsParam = $this->addDbObjectRelationParam($object, $dbObjectsParam);
        }

        // get all needed db object
        foreach ($dbObjectsParam as $class => $identifiers) {
            $dbObjects = $this->entityManager->getRepository($class)->findBy([$this->entitiesToProcess[$class]['identifier'] => $identifiers]);
            foreach ($dbObjects as $dbObject) {
                $toReturn[$class][$this->accessor->getValue($dbObject, $this->entitiesToProcess[$class]['identifier'])] = $dbObject;
            }
        }

        return $toReturn;
    }

    /** Look relation of object and add param in $dbObjectsParam */
    private function addDbObjectRelationParam($object, $dbObjectsParam): array
    {
        $objectClass = get_class($object);
        foreach ($this->entitiesToProcess[$objectClass]['properties'] as $property) {
            $propertyValue = $this->accessor->getValue($object, $property);
            if ($this->isEntityRelation($propertyValue)) {
                $relation = $propertyValue; //better understanding

                $dbObjectsParam = $this->addRelationParam($relation, $dbObjectsParam);
            } elseif ($propertyValue instanceof \Traversable) {
                foreach ($propertyValue as $v) {
                    if ($this->isEntityRelation($v)) {
                        $dbObjectsParam = $this->addRelationParam($v, $dbObjectsParam);
                    }
                }
            }
        }

        return $dbObjectsParam;
    }

    private function addRelationParam($object, array $dbObjectsParam): array
    {
        if (!isset($this->entitiesToProcess[get_class($object)])) {
            throw new EntityTypeNotHandledException(get_class($object));
        }
        $relationIdentifier = $this->accessor->getValue($object, $this->entitiesToProcess[get_class($object)]['identifier']);
        if (!isset($this->references[$relationIdentifier])) {
            $dbObjectsParam = ArrayUtils::addMultidimensionalArrayValue($dbObjectsParam, get_class($object), $relationIdentifier);
            $dbObjectsParam = $this->addDbObjectRelationParam($object, $dbObjectsParam);
        }

        return $dbObjectsParam;
    }
}


1		<?php
2
3		namespace Smart\EtlBundle\Loader;
4
5		use Doctrine\ORM\EntityManager;
6		use Smart\EtlBundle\Entity\ImportableInterface;
7		use Smart\EtlBundle\Exception\Loader\EntityTypeNotHandledException;
8		use Smart\EtlBundle\Exception\Loader\EntityAlreadyRegisteredException;
9		use Smart\EtlBundle\Exception\Loader\LoaderException;
10		use Smart\EtlBundle\Exception\Loader\LoadUnvalidObjectsException;
11		use Smart\EtlBundle\Utils\ArrayUtils;
12		use Symfony\Component\PropertyAccess\PropertyAccess;
13		use Symfony\Component\PropertyAccess\PropertyAccessor;
14		use Symfony\Component\Validator\Validator\ValidatorInterface;
15
16		/**
17		* Nicolas Bastien <[email protected]>
18		*/
19		class DoctrineInsertUpdateLoader implements LoaderInterface
20		{
21		const VALIDATION_GROUPS = 'smart_etl_loader';
22		const BATCH_SIZE = 30;
23
24		/**
25		* @var EntityManager
26		*/
27		protected $entityManager;
28
29		/**
30		* @var array
31		*/
32		protected $references;
33
34		/**
35		* @var PropertyAccessor
36		*/
37		protected $accessor;
38
39		/**
40		* List of entities to extract
41	1	* [
42		* 'class' => []
43	1	* ]
44	1	* @var array
45	1	*/
46		protected $entitiesToProcess = [];
47
48		/**
49		* @var array
50		*/
51		protected $loadLogs = [];
52
53		/**
54	1	* @var array keep validation errors for each process object with his own associative data index
55		*/
56	1	protected $arrayValidationErrors = [];
57
58		/**
59		* @var mixed
60	1	*/
61	1	protected $processKey = null;
62	1
63	1	protected ValidatorInterface $validator;
64	1
65		/**
66		* @param ValidatorInterface $validator
67	1	*/
68		public function __construct($entityManager, ValidatorInterface $validator)
69		{
70		$this->entityManager = $entityManager;
71		$this->validator = $validator;
72		$this->accessor = PropertyAccess::createPropertyAccessor();
73	1	}
74
75	1	/**
76		* @param string $entityClass
77	1	* @param string $identifierProperty : if null this entity will be always insert
78	1	* @param array $entityProperties properties to synchronize
79		* @return $this
80	1	*/
81	1	public function addEntityToProcess($entityClass, $identifierProperty, array $entityProperties = [])
82		{
83		if (isset($this->entitiesToProcess[$entityClass])) {
84		throw new EntityAlreadyRegisteredException($entityClass);
85		}
86	1
87		$this->entitiesToProcess[$entityClass] = [
88		'class' => $entityClass,
89		'identifier' => $identifierProperty,
90		'properties' => $entityProperties
91		];
92
93		return $this;
94	1	}
95
96	1	/**
97		* @throws \Exception
98		*/
99	1	public function load(array $data)
100		{
101		$this->entityManager->beginTransaction();
102	1	try {
103	1	$index = 1;
104	1	$dbObjects = $this->getDbObjects($data);
105	1
106		foreach ($data as $key => $object) {
107	1	$this->processKey = $key;
108		$this->processObject($object, $dbObjects);
109
110	1	if (($index % self::BATCH_SIZE) === 0) {
111	1	$this->entityManager->flush();
112		}
113	1
114		$index++;
115	1	}
116	1
117		if (count($this->arrayValidationErrors) > 0) {
118	1	throw new LoadUnvalidObjectsException($this->arrayValidationErrors);
119		}
120	1
121	1	$this->entityManager->flush();
122	1	$this->entityManager->commit();
123	1	} catch (\Exception $e) {
124		$this->entityManager->rollback();
125		if ($e instanceof LoaderException) {
126	1	throw $e;
127	1	}
128
129	1	throw new \Exception('EXCEPTION LOADER : ' . $e->getMessage());
130		}
131	1	}
132
133		/**
134	1	* @param ImportableInterface $object
135	1	* @return ImportableInterface
136		* @throws \Exception
137		* @throws \TypeError
138		*/
139		protected function processObject($object, array $dbObjects)
140		{
141		$objectClass = get_class($object);
142	1	if (!isset($this->entitiesToProcess[$objectClass])) {
143	1	throw new EntityTypeNotHandledException($objectClass);
144	1	}
145
146	1	$validationErrors = $this->validator->validate($object, null, self::VALIDATION_GROUPS);
147	1	if ($validationErrors->count() > 0) {
148	1	$this->arrayValidationErrors[$this->processKey] = $validationErrors;
149
150	1	return null;
151	1	}
152	1
153		$identifier = $this->accessor->getValue($object, $this->entitiesToProcess[$objectClass]['identifier']);
154		//Replace relations by their reference
155	1	foreach ($this->entitiesToProcess[$objectClass]['properties'] as $property) {
156	1	$propertyValue = $this->accessor->getValue($object, $property);
157		if ($this->isEntityRelation($propertyValue)) {
158	1	$relation = $propertyValue; //better understanding
159	1
160		$relationIdentifier = $this->accessor->getValue($relation, $this->entitiesToProcess[get_class($relation)]['identifier']);
161	1	if (!isset($this->references[$relationIdentifier])) {
162		//new relation should be processed before
163		$this->processObject($relation, $dbObjects);
164	1	}
165		$this->accessor->setValue(
166		$object,
167		$property,
168		$this->references[$relationIdentifier]
169		);
170		} elseif ($propertyValue instanceof \Traversable) {
171		foreach ($propertyValue as $k => $v) {
172		if ($this->isEntityRelation($v)) {
173	1	$relationIdentifier = $this->accessor->getValue($v, $this->entitiesToProcess[get_class($v)]['identifier']);
174		if (!isset($this->references[$relationIdentifier])) {
175	1	//new relation should be processed before
176		$this->processObject($v, $dbObjects);
177		}
178		$propertyValue[$k] = $this->references[$relationIdentifier];
179		}
180		}
181		$this->accessor->setValue(
182		$object,
183		$property,
184		$propertyValue
185		);
186		}
187		}
188
189		$dbObject = null;
190
191		if (isset($dbObjects[$objectClass]) && isset($dbObjects[$objectClass][$identifier])) {
192		$dbObject = $dbObjects[$objectClass][$identifier];
193		}
194		if ($dbObject === null) {
195		if (!$object->isImported()) {
196		$object->setImportedAt(new \DateTime());
197		}
198		$this->entityManager->persist($object);
199		if (!is_null($identifier)) {
200		$this->references[$identifier] = $object;
201		}
202
203		if (isset($this->loadLogs[$objectClass])) {
204		$this->loadLogs[$objectClass]['nb_created']++;
205		} else {
206		$this->loadLogs[$objectClass] = [
207		'nb_created' => 1,
208		'nb_updated' => 0,
209		];
210		}
211		} else {
212		// todo validate if there is no change (if so do not increase the nb_updated)
213		foreach ($this->entitiesToProcess[$objectClass]['properties'] as $property) {
214		$this->accessor->setValue($dbObject, $property, $this->accessor->getValue($object, $property));
215		}
216		if (!$dbObject->isImported()) {
217		$dbObject->setImportedAt(new \DateTime());
218		}
219		$this->references[$identifier] = $dbObject;
220
221		if (isset($this->loadLogs[$objectClass])) {
222		$this->loadLogs[$objectClass]['nb_updated']++;
223		} else {
224		$this->loadLogs[$objectClass] = [
225		'nb_created' => 0,
226		'nb_updated' => 1,
227		];
228		}
229		}
230
231		return $object;
232		}
233
234		/**
235		* Check if $propertyValue is an entity relation to process
236		*
237		* @param mixed $propertyValue
238		* @return bool
239		*/
240		protected function isEntityRelation($propertyValue)
241		{
242		return (is_object($propertyValue) && !($propertyValue instanceof \DateTime) && !($propertyValue instanceof \Traversable));
243		}
244
245		public function getLogs(): array
246		{
247		return $this->loadLogs;
248		}
249
250		public function clearLogs(): void
251		{
252		$this->loadLogs = [];
253		}
254
255		private function getDbObjects(array $datas): array
256		{
257		$dbObjectsParam = [];
258		$toReturn = [];
259
260		// construct of array with all db object identifier needed
261		foreach ($datas as $object) {
262		$objectClass = get_class($object);
263
264		$dbObjectsParam = ArrayUtils::addMultidimensionalArrayValue($dbObjectsParam, $objectClass, $this->accessor->getValue($object, $this->entitiesToProcess[$objectClass]['identifier']));
265		$dbObjectsParam = $this->addDbObjectRelationParam($object, $dbObjectsParam);
266		}
267
268		// get all needed db object
269		foreach ($dbObjectsParam as $class => $identifiers) {
270		$dbObjects = $this->entityManager->getRepository($class)->findBy([$this->entitiesToProcess[$class]['identifier'] => $identifiers]);
271		foreach ($dbObjects as $dbObject) {
272		$toReturn[$class][$this->accessor->getValue($dbObject, $this->entitiesToProcess[$class]['identifier'])] = $dbObject;
273		}
274		}
275
276		return $toReturn;
277		}
278
279		/** Look relation of object and add param in $dbObjectsParam */
280		private function addDbObjectRelationParam($object, $dbObjectsParam): array
281		{
282		$objectClass = get_class($object);
283		foreach ($this->entitiesToProcess[$objectClass]['properties'] as $property) {
284		$propertyValue = $this->accessor->getValue($object, $property);
285		if ($this->isEntityRelation($propertyValue)) {
286		$relation = $propertyValue; //better understanding
287
288		$dbObjectsParam = $this->addRelationParam($relation, $dbObjectsParam);
289		} elseif ($propertyValue instanceof \Traversable) {
290		foreach ($propertyValue as $v) {
291		if ($this->isEntityRelation($v)) {
292		$dbObjectsParam = $this->addRelationParam($v, $dbObjectsParam);
293		}
294		}
295		}
296		}
297
298		return $dbObjectsParam;
299		}
300
301		private function addRelationParam($object, array $dbObjectsParam): array
302		{
303		if (!isset($this->entitiesToProcess[get_class($object)])) {
304		throw new EntityTypeNotHandledException(get_class($object));
305		}
306		$relationIdentifier = $this->accessor->getValue($object, $this->entitiesToProcess[get_class($object)]['identifier']);
307		if (!isset($this->references[$relationIdentifier])) {
308		$dbObjectsParam = ArrayUtils::addMultidimensionalArrayValue($dbObjectsParam, get_class($object), $relationIdentifier);
309		$dbObjectsParam = $this->addDbObjectRelationParam($object, $dbObjectsParam);
310		}
311
312		return $dbObjectsParam;
313		}
314		}
315

smartbooster / etl-bundle

Pull Request — master (#35)

DoctrineInsertUpdateLoader::clearLogs() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like