Importer::skipBOM()   A
last analyzed

Complexity

Conditions 2
Paths 2

Size

Total Lines 5
Code Lines 2

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 3
CRAP Score 2

Importance

Changes 0
Metric Value
cc 2
eloc 2
c 0
b 0
f 0
nc 2
nop 1
dl 0
loc 5
ccs 3
cts 3
cp 1
crap 2
rs 10
1
<?php
2
3
declare(strict_types=1);
4
5
namespace Application\Service;
6
7
use Application\Enum\Membership;
8
use Application\Enum\ProductType;
9
use Application\Model\Organization;
10
use Application\Model\User;
11
use Application\Repository\OrganizationRepository;
12
use Doctrine\DBAL\Connection;
13
use Ecodev\Felix\Api\Exception;
14
use Ecodev\Felix\Api\ExceptionWithoutMailLogging;
15
use Laminas\Validator\EmailAddress;
16
use Throwable;
17
18
/**
19
 * Service to import users from CSV with maximal performance.
20
 *
21
 * Users are never deleted, even though technically they should be, to limit the loss of
22
 * data in case of human error in the incoming files. Because it could mean losing all historic
23
 * of purchases.
24
 *
25
 * On the other hand, organizations are **always** deleted, because they don't have any related objects,
26
 * and they are not editable (not even visible) in any way in the app.
27
 */
28
class Importer
29
{
30
    private int $lineNumber = 0;
31
32
    private int $lastReview = 0;
33
34
    private array $reviewByNumber = [];
35
36
    private array $countryByName = [];
37
38
    private Connection $connection;
39
40
    private int $updatedUsers = 0;
41
42
    private int $updatedOrganizations = 0;
43
44
    private int $deletedOrganizations = 0;
45
46
    private array $seenEmails = [];
47
48
    private array $seenPatterns = [];
49
50
    private ?int $currentUser = null;
51
52
    private array $errors = [];
53
54
    private array $usersParams = [];
55
56
    private array $organizationsParams = [];
57
58 18
    public function import(string $filename): array
59
    {
60 18
        $start = microtime(true);
61 18
        $this->connection = _em()->getConnection();
62 18
        $this->fetchReviews();
63 18
        $this->fetchLastReview();
64 18
        $this->fetchCountries();
65 18
        $this->currentUser = User::getCurrent() ? User::getCurrent()->getId() : null;
66 18
        $this->updatedUsers = 0;
67 18
        $this->updatedOrganizations = 0;
68 18
        $this->deletedOrganizations = 0;
69 18
        $this->seenEmails = [];
70 18
        $this->seenPatterns = [];
71
72 18
        if (!file_exists($filename)) {
73 1
            throw new Exception('File not found: ' . $filename);
74
        }
75
76 17
        $file = fopen($filename, 'rb');
77 17
        if ($file === false) {
78
            throw new Exception('Could not read file: ' . $filename);
79
        }
80
81 17
        $this->skipBOM($file);
82
83
        try {
84 17
            $this->connection->beginTransaction();
85 17
            $this->read($file);
86
87 17
            $this->markToDelete();
88 17
            $this->updateAllUsers();
89 17
            $this->updateAllOrganizations();
90 17
            $this->deleteOldOrganizations();
91
92 17
            if ($this->errors) {
93 12
                throw new ExceptionWithoutMailLogging(implode(PHP_EOL, $this->errors));
94
            }
95
96
            // Give user automatic access via organization
97
            /** @var OrganizationRepository $organizationRepository */
98 5
            $organizationRepository = _em()->getRepository(Organization::class);
99 5
            $organizationRepository->applyOrganizationAccesses();
100
101 5
            $this->connection->commit();
102 12
        } catch (Throwable $exception) {
103 12
            $this->connection->rollBack();
104
105 12
            throw $exception;
106
        } finally {
107 17
            fclose($file);
108
        }
109
110 5
        $totalUsers = (int) $this->connection->fetchOne('SELECT COUNT(*) FROM user');
111 5
        $totalOrganizations = (int) $this->connection->fetchOne('SELECT COUNT(*) FROM organization');
112
113 5
        $time = round(microtime(true) - $start, 1);
114
115 5
        return [
116 5
            'updatedUsers' => $this->updatedUsers,
117 5
            'updatedOrganizations' => $this->updatedOrganizations,
118 5
            'deletedOrganizations' => $this->deletedOrganizations,
119 5
            'totalUsers' => $totalUsers,
120 5
            'totalOrganizations' => $totalOrganizations,
121 5
            'totalLines' => $this->lineNumber,
122 5
            'time' => $time,
123 5
        ];
124
    }
125
126 18
    private function fetchReviews(): void
127
    {
128 18
        $records = $this->connection->fetchAllAssociative('SELECT id, review_number FROM product WHERE review_number IS NOT NULL');
129
130 18
        $this->reviewByNumber = [];
131 18
        foreach ($records as $r) {
132 18
            $this->reviewByNumber[$r['review_number']] = $r['id'];
133
        }
134
    }
135
136 18
    private function fetchLastReview(): void
137
    {
138 18
        $records = $this->connection->fetchAllAssociative('SELECT id, review_number FROM product WHERE review_number IS NOT NULL AND is_active = 1 ORDER BY review_number DESC LIMIT 1');
139 18
        $this->lastReview = (int) $records[0]['review_number'];
140
    }
141
142 18
    private function fetchCountries(): void
143
    {
144 18
        $records = $this->connection->fetchAllAssociative('SELECT id, LOWER(name) AS name FROM country');
145
146 18
        $this->countryByName = [];
147 18
        foreach ($records as $r) {
148 18
            $r['name'] = $this->toUpper($r['name']);
149 18
            $this->countryByName[$r['name']] = $r;
150
        }
151
    }
152
153
    /**
154
     * @param resource $file
155
     */
156 17
    private function skipBOM($file): void
157
    {
158
        // Consume BOM, but if not BOM, rewind to beginning
159 17
        if (fgets($file, 4) !== "\xEF\xBB\xBF") {
160 15
            rewind($file);
161
        }
162
    }
163
164
    /**
165
     * @param resource $file
166
     */
167 17
    private function read($file): void
168
    {
169 17
        $this->lineNumber = 0;
170 17
        $expectedColumnCount = 14;
171 17
        while ($line = fgetcsv($file, 0, "\t")) {
172 17
            ++$this->lineNumber;
173
174 17
            $actualColumnCount = count($line);
175 17
            if ($actualColumnCount !== $expectedColumnCount) {
176 1
                $this->throw("Doit avoir exactement $expectedColumnCount colonnes, mais en a " . $actualColumnCount);
177
178 1
                continue;
179
            }
180
181
            // un-escape all fields
182 16
            $line = array_map(fn ($r) => html_entity_decode($r), $line);
183
184 16
            [
185 16
                $email,
186 16
                $pattern,
187 16
                $subscriptionType,
188 16
                $lastReviewNumber,
189 16
                $ignored,
190 16
                $firstName,
191 16
                $lastName,
192 16
                $street,
193 16
                $street2,
194 16
                $postcode,
195 16
                $locality,
196 16
                $country,
197 16
                $phone,
198 16
                $membership,
199 16
            ] = $line;
200
201 16
            $email = trim($email);
202 16
            $pattern = trim($pattern);
203
204 16
            if (!$email && !$pattern) {
205 1
                $this->throw('Il faut soit un email, soit un pattern, mais aucun existe');
206
207 1
                continue;
208
            }
209
210 15
            $lastReviewId = $this->readReviewId($lastReviewNumber);
211
212 15
            if ($email) {
213 12
                $this->assertEmail($email);
214 12
                $membership = $this->readMembership($membership);
215 12
                $country = $this->readCountryId($country);
216 12
                $subscriptionType = $this->readSubscriptionType($subscriptionType);
217
218 12
                $this->updateUser(
219 12
                    $email,
220 12
                    $subscriptionType?->value,
221 12
                    $lastReviewId,
222 12
                    $membership->value,
223 12
                    $firstName,
224 12
                    $lastName,
225 12
                    trim(implode(' ', [$street, $street2])),
226 12
                    $postcode,
227 12
                    $locality,
228 12
                    $country,
229 12
                    $phone
230 12
                );
231
            }
232
233 15
            if ($pattern) {
234 5
                $this->assertPattern($pattern);
235
236 5
                $this->updateOrganization(
237 5
                    $pattern,
238 5
                    $lastReviewId
239 5
                );
240
            }
241
        }
242
    }
243
244 12
    private function assertEmail(string $email): void
245
    {
246 12
        $validator = new EmailAddress();
247 12
        if (!$validator->isValid($email)) {
248 1
            $this->throw('Ce n\'est pas une addresse email valide : "' . $email . '"');
249
250 1
            return;
251
        }
252
253 11
        if (array_key_exists($email, $this->seenEmails)) {
254 1
            $this->throw('L\'email "' . $email . '" est dupliqué et a déjà été vu à la ligne ' . $this->seenEmails[$email]);
255
256 1
            return;
257
        }
258
259 11
        $this->seenEmails[$email] = $this->lineNumber;
260
    }
261
262 5
    private function assertPattern(string $pattern): void
263
    {
264 5
        if (@preg_match('~' . $pattern . '~', '') === false) {
265 1
            $this->throw('Ce n\'est pas une expression régulière valide : "' . $pattern . '"');
266
267 1
            return;
268
        }
269
270 4
        if (preg_match('~^\|~', $pattern) || preg_match('~\|$~', $pattern)) {
271 1
            $this->throw('L\'expression régulière ne peut pas commencer ou terminer par `|`, car c\'est trop dangeureux: "' . $pattern . '"');
272
273 1
            return;
274
        }
275
276 3
        if (array_key_exists($pattern, $this->seenPatterns)) {
277 1
            $this->throw('Le pattern "' . $pattern . '" est dupliqué et a déjà été vu à la ligne ' . $this->seenPatterns[$pattern]);
278
279 1
            return;
280
        }
281
282 3
        $count = $this->connection->executeQuery('SELECT COUNT(*) FROM user WHERE email REGEXP :pattern', ['pattern' => $pattern])->fetchOne();
283 3
        $maximumUserPerPattern = 100;
284 3
        if ($count > $maximumUserPerPattern) {
285
            $this->throw(
286
                <<<STRING
287
                    L'expression régulière "$pattern" affecte $count utilisateurs, ce qui est supérieur à la limite de $maximumUserPerPattern. Si c'est vraiment voulu, il faut contacter Ecodev.
288
                    STRING
289
            );
290
        }
291
292 3
        $this->seenPatterns[$pattern] = $this->lineNumber;
293
    }
294
295 15
    private function readReviewId(string $reviewNumber): ?int
296
    {
297 15
        if (!$reviewNumber) {
298 11
            return null;
299
        }
300
301 6
        if (!preg_match('~^\d+$~', $reviewNumber)) {
302 1
            $this->throw('Un numéro de revue doit être entièrement numérique, mais est : "' . $reviewNumber . '"');
303
304 1
            return null;
305
        }
306
307 5
        $reviewNumberNumeric = (int) $reviewNumber;
308 5
        if (!array_key_exists($reviewNumberNumeric, $this->reviewByNumber)) {
309 2
            $this->throw('Revue introuvable pour le numéro de revue : ' . $reviewNumber);
310
311 2
            return null;
312
        }
313
314 3
        return $reviewNumberNumeric >= $this->lastReview ? $this->reviewByNumber[$reviewNumberNumeric] : null;
315
    }
316
317 12
    private function readCountryId(string $country): ?int
318
    {
319 12
        if (!$country) {
320 9
            return null;
321
        }
322
323
        // Case insensitive match
324 5
        $upper = $this->toUpper($country);
325 5
        if (array_key_exists($upper, $this->countryByName)) {
326 3
            return $this->countryByName[$upper]['id'];
327
        }
328
329
        // Suggest our best guess, so user can fix their data without lookup up countries manually
330 2
        $best = 0;
331 2
        $bestGuess = 0;
332 2
        foreach ($this->countryByName as $r) {
333 2
            similar_text($upper, $r['name'], $percent);
334 2
            if ($percent > $best) {
335 2
                $best = $percent;
336 2
                $bestGuess = $r;
337
            }
338
        }
339
340 2
        $this->throw('Pays "' . $country . '" introuvable. Vouliez-vous dire "' . $bestGuess['name'] . '" ?');
341
342 2
        return null;
343
    }
344
345 12
    private function throw(string $message): void
346
    {
347 12
        $this->errors[] = ('A la ligne ' . $this->lineNumber . ' : ' . $message);
348
    }
349
350 17
    private function deleteOldOrganizations(): void
351
    {
352 17
        $sql = 'DELETE FROM organization WHERE should_delete';
353 17
        $this->deletedOrganizations += $this->connection->executeStatement($sql);
354
    }
355
356 12
    private function readMembership($membership): Membership
357
    {
358 12
        if ($membership === '1') {
359 1
            return Membership::Member;
360
        }
361
362 12
        return Membership::None;
363
    }
364
365 12
    private function readSubscriptionType(string $subscriptionType): ?ProductType
366
    {
367 12
        if (!$subscriptionType) {
368 8
            return null;
369
        }
370
371 6
        if ($subscriptionType === 'Web') {
372 2
            return ProductType::Digital;
373
        }
374
375 5
        if ($subscriptionType === 'Papier') {
376 4
            return ProductType::Paper;
377
        }
378
379 2
        if ($subscriptionType === 'Papier/web') {
380 1
            return ProductType::Both;
381
        }
382
383 1
        $this->throw('Le subscriptionType est invalide : "' . $subscriptionType . '"');
384
385 1
        return null;
386
    }
387
388 12
    private function updateUser(...$args): void
389
    {
390 12
        $params = $args;
391 12
        $params[] = false; // web_temporary_access
392 12
        $params[] = false; // should_delete
393 12
        $params[] = ''; // password
394 12
        $params[] = $this->currentUser;
395
396 12
        array_push($this->usersParams, ...$params);
397 12
        ++$this->updatedUsers;
398
    }
399
400 17
    private function updateAllUsers(): void
401
    {
402 17
        if (!$this->updatedUsers) {
403 5
            return;
404
        }
405
406 12
        $placeholders = $this->placeholders($this->updatedUsers, '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NOW())');
407
408 12
        $sql = 'INSERT INTO user (
409
                            email,
410
                            subscription_type,
411
                            subscription_last_review_id,
412
                            membership,
413
                            first_name,
414
                            last_name,
415
                            street,
416
                            postcode,
417
                            locality,
418
                            country_id,
419
                            phone,
420
                            web_temporary_access,
421
                            should_delete,
422
                            password,
423
                            creator_id,
424
                            creation_date
425
                        )
426 12
                        VALUES ' . $placeholders . '
427
                        ON DUPLICATE KEY UPDATE
428
                            email = VALUES(email),
429
                            subscription_type = VALUES(subscription_type),
430
                            subscription_last_review_id = VALUES(subscription_last_review_id),
431
                            membership = VALUES(membership),
432
                            first_name = VALUES(first_name),
433
                            last_name = VALUES(last_name),
434
                            street = VALUES(street),
435
                            postcode = VALUES(postcode),
436
                            locality = VALUES(locality),
437
                            country_id = VALUES(country_id),
438
                            phone = VALUES(phone),
439
                            web_temporary_access = VALUES(web_temporary_access),
440
                            should_delete = VALUES(should_delete),
441
                            updater_id = VALUES(creator_id),
442 12
                            update_date = NOW()';
443
444 12
        $this->connection->executeStatement($sql, $this->usersParams);
445
    }
446
447 5
    private function updateOrganization(...$args): void
448
    {
449 5
        $params = $args;
450 5
        $params[] = false; // should_delete
451 5
        $params[] = $this->currentUser;
452
453 5
        array_push($this->organizationsParams, ...$params);
454
455 5
        ++$this->updatedOrganizations;
456
    }
457
458 17
    private function updateAllOrganizations(): void
459
    {
460 17
        if (!$this->updatedOrganizations) {
461 12
            return;
462
        }
463
464 5
        $placeholders = $this->placeholders($this->updatedOrganizations, '(?, ?, ?, ?, NOW())');
465
466 5
        $sql = 'INSERT INTO organization (pattern, subscription_last_review_id, should_delete, creator_id, creation_date)
467 5
                        VALUES ' . $placeholders . '
468
                        ON DUPLICATE KEY UPDATE
469
                        pattern = VALUES(pattern),
470
                        subscription_last_review_id = VALUES(subscription_last_review_id),
471
                        updater_id = VALUES(creator_id),
472
                        should_delete = VALUES(should_delete),
473 5
                        update_date = NOW()';
474
475 5
        $this->connection->executeStatement($sql, $this->organizationsParams);
476
    }
477
478 17
    private function markToDelete(): void
479
    {
480 17
        $this->connection->executeStatement('UPDATE user SET should_delete = 1');
481 17
        $this->connection->executeStatement('UPDATE organization SET should_delete = 1');
482
    }
483
484
    /**
485
     * To upper without any accent.
486
     */
487 18
    private function toUpper(string $name): string
488
    {
489 18
        $withoutAccent = iconv('UTF-8', 'ASCII//TRANSLIT', mb_strtolower($name));
490
491 18
        return trim(mb_strtoupper($withoutAccent));
492
    }
493
494 15
    private function placeholders(int $count, string $placeholder): string
495
    {
496 15
        return implode(',' . PHP_EOL, array_fill(0, $count, $placeholder));
497
    }
498
}
499