1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
declare(strict_types=1); |
4
|
|
|
|
5
|
|
|
namespace Application\Service; |
6
|
|
|
|
7
|
|
|
use Application\Enum\Membership; |
8
|
|
|
use Application\Enum\ProductType; |
9
|
|
|
use Application\Model\Organization; |
10
|
|
|
use Application\Model\User; |
11
|
|
|
use Application\Repository\OrganizationRepository; |
12
|
|
|
use Doctrine\DBAL\Connection; |
13
|
|
|
use Ecodev\Felix\Api\Exception; |
14
|
|
|
use Ecodev\Felix\Api\ExceptionWithoutMailLogging; |
15
|
|
|
use Laminas\Validator\EmailAddress; |
16
|
|
|
use Throwable; |
17
|
|
|
|
18
|
|
|
/** |
19
|
|
|
* Service to import users from CSV with maximal performance. |
20
|
|
|
* |
21
|
|
|
* Users are never deleted, even though technically they should be, to limit the loss of |
22
|
|
|
* data in case of human error in the incoming files. Because it could mean losing all historic |
23
|
|
|
* of purchases. |
24
|
|
|
* |
25
|
|
|
* On the other hand, organizations are **always** deleted, because they don't have any related objects, |
26
|
|
|
* and they are not editable (not even visible) in any way in the app. |
27
|
|
|
*/ |
28
|
|
|
class Importer |
29
|
|
|
{ |
30
|
|
|
private int $lineNumber = 0; |
31
|
|
|
|
32
|
|
|
private int $lastReview = 0; |
33
|
|
|
|
34
|
|
|
private array $reviewByNumber = []; |
35
|
|
|
|
36
|
|
|
private array $countryByName = []; |
37
|
|
|
|
38
|
|
|
private Connection $connection; |
39
|
|
|
|
40
|
|
|
private int $updatedUsers = 0; |
41
|
|
|
|
42
|
|
|
private int $updatedOrganizations = 0; |
43
|
|
|
|
44
|
|
|
private int $deletedOrganizations = 0; |
45
|
|
|
|
46
|
|
|
private array $seenEmails = []; |
47
|
|
|
|
48
|
|
|
private array $seenPatterns = []; |
49
|
|
|
|
50
|
|
|
private ?int $currentUser = null; |
51
|
|
|
|
52
|
|
|
private array $errors = []; |
53
|
|
|
|
54
|
|
|
private array $usersParams = []; |
55
|
|
|
|
56
|
|
|
private array $organizationsParams = []; |
57
|
|
|
|
58
|
18 |
|
public function import(string $filename): array |
59
|
|
|
{ |
60
|
18 |
|
$start = microtime(true); |
61
|
18 |
|
$this->connection = _em()->getConnection(); |
62
|
18 |
|
$this->fetchReviews(); |
63
|
18 |
|
$this->fetchLastReview(); |
64
|
18 |
|
$this->fetchCountries(); |
65
|
18 |
|
$this->currentUser = User::getCurrent() ? User::getCurrent()->getId() : null; |
66
|
18 |
|
$this->updatedUsers = 0; |
67
|
18 |
|
$this->updatedOrganizations = 0; |
68
|
18 |
|
$this->deletedOrganizations = 0; |
69
|
18 |
|
$this->seenEmails = []; |
70
|
18 |
|
$this->seenPatterns = []; |
71
|
|
|
|
72
|
18 |
|
if (!file_exists($filename)) { |
73
|
1 |
|
throw new Exception('File not found: ' . $filename); |
74
|
|
|
} |
75
|
|
|
|
76
|
17 |
|
$file = fopen($filename, 'rb'); |
77
|
17 |
|
if ($file === false) { |
78
|
|
|
throw new Exception('Could not read file: ' . $filename); |
79
|
|
|
} |
80
|
|
|
|
81
|
17 |
|
$this->skipBOM($file); |
82
|
|
|
|
83
|
|
|
try { |
84
|
17 |
|
$this->connection->beginTransaction(); |
85
|
17 |
|
$this->read($file); |
86
|
|
|
|
87
|
17 |
|
$this->markToDelete(); |
88
|
17 |
|
$this->updateAllUsers(); |
89
|
17 |
|
$this->updateAllOrganizations(); |
90
|
17 |
|
$this->deleteOldOrganizations(); |
91
|
|
|
|
92
|
17 |
|
if ($this->errors) { |
93
|
12 |
|
throw new ExceptionWithoutMailLogging(implode(PHP_EOL, $this->errors)); |
94
|
|
|
} |
95
|
|
|
|
96
|
|
|
// Give user automatic access via organization |
97
|
|
|
/** @var OrganizationRepository $organizationRepository */ |
98
|
5 |
|
$organizationRepository = _em()->getRepository(Organization::class); |
99
|
5 |
|
$organizationRepository->applyOrganizationAccesses(); |
100
|
|
|
|
101
|
5 |
|
$this->connection->commit(); |
102
|
12 |
|
} catch (Throwable $exception) { |
103
|
12 |
|
$this->connection->rollBack(); |
104
|
|
|
|
105
|
12 |
|
throw $exception; |
106
|
|
|
} finally { |
107
|
17 |
|
fclose($file); |
108
|
|
|
} |
109
|
|
|
|
110
|
5 |
|
$totalUsers = (int) $this->connection->fetchOne('SELECT COUNT(*) FROM user'); |
111
|
5 |
|
$totalOrganizations = (int) $this->connection->fetchOne('SELECT COUNT(*) FROM organization'); |
112
|
|
|
|
113
|
5 |
|
$time = round(microtime(true) - $start, 1); |
114
|
|
|
|
115
|
5 |
|
return [ |
116
|
5 |
|
'updatedUsers' => $this->updatedUsers, |
117
|
5 |
|
'updatedOrganizations' => $this->updatedOrganizations, |
118
|
5 |
|
'deletedOrganizations' => $this->deletedOrganizations, |
119
|
5 |
|
'totalUsers' => $totalUsers, |
120
|
5 |
|
'totalOrganizations' => $totalOrganizations, |
121
|
5 |
|
'totalLines' => $this->lineNumber, |
122
|
5 |
|
'time' => $time, |
123
|
5 |
|
]; |
124
|
|
|
} |
125
|
|
|
|
126
|
18 |
|
private function fetchReviews(): void |
127
|
|
|
{ |
128
|
18 |
|
$records = $this->connection->fetchAllAssociative('SELECT id, review_number FROM product WHERE review_number IS NOT NULL'); |
129
|
|
|
|
130
|
18 |
|
$this->reviewByNumber = []; |
131
|
18 |
|
foreach ($records as $r) { |
132
|
18 |
|
$this->reviewByNumber[$r['review_number']] = $r['id']; |
133
|
|
|
} |
134
|
|
|
} |
135
|
|
|
|
136
|
18 |
|
private function fetchLastReview(): void |
137
|
|
|
{ |
138
|
18 |
|
$records = $this->connection->fetchAllAssociative('SELECT id, review_number FROM product WHERE review_number IS NOT NULL AND is_active = 1 ORDER BY review_number DESC LIMIT 1'); |
139
|
18 |
|
$this->lastReview = (int) $records[0]['review_number']; |
140
|
|
|
} |
141
|
|
|
|
142
|
18 |
|
private function fetchCountries(): void |
143
|
|
|
{ |
144
|
18 |
|
$records = $this->connection->fetchAllAssociative('SELECT id, LOWER(name) AS name FROM country'); |
145
|
|
|
|
146
|
18 |
|
$this->countryByName = []; |
147
|
18 |
|
foreach ($records as $r) { |
148
|
18 |
|
$r['name'] = $this->toUpper($r['name']); |
149
|
18 |
|
$this->countryByName[$r['name']] = $r; |
150
|
|
|
} |
151
|
|
|
} |
152
|
|
|
|
153
|
|
|
/** |
154
|
|
|
* @param resource $file |
155
|
|
|
*/ |
156
|
17 |
|
private function skipBOM($file): void |
157
|
|
|
{ |
158
|
|
|
// Consume BOM, but if not BOM, rewind to beginning |
159
|
17 |
|
if (fgets($file, 4) !== "\xEF\xBB\xBF") { |
160
|
15 |
|
rewind($file); |
161
|
|
|
} |
162
|
|
|
} |
163
|
|
|
|
164
|
|
|
/** |
165
|
|
|
* @param resource $file |
166
|
|
|
*/ |
167
|
17 |
|
private function read($file): void |
168
|
|
|
{ |
169
|
17 |
|
$this->lineNumber = 0; |
170
|
17 |
|
$expectedColumnCount = 14; |
171
|
17 |
|
while ($line = fgetcsv($file, 0, "\t")) { |
172
|
17 |
|
++$this->lineNumber; |
173
|
|
|
|
174
|
17 |
|
$actualColumnCount = count($line); |
175
|
17 |
|
if ($actualColumnCount !== $expectedColumnCount) { |
176
|
1 |
|
$this->throw("Doit avoir exactement $expectedColumnCount colonnes, mais en a " . $actualColumnCount); |
177
|
|
|
|
178
|
1 |
|
continue; |
179
|
|
|
} |
180
|
|
|
|
181
|
|
|
// un-escape all fields |
182
|
16 |
|
$line = array_map(fn ($r) => html_entity_decode($r), $line); |
183
|
|
|
|
184
|
16 |
|
[ |
185
|
16 |
|
$email, |
186
|
16 |
|
$pattern, |
187
|
16 |
|
$subscriptionType, |
188
|
16 |
|
$lastReviewNumber, |
189
|
16 |
|
$ignored, |
190
|
16 |
|
$firstName, |
191
|
16 |
|
$lastName, |
192
|
16 |
|
$street, |
193
|
16 |
|
$street2, |
194
|
16 |
|
$postcode, |
195
|
16 |
|
$locality, |
196
|
16 |
|
$country, |
197
|
16 |
|
$phone, |
198
|
16 |
|
$membership, |
199
|
16 |
|
] = $line; |
200
|
|
|
|
201
|
16 |
|
$email = trim($email); |
202
|
16 |
|
$pattern = trim($pattern); |
203
|
|
|
|
204
|
16 |
|
if (!$email && !$pattern) { |
205
|
1 |
|
$this->throw('Il faut soit un email, soit un pattern, mais aucun existe'); |
206
|
|
|
|
207
|
1 |
|
continue; |
208
|
|
|
} |
209
|
|
|
|
210
|
15 |
|
$lastReviewId = $this->readReviewId($lastReviewNumber); |
211
|
|
|
|
212
|
15 |
|
if ($email) { |
213
|
12 |
|
$this->assertEmail($email); |
214
|
12 |
|
$membership = $this->readMembership($membership); |
215
|
12 |
|
$country = $this->readCountryId($country); |
216
|
12 |
|
$subscriptionType = $this->readSubscriptionType($subscriptionType); |
217
|
|
|
|
218
|
12 |
|
$this->updateUser( |
219
|
12 |
|
$email, |
220
|
12 |
|
$subscriptionType?->value, |
221
|
12 |
|
$lastReviewId, |
222
|
12 |
|
$membership->value, |
223
|
12 |
|
$firstName, |
224
|
12 |
|
$lastName, |
225
|
12 |
|
trim(implode(' ', [$street, $street2])), |
226
|
12 |
|
$postcode, |
227
|
12 |
|
$locality, |
228
|
12 |
|
$country, |
229
|
12 |
|
$phone |
230
|
12 |
|
); |
231
|
|
|
} |
232
|
|
|
|
233
|
15 |
|
if ($pattern) { |
234
|
5 |
|
$this->assertPattern($pattern); |
235
|
|
|
|
236
|
5 |
|
$this->updateOrganization( |
237
|
5 |
|
$pattern, |
238
|
5 |
|
$lastReviewId |
239
|
5 |
|
); |
240
|
|
|
} |
241
|
|
|
} |
242
|
|
|
} |
243
|
|
|
|
244
|
12 |
|
private function assertEmail(string $email): void |
245
|
|
|
{ |
246
|
12 |
|
$validator = new EmailAddress(); |
247
|
12 |
|
if (!$validator->isValid($email)) { |
248
|
1 |
|
$this->throw('Ce n\'est pas une addresse email valide : "' . $email . '"'); |
249
|
|
|
|
250
|
1 |
|
return; |
251
|
|
|
} |
252
|
|
|
|
253
|
11 |
|
if (array_key_exists($email, $this->seenEmails)) { |
254
|
1 |
|
$this->throw('L\'email "' . $email . '" est dupliqué et a déjà été vu à la ligne ' . $this->seenEmails[$email]); |
255
|
|
|
|
256
|
1 |
|
return; |
257
|
|
|
} |
258
|
|
|
|
259
|
11 |
|
$this->seenEmails[$email] = $this->lineNumber; |
260
|
|
|
} |
261
|
|
|
|
262
|
5 |
|
private function assertPattern(string $pattern): void |
263
|
|
|
{ |
264
|
5 |
|
if (@preg_match('~' . $pattern . '~', '') === false) { |
265
|
1 |
|
$this->throw('Ce n\'est pas une expression régulière valide : "' . $pattern . '"'); |
266
|
|
|
|
267
|
1 |
|
return; |
268
|
|
|
} |
269
|
|
|
|
270
|
4 |
|
if (preg_match('~^\|~', $pattern) || preg_match('~\|$~', $pattern)) { |
271
|
1 |
|
$this->throw('L\'expression régulière ne peut pas commencer ou terminer par `|`, car c\'est trop dangeureux: "' . $pattern . '"'); |
272
|
|
|
|
273
|
1 |
|
return; |
274
|
|
|
} |
275
|
|
|
|
276
|
3 |
|
if (array_key_exists($pattern, $this->seenPatterns)) { |
277
|
1 |
|
$this->throw('Le pattern "' . $pattern . '" est dupliqué et a déjà été vu à la ligne ' . $this->seenPatterns[$pattern]); |
278
|
|
|
|
279
|
1 |
|
return; |
280
|
|
|
} |
281
|
|
|
|
282
|
3 |
|
$count = $this->connection->executeQuery('SELECT COUNT(*) FROM user WHERE email REGEXP :pattern', ['pattern' => $pattern])->fetchOne(); |
283
|
3 |
|
$maximumUserPerPattern = 100; |
284
|
3 |
|
if ($count > $maximumUserPerPattern) { |
285
|
|
|
$this->throw( |
286
|
|
|
<<<STRING |
287
|
|
|
L'expression régulière "$pattern" affecte $count utilisateurs, ce qui est supérieur à la limite de $maximumUserPerPattern. Si c'est vraiment voulu, il faut contacter Ecodev. |
288
|
|
|
STRING |
289
|
|
|
); |
290
|
|
|
} |
291
|
|
|
|
292
|
3 |
|
$this->seenPatterns[$pattern] = $this->lineNumber; |
293
|
|
|
} |
294
|
|
|
|
295
|
15 |
|
private function readReviewId(string $reviewNumber): ?int |
296
|
|
|
{ |
297
|
15 |
|
if (!$reviewNumber) { |
298
|
11 |
|
return null; |
299
|
|
|
} |
300
|
|
|
|
301
|
6 |
|
if (!preg_match('~^\d+$~', $reviewNumber)) { |
302
|
1 |
|
$this->throw('Un numéro de revue doit être entièrement numérique, mais est : "' . $reviewNumber . '"'); |
303
|
|
|
|
304
|
1 |
|
return null; |
305
|
|
|
} |
306
|
|
|
|
307
|
5 |
|
$reviewNumberNumeric = (int) $reviewNumber; |
308
|
5 |
|
if (!array_key_exists($reviewNumberNumeric, $this->reviewByNumber)) { |
309
|
2 |
|
$this->throw('Revue introuvable pour le numéro de revue : ' . $reviewNumber); |
310
|
|
|
|
311
|
2 |
|
return null; |
312
|
|
|
} |
313
|
|
|
|
314
|
3 |
|
return $reviewNumberNumeric >= $this->lastReview ? $this->reviewByNumber[$reviewNumberNumeric] : null; |
315
|
|
|
} |
316
|
|
|
|
317
|
12 |
|
private function readCountryId(string $country): ?int |
318
|
|
|
{ |
319
|
12 |
|
if (!$country) { |
320
|
9 |
|
return null; |
321
|
|
|
} |
322
|
|
|
|
323
|
|
|
// Case insensitive match |
324
|
5 |
|
$upper = $this->toUpper($country); |
325
|
5 |
|
if (array_key_exists($upper, $this->countryByName)) { |
326
|
3 |
|
return $this->countryByName[$upper]['id']; |
327
|
|
|
} |
328
|
|
|
|
329
|
|
|
// Suggest our best guess, so user can fix their data without lookup up countries manually |
330
|
2 |
|
$best = 0; |
331
|
2 |
|
$bestGuess = 0; |
332
|
2 |
|
foreach ($this->countryByName as $r) { |
333
|
2 |
|
similar_text($upper, $r['name'], $percent); |
334
|
2 |
|
if ($percent > $best) { |
335
|
2 |
|
$best = $percent; |
336
|
2 |
|
$bestGuess = $r; |
337
|
|
|
} |
338
|
|
|
} |
339
|
|
|
|
340
|
2 |
|
$this->throw('Pays "' . $country . '" introuvable. Vouliez-vous dire "' . $bestGuess['name'] . '" ?'); |
341
|
|
|
|
342
|
2 |
|
return null; |
343
|
|
|
} |
344
|
|
|
|
345
|
12 |
|
private function throw(string $message): void |
346
|
|
|
{ |
347
|
12 |
|
$this->errors[] = ('A la ligne ' . $this->lineNumber . ' : ' . $message); |
348
|
|
|
} |
349
|
|
|
|
350
|
17 |
|
private function deleteOldOrganizations(): void |
351
|
|
|
{ |
352
|
17 |
|
$sql = 'DELETE FROM organization WHERE should_delete'; |
353
|
17 |
|
$this->deletedOrganizations += $this->connection->executeStatement($sql); |
354
|
|
|
} |
355
|
|
|
|
356
|
12 |
|
private function readMembership($membership): Membership |
357
|
|
|
{ |
358
|
12 |
|
if ($membership === '1') { |
359
|
1 |
|
return Membership::Member; |
360
|
|
|
} |
361
|
|
|
|
362
|
12 |
|
return Membership::None; |
363
|
|
|
} |
364
|
|
|
|
365
|
12 |
|
private function readSubscriptionType(string $subscriptionType): ?ProductType |
366
|
|
|
{ |
367
|
12 |
|
if (!$subscriptionType) { |
368
|
8 |
|
return null; |
369
|
|
|
} |
370
|
|
|
|
371
|
6 |
|
if ($subscriptionType === 'Web') { |
372
|
2 |
|
return ProductType::Digital; |
373
|
|
|
} |
374
|
|
|
|
375
|
5 |
|
if ($subscriptionType === 'Papier') { |
376
|
4 |
|
return ProductType::Paper; |
377
|
|
|
} |
378
|
|
|
|
379
|
2 |
|
if ($subscriptionType === 'Papier/web') { |
380
|
1 |
|
return ProductType::Both; |
381
|
|
|
} |
382
|
|
|
|
383
|
1 |
|
$this->throw('Le subscriptionType est invalide : "' . $subscriptionType . '"'); |
384
|
|
|
|
385
|
1 |
|
return null; |
386
|
|
|
} |
387
|
|
|
|
388
|
12 |
|
private function updateUser(...$args): void |
389
|
|
|
{ |
390
|
12 |
|
$params = $args; |
391
|
12 |
|
$params[] = false; // web_temporary_access |
392
|
12 |
|
$params[] = false; // should_delete |
393
|
12 |
|
$params[] = ''; // password |
394
|
12 |
|
$params[] = $this->currentUser; |
395
|
|
|
|
396
|
12 |
|
array_push($this->usersParams, ...$params); |
397
|
12 |
|
++$this->updatedUsers; |
398
|
|
|
} |
399
|
|
|
|
400
|
17 |
|
private function updateAllUsers(): void |
401
|
|
|
{ |
402
|
17 |
|
if (!$this->updatedUsers) { |
403
|
5 |
|
return; |
404
|
|
|
} |
405
|
|
|
|
406
|
12 |
|
$placeholders = $this->placeholders($this->updatedUsers, '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NOW())'); |
407
|
|
|
|
408
|
12 |
|
$sql = 'INSERT INTO user ( |
409
|
|
|
email, |
410
|
|
|
subscription_type, |
411
|
|
|
subscription_last_review_id, |
412
|
|
|
membership, |
413
|
|
|
first_name, |
414
|
|
|
last_name, |
415
|
|
|
street, |
416
|
|
|
postcode, |
417
|
|
|
locality, |
418
|
|
|
country_id, |
419
|
|
|
phone, |
420
|
|
|
web_temporary_access, |
421
|
|
|
should_delete, |
422
|
|
|
password, |
423
|
|
|
creator_id, |
424
|
|
|
creation_date |
425
|
|
|
) |
426
|
12 |
|
VALUES ' . $placeholders . ' |
427
|
|
|
ON DUPLICATE KEY UPDATE |
428
|
|
|
email = VALUES(email), |
429
|
|
|
subscription_type = VALUES(subscription_type), |
430
|
|
|
subscription_last_review_id = VALUES(subscription_last_review_id), |
431
|
|
|
membership = VALUES(membership), |
432
|
|
|
first_name = VALUES(first_name), |
433
|
|
|
last_name = VALUES(last_name), |
434
|
|
|
street = VALUES(street), |
435
|
|
|
postcode = VALUES(postcode), |
436
|
|
|
locality = VALUES(locality), |
437
|
|
|
country_id = VALUES(country_id), |
438
|
|
|
phone = VALUES(phone), |
439
|
|
|
web_temporary_access = VALUES(web_temporary_access), |
440
|
|
|
should_delete = VALUES(should_delete), |
441
|
|
|
updater_id = VALUES(creator_id), |
442
|
12 |
|
update_date = NOW()'; |
443
|
|
|
|
444
|
12 |
|
$this->connection->executeStatement($sql, $this->usersParams); |
445
|
|
|
} |
446
|
|
|
|
447
|
5 |
|
private function updateOrganization(...$args): void |
448
|
|
|
{ |
449
|
5 |
|
$params = $args; |
450
|
5 |
|
$params[] = false; // should_delete |
451
|
5 |
|
$params[] = $this->currentUser; |
452
|
|
|
|
453
|
5 |
|
array_push($this->organizationsParams, ...$params); |
454
|
|
|
|
455
|
5 |
|
++$this->updatedOrganizations; |
456
|
|
|
} |
457
|
|
|
|
458
|
17 |
|
private function updateAllOrganizations(): void |
459
|
|
|
{ |
460
|
17 |
|
if (!$this->updatedOrganizations) { |
461
|
12 |
|
return; |
462
|
|
|
} |
463
|
|
|
|
464
|
5 |
|
$placeholders = $this->placeholders($this->updatedOrganizations, '(?, ?, ?, ?, NOW())'); |
465
|
|
|
|
466
|
5 |
|
$sql = 'INSERT INTO organization (pattern, subscription_last_review_id, should_delete, creator_id, creation_date) |
467
|
5 |
|
VALUES ' . $placeholders . ' |
468
|
|
|
ON DUPLICATE KEY UPDATE |
469
|
|
|
pattern = VALUES(pattern), |
470
|
|
|
subscription_last_review_id = VALUES(subscription_last_review_id), |
471
|
|
|
updater_id = VALUES(creator_id), |
472
|
|
|
should_delete = VALUES(should_delete), |
473
|
5 |
|
update_date = NOW()'; |
474
|
|
|
|
475
|
5 |
|
$this->connection->executeStatement($sql, $this->organizationsParams); |
476
|
|
|
} |
477
|
|
|
|
478
|
17 |
|
private function markToDelete(): void |
479
|
|
|
{ |
480
|
17 |
|
$this->connection->executeStatement('UPDATE user SET should_delete = 1'); |
481
|
17 |
|
$this->connection->executeStatement('UPDATE organization SET should_delete = 1'); |
482
|
|
|
} |
483
|
|
|
|
484
|
|
|
/** |
485
|
|
|
* To upper without any accent. |
486
|
|
|
*/ |
487
|
18 |
|
private function toUpper(string $name): string |
488
|
|
|
{ |
489
|
18 |
|
$withoutAccent = iconv('UTF-8', 'ASCII//TRANSLIT', mb_strtolower($name)); |
490
|
|
|
|
491
|
18 |
|
return trim(mb_strtoupper($withoutAccent)); |
492
|
|
|
} |
493
|
|
|
|
494
|
15 |
|
private function placeholders(int $count, string $placeholder): string |
495
|
|
|
{ |
496
|
15 |
|
return implode(',' . PHP_EOL, array_fill(0, $count, $placeholder)); |
497
|
|
|
} |
498
|
|
|
} |
499
|
|
|
|