Completed
Push — master ( 618dc8...d5c26d )
by Aimeos
02:29
created

Standard::import()   B

Complexity

Conditions 8
Paths 77

Size

Total Lines 53
Code Lines 29

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
dl 0
loc 53
rs 7.1199
c 0
b 0
f 0
cc 8
eloc 29
nc 77
nop 5

How to fix   Long Method   

Long Method

Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.

For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.

Commonly applied refactorings include:

1
<?php
2
3
/**
4
 * @license LGPLv3, http://opensource.org/licenses/LGPL-3.0
5
 * @copyright Aimeos (aimeos.org), 2015-2016
6
 * @package Controller
7
 * @subpackage Jobs
8
 */
9
10
11
namespace Aimeos\Controller\Jobs\Product\Import\Csv;
12
13
14
/**
15
 * Job controller for CSV product imports.
16
 *
17
 * @package Controller
18
 * @subpackage Jobs
19
 */
20
class Standard
21
	extends \Aimeos\Controller\Common\Product\Import\Csv\Base
0 ignored issues
show
Coding Style introduced by
The extends keyword must be on the same line as the class name
Loading history...
Coding Style introduced by
Expected 0 spaces between "Base" and comma; 1 found
Loading history...
22
	implements \Aimeos\Controller\Jobs\Iface
0 ignored issues
show
Coding Style introduced by
The implements keyword must be on the same line as the class name
Loading history...
23
{
24
	/**
25
	 * Returns the localized name of the job.
26
	 *
27
	 * @return string Name of the job
28
	 */
29
	public function getName()
30
	{
31
		return $this->getContext()->getI18n()->dt( 'controller/jobs', 'Product import CSV' );
32
	}
33
34
35
	/**
36
	 * Returns the localized description of the job.
37
	 *
38
	 * @return string Description of the job
39
	 */
40
	public function getDescription()
41
	{
42
		return $this->getContext()->getI18n()->dt( 'controller/jobs', 'Imports new and updates existing products from CSV files' );
43
	}
44
45
46
	/**
47
	 * Executes the job.
48
	 *
49
	 * @throws \Aimeos\Controller\Jobs\Exception If an error occurs
50
	 */
51
	public function run()
52
	{
53
		$total = $errors = 0;
54
		$context = $this->getContext();
55
		$config = $context->getConfig();
56
		$logger = $context->getLogger();
57
		$domains = array( 'attribute', 'media', 'price', 'product', 'text' );
58
		$mappings = $this->getDefaultMapping();
59
60
61
		/** controller/common/product/import/csv/domains
62
		 * List of item domain names that should be retrieved along with the product items
63
		 *
64
		 * For efficient processing, the items associated to the products can be
65
		 * fetched to, minimizing the number of database queries required. To be
66
		 * most effective, the list of item domain names should be used in the
67
		 * mapping configuration too, so the retrieved items will be used during
68
		 * the import.
69
		 *
70
		 * @param array Associative list of MShop item domain names
71
		 * @since 2015.05
72
		 * @category Developer
73
		 * @see controller/common/product/import/csv/mapping
74
		 * @see controller/common/product/import/csv/converter
75
		 * @see controller/common/product/import/csv/max-size
76
		 */
77
		$domains = $config->get( 'controller/common/product/import/csv/domains', $domains );
78
79
		/** controller/jobs/product/import/csv/domains
80
		 * List of item domain names that should be retrieved along with the product items
81
		 *
82
		 * This configuration setting overwrites the shared option
83
		 * "controller/common/product/import/csv/domains" if you need a
84
		 * specific setting for the job controller. Otherwise, you should
85
		 * use the shared option for consistency.
86
		 *
87
		 * @param array Associative list of MShop item domain names
88
		 * @since 2015.05
89
		 * @category Developer
90
		 * @see controller/jobs/product/import/csv/mapping
91
		 * @see controller/jobs/product/import/csv/skip-lines
92
		 * @see controller/jobs/product/import/csv/converter
93
		 * @see controller/jobs/product/import/csv/strict
94
		 * @see controller/jobs/product/import/csv/backup
95
		 * @see controller/common/product/import/csv/max-size
96
		 */
97
		$domains = $config->get( 'controller/jobs/product/import/csv/domains', $domains );
98
99
100
		/** controller/common/product/import/csv/mapping
101
		 * List of mappings between the position in the CSV file and item keys
102
		 *
103
		 * The importer have to know which data is at which position in the CSV
104
		 * file. Therefore, you need to specify a mapping between each position
105
		 * and the MShop domain item key (e.g. "product.code") it represents.
106
		 *
107
		 * You can use all domain item keys which are used in the fromArray()
108
		 * methods of the item classes. The "*.type" item keys will be
109
		 * automatically converted to their "*.typeid" representation. You only
110
		 * need to make sure that the corresponding type is available in the
111
		 * database.
112
		 *
113
		 * These mappings are grouped together by their processor names, which
114
		 * are responsible for importing the data, e.g. all mappings in "item"
115
		 * will be processed by the base product importer while the mappings in
116
		 * "text" will be imported by the text processor.
117
		 *
118
		 * @param array Associative list of processor names and lists of key/position pairs
119
		 * @since 2015.05
120
		 * @category Developer
121
		 * @see controller/common/product/import/csv/domains
122
		 * @see controller/common/product/import/csv/converter
123
		 * @see controller/common/product/import/csv/max-size
124
		 */
125
		$mappings = $config->get( 'controller/common/product/import/csv/mapping', $mappings );
126
127
		/** controller/jobs/product/import/csv/mapping
128
		 * List of mappings between the position in the CSV file and item keys
129
		 *
130
		 * This configuration setting overwrites the shared option
131
		 * "controller/common/product/import/csv/mapping" if you need a
132
		 * specific setting for the job controller. Otherwise, you should
133
		 * use the shared option for consistency.
134
		 *
135
		 * @param array Associative list of processor names and lists of key/position pairs
136
		 * @since 2015.05
137
		 * @category Developer
138
		 * @see controller/jobs/product/import/csv/domains
139
		 * @see controller/jobs/product/import/csv/skip-lines
140
		 * @see controller/jobs/product/import/csv/converter
141
		 * @see controller/jobs/product/import/csv/strict
142
		 * @see controller/jobs/product/import/csv/backup
143
		 * @see controller/common/product/import/csv/max-size
144
		 */
145
		$mappings = $config->get( 'controller/jobs/product/import/csv/mapping', $mappings );
146
147
148
		/** controller/common/product/import/csv/converter
149
		 * List of converter names for the values at the position in the CSV file
150
		 *
151
		 * Not all data in the CSV file is already in the required format. Maybe
152
		 * the text encoding isn't UTF-8, the date is not in ISO format or something
153
		 * similar. In order to convert the data before it's imported, you can
154
		 * specify a list of converter objects that should be applied to the data
155
		 * from the CSV file.
156
		 *
157
		 * To each field in the CSV file, you can apply one or more converters,
158
		 * e.g. to encode a Latin text to UTF8 for the second CSV field:
159
		 *
160
		 *  array( 1 => 'Text/LatinUTF8' )
161
		 *
162
		 * Similarly, you can also apply several converters at once to the same
163
		 * field:
164
		 *
165
		 *  array( 1 => array( 'Text/LatinUTF8', 'DateTime/EnglishISO' ) )
166
		 *
167
		 * It would convert the data of the second CSV field first to UTF-8 and
168
		 * afterwards try to translate it to an ISO date format.
169
		 *
170
		 * The available converter objects are named "\Aimeos\MW\Convert\<type>_<conversion>"
171
		 * where <type> is the data type and <conversion> the way of the conversion.
172
		 * In the configuration, the type and conversion must be separated by a
173
		 * slash (<type>/<conversion>).
174
		 *
175
		 * '''Note:''' Keep in mind that the position of the CSV fields start at
176
		 * zero (0). If you only need to convert a few fields, you don't have to
177
		 * configure all fields. Only specify the positions in the array you
178
		 * really need!
179
		 *
180
		 * @param array Associative list of position/converter name (or list of names) pairs
181
		 * @since 2015.05
182
		 * @category Developer
183
		 * @see controller/common/product/import/csv/domains
184
		 * @see controller/common/product/import/csv/mapping
185
		 * @see controller/common/product/import/csv/max-size
186
		 */
187
		$converters = $config->get( 'controller/common/product/import/csv/converter', array() );
188
189
		/** controller/jobs/product/import/csv/converter
190
		 * List of converter names for the values at the position in the CSV file
191
		 *
192
		 * This configuration setting overwrites the shared option
193
		 * "controller/common/product/import/csv/converter" if you need a
194
		 * specific setting for the job controller. Otherwise, you should
195
		 * use the shared option for consistency.
196
		 *
197
		 * @param array Associative list of position/converter name (or list of names) pairs
198
		 * @since 2015.05
199
		 * @category Developer
200
		 * @see controller/jobs/product/import/csv/domains
201
		 * @see controller/jobs/product/import/csv/mapping
202
		 * @see controller/jobs/product/import/csv/skip-lines
203
		 * @see controller/jobs/product/import/csv/strict
204
		 * @see controller/jobs/product/import/csv/backup
205
		 * @see controller/common/product/import/csv/max-size
206
		 */
207
		$converters = $config->get( 'controller/jobs/product/import/csv/converter', $converters );
208
209
210
		/** controller/common/product/import/csv/max-size
211
		 * Maximum number of CSV rows to import at once
212
		 *
213
		 * It's more efficient to read and import more than one row at a time
214
		 * to speed up the import. Usually, the bigger the chunk that is imported
215
		 * at once, the less time the importer will need. The downside is that
216
		 * the amount of memory required by the import process will increase as
217
		 * well. Therefore, it's a trade-off between memory consumption and
218
		 * import speed.
219
		 *
220
		 * @param integer Number of rows
221
		 * @since 2015.05
222
		 * @category Developer
223
		 * @see controller/common/product/import/csv/domains
224
		 * @see controller/common/product/import/csv/mapping
225
		 * @see controller/common/product/import/csv/converter
226
		 */
227
		$maxcnt = (int) $config->get( 'controller/common/product/import/csv/max-size', 1000 );
228
229
230
		/** controller/jobs/product/import/csv/skip-lines
231
		 * Number of rows skipped in front of each CSV files
232
		 *
233
		 * Some CSV files contain header information describing the content of
234
		 * the column values. These data is for informational purpose only and
235
		 * can't be imported into the database. Using this option, you can
236
		 * define the number of lines that should be left out before the import
237
		 * begins.
238
		 *
239
		 * @param integer Number of rows
240
		 * @since 2015.08
241
		 * @category Developer
242
		 * @see controller/jobs/product/import/csv/domains
243
		 * @see controller/jobs/product/import/csv/mapping
244
		 * @see controller/jobs/product/import/csv/converter
245
		 * @see controller/jobs/product/import/csv/strict
246
		 * @see controller/jobs/product/import/csv/backup
247
		 * @see controller/common/product/import/csv/max-size
248
		 */
249
		$skiplines = (int) $config->get( 'controller/jobs/product/import/csv/skip-lines', 0 );
250
251
252
		/** controller/jobs/product/import/csv/strict
253
		 * Log all columns from the file that are not mapped and therefore not imported
254
		 *
255
		 * Depending on the mapping, there can be more columns in the CSV file
256
		 * than those which will be imported. This can be by purpose if you want
257
		 * to import only selected columns or if you've missed to configure one
258
		 * or more columns. This configuration option will log all columns that
259
		 * have not been imported if set to true. Otherwise, the left over fields
260
		 * in the imported line will be silently ignored.
261
		 *
262
		 * @param boolen True if not imported columns should be logged, false if not
263
		 * @since 2015.08
264
		 * @category User
265
		 * @category Developer
266
		 * @see controller/jobs/product/import/csv/domains
267
		 * @see controller/jobs/product/import/csv/mapping
268
		 * @see controller/jobs/product/import/csv/skip-lines
269
		 * @see controller/jobs/product/import/csv/converter
270
		 * @see controller/jobs/product/import/csv/backup
271
		 * @see controller/common/product/import/csv/max-size
272
		 */
273
		$strict = (bool) $config->get( 'controller/jobs/product/import/csv/strict', true );
274
275
276
		/** controller/jobs/product/import/csv/backup
277
		 * Name of the backup for sucessfully imported files
278
		 *
279
		 * After a CSV file was imported successfully, you can move it to another
280
		 * location, so it won't be imported again and isn't overwritten by the
281
		 * next file that is stored at the same location in the file system.
282
		 *
283
		 * You should use an absolute path to be sure but can be relative path
284
		 * if you absolutely know from where the job will be executed from. The
285
		 * name of the new backup location can contain placeholders understood
286
		 * by the PHP strftime() function to create dynamic paths, e.g. "backup/%Y-%m-%d"
287
		 * which would create "backup/2000-01-01". For more information about the
288
		 * strftime() placeholders, please have a look into the PHP documentation of
289
		 * the {@link http://php.net/manual/en/function.strftime.php strftime() function}.
290
		 *
291
		 * '''Note:''' If no backup name is configured, the file or directory
292
		 * won't be moved away. Please make also sure that the parent directory
293
		 * and the new directory are writable so the file or directory could be
294
		 * moved.
295
		 *
296
		 * @param integer Name of the backup file, optionally with date/time placeholders
297
		 * @since 2015.05
298
		 * @category Developer
299
		 * @see controller/jobs/product/import/csv/domains
300
		 * @see controller/jobs/product/import/csv/mapping
301
		 * @see controller/jobs/product/import/csv/skip-lines
302
		 * @see controller/jobs/product/import/csv/converter
303
		 * @see controller/jobs/product/import/csv/strict
304
		 * @see controller/common/product/import/csv/max-size
305
		 */
306
		$backup = $config->get( 'controller/jobs/product/import/csv/backup' );
307
308
309
		if( !isset( $mappings['item'] ) || !is_array( $mappings['item'] ) )
310
		{
311
			$msg = sprintf( 'Required mapping key "%1$s" is missing or contains no array', 'item' );
312
			throw new \Aimeos\Controller\Jobs\Exception( $msg );
313
		}
314
315
		try
316
		{
317
			$procMappings = $mappings;
318
			unset( $procMappings['item'] );
319
320
			$codePos = $this->getCodePosition( $mappings['item'] );
321
			$convlist = $this->getConverterList( $converters );
322
			$processor = $this->getProcessors( $procMappings );
323
			$container = $this->getContainer();
324
			$path = $container->getName();
325
326
			$msg = sprintf( 'Started product import from "%1$s" (%2$s)', $path, __CLASS__ );
327
			$logger->log( $msg, \Aimeos\MW\Logger\Base::NOTICE );
328
329
			foreach( $container as $content )
330
			{
331
				$name = $content->getName();
332
333
				for( $i = 0; $i < $skiplines; $i++ ) {
334
					$content->next();
335
				}
336
337
				while( ( $data = $this->getData( $content, $maxcnt, $codePos ) ) !== array() )
338
				{
339
					$data = $this->convertData( $convlist, $data );
340
					$products = $this->getProducts( array_keys( $data ), $domains );
341
					$errcnt = $this->import( $products, $data, $mappings['item'], $processor, $strict );
342
					$chunkcnt = count( $data );
343
344
					$msg = 'Imported product lines from "%1$s": %2$d/%3$d (%4$s)';
345
					$logger->log( sprintf( $msg, $name, $chunkcnt - $errcnt, $chunkcnt, __CLASS__ ), \Aimeos\MW\Logger\Base::NOTICE );
346
347
					$errors += $errcnt;
348
					$total += $chunkcnt;
349
					unset( $products, $data );
350
				}
351
			}
352
353
			$container->close();
354
		}
355
		catch( \Exception $e )
356
		{
357
			$logger->log( 'Product import error: ' . $e->getMessage() );
358
			$logger->log( $e->getTraceAsString() );
359
360
			throw new \Aimeos\Controller\Jobs\Exception( $e->getMessage() );
361
		}
362
363
		$msg = 'Finished product import from "%1$s": %2$d successful, %3$s errors, %4$s total (%5$s)';
364
		$logger->log( sprintf( $msg, $path, $total - $errors, $errors, $total, __CLASS__ ), \Aimeos\MW\Logger\Base::NOTICE );
365
366
		if( $errors > 0 )
367
		{
368
			$msg = sprintf( 'Invalid product lines in "%1$s": %2$d/%3$d', $path, $errors, $total );
369
			throw new \Aimeos\Controller\Jobs\Exception( $msg );
370
		}
371
372
		if( !empty( $backup ) && @rename( $path, strftime( $backup ) ) === false ) {
373
			throw new \Aimeos\Controller\Jobs\Exception( sprintf( 'Unable to move imported file' ) );
374
		}
375
	}
376
377
378
	/**
379
	 * Returns the position of the "product.code" column from the product item mapping
380
	 *
381
	 * @param array $mapping Mapping of the "item" columns with position as key and code as value
382
	 * @return integer Position of the "product.code" column
383
	 * @throws \Aimeos\Controller\Jobs\Exception If no mapping for "product.code" is found
384
	 */
385
	protected function getCodePosition( array $mapping )
386
	{
387
		foreach( $mapping as $pos => $key )
388
		{
389
			if( $key === 'product.code' ) {
390
				return $pos;
391
			}
392
		}
393
394
		throw new \Aimeos\Controller\Jobs\Exception( sprintf( 'No "product.code" column in CSV mapping found' ) );
395
	}
396
397
398
	/**
399
	 * Opens and returns the container which includes the product data
400
	 *
401
	 * @return \Aimeos\MW\Container\Iface Container object
402
	 */
403
	protected function getContainer()
404
	{
405
		$config = $this->getContext()->getConfig();
406
407
		/** controller/jobs/product/import/csv/location
408
		 * File or directory where the content is stored which should be imported
409
		 *
410
		 * You need to configure the file or directory that acts as container
411
		 * for the CSV files that should be imported. It should be an absolute
412
		 * path to be sure but can be relative path if you absolutely know from
413
		 * where the job will be executed from.
414
		 *
415
		 * The path can point to any supported container format as long as the
416
		 * content is in CSV format, e.g.
417
		 * * Directory container / CSV file
418
		 * * Zip container / compressed CSV file
419
		 * * PHPExcel container / PHPExcel sheet
420
		 *
421
		 * @param string Absolute file or directory path
422
		 * @since 2015.05
423
		 * @category Developer
424
		 * @category User
425
		 * @see controller/jobs/product/import/csv/container/type
426
		 * @see controller/jobs/product/import/csv/container/content
427
		 * @see controller/jobs/product/import/csv/container/options
428
		 */
429
		$location = $config->get( 'controller/jobs/product/import/csv/location', '.' );
430
431
		/** controller/jobs/product/import/csv/container/type
432
		 * Nave of the container type to read the data from
433
		 *
434
		 * The container type tells the importer how it should retrieve the data.
435
		 * There are currently three container types that support the necessary
436
		 * CSV content:
437
		 * * Directory
438
		 * * Zip
439
		 * * PHPExcel
440
		 *
441
		 * '''Note:''' For the PHPExcel container, you need to install the
442
		 * "ai-container" extension.
443
		 *
444
		 * @param string Container type name
445
		 * @since 2015.05
446
		 * @category Developer
447
		 * @category User
448
		 * @see controller/jobs/product/import/csv/location
449
		 * @see controller/jobs/product/import/csv/container/content
450
		 * @see controller/jobs/product/import/csv/container/options
451
		 */
452
		$container = $config->get( 'controller/jobs/product/import/csv/container/type', 'Directory' );
453
454
		/** controller/jobs/product/import/csv/container/content
455
		 * Name of the content type inside the container to read the data from
456
		 *
457
		 * The content type must always be a CSV-like format and there are
458
		 * currently two format types that are supported:
459
		 * * CSV
460
		 * * PHPExcel
461
		 *
462
		 * '''Note:''' for the PHPExcel content type, you need to install the
463
		 * "ai-container" extension.
464
		 *
465
		 * @param array Content type name
466
		 * @since 2015.05
467
		 * @category Developer
468
		 * @category User
469
		 * @see controller/jobs/product/import/csv/location
470
		 * @see controller/jobs/product/import/csv/container/type
471
		 * @see controller/jobs/product/import/csv/container/options
472
		 */
473
		$content = $config->get( 'controller/jobs/product/import/csv/container/content', 'CSV' );
474
475
		/** controller/jobs/product/import/csv/container/options
476
		 * List of file container options for the product import files
477
		 *
478
		 * Some container/content type allow you to hand over additional settings
479
		 * for configuration. Please have a look at the article about
480
		 * {@link http://aimeos.org/docs/Developers/Utility/Create_and_read_files container/content files}
481
		 * for more information.
482
		 *
483
		 * @param array Associative list of option name/value pairs
484
		 * @since 2015.05
485
		 * @category Developer
486
		 * @category User
487
		 * @see controller/jobs/product/import/csv/location
488
		 * @see controller/jobs/product/import/csv/container/content
489
		 * @see controller/jobs/product/import/csv/container/type
490
		 */
491
		$options = $config->get( 'controller/jobs/product/import/csv/container/options', array() );
492
493
		return \Aimeos\MW\Container\Factory::getContainer( $location, $container, $content, $options );
494
	}
495
496
497
	/**
498
	 * Imports the CSV data and creates new products or updates existing ones
499
	 *
500
	 * @param array $products List of products items implementing \Aimeos\MShop\Product\Item\Iface
501
	 * @param array $data Associative list of import data as index/value pairs
502
	 * @param array $mapping Associative list of positions and domain item keys
503
	 * @param \Aimeos\Controller\Common\Product\Import\Csv\Processor\Iface $processor Processor object
504
	 * @param boolean $strict Log columns not mapped or silently ignore them
505
	 * @return integer Number of products that couldn't be imported
506
	 * @throws \Aimeos\Controller\Jobs\Exception
507
	 */
508
	protected function import( array $products, array $data, array $mapping,
509
		\Aimeos\Controller\Common\Product\Import\Csv\Processor\Iface $processor, $strict )
510
	{
511
		$errors = 0;
512
		$context = $this->getContext();
513
		$manager = \Aimeos\MShop\Factory::createManager( $context, 'product' );
514
515
		foreach( $data as $code => $list )
516
		{
517
			$manager->begin();
518
519
			try
520
			{
521
				if( isset( $products[$code] ) ) {
522
					$product = $products[$code];
523
				} else {
524
					$product = $manager->createItem();
525
				}
526
527
				$map = $this->getMappedChunk( $list, $mapping );
528
529
				if( isset( $map[0] ) )
530
				{
531
					$map = $map[0]; // there can only be one chunk for the base product data
532
533
					$typecode = ( isset( $map['product.type'] ) ? $map['product.type'] : 'default' );
534
					$map['product.typeid'] = $this->getTypeId( 'product/type', 'product', $typecode );
535
536
					$product->fromArray( $this->addItemDefaults( $map ) );
537
					$manager->saveItem( $product );
538
539
					$list = $processor->process( $product, $list );
540
				}
541
542
				$manager->commit();
543
			}
544
			catch( \Exception $e )
545
			{
546
				$manager->rollback();
547
548
				$msg = sprintf( 'Unable to import product with code "%1$s": %2$s', $code, $e->getMessage() );
549
				$context->getLogger()->log( $msg );
550
551
				$errors++;
552
			}
553
554
			if( $strict && !empty( $list ) ) {
555
				$context->getLogger()->log( 'Not imported: ' . print_r( $list, true ) );
556
			}
557
		}
558
559
		return $errors;
560
	}
561
562
563
	/**
564
	 * Adds the product item default values and returns the resulting array
565
	 *
566
	 * @param array $list Associative list of domain item keys and their values, e.g. "product.status" => 1
567
	 * @return array Given associative list enriched by default values if they were not already set
568
	 */
569
	protected function addItemDefaults( array $list )
570
	{
571
		if( !isset( $list['product.status'] ) ) {
572
			$list['product.status'] = 1;
573
		}
574
575
		return $list;
576
	}
577
}
578