1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* Utility class to facilitate complex CSV-imports by defining column-mappings |
4
|
|
|
* and custom converters. |
5
|
|
|
* |
6
|
|
|
* Uses the fgetcsv() function to process CSV input. Accepts a file-handler as |
7
|
|
|
* input. |
8
|
|
|
* |
9
|
|
|
* @see http://tools.ietf.org/html/rfc4180 |
10
|
|
|
* |
11
|
|
|
* @package framework |
12
|
|
|
* @subpackage bulkloading |
13
|
|
|
* |
14
|
|
|
* @todo Support for deleting existing records not matched in the import |
15
|
|
|
* (through relation checks) |
16
|
|
|
*/ |
17
|
|
|
class CsvBulkLoader extends BulkLoader { |
18
|
|
|
|
19
|
|
|
/** |
20
|
|
|
* Delimiter character (Default: comma). |
21
|
|
|
* |
22
|
|
|
* @var string |
23
|
|
|
*/ |
24
|
|
|
public $delimiter = ','; |
25
|
|
|
|
26
|
|
|
/** |
27
|
|
|
* Enclosure character (Default: doublequote) |
28
|
|
|
* |
29
|
|
|
* @var string |
30
|
|
|
*/ |
31
|
|
|
public $enclosure = '"'; |
32
|
|
|
|
33
|
|
|
/** |
34
|
|
|
* Identifies if csv the has a header row. |
35
|
|
|
* |
36
|
|
|
* @var boolean |
37
|
|
|
*/ |
38
|
|
|
public $hasHeaderRow = true; |
39
|
|
|
|
40
|
|
|
/** |
41
|
|
|
* Number of lines to split large CSV files into. |
42
|
|
|
* |
43
|
|
|
* @var int |
44
|
|
|
* |
45
|
|
|
* @config |
46
|
|
|
*/ |
47
|
|
|
private static $lines = 1000; |
48
|
|
|
|
49
|
|
|
/** |
50
|
|
|
* @inheritDoc |
51
|
|
|
*/ |
52
|
|
|
public function preview($filepath) { |
53
|
|
|
return $this->processAll($filepath, true); |
54
|
|
|
} |
55
|
|
|
|
56
|
|
|
/** |
57
|
|
|
* @param string $filepath |
58
|
|
|
* @param boolean $preview |
59
|
|
|
* |
60
|
|
|
* @return null|BulkLoader_Result |
61
|
|
|
*/ |
62
|
|
|
protected function processAll($filepath, $preview = false) { |
63
|
|
|
$filepath = Director::getAbsFile($filepath); |
64
|
|
|
$files = $this->splitFile($filepath); |
65
|
|
|
|
66
|
|
|
$result = null; |
67
|
|
|
$last = null; |
68
|
|
|
|
69
|
|
|
try { |
70
|
|
|
foreach ($files as $file) { |
71
|
|
|
$last = $file; |
72
|
|
|
|
73
|
|
|
$next = $this->processChunk($file, $preview); |
74
|
|
|
|
75
|
|
|
if ($result instanceof BulkLoader_Result) { |
76
|
|
|
$result->merge($next); |
77
|
|
|
} else { |
78
|
|
|
$result = $next; |
79
|
|
|
} |
80
|
|
|
|
81
|
|
|
@unlink($file); |
|
|
|
|
82
|
|
|
} |
83
|
|
|
} catch (Exception $e) { |
84
|
|
|
$failedMessage = sprintf("Failed to parse %s", $last); |
85
|
|
|
if (Director::isDev()) { |
86
|
|
|
$failedMessage = sprintf($failedMessage . " because %s", $e->getMessage()); |
87
|
|
|
} |
88
|
|
|
print $failedMessage . PHP_EOL; |
89
|
|
|
} |
90
|
|
|
|
91
|
|
|
return $result; |
92
|
|
|
} |
93
|
|
|
|
94
|
|
|
/** |
95
|
|
|
* Splits a large file up into many smaller files. |
96
|
|
|
* |
97
|
|
|
* @param string $path Path to large file to split |
98
|
|
|
* @param int $lines Number of lines per file |
99
|
|
|
* |
100
|
|
|
* @return array List of file paths |
101
|
|
|
*/ |
102
|
|
|
protected function splitFile($path, $lines = null) { |
103
|
|
|
$previous = ini_get('auto_detect_line_endings'); |
104
|
|
|
|
105
|
|
|
ini_set('auto_detect_line_endings', true); |
106
|
|
|
|
107
|
|
|
if (!is_int($lines)) { |
108
|
|
|
$lines = $this->config()->get("lines"); |
109
|
|
|
} |
110
|
|
|
|
111
|
|
|
$new = $this->getNewSplitFileName(); |
112
|
|
|
|
113
|
|
|
$to = fopen($new, 'w+'); |
114
|
|
|
$from = fopen($path, 'r'); |
115
|
|
|
|
116
|
|
|
$header = null; |
117
|
|
|
|
118
|
|
|
if ($this->hasHeaderRow) { |
119
|
|
|
$header = fgets($from); |
120
|
|
|
fwrite($to, $header); |
121
|
|
|
} |
122
|
|
|
|
123
|
|
|
$files = array(); |
124
|
|
|
$files[] = $new; |
125
|
|
|
|
126
|
|
|
$count = 0; |
127
|
|
|
|
128
|
|
|
while (!feof($from)) { |
129
|
|
|
fwrite($to, fgets($from)); |
130
|
|
|
|
131
|
|
|
$count++; |
132
|
|
|
|
133
|
|
|
if ($count >= $lines) { |
134
|
|
|
fclose($to); |
135
|
|
|
|
136
|
|
|
// get a new temporary file name, to write the next lines to |
137
|
|
|
$new = $this->getNewSplitFileName(); |
138
|
|
|
|
139
|
|
|
$to = fopen($new, 'w+'); |
140
|
|
|
|
141
|
|
|
if ($this->hasHeaderRow) { |
142
|
|
|
// add the headers to the new file |
143
|
|
|
fwrite($to, $header); |
144
|
|
|
} |
145
|
|
|
|
146
|
|
|
$files[] = $new; |
147
|
|
|
|
148
|
|
|
$count = 0; |
149
|
|
|
} |
150
|
|
|
} |
151
|
|
|
|
152
|
|
|
fclose($to); |
153
|
|
|
|
154
|
|
|
ini_set('auto_detect_line_endings', $previous); |
155
|
|
|
|
156
|
|
|
return $files; |
157
|
|
|
} |
158
|
|
|
|
159
|
|
|
/** |
160
|
|
|
* @return string |
161
|
|
|
*/ |
162
|
|
|
protected function getNewSplitFileName() { |
163
|
|
|
return TEMP_FOLDER . '/' . uniqid('BulkLoader', true) . '.csv'; |
164
|
|
|
} |
165
|
|
|
|
166
|
|
|
/** |
167
|
|
|
* @param string $filepath |
168
|
|
|
* @param boolean $preview |
169
|
|
|
* |
170
|
|
|
* @return BulkLoader_Result |
171
|
|
|
*/ |
172
|
|
|
protected function processChunk($filepath, $preview = false) { |
173
|
|
|
$results = new BulkLoader_Result(); |
174
|
|
|
|
175
|
|
|
$csv = new CSVParser( |
176
|
|
|
$filepath, |
177
|
|
|
$this->delimiter, |
178
|
|
|
$this->enclosure |
179
|
|
|
); |
180
|
|
|
|
181
|
|
|
// ColumnMap has two uses, depending on whether hasHeaderRow is set |
182
|
|
|
if($this->columnMap) { |
|
|
|
|
183
|
|
|
// if the map goes to a callback, use the same key value as the map |
184
|
|
|
// value, rather than function name as multiple keys may use the |
185
|
|
|
// same callback |
186
|
|
|
foreach($this->columnMap as $k => $v) { |
187
|
|
|
if(strpos($v, "->") === 0) { |
188
|
|
|
$map[$k] = $k; |
|
|
|
|
189
|
|
|
} else { |
190
|
|
|
$map[$k] = $v; |
|
|
|
|
191
|
|
|
} |
192
|
|
|
} |
193
|
|
|
|
194
|
|
|
if($this->hasHeaderRow) { |
195
|
|
|
$csv->mapColumns($map); |
196
|
|
|
} else { |
197
|
|
|
$csv->provideHeaderRow($map); |
198
|
|
|
} |
199
|
|
|
} |
200
|
|
|
|
201
|
|
|
foreach($csv as $row) { |
202
|
|
|
$this->processRecord($row, $this->columnMap, $results, $preview); |
203
|
|
|
} |
204
|
|
|
|
205
|
|
|
return $results; |
206
|
|
|
} |
207
|
|
|
|
208
|
|
|
/** |
209
|
|
|
* @todo Better messages for relation checks and duplicate detection |
210
|
|
|
* Note that columnMap isn't used. |
211
|
|
|
* |
212
|
|
|
* @param array $record |
213
|
|
|
* @param array $columnMap |
214
|
|
|
* @param BulkLoader_Result $results |
215
|
|
|
* @param boolean $preview |
216
|
|
|
* |
217
|
|
|
* @return int |
218
|
|
|
*/ |
219
|
|
|
protected function processRecord($record, $columnMap, &$results, $preview = false) { |
220
|
|
|
$class = $this->objectClass; |
221
|
|
|
|
222
|
|
|
// find existing object, or create new one |
223
|
|
|
$existingObj = $this->findExistingObject($record, $columnMap); |
|
|
|
|
224
|
|
|
$obj = ($existingObj) ? $existingObj : new $class(); |
225
|
|
|
|
226
|
|
|
// first run: find/create any relations and store them on the object |
227
|
|
|
// we can't combine runs, as other columns might rely on the relation being present |
228
|
|
|
$relations = array(); |
229
|
|
|
foreach($record as $fieldName => $val) { |
230
|
|
|
// don't bother querying of value is not set |
231
|
|
|
if($this->isNullValue($val)) continue; |
232
|
|
|
|
233
|
|
|
// checking for existing relations |
234
|
|
|
if(isset($this->relationCallbacks[$fieldName])) { |
235
|
|
|
// trigger custom search method for finding a relation based on the given value |
236
|
|
|
// and write it back to the relation (or create a new object) |
237
|
|
|
$relationName = $this->relationCallbacks[$fieldName]['relationname']; |
238
|
|
|
if($this->hasMethod($this->relationCallbacks[$fieldName]['callback'])) { |
239
|
|
|
$relationObj = $this->{$this->relationCallbacks[$fieldName]['callback']}($obj, $val, $record); |
240
|
|
|
} elseif($obj->hasMethod($this->relationCallbacks[$fieldName]['callback'])) { |
241
|
|
|
$relationObj = $obj->{$this->relationCallbacks[$fieldName]['callback']}($val, $record); |
242
|
|
|
} |
243
|
|
|
if(!$relationObj || !$relationObj->exists()) { |
|
|
|
|
244
|
|
|
$relationClass = $obj->hasOneComponent($relationName); |
245
|
|
|
$relationObj = new $relationClass(); |
246
|
|
|
//write if we aren't previewing |
247
|
|
|
if (!$preview) $relationObj->write(); |
248
|
|
|
} |
249
|
|
|
$obj->{"{$relationName}ID"} = $relationObj->ID; |
250
|
|
|
//write if we are not previewing |
251
|
|
|
if (!$preview) { |
252
|
|
|
$obj->write(); |
253
|
|
|
$obj->flushCache(); // avoid relation caching confusion |
254
|
|
|
} |
255
|
|
|
|
256
|
|
|
} elseif(strpos($fieldName, '.') !== false) { |
257
|
|
|
// we have a relation column with dot notation |
258
|
|
|
list($relationName, $columnName) = explode('.', $fieldName); |
|
|
|
|
259
|
|
|
// always gives us an component (either empty or existing) |
260
|
|
|
$relationObj = $obj->getComponent($relationName); |
261
|
|
|
if (!$preview) $relationObj->write(); |
262
|
|
|
$obj->{"{$relationName}ID"} = $relationObj->ID; |
263
|
|
|
|
264
|
|
|
//write if we are not previewing |
265
|
|
|
if (!$preview) { |
266
|
|
|
$obj->write(); |
267
|
|
|
$obj->flushCache(); // avoid relation caching confusion |
268
|
|
|
} |
269
|
|
|
} |
270
|
|
|
} |
271
|
|
|
|
272
|
|
|
// second run: save data |
273
|
|
|
|
274
|
|
|
foreach($record as $fieldName => $val) { |
275
|
|
|
// break out of the loop if we are previewing |
276
|
|
|
if ($preview) { |
277
|
|
|
break; |
278
|
|
|
} |
279
|
|
|
|
280
|
|
|
// look up the mapping to see if this needs to map to callback |
281
|
|
|
$mapped = $this->columnMap && isset($this->columnMap[$fieldName]); |
|
|
|
|
282
|
|
|
|
283
|
|
|
if($mapped && strpos($this->columnMap[$fieldName], '->') === 0) { |
284
|
|
|
$funcName = substr($this->columnMap[$fieldName], 2); |
285
|
|
|
|
286
|
|
|
$this->$funcName($obj, $val, $record); |
287
|
|
|
} else if($obj->hasMethod("import{$fieldName}")) { |
288
|
|
|
$obj->{"import{$fieldName}"}($val, $record); |
289
|
|
|
} else { |
290
|
|
|
$obj->update(array($fieldName => $val)); |
291
|
|
|
} |
292
|
|
|
} |
293
|
|
|
|
294
|
|
|
// write record |
295
|
|
|
$id = ($preview) ? 0 : $obj->write(); |
296
|
|
|
|
297
|
|
|
// @todo better message support |
298
|
|
|
$message = ''; |
299
|
|
|
|
300
|
|
|
// save to results |
301
|
|
|
if($existingObj) { |
302
|
|
|
$results->addUpdated($obj, $message); |
303
|
|
|
} else { |
304
|
|
|
$results->addCreated($obj, $message); |
305
|
|
|
} |
306
|
|
|
|
307
|
|
|
$objID = $obj->ID; |
308
|
|
|
|
309
|
|
|
$obj->destroy(); |
310
|
|
|
|
311
|
|
|
// memory usage |
312
|
|
|
unset($existingObj); |
313
|
|
|
unset($obj); |
314
|
|
|
|
315
|
|
|
return $objID; |
316
|
|
|
} |
317
|
|
|
|
318
|
|
|
/** |
319
|
|
|
* Find an existing objects based on one or more uniqueness columns |
320
|
|
|
* specified via {@link self::$duplicateChecks}. |
321
|
|
|
* |
322
|
|
|
* @param array $record CSV data column |
323
|
|
|
* |
324
|
|
|
* @return mixed |
325
|
|
|
*/ |
326
|
|
|
public function findExistingObject($record) { |
327
|
|
|
$SNG_objectClass = singleton($this->objectClass); |
328
|
|
|
// checking for existing records (only if not already found) |
329
|
|
|
|
330
|
|
|
foreach($this->duplicateChecks as $fieldName => $duplicateCheck) { |
331
|
|
|
if(is_string($duplicateCheck)) { |
332
|
|
|
|
333
|
|
|
// Skip current duplicate check if field value is empty |
334
|
|
|
if(empty($record[$duplicateCheck])) continue; |
335
|
|
|
|
336
|
|
|
// Check existing record with this value |
337
|
|
|
$dbFieldValue = $record[$duplicateCheck]; |
338
|
|
|
$existingRecord = DataObject::get($this->objectClass) |
339
|
|
|
->filter($duplicateCheck, $dbFieldValue) |
340
|
|
|
->first(); |
341
|
|
|
|
342
|
|
|
if($existingRecord) return $existingRecord; |
343
|
|
|
} elseif(is_array($duplicateCheck) && isset($duplicateCheck['callback'])) { |
344
|
|
|
if($this->hasMethod($duplicateCheck['callback'])) { |
345
|
|
|
$existingRecord = $this->{$duplicateCheck['callback']}($record[$fieldName], $record); |
346
|
|
|
} elseif($SNG_objectClass->hasMethod($duplicateCheck['callback'])) { |
347
|
|
|
$existingRecord = $SNG_objectClass->{$duplicateCheck['callback']}($record[$fieldName], $record); |
348
|
|
|
} else { |
349
|
|
|
user_error("CsvBulkLoader::processRecord():" |
350
|
|
|
. " {$duplicateCheck['callback']} not found on importer or object class.", E_USER_ERROR); |
351
|
|
|
} |
352
|
|
|
|
353
|
|
|
if($existingRecord) { |
354
|
|
|
return $existingRecord; |
|
|
|
|
355
|
|
|
} |
356
|
|
|
} else { |
357
|
|
|
user_error('CsvBulkLoader::processRecord(): Wrong format for $duplicateChecks', E_USER_ERROR); |
358
|
|
|
} |
359
|
|
|
} |
360
|
|
|
|
361
|
|
|
return false; |
362
|
|
|
} |
363
|
|
|
|
364
|
|
|
/** |
365
|
|
|
* Determine whether any loaded files should be parsed with a |
366
|
|
|
* header-row (otherwise we rely on {@link self::$columnMap}. |
367
|
|
|
* |
368
|
|
|
* @return boolean |
369
|
|
|
*/ |
370
|
|
|
public function hasHeaderRow() { |
371
|
|
|
return ($this->hasHeaderRow || isset($this->columnMap)); |
372
|
|
|
} |
373
|
|
|
} |
374
|
|
|
|
If you suppress an error, we recommend checking for the error condition explicitly: