silverstripe /
silverstripe-framework
| Total Complexity | 68 |
| Total Lines | 450 |
| Duplicated Lines | 0 % |
| Changes | 0 | ||
Complex classes like CsvBulkLoader often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use CsvBulkLoader, and based on these observations, apply Extract Interface, too.
| 1 | <?php |
||
| 22 | class CsvBulkLoader extends BulkLoader |
||
| 23 | { |
||
| 24 | |||
| 25 | /** |
||
| 26 | * Delimiter character (Default: comma). |
||
| 27 | * |
||
| 28 | * @var string |
||
| 29 | */ |
||
| 30 | public $delimiter = ','; |
||
| 31 | |||
| 32 | /** |
||
| 33 | * Enclosure character (Default: doublequote) |
||
| 34 | * |
||
| 35 | * @var string |
||
| 36 | */ |
||
| 37 | public $enclosure = '"'; |
||
| 38 | |||
| 39 | /** |
||
| 40 | * Identifies if csv the has a header row. |
||
| 41 | * |
||
| 42 | * @var boolean |
||
| 43 | */ |
||
| 44 | public $hasHeaderRow = true; |
||
| 45 | |||
| 46 | /** |
||
| 47 | * Number of lines to split large CSV files into. |
||
| 48 | * |
||
| 49 | * @var int |
||
| 50 | * |
||
| 51 | * @config |
||
| 52 | */ |
||
| 53 | private static $lines = 1000; |
||
| 54 | |||
| 55 | /** |
||
| 56 | * @inheritDoc |
||
| 57 | */ |
||
| 58 | public function preview($filepath) |
||
| 61 | } |
||
| 62 | |||
| 63 | /** |
||
| 64 | * @param string $filepath |
||
| 65 | * @param boolean $preview |
||
| 66 | * |
||
| 67 | * @return null|BulkLoader_Result |
||
| 68 | */ |
||
| 69 | protected function processAll($filepath, $preview = false) |
||
| 70 | { |
||
| 71 | $previousDetectLE = ini_get('auto_detect_line_endings'); |
||
| 72 | |||
| 73 | ini_set('auto_detect_line_endings', true); |
||
|
Bug
introduced
by
|
|||
| 74 | $result = BulkLoader_Result::create(); |
||
| 75 | |||
| 76 | try { |
||
| 77 | $filepath = Director::getAbsFile($filepath); |
||
| 78 | $csvReader = Reader::createFromPath($filepath, 'r'); |
||
| 79 | $csvReader->setDelimiter($this->delimiter); |
||
| 80 | |||
| 81 | // league/csv 9 |
||
| 82 | if (method_exists($csvReader, 'skipInputBOM')) { |
||
| 83 | $csvReader->skipInputBOM(); |
||
| 84 | // league/csv 8 |
||
| 85 | } else { |
||
| 86 | $csvReader->stripBom(true); |
||
| 87 | } |
||
| 88 | |||
| 89 | $tabExtractor = function ($row, $rowOffset) { |
||
| 90 | foreach ($row as &$item) { |
||
| 91 | // [SS-2017-007] Ensure all cells with leading tab and then [@=+] have the tab removed on import |
||
| 92 | if (preg_match("/^\t[\-@=\+]+.*/", $item)) { |
||
| 93 | $item = ltrim($item, "\t"); |
||
| 94 | } |
||
| 95 | } |
||
| 96 | return $row; |
||
| 97 | }; |
||
| 98 | |||
| 99 | if ($this->columnMap) { |
||
| 100 | $headerMap = $this->getNormalisedColumnMap(); |
||
| 101 | |||
| 102 | $remapper = function ($row, $rowOffset) use ($headerMap, $tabExtractor) { |
||
| 103 | $row = $tabExtractor($row, $rowOffset); |
||
| 104 | foreach ($headerMap as $column => $renamedColumn) { |
||
| 105 | if ($column == $renamedColumn) { |
||
| 106 | continue; |
||
| 107 | } |
||
| 108 | if (array_key_exists($column, $row)) { |
||
| 109 | if (strpos($renamedColumn, '_ignore_') !== 0) { |
||
| 110 | $row[$renamedColumn] = $row[$column]; |
||
| 111 | } |
||
| 112 | unset($row[$column]); |
||
| 113 | } |
||
| 114 | } |
||
| 115 | return $row; |
||
| 116 | }; |
||
| 117 | } else { |
||
| 118 | $remapper = $tabExtractor; |
||
| 119 | } |
||
| 120 | |||
| 121 | if ($this->hasHeaderRow) { |
||
| 122 | if (method_exists($csvReader, 'fetchAssoc')) { |
||
| 123 | $rows = $csvReader->fetchAssoc(0, $remapper); |
||
| 124 | } else { |
||
| 125 | $csvReader->setHeaderOffset(0); |
||
| 126 | $rows = new MapIterator($csvReader->getRecords(), $remapper); |
||
| 127 | } |
||
| 128 | } elseif ($this->columnMap) { |
||
| 129 | if (method_exists($csvReader, 'fetchAssoc')) { |
||
| 130 | $rows = $csvReader->fetchAssoc($headerMap, $remapper); |
||
| 131 | } else { |
||
| 132 | $rows = new MapIterator($csvReader->getRecords($headerMap), $remapper); |
||
| 133 | } |
||
| 134 | } |
||
| 135 | |||
| 136 | foreach ($rows as $row) { |
||
| 137 | $this->processRecord($row, $this->columnMap, $result, $preview); |
||
| 138 | } |
||
| 139 | } catch (\Exception $e) { |
||
| 140 | $failedMessage = sprintf("Failed to parse %s", $filepath); |
||
| 141 | if (Director::isDev()) { |
||
| 142 | $failedMessage = sprintf($failedMessage . " because %s", $e->getMessage()); |
||
| 143 | } |
||
| 144 | print $failedMessage . PHP_EOL; |
||
| 145 | } finally { |
||
| 146 | ini_set('auto_detect_line_endings', $previousDetectLE); |
||
| 147 | } |
||
| 148 | return $result; |
||
| 149 | } |
||
| 150 | |||
| 151 | protected function getNormalisedColumnMap() |
||
| 152 | { |
||
| 153 | $map = []; |
||
| 154 | foreach ($this->columnMap as $column => $newColumn) { |
||
| 155 | if (strpos($newColumn, "->") === 0) { |
||
| 156 | $map[$column] = $column; |
||
| 157 | } elseif (is_null($newColumn)) { |
||
| 158 | // the column map must consist of unique scalar values |
||
| 159 | // `null` can be present multiple times and is not scalar |
||
| 160 | // so we name it in a standard way so we can remove it later |
||
| 161 | $map[$column] = '_ignore_' . $column; |
||
| 162 | } else { |
||
| 163 | $map[$column] = $newColumn; |
||
| 164 | } |
||
| 165 | } |
||
| 166 | return $map; |
||
| 167 | } |
||
| 168 | |||
| 169 | /** |
||
| 170 | * Splits a large file up into many smaller files. |
||
| 171 | * |
||
| 172 | * @param string $path Path to large file to split |
||
| 173 | * @param int $lines Number of lines per file |
||
| 174 | * |
||
| 175 | * @return array List of file paths |
||
| 176 | */ |
||
| 177 | protected function splitFile($path, $lines = null) |
||
| 178 | { |
||
| 179 | Deprecation::notice('5.0', 'splitFile is deprecated, please process files using a stream'); |
||
| 180 | $previous = ini_get('auto_detect_line_endings'); |
||
| 181 | |||
| 182 | ini_set('auto_detect_line_endings', true); |
||
| 183 | |||
| 184 | if (!is_int($lines)) { |
||
| 185 | $lines = $this->config()->get("lines"); |
||
| 186 | } |
||
| 187 | |||
| 188 | $new = $this->getNewSplitFileName(); |
||
| 189 | |||
| 190 | $to = fopen($new, 'w+'); |
||
| 191 | $from = fopen($path, 'r'); |
||
| 192 | |||
| 193 | $header = null; |
||
| 194 | |||
| 195 | if ($this->hasHeaderRow) { |
||
| 196 | $header = fgets($from); |
||
| 197 | fwrite($to, $header); |
||
| 198 | } |
||
| 199 | |||
| 200 | $files = []; |
||
| 201 | $files[] = $new; |
||
| 202 | |||
| 203 | $count = 0; |
||
| 204 | |||
| 205 | while (!feof($from)) { |
||
| 206 | fwrite($to, fgets($from)); |
||
| 207 | |||
| 208 | $count++; |
||
| 209 | |||
| 210 | if ($count >= $lines) { |
||
| 211 | fclose($to); |
||
| 212 | |||
| 213 | // get a new temporary file name, to write the next lines to |
||
| 214 | $new = $this->getNewSplitFileName(); |
||
| 215 | |||
| 216 | $to = fopen($new, 'w+'); |
||
| 217 | |||
| 218 | if ($this->hasHeaderRow) { |
||
| 219 | // add the headers to the new file |
||
| 220 | fwrite($to, $header); |
||
| 221 | } |
||
| 222 | |||
| 223 | $files[] = $new; |
||
| 224 | |||
| 225 | $count = 0; |
||
| 226 | } |
||
| 227 | } |
||
| 228 | |||
| 229 | fclose($to); |
||
| 230 | |||
| 231 | ini_set('auto_detect_line_endings', $previous); |
||
| 232 | |||
| 233 | return $files; |
||
| 234 | } |
||
| 235 | |||
| 236 | /** |
||
| 237 | * @return string |
||
| 238 | */ |
||
| 239 | protected function getNewSplitFileName() |
||
| 243 | } |
||
| 244 | |||
| 245 | /** |
||
| 246 | * @param string $filepath |
||
| 247 | * @param boolean $preview |
||
| 248 | * |
||
| 249 | * @return BulkLoader_Result |
||
| 250 | */ |
||
| 251 | protected function processChunk($filepath, $preview = false) |
||
| 288 | } |
||
| 289 | |||
| 290 | /** |
||
| 291 | * @todo Better messages for relation checks and duplicate detection |
||
| 292 | * Note that columnMap isn't used. |
||
| 293 | * |
||
| 294 | * @param array $record |
||
| 295 | * @param array $columnMap |
||
| 296 | * @param BulkLoader_Result $results |
||
| 297 | * @param boolean $preview |
||
| 298 | * |
||
| 299 | * @return int |
||
| 300 | */ |
||
| 301 | protected function processRecord($record, $columnMap, &$results, $preview = false) |
||
| 408 | } |
||
| 409 | |||
| 410 | /** |
||
| 411 | * Find an existing objects based on one or more uniqueness columns |
||
| 412 | * specified via {@link self::$duplicateChecks}. |
||
| 413 | * |
||
| 414 | * @todo support $columnMap |
||
| 415 | * |
||
| 416 | * @param array $record CSV data column |
||
| 417 | * @param array $columnMap |
||
| 418 | * @return DataObject |
||
| 419 | */ |
||
| 420 | public function findExistingObject($record, $columnMap = []) |
||
| 461 | } |
||
| 462 | |||
| 463 | /** |
||
| 464 | * Determine whether any loaded files should be parsed with a |
||
| 465 | * header-row (otherwise we rely on {@link self::$columnMap}. |
||
| 466 | * |
||
| 467 | * @return boolean |
||
| 468 | */ |
||
| 469 | public function hasHeaderRow() |
||
| 472 | } |
||
| 473 | } |
||
| 474 |
Loading history...
