owncloud /
core
Duplicate code is one of the most pungent code smells. A rule that is often used is to re-structure code once it is duplicated in three or more places.
Common duplication problems, and corresponding solutions are:
Complex classes like Scanner often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes. You can also have a look at the cohesion graph to spot any un-connected, or weakly-connected components.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use Scanner, and based on these observations, apply Extract Interface, too.
| 1 | <?php |
||
| 56 | class Scanner extends BasicEmitter implements IScanner { |
||
| 57 | /** |
||
| 58 | * @var \OC\Files\Storage\Storage $storage |
||
| 59 | */ |
||
| 60 | protected $storage; |
||
| 61 | |||
| 62 | /** |
||
| 63 | * @var string $storageId |
||
| 64 | */ |
||
| 65 | protected $storageId; |
||
| 66 | |||
| 67 | /** |
||
| 68 | * @var \OC\Files\Cache\Cache $cache |
||
| 69 | */ |
||
| 70 | protected $cache; |
||
| 71 | |||
| 72 | /** |
||
| 73 | * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache |
||
| 74 | */ |
||
| 75 | protected $cacheActive; |
||
| 76 | |||
| 77 | /** |
||
| 78 | * @var bool $useTransactions whether to use transactions |
||
| 79 | */ |
||
| 80 | protected $useTransactions = true; |
||
| 81 | |||
| 82 | /** |
||
| 83 | * @var \OCP\Lock\ILockingProvider |
||
| 84 | */ |
||
| 85 | protected $lockingProvider; |
||
| 86 | |||
| 87 | public function __construct(\OC\Files\Storage\Storage $storage) { |
||
| 94 | |||
| 95 | /** |
||
| 96 | * Whether to wrap the scanning of a folder in a database transaction |
||
| 97 | * On default transactions are used |
||
| 98 | * |
||
| 99 | * @param bool $useTransactions |
||
| 100 | */ |
||
| 101 | public function setUseTransactions($useTransactions) { |
||
| 104 | |||
| 105 | /** |
||
| 106 | * get all the metadata of a file or folder |
||
| 107 | * * |
||
| 108 | * |
||
| 109 | * @param string $path |
||
| 110 | * @return array an array of metadata of the file |
||
| 111 | */ |
||
| 112 | protected function getData($path) { |
||
| 119 | |||
| 120 | /** |
||
| 121 | * scan a single file and store it in the cache |
||
| 122 | * |
||
| 123 | * @param string $file |
||
| 124 | * @param int $reuseExisting |
||
| 125 | * @param int $parentId |
||
| 126 | * @param array | null $cacheData existing data in the cache for the file to be scanned |
||
| 127 | * @param bool $lock set to false to disable getting an additional read lock during scanning |
||
| 128 | * @return array an array of metadata of the scanned file |
||
| 129 | * @throws \OC\ServerNotAvailableException |
||
| 130 | * @throws \OCP\Lock\LockedException |
||
| 131 | */ |
||
| 132 | public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) { |
||
| 133 | |||
| 134 | // only proceed if $file is not a partial file nor a blacklisted file |
||
| 135 | if (!self::isPartialFile($file) and !Filesystem::isFileBlacklisted($file)) { |
||
| 136 | |||
| 137 | //acquire a lock |
||
| 138 | if ($lock) { |
||
| 139 | if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
||
| 140 | $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
||
| 141 | } |
||
| 142 | } |
||
| 143 | |||
| 144 | try { |
||
| 145 | $data = $this->getData($file); |
||
| 146 | } catch (ForbiddenException $e) { |
||
| 147 | return null; |
||
| 148 | } |
||
| 149 | |||
| 150 | if ($data) { |
||
| 151 | |||
| 152 | // pre-emit only if it was a file. By that we avoid counting/treating folders as files |
||
| 153 | View Code Duplication | if ($data['mimetype'] !== 'httpd/unix-directory') { |
|
|
|
|||
| 154 | $this->emit('\OC\Files\Cache\Scanner', 'scanFile', array($file, $this->storageId)); |
||
| 155 | \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', array('path' => $file, 'storage' => $this->storageId)); |
||
| 156 | } |
||
| 157 | |||
| 158 | $parent = dirname($file); |
||
| 159 | if ($parent === '.' or $parent === '/') { |
||
| 160 | $parent = ''; |
||
| 161 | } |
||
| 162 | if ($parentId === -1) { |
||
| 163 | $parentId = $this->cache->getId($parent); |
||
| 164 | } |
||
| 165 | |||
| 166 | // scan the parent if it's not in the cache (id -1) and the current file is not the root folder |
||
| 167 | if ($file and $parentId === -1) { |
||
| 168 | $parentData = $this->scanFile($parent); |
||
| 169 | $parentId = $parentData['fileid']; |
||
| 170 | } |
||
| 171 | if ($parent) { |
||
| 172 | $data['parent'] = $parentId; |
||
| 173 | } |
||
| 174 | if (is_null($cacheData)) { |
||
| 175 | /** @var CacheEntry $cacheData */ |
||
| 176 | $cacheData = $this->cache->get($file); |
||
| 177 | } |
||
| 178 | if ($cacheData and $reuseExisting and isset($cacheData['fileid'])) { |
||
| 179 | // prevent empty etag |
||
| 180 | if (empty($cacheData['etag'])) { |
||
| 181 | $etag = $data['etag']; |
||
| 182 | } else { |
||
| 183 | $etag = $cacheData['etag']; |
||
| 184 | } |
||
| 185 | $fileId = $cacheData['fileid']; |
||
| 186 | $data['fileid'] = $fileId; |
||
| 187 | // only reuse data if the file hasn't explicitly changed |
||
| 188 | if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) { |
||
| 189 | $data['mtime'] = $cacheData['mtime']; |
||
| 190 | if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { |
||
| 191 | $data['size'] = $cacheData['size']; |
||
| 192 | } |
||
| 193 | if ($reuseExisting & self::REUSE_ETAG) { |
||
| 194 | $data['etag'] = $etag; |
||
| 195 | } |
||
| 196 | } |
||
| 197 | // Only update metadata that has changed |
||
| 198 | $newData = array_diff_assoc($data, $cacheData->getData()); |
||
| 199 | } else { |
||
| 200 | $newData = $data; |
||
| 201 | $fileId = -1; |
||
| 202 | } |
||
| 203 | if (!empty($newData)) { |
||
| 204 | // Reset the checksum if the data has changed |
||
| 205 | $newData['checksum'] = ''; |
||
| 206 | $data['fileid'] = $this->addToCache($file, $newData, $fileId); |
||
| 207 | } |
||
| 208 | if (isset($cacheData['size'])) { |
||
| 209 | $data['oldSize'] = $cacheData['size']; |
||
| 210 | } else { |
||
| 211 | $data['oldSize'] = 0; |
||
| 212 | } |
||
| 213 | |||
| 214 | if (isset($cacheData['encrypted'])) { |
||
| 215 | $data['encrypted'] = $cacheData['encrypted']; |
||
| 216 | } |
||
| 217 | |||
| 218 | // post-emit only if it was a file. By that we avoid counting/treating folders as files |
||
| 219 | View Code Duplication | if ($data['mimetype'] !== 'httpd/unix-directory') { |
|
| 220 | $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId)); |
||
| 221 | \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId)); |
||
| 222 | } |
||
| 223 | |||
| 224 | } else { |
||
| 225 | $this->removeFromCache($file); |
||
| 226 | } |
||
| 227 | |||
| 228 | //release the acquired lock |
||
| 229 | if ($lock) { |
||
| 230 | if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
||
| 231 | $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
||
| 232 | } |
||
| 233 | } |
||
| 234 | |||
| 235 | if ($data && !isset($data['encrypted'])) { |
||
| 236 | $data['encrypted'] = false; |
||
| 237 | } |
||
| 238 | return $data; |
||
| 239 | } |
||
| 240 | |||
| 241 | return null; |
||
| 242 | } |
||
| 243 | |||
| 244 | protected function removeFromCache($path) { |
||
| 251 | |||
| 252 | /** |
||
| 253 | * @param string $path |
||
| 254 | * @param array $data |
||
| 255 | * @param int $fileId |
||
| 256 | * @return int the id of the added file |
||
| 257 | */ |
||
| 258 | protected function addToCache($path, $data, $fileId = -1) { |
||
| 272 | |||
| 273 | /** |
||
| 274 | * @param string $path |
||
| 275 | * @param array $data |
||
| 276 | * @param int $fileId |
||
| 277 | */ |
||
| 278 | protected function updateCache($path, $data, $fileId = -1) { |
||
| 289 | |||
| 290 | /** |
||
| 291 | * scan a folder and all it's children |
||
| 292 | * |
||
| 293 | * @param string $path |
||
| 294 | * @param bool $recursive |
||
| 295 | * @param int $reuse |
||
| 296 | * @param bool $lock set to false to disable getting an additional read lock during scanning |
||
| 297 | * @return array an array of the meta data of the scanned file or folder |
||
| 298 | */ |
||
| 299 | public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { |
||
| 322 | |||
| 323 | /** |
||
| 324 | * Get the children currently in the cache |
||
| 325 | * |
||
| 326 | * @param int $folderId |
||
| 327 | * @return array[] |
||
| 328 | */ |
||
| 329 | protected function getExistingChildren($folderId) { |
||
| 337 | |||
| 338 | /** |
||
| 339 | * Get the children from the storage |
||
| 340 | * |
||
| 341 | * @param string $folder |
||
| 342 | * @return string[] |
||
| 343 | */ |
||
| 344 | protected function getNewChildren($folder) { |
||
| 357 | |||
| 358 | /** |
||
| 359 | * scan all the files and folders in a folder |
||
| 360 | * |
||
| 361 | * @param string $path |
||
| 362 | * @param bool $recursive |
||
| 363 | * @param int $reuse |
||
| 364 | * @param int $folderId id for the folder to be scanned |
||
| 365 | * @param bool $lock set to false to disable getting an additional read lock during scanning |
||
| 366 | * @return int the size of the scanned folder or -1 if the size is unknown at this stage |
||
| 367 | */ |
||
| 368 | protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) { |
||
| 369 | View Code Duplication | if ($reuse === -1) { |
|
| 370 | $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; |
||
| 371 | } |
||
| 372 | $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', array($path, $this->storageId)); |
||
| 373 | $size = 0; |
||
| 374 | if (!is_null($folderId)) { |
||
| 375 | $folderId = $this->cache->getId($path); |
||
| 376 | } |
||
| 377 | $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size); |
||
| 378 | |||
| 379 | foreach ($childQueue as $child => $childId) { |
||
| 380 | $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock); |
||
| 381 | if ($childSize === -1) { |
||
| 382 | $size = -1; |
||
| 383 | } else if ($size !== -1) { |
||
| 384 | $size += $childSize; |
||
| 385 | } |
||
| 386 | } |
||
| 387 | if ($this->cacheActive) { |
||
| 388 | $this->cache->update($folderId, array('size' => $size)); |
||
| 389 | } |
||
| 390 | $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', array($path, $this->storageId)); |
||
| 391 | return $size; |
||
| 392 | } |
||
| 393 | |||
| 394 | private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) { |
||
| 395 | // we put this in it's own function so it cleans up the memory before we start recursing |
||
| 396 | $existingChildren = $this->getExistingChildren($folderId); |
||
| 397 | $newChildren = $this->getNewChildren($path); |
||
| 398 | |||
| 399 | if ($this->useTransactions) { |
||
| 400 | \OC::$server->getDatabaseConnection()->beginTransaction(); |
||
| 401 | } |
||
| 402 | |||
| 403 | $exceptionOccurred = false; |
||
| 404 | $childQueue = []; |
||
| 405 | foreach ($newChildren as $file) { |
||
| 406 | $child = ($path) ? $path . '/' . $file : $file; |
||
| 407 | try { |
||
| 408 | $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : null; |
||
| 409 | $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock); |
||
| 410 | if ($data) { |
||
| 411 | if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) { |
||
| 412 | $childQueue[$child] = $data['fileid']; |
||
| 413 | } else if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE_INCOMPLETE and $data['size'] === -1) { |
||
| 414 | // only recurse into folders which aren't fully scanned |
||
| 415 | $childQueue[$child] = $data['fileid']; |
||
| 416 | } else if ($data['size'] === -1) { |
||
| 417 | $size = -1; |
||
| 418 | } else if ($size !== -1) { |
||
| 419 | $size += $data['size']; |
||
| 420 | } |
||
| 421 | } |
||
| 422 | } catch (\Doctrine\DBAL\DBALException $ex) { |
||
| 423 | // might happen if inserting duplicate while a scanning |
||
| 424 | // process is running in parallel |
||
| 425 | // log and ignore |
||
| 426 | \OCP\Util::writeLog('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OCP\Util::DEBUG); |
||
| 427 | $exceptionOccurred = true; |
||
| 428 | } catch (\OCP\Lock\LockedException $e) { |
||
| 429 | if ($this->useTransactions) { |
||
| 430 | \OC::$server->getDatabaseConnection()->rollback(); |
||
| 431 | } |
||
| 432 | throw $e; |
||
| 433 | } |
||
| 434 | } |
||
| 435 | $removedChildren = \array_diff(array_keys($existingChildren), $newChildren); |
||
| 436 | foreach ($removedChildren as $childName) { |
||
| 437 | $child = ($path) ? $path . '/' . $childName : $childName; |
||
| 438 | $this->removeFromCache($child); |
||
| 439 | } |
||
| 440 | if ($this->useTransactions) { |
||
| 441 | \OC::$server->getDatabaseConnection()->commit(); |
||
| 442 | } |
||
| 443 | if ($exceptionOccurred) { |
||
| 444 | // It might happen that the parallel scan process has already |
||
| 445 | // inserted mimetypes but those weren't available yet inside the transaction |
||
| 446 | // To make sure to have the updated mime types in such cases, |
||
| 447 | // we reload them here |
||
| 448 | \OC::$server->getMimeTypeLoader()->reset(); |
||
| 449 | } |
||
| 450 | return $childQueue; |
||
| 451 | } |
||
| 452 | |||
| 453 | /** |
||
| 454 | * check if the file should be ignored when scanning |
||
| 455 | * NOTE: files with a '.part' extension are ignored as well! |
||
| 456 | * prevents unfinished put requests to be scanned |
||
| 457 | * |
||
| 458 | * @param string $file |
||
| 459 | * @return boolean |
||
| 460 | */ |
||
| 461 | public static function isPartialFile($file) { |
||
| 471 | |||
| 472 | /** |
||
| 473 | * walk over any folders that are not fully scanned yet and scan them |
||
| 474 | */ |
||
| 475 | public function backgroundScan() { |
||
| 476 | if (!$this->cache->inCache('')) { |
||
| 477 | $this->runBackgroundScanJob(function () { |
||
| 478 | $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG); |
||
| 479 | }, ''); |
||
| 480 | } else { |
||
| 481 | $lastPath = null; |
||
| 482 | while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { |
||
| 483 | $this->runBackgroundScanJob(function() use ($path) { |
||
| 484 | $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE); |
||
| 485 | }, $path); |
||
| 486 | // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() |
||
| 487 | // to make this possible |
||
| 488 | $lastPath = $path; |
||
| 489 | } |
||
| 490 | } |
||
| 491 | } |
||
| 492 | |||
| 493 | private function runBackgroundScanJob(callable $callback, $path) { |
||
| 510 | |||
| 511 | /** |
||
| 512 | * Set whether the cache is affected by scan operations |
||
| 513 | * |
||
| 514 | * @param boolean $active The active state of the cache |
||
| 515 | */ |
||
| 516 | public function setCacheActive($active) { |
||
| 519 | } |
||
| 520 |
Loading history...
Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation.
You can also find more detailed suggestions in the “Code” section of your repository.