nextcloud /
server
@@ -33,150 +33,150 @@ |
||
| 33 | 33 | */ |
| 34 | 34 | class Loader implements IMimeTypeLoader { |
| 35 | 35 | |
| 36 | - /** @var IDBConnection */ |
|
| 37 | - private $dbConnection; |
|
| 38 | - |
|
| 39 | - /** @var array [id => mimetype] */ |
|
| 40 | - protected $mimetypes; |
|
| 41 | - |
|
| 42 | - /** @var array [mimetype => id] */ |
|
| 43 | - protected $mimetypeIds; |
|
| 44 | - |
|
| 45 | - /** |
|
| 46 | - * @param IDBConnection $dbConnection |
|
| 47 | - */ |
|
| 48 | - public function __construct(IDBConnection $dbConnection) { |
|
| 49 | - $this->dbConnection = $dbConnection; |
|
| 50 | - $this->mimetypes = []; |
|
| 51 | - $this->mimetypeIds = []; |
|
| 52 | - } |
|
| 53 | - |
|
| 54 | - /** |
|
| 55 | - * Get a mimetype from its ID |
|
| 56 | - * |
|
| 57 | - * @param int $id |
|
| 58 | - * @return string|null |
|
| 59 | - */ |
|
| 60 | - public function getMimetypeById($id) { |
|
| 61 | - if (!$this->mimetypes) { |
|
| 62 | - $this->loadMimetypes(); |
|
| 63 | - } |
|
| 64 | - if (isset($this->mimetypes[$id])) { |
|
| 65 | - return $this->mimetypes[$id]; |
|
| 66 | - } |
|
| 67 | - return null; |
|
| 68 | - } |
|
| 69 | - |
|
| 70 | - /** |
|
| 71 | - * Get a mimetype ID, adding the mimetype to the DB if it does not exist |
|
| 72 | - * |
|
| 73 | - * @param string $mimetype |
|
| 74 | - * @return int |
|
| 75 | - */ |
|
| 76 | - public function getId($mimetype) { |
|
| 77 | - if (!$this->mimetypeIds) { |
|
| 78 | - $this->loadMimetypes(); |
|
| 79 | - } |
|
| 80 | - if (isset($this->mimetypeIds[$mimetype])) { |
|
| 81 | - return $this->mimetypeIds[$mimetype]; |
|
| 82 | - } |
|
| 83 | - return $this->store($mimetype); |
|
| 84 | - } |
|
| 85 | - |
|
| 86 | - /** |
|
| 87 | - * Test if a mimetype exists in the database |
|
| 88 | - * |
|
| 89 | - * @param string $mimetype |
|
| 90 | - * @return bool |
|
| 91 | - */ |
|
| 92 | - public function exists($mimetype) { |
|
| 93 | - if (!$this->mimetypeIds) { |
|
| 94 | - $this->loadMimetypes(); |
|
| 95 | - } |
|
| 96 | - return isset($this->mimetypeIds[$mimetype]); |
|
| 97 | - } |
|
| 98 | - |
|
| 99 | - /** |
|
| 100 | - * Clear all loaded mimetypes, allow for re-loading |
|
| 101 | - */ |
|
| 102 | - public function reset() { |
|
| 103 | - $this->mimetypes = []; |
|
| 104 | - $this->mimetypeIds = []; |
|
| 105 | - } |
|
| 106 | - |
|
| 107 | - /** |
|
| 108 | - * Store a mimetype in the DB |
|
| 109 | - * |
|
| 110 | - * @param string $mimetype |
|
| 111 | - * @param int inserted ID |
|
| 112 | - */ |
|
| 113 | - protected function store($mimetype) { |
|
| 114 | - try { |
|
| 115 | - $qb = $this->dbConnection->getQueryBuilder(); |
|
| 116 | - $qb->insert('mimetypes') |
|
| 117 | - ->values([ |
|
| 118 | - 'mimetype' => $qb->createNamedParameter($mimetype) |
|
| 119 | - ]); |
|
| 120 | - $qb->execute(); |
|
| 121 | - } catch (UniqueConstraintViolationException $e) { |
|
| 122 | - if ($this->dbConnection->inTransaction()) { |
|
| 123 | - // if we're inside a transaction we can't recover safely |
|
| 124 | - throw $e; |
|
| 125 | - } |
|
| 126 | - // something inserted it before us |
|
| 127 | - } |
|
| 128 | - |
|
| 129 | - $fetch = $this->dbConnection->getQueryBuilder(); |
|
| 130 | - $fetch->select('id') |
|
| 131 | - ->from('mimetypes') |
|
| 132 | - ->where( |
|
| 133 | - $fetch->expr()->eq('mimetype', $fetch->createNamedParameter($mimetype) |
|
| 134 | - )); |
|
| 135 | - $row = $fetch->execute()->fetch(); |
|
| 136 | - |
|
| 137 | - $this->mimetypes[$row['id']] = $mimetype; |
|
| 138 | - $this->mimetypeIds[$mimetype] = $row['id']; |
|
| 139 | - return $row['id']; |
|
| 140 | - } |
|
| 141 | - |
|
| 142 | - /** |
|
| 143 | - * Load all mimetypes from DB |
|
| 144 | - */ |
|
| 145 | - private function loadMimetypes() { |
|
| 146 | - $qb = $this->dbConnection->getQueryBuilder(); |
|
| 147 | - $qb->select('id', 'mimetype') |
|
| 148 | - ->from('mimetypes'); |
|
| 149 | - $results = $qb->execute()->fetchAll(); |
|
| 150 | - |
|
| 151 | - foreach ($results as $row) { |
|
| 152 | - $this->mimetypes[$row['id']] = $row['mimetype']; |
|
| 153 | - $this->mimetypeIds[$row['mimetype']] = $row['id']; |
|
| 154 | - } |
|
| 155 | - } |
|
| 156 | - |
|
| 157 | - /** |
|
| 158 | - * Update filecache mimetype based on file extension |
|
| 159 | - * |
|
| 160 | - * @param string $ext file extension |
|
| 161 | - * @param int $mimeTypeId |
|
| 162 | - * @return int number of changed rows |
|
| 163 | - */ |
|
| 164 | - public function updateFilecache($ext, $mimeTypeId) { |
|
| 165 | - $folderMimeTypeId = $this->getId('httpd/unix-directory'); |
|
| 166 | - $update = $this->dbConnection->getQueryBuilder(); |
|
| 167 | - $update->update('filecache') |
|
| 168 | - ->set('mimetype', $update->createNamedParameter($mimeTypeId)) |
|
| 169 | - ->where($update->expr()->neq( |
|
| 170 | - 'mimetype', $update->createNamedParameter($mimeTypeId) |
|
| 171 | - )) |
|
| 172 | - ->andWhere($update->expr()->neq( |
|
| 173 | - 'mimetype', $update->createNamedParameter($folderMimeTypeId) |
|
| 174 | - )) |
|
| 175 | - ->andWhere($update->expr()->like( |
|
| 176 | - $update->createFunction('LOWER(' . $update->getColumnName('name') . ')'), |
|
| 177 | - $update->createNamedParameter('%' . $this->dbConnection->escapeLikeParameter('.' . $ext)) |
|
| 178 | - )); |
|
| 179 | - return $update->execute(); |
|
| 180 | - } |
|
| 36 | + /** @var IDBConnection */ |
|
| 37 | + private $dbConnection; |
|
| 38 | + |
|
| 39 | + /** @var array [id => mimetype] */ |
|
| 40 | + protected $mimetypes; |
|
| 41 | + |
|
| 42 | + /** @var array [mimetype => id] */ |
|
| 43 | + protected $mimetypeIds; |
|
| 44 | + |
|
| 45 | + /** |
|
| 46 | + * @param IDBConnection $dbConnection |
|
| 47 | + */ |
|
| 48 | + public function __construct(IDBConnection $dbConnection) { |
|
| 49 | + $this->dbConnection = $dbConnection; |
|
| 50 | + $this->mimetypes = []; |
|
| 51 | + $this->mimetypeIds = []; |
|
| 52 | + } |
|
| 53 | + |
|
| 54 | + /** |
|
| 55 | + * Get a mimetype from its ID |
|
| 56 | + * |
|
| 57 | + * @param int $id |
|
| 58 | + * @return string|null |
|
| 59 | + */ |
|
| 60 | + public function getMimetypeById($id) { |
|
| 61 | + if (!$this->mimetypes) { |
|
| 62 | + $this->loadMimetypes(); |
|
| 63 | + } |
|
| 64 | + if (isset($this->mimetypes[$id])) { |
|
| 65 | + return $this->mimetypes[$id]; |
|
| 66 | + } |
|
| 67 | + return null; |
|
| 68 | + } |
|
| 69 | + |
|
| 70 | + /** |
|
| 71 | + * Get a mimetype ID, adding the mimetype to the DB if it does not exist |
|
| 72 | + * |
|
| 73 | + * @param string $mimetype |
|
| 74 | + * @return int |
|
| 75 | + */ |
|
| 76 | + public function getId($mimetype) { |
|
| 77 | + if (!$this->mimetypeIds) { |
|
| 78 | + $this->loadMimetypes(); |
|
| 79 | + } |
|
| 80 | + if (isset($this->mimetypeIds[$mimetype])) { |
|
| 81 | + return $this->mimetypeIds[$mimetype]; |
|
| 82 | + } |
|
| 83 | + return $this->store($mimetype); |
|
| 84 | + } |
|
| 85 | + |
|
| 86 | + /** |
|
| 87 | + * Test if a mimetype exists in the database |
|
| 88 | + * |
|
| 89 | + * @param string $mimetype |
|
| 90 | + * @return bool |
|
| 91 | + */ |
|
| 92 | + public function exists($mimetype) { |
|
| 93 | + if (!$this->mimetypeIds) { |
|
| 94 | + $this->loadMimetypes(); |
|
| 95 | + } |
|
| 96 | + return isset($this->mimetypeIds[$mimetype]); |
|
| 97 | + } |
|
| 98 | + |
|
| 99 | + /** |
|
| 100 | + * Clear all loaded mimetypes, allow for re-loading |
|
| 101 | + */ |
|
| 102 | + public function reset() { |
|
| 103 | + $this->mimetypes = []; |
|
| 104 | + $this->mimetypeIds = []; |
|
| 105 | + } |
|
| 106 | + |
|
| 107 | + /** |
|
| 108 | + * Store a mimetype in the DB |
|
| 109 | + * |
|
| 110 | + * @param string $mimetype |
|
| 111 | + * @param int inserted ID |
|
| 112 | + */ |
|
| 113 | + protected function store($mimetype) { |
|
| 114 | + try { |
|
| 115 | + $qb = $this->dbConnection->getQueryBuilder(); |
|
| 116 | + $qb->insert('mimetypes') |
|
| 117 | + ->values([ |
|
| 118 | + 'mimetype' => $qb->createNamedParameter($mimetype) |
|
| 119 | + ]); |
|
| 120 | + $qb->execute(); |
|
| 121 | + } catch (UniqueConstraintViolationException $e) { |
|
| 122 | + if ($this->dbConnection->inTransaction()) { |
|
| 123 | + // if we're inside a transaction we can't recover safely |
|
| 124 | + throw $e; |
|
| 125 | + } |
|
| 126 | + // something inserted it before us |
|
| 127 | + } |
|
| 128 | + |
|
| 129 | + $fetch = $this->dbConnection->getQueryBuilder(); |
|
| 130 | + $fetch->select('id') |
|
| 131 | + ->from('mimetypes') |
|
| 132 | + ->where( |
|
| 133 | + $fetch->expr()->eq('mimetype', $fetch->createNamedParameter($mimetype) |
|
| 134 | + )); |
|
| 135 | + $row = $fetch->execute()->fetch(); |
|
| 136 | + |
|
| 137 | + $this->mimetypes[$row['id']] = $mimetype; |
|
| 138 | + $this->mimetypeIds[$mimetype] = $row['id']; |
|
| 139 | + return $row['id']; |
|
| 140 | + } |
|
| 141 | + |
|
| 142 | + /** |
|
| 143 | + * Load all mimetypes from DB |
|
| 144 | + */ |
|
| 145 | + private function loadMimetypes() { |
|
| 146 | + $qb = $this->dbConnection->getQueryBuilder(); |
|
| 147 | + $qb->select('id', 'mimetype') |
|
| 148 | + ->from('mimetypes'); |
|
| 149 | + $results = $qb->execute()->fetchAll(); |
|
| 150 | + |
|
| 151 | + foreach ($results as $row) { |
|
| 152 | + $this->mimetypes[$row['id']] = $row['mimetype']; |
|
| 153 | + $this->mimetypeIds[$row['mimetype']] = $row['id']; |
|
| 154 | + } |
|
| 155 | + } |
|
| 156 | + |
|
| 157 | + /** |
|
| 158 | + * Update filecache mimetype based on file extension |
|
| 159 | + * |
|
| 160 | + * @param string $ext file extension |
|
| 161 | + * @param int $mimeTypeId |
|
| 162 | + * @return int number of changed rows |
|
| 163 | + */ |
|
| 164 | + public function updateFilecache($ext, $mimeTypeId) { |
|
| 165 | + $folderMimeTypeId = $this->getId('httpd/unix-directory'); |
|
| 166 | + $update = $this->dbConnection->getQueryBuilder(); |
|
| 167 | + $update->update('filecache') |
|
| 168 | + ->set('mimetype', $update->createNamedParameter($mimeTypeId)) |
|
| 169 | + ->where($update->expr()->neq( |
|
| 170 | + 'mimetype', $update->createNamedParameter($mimeTypeId) |
|
| 171 | + )) |
|
| 172 | + ->andWhere($update->expr()->neq( |
|
| 173 | + 'mimetype', $update->createNamedParameter($folderMimeTypeId) |
|
| 174 | + )) |
|
| 175 | + ->andWhere($update->expr()->like( |
|
| 176 | + $update->createFunction('LOWER(' . $update->getColumnName('name') . ')'), |
|
| 177 | + $update->createNamedParameter('%' . $this->dbConnection->escapeLikeParameter('.' . $ext)) |
|
| 178 | + )); |
|
| 179 | + return $update->execute(); |
|
| 180 | + } |
|
| 181 | 181 | |
| 182 | 182 | } |
@@ -54,497 +54,497 @@ |
||
| 54 | 54 | * @package OC\Files\Cache |
| 55 | 55 | */ |
| 56 | 56 | class Scanner extends BasicEmitter implements IScanner { |
| 57 | - /** |
|
| 58 | - * @var \OC\Files\Storage\Storage $storage |
|
| 59 | - */ |
|
| 60 | - protected $storage; |
|
| 61 | - |
|
| 62 | - /** |
|
| 63 | - * @var string $storageId |
|
| 64 | - */ |
|
| 65 | - protected $storageId; |
|
| 66 | - |
|
| 67 | - /** |
|
| 68 | - * @var \OC\Files\Cache\Cache $cache |
|
| 69 | - */ |
|
| 70 | - protected $cache; |
|
| 71 | - |
|
| 72 | - /** |
|
| 73 | - * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache |
|
| 74 | - */ |
|
| 75 | - protected $cacheActive; |
|
| 76 | - |
|
| 77 | - /** |
|
| 78 | - * @var bool $useTransactions whether to use transactions |
|
| 79 | - */ |
|
| 80 | - protected $useTransactions = true; |
|
| 81 | - |
|
| 82 | - /** |
|
| 83 | - * @var \OCP\Lock\ILockingProvider |
|
| 84 | - */ |
|
| 85 | - protected $lockingProvider; |
|
| 86 | - |
|
| 87 | - public function __construct(\OC\Files\Storage\Storage $storage) { |
|
| 88 | - $this->storage = $storage; |
|
| 89 | - $this->storageId = $this->storage->getId(); |
|
| 90 | - $this->cache = $storage->getCache(); |
|
| 91 | - $this->cacheActive = !\OC::$server->getConfig()->getSystemValue('filesystem_cache_readonly', false); |
|
| 92 | - $this->lockingProvider = \OC::$server->getLockingProvider(); |
|
| 93 | - } |
|
| 94 | - |
|
| 95 | - /** |
|
| 96 | - * Whether to wrap the scanning of a folder in a database transaction |
|
| 97 | - * On default transactions are used |
|
| 98 | - * |
|
| 99 | - * @param bool $useTransactions |
|
| 100 | - */ |
|
| 101 | - public function setUseTransactions($useTransactions) { |
|
| 102 | - $this->useTransactions = $useTransactions; |
|
| 103 | - } |
|
| 104 | - |
|
| 105 | - /** |
|
| 106 | - * get all the metadata of a file or folder |
|
| 107 | - * * |
|
| 108 | - * |
|
| 109 | - * @param string $path |
|
| 110 | - * @return array an array of metadata of the file |
|
| 111 | - */ |
|
| 112 | - protected function getData($path) { |
|
| 113 | - $data = $this->storage->getMetaData($path); |
|
| 114 | - if (is_null($data)) { |
|
| 115 | - \OCP\Util::writeLog('OC\Files\Cache\Scanner', "!!! Path '$path' is not accessible or present !!!", \OCP\Util::DEBUG); |
|
| 116 | - } |
|
| 117 | - return $data; |
|
| 118 | - } |
|
| 119 | - |
|
| 120 | - /** |
|
| 121 | - * scan a single file and store it in the cache |
|
| 122 | - * |
|
| 123 | - * @param string $file |
|
| 124 | - * @param int $reuseExisting |
|
| 125 | - * @param int $parentId |
|
| 126 | - * @param array | null $cacheData existing data in the cache for the file to be scanned |
|
| 127 | - * @param bool $lock set to false to disable getting an additional read lock during scanning |
|
| 128 | - * @return array an array of metadata of the scanned file |
|
| 129 | - * @throws \OC\ServerNotAvailableException |
|
| 130 | - * @throws \OCP\Lock\LockedException |
|
| 131 | - */ |
|
| 132 | - public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) { |
|
| 133 | - if ($file !== '') { |
|
| 134 | - try { |
|
| 135 | - $this->storage->verifyPath(dirname($file), basename($file)); |
|
| 136 | - } catch (\Exception $e) { |
|
| 137 | - return null; |
|
| 138 | - } |
|
| 139 | - } |
|
| 140 | - // only proceed if $file is not a partial file nor a blacklisted file |
|
| 141 | - if (!self::isPartialFile($file) and !Filesystem::isFileBlacklisted($file)) { |
|
| 142 | - |
|
| 143 | - //acquire a lock |
|
| 144 | - if ($lock) { |
|
| 145 | - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 146 | - $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 147 | - } |
|
| 148 | - } |
|
| 149 | - |
|
| 150 | - try { |
|
| 151 | - $data = $this->getData($file); |
|
| 152 | - } catch (ForbiddenException $e) { |
|
| 153 | - if ($lock) { |
|
| 154 | - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 155 | - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 156 | - } |
|
| 157 | - } |
|
| 158 | - |
|
| 159 | - return null; |
|
| 160 | - } |
|
| 161 | - |
|
| 162 | - try { |
|
| 163 | - if ($data) { |
|
| 164 | - |
|
| 165 | - // pre-emit only if it was a file. By that we avoid counting/treating folders as files |
|
| 166 | - if ($data['mimetype'] !== 'httpd/unix-directory') { |
|
| 167 | - $this->emit('\OC\Files\Cache\Scanner', 'scanFile', array($file, $this->storageId)); |
|
| 168 | - \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', array('path' => $file, 'storage' => $this->storageId)); |
|
| 169 | - } |
|
| 170 | - |
|
| 171 | - $parent = dirname($file); |
|
| 172 | - if ($parent === '.' or $parent === '/') { |
|
| 173 | - $parent = ''; |
|
| 174 | - } |
|
| 175 | - if ($parentId === -1) { |
|
| 176 | - $parentId = $this->cache->getParentId($file); |
|
| 177 | - } |
|
| 178 | - |
|
| 179 | - // scan the parent if it's not in the cache (id -1) and the current file is not the root folder |
|
| 180 | - if ($file and $parentId === -1) { |
|
| 181 | - $parentData = $this->scanFile($parent); |
|
| 182 | - if (!$parentData) { |
|
| 183 | - return null; |
|
| 184 | - } |
|
| 185 | - $parentId = $parentData['fileid']; |
|
| 186 | - } |
|
| 187 | - if ($parent) { |
|
| 188 | - $data['parent'] = $parentId; |
|
| 189 | - } |
|
| 190 | - if (is_null($cacheData)) { |
|
| 191 | - /** @var CacheEntry $cacheData */ |
|
| 192 | - $cacheData = $this->cache->get($file); |
|
| 193 | - } |
|
| 194 | - if ($cacheData and $reuseExisting and isset($cacheData['fileid'])) { |
|
| 195 | - // prevent empty etag |
|
| 196 | - if (empty($cacheData['etag'])) { |
|
| 197 | - $etag = $data['etag']; |
|
| 198 | - } else { |
|
| 199 | - $etag = $cacheData['etag']; |
|
| 200 | - } |
|
| 201 | - $fileId = $cacheData['fileid']; |
|
| 202 | - $data['fileid'] = $fileId; |
|
| 203 | - // only reuse data if the file hasn't explicitly changed |
|
| 204 | - if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) { |
|
| 205 | - $data['mtime'] = $cacheData['mtime']; |
|
| 206 | - if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { |
|
| 207 | - $data['size'] = $cacheData['size']; |
|
| 208 | - } |
|
| 209 | - if ($reuseExisting & self::REUSE_ETAG) { |
|
| 210 | - $data['etag'] = $etag; |
|
| 211 | - } |
|
| 212 | - } |
|
| 213 | - // Only update metadata that has changed |
|
| 214 | - $newData = array_diff_assoc($data, $cacheData->getData()); |
|
| 215 | - } else { |
|
| 216 | - $newData = $data; |
|
| 217 | - $fileId = -1; |
|
| 218 | - } |
|
| 219 | - if (!empty($newData)) { |
|
| 220 | - // Reset the checksum if the data has changed |
|
| 221 | - $newData['checksum'] = ''; |
|
| 222 | - $data['fileid'] = $this->addToCache($file, $newData, $fileId); |
|
| 223 | - } |
|
| 224 | - if (isset($cacheData['size'])) { |
|
| 225 | - $data['oldSize'] = $cacheData['size']; |
|
| 226 | - } else { |
|
| 227 | - $data['oldSize'] = 0; |
|
| 228 | - } |
|
| 229 | - |
|
| 230 | - if (isset($cacheData['encrypted'])) { |
|
| 231 | - $data['encrypted'] = $cacheData['encrypted']; |
|
| 232 | - } |
|
| 233 | - |
|
| 234 | - // post-emit only if it was a file. By that we avoid counting/treating folders as files |
|
| 235 | - if ($data['mimetype'] !== 'httpd/unix-directory') { |
|
| 236 | - $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId)); |
|
| 237 | - \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId)); |
|
| 238 | - } |
|
| 239 | - |
|
| 240 | - } else { |
|
| 241 | - $this->removeFromCache($file); |
|
| 242 | - } |
|
| 243 | - } catch (\Exception $e) { |
|
| 244 | - if ($lock) { |
|
| 245 | - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 246 | - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 247 | - } |
|
| 248 | - } |
|
| 249 | - throw $e; |
|
| 250 | - } |
|
| 251 | - |
|
| 252 | - //release the acquired lock |
|
| 253 | - if ($lock) { |
|
| 254 | - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 255 | - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 256 | - } |
|
| 257 | - } |
|
| 258 | - |
|
| 259 | - if ($data && !isset($data['encrypted'])) { |
|
| 260 | - $data['encrypted'] = false; |
|
| 261 | - } |
|
| 262 | - return $data; |
|
| 263 | - } |
|
| 264 | - |
|
| 265 | - return null; |
|
| 266 | - } |
|
| 267 | - |
|
| 268 | - protected function removeFromCache($path) { |
|
| 269 | - \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $path)); |
|
| 270 | - $this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', array($path)); |
|
| 271 | - if ($this->cacheActive) { |
|
| 272 | - $this->cache->remove($path); |
|
| 273 | - } |
|
| 274 | - } |
|
| 275 | - |
|
| 276 | - /** |
|
| 277 | - * @param string $path |
|
| 278 | - * @param array $data |
|
| 279 | - * @param int $fileId |
|
| 280 | - * @return int the id of the added file |
|
| 281 | - */ |
|
| 282 | - protected function addToCache($path, $data, $fileId = -1) { |
|
| 283 | - if (isset($data['scan_permissions'])) { |
|
| 284 | - $data['permissions'] = $data['scan_permissions']; |
|
| 285 | - } |
|
| 286 | - \OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data)); |
|
| 287 | - $this->emit('\OC\Files\Cache\Scanner', 'addToCache', array($path, $this->storageId, $data)); |
|
| 288 | - if ($this->cacheActive) { |
|
| 289 | - if ($fileId !== -1) { |
|
| 290 | - $this->cache->update($fileId, $data); |
|
| 291 | - return $fileId; |
|
| 292 | - } else { |
|
| 293 | - return $this->cache->put($path, $data); |
|
| 294 | - } |
|
| 295 | - } else { |
|
| 296 | - return -1; |
|
| 297 | - } |
|
| 298 | - } |
|
| 299 | - |
|
| 300 | - /** |
|
| 301 | - * @param string $path |
|
| 302 | - * @param array $data |
|
| 303 | - * @param int $fileId |
|
| 304 | - */ |
|
| 305 | - protected function updateCache($path, $data, $fileId = -1) { |
|
| 306 | - \OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data)); |
|
| 307 | - $this->emit('\OC\Files\Cache\Scanner', 'updateCache', array($path, $this->storageId, $data)); |
|
| 308 | - if ($this->cacheActive) { |
|
| 309 | - if ($fileId !== -1) { |
|
| 310 | - $this->cache->update($fileId, $data); |
|
| 311 | - } else { |
|
| 312 | - $this->cache->put($path, $data); |
|
| 313 | - } |
|
| 314 | - } |
|
| 315 | - } |
|
| 316 | - |
|
| 317 | - /** |
|
| 318 | - * scan a folder and all it's children |
|
| 319 | - * |
|
| 320 | - * @param string $path |
|
| 321 | - * @param bool $recursive |
|
| 322 | - * @param int $reuse |
|
| 323 | - * @param bool $lock set to false to disable getting an additional read lock during scanning |
|
| 324 | - * @return array an array of the meta data of the scanned file or folder |
|
| 325 | - */ |
|
| 326 | - public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { |
|
| 327 | - if ($reuse === -1) { |
|
| 328 | - $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; |
|
| 329 | - } |
|
| 330 | - if ($lock) { |
|
| 331 | - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 332 | - $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); |
|
| 333 | - $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 334 | - } |
|
| 335 | - } |
|
| 336 | - $data = $this->scanFile($path, $reuse, -1, null, $lock); |
|
| 337 | - if ($data and $data['mimetype'] === 'httpd/unix-directory') { |
|
| 338 | - $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock); |
|
| 339 | - $data['size'] = $size; |
|
| 340 | - } |
|
| 341 | - if ($lock) { |
|
| 342 | - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 343 | - $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 344 | - $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); |
|
| 345 | - } |
|
| 346 | - } |
|
| 347 | - return $data; |
|
| 348 | - } |
|
| 349 | - |
|
| 350 | - /** |
|
| 351 | - * Get the children currently in the cache |
|
| 352 | - * |
|
| 353 | - * @param int $folderId |
|
| 354 | - * @return array[] |
|
| 355 | - */ |
|
| 356 | - protected function getExistingChildren($folderId) { |
|
| 357 | - $existingChildren = array(); |
|
| 358 | - $children = $this->cache->getFolderContentsById($folderId); |
|
| 359 | - foreach ($children as $child) { |
|
| 360 | - $existingChildren[$child['name']] = $child; |
|
| 361 | - } |
|
| 362 | - return $existingChildren; |
|
| 363 | - } |
|
| 364 | - |
|
| 365 | - /** |
|
| 366 | - * Get the children from the storage |
|
| 367 | - * |
|
| 368 | - * @param string $folder |
|
| 369 | - * @return string[] |
|
| 370 | - */ |
|
| 371 | - protected function getNewChildren($folder) { |
|
| 372 | - $children = array(); |
|
| 373 | - if ($dh = $this->storage->opendir($folder)) { |
|
| 374 | - if (is_resource($dh)) { |
|
| 375 | - while (($file = readdir($dh)) !== false) { |
|
| 376 | - if (!Filesystem::isIgnoredDir($file)) { |
|
| 377 | - $children[] = trim(\OC\Files\Filesystem::normalizePath($file), '/'); |
|
| 378 | - } |
|
| 379 | - } |
|
| 380 | - } |
|
| 381 | - } |
|
| 382 | - return $children; |
|
| 383 | - } |
|
| 384 | - |
|
| 385 | - /** |
|
| 386 | - * scan all the files and folders in a folder |
|
| 387 | - * |
|
| 388 | - * @param string $path |
|
| 389 | - * @param bool $recursive |
|
| 390 | - * @param int $reuse |
|
| 391 | - * @param int $folderId id for the folder to be scanned |
|
| 392 | - * @param bool $lock set to false to disable getting an additional read lock during scanning |
|
| 393 | - * @return int the size of the scanned folder or -1 if the size is unknown at this stage |
|
| 394 | - */ |
|
| 395 | - protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) { |
|
| 396 | - if ($reuse === -1) { |
|
| 397 | - $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; |
|
| 398 | - } |
|
| 399 | - $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', array($path, $this->storageId)); |
|
| 400 | - $size = 0; |
|
| 401 | - if (!is_null($folderId)) { |
|
| 402 | - $folderId = $this->cache->getId($path); |
|
| 403 | - } |
|
| 404 | - $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size); |
|
| 405 | - |
|
| 406 | - foreach ($childQueue as $child => $childId) { |
|
| 407 | - $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock); |
|
| 408 | - if ($childSize === -1) { |
|
| 409 | - $size = -1; |
|
| 410 | - } else if ($size !== -1) { |
|
| 411 | - $size += $childSize; |
|
| 412 | - } |
|
| 413 | - } |
|
| 414 | - if ($this->cacheActive) { |
|
| 415 | - $this->cache->update($folderId, array('size' => $size)); |
|
| 416 | - } |
|
| 417 | - $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', array($path, $this->storageId)); |
|
| 418 | - return $size; |
|
| 419 | - } |
|
| 420 | - |
|
| 421 | - private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) { |
|
| 422 | - // we put this in it's own function so it cleans up the memory before we start recursing |
|
| 423 | - $existingChildren = $this->getExistingChildren($folderId); |
|
| 424 | - $newChildren = $this->getNewChildren($path); |
|
| 425 | - |
|
| 426 | - if ($this->useTransactions) { |
|
| 427 | - \OC::$server->getDatabaseConnection()->beginTransaction(); |
|
| 428 | - } |
|
| 429 | - |
|
| 430 | - $exceptionOccurred = false; |
|
| 431 | - $childQueue = []; |
|
| 432 | - foreach ($newChildren as $file) { |
|
| 433 | - $child = ($path) ? $path . '/' . $file : $file; |
|
| 434 | - try { |
|
| 435 | - $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : null; |
|
| 436 | - $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock); |
|
| 437 | - if ($data) { |
|
| 438 | - if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) { |
|
| 439 | - $childQueue[$child] = $data['fileid']; |
|
| 440 | - } else if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE_INCOMPLETE and $data['size'] === -1) { |
|
| 441 | - // only recurse into folders which aren't fully scanned |
|
| 442 | - $childQueue[$child] = $data['fileid']; |
|
| 443 | - } else if ($data['size'] === -1) { |
|
| 444 | - $size = -1; |
|
| 445 | - } else if ($size !== -1) { |
|
| 446 | - $size += $data['size']; |
|
| 447 | - } |
|
| 448 | - } |
|
| 449 | - } catch (\Doctrine\DBAL\DBALException $ex) { |
|
| 450 | - // might happen if inserting duplicate while a scanning |
|
| 451 | - // process is running in parallel |
|
| 452 | - // log and ignore |
|
| 453 | - if ($this->useTransactions) { |
|
| 454 | - \OC::$server->getDatabaseConnection()->rollback(); |
|
| 455 | - \OC::$server->getDatabaseConnection()->beginTransaction(); |
|
| 456 | - } |
|
| 457 | - \OCP\Util::writeLog('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OCP\Util::DEBUG); |
|
| 458 | - $exceptionOccurred = true; |
|
| 459 | - } catch (\OCP\Lock\LockedException $e) { |
|
| 460 | - if ($this->useTransactions) { |
|
| 461 | - \OC::$server->getDatabaseConnection()->rollback(); |
|
| 462 | - } |
|
| 463 | - throw $e; |
|
| 464 | - } |
|
| 465 | - } |
|
| 466 | - $removedChildren = \array_diff(array_keys($existingChildren), $newChildren); |
|
| 467 | - foreach ($removedChildren as $childName) { |
|
| 468 | - $child = ($path) ? $path . '/' . $childName : $childName; |
|
| 469 | - $this->removeFromCache($child); |
|
| 470 | - } |
|
| 471 | - if ($this->useTransactions) { |
|
| 472 | - \OC::$server->getDatabaseConnection()->commit(); |
|
| 473 | - } |
|
| 474 | - if ($exceptionOccurred) { |
|
| 475 | - // It might happen that the parallel scan process has already |
|
| 476 | - // inserted mimetypes but those weren't available yet inside the transaction |
|
| 477 | - // To make sure to have the updated mime types in such cases, |
|
| 478 | - // we reload them here |
|
| 479 | - \OC::$server->getMimeTypeLoader()->reset(); |
|
| 480 | - } |
|
| 481 | - return $childQueue; |
|
| 482 | - } |
|
| 483 | - |
|
| 484 | - /** |
|
| 485 | - * check if the file should be ignored when scanning |
|
| 486 | - * NOTE: files with a '.part' extension are ignored as well! |
|
| 487 | - * prevents unfinished put requests to be scanned |
|
| 488 | - * |
|
| 489 | - * @param string $file |
|
| 490 | - * @return boolean |
|
| 491 | - */ |
|
| 492 | - public static function isPartialFile($file) { |
|
| 493 | - if (pathinfo($file, PATHINFO_EXTENSION) === 'part') { |
|
| 494 | - return true; |
|
| 495 | - } |
|
| 496 | - if (strpos($file, '.part/') !== false) { |
|
| 497 | - return true; |
|
| 498 | - } |
|
| 499 | - |
|
| 500 | - return false; |
|
| 501 | - } |
|
| 502 | - |
|
| 503 | - /** |
|
| 504 | - * walk over any folders that are not fully scanned yet and scan them |
|
| 505 | - */ |
|
| 506 | - public function backgroundScan() { |
|
| 507 | - if (!$this->cache->inCache('')) { |
|
| 508 | - $this->runBackgroundScanJob(function () { |
|
| 509 | - $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG); |
|
| 510 | - }, ''); |
|
| 511 | - } else { |
|
| 512 | - $lastPath = null; |
|
| 513 | - while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { |
|
| 514 | - $this->runBackgroundScanJob(function () use ($path) { |
|
| 515 | - $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE); |
|
| 516 | - }, $path); |
|
| 517 | - // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() |
|
| 518 | - // to make this possible |
|
| 519 | - $lastPath = $path; |
|
| 520 | - } |
|
| 521 | - } |
|
| 522 | - } |
|
| 523 | - |
|
| 524 | - private function runBackgroundScanJob(callable $callback, $path) { |
|
| 525 | - try { |
|
| 526 | - $callback(); |
|
| 527 | - \OC_Hook::emit('Scanner', 'correctFolderSize', array('path' => $path)); |
|
| 528 | - if ($this->cacheActive && $this->cache instanceof Cache) { |
|
| 529 | - $this->cache->correctFolderSize($path); |
|
| 530 | - } |
|
| 531 | - } catch (\OCP\Files\StorageInvalidException $e) { |
|
| 532 | - // skip unavailable storages |
|
| 533 | - } catch (\OCP\Files\StorageNotAvailableException $e) { |
|
| 534 | - // skip unavailable storages |
|
| 535 | - } catch (\OCP\Files\ForbiddenException $e) { |
|
| 536 | - // skip forbidden storages |
|
| 537 | - } catch (\OCP\Lock\LockedException $e) { |
|
| 538 | - // skip unavailable storages |
|
| 539 | - } |
|
| 540 | - } |
|
| 541 | - |
|
| 542 | - /** |
|
| 543 | - * Set whether the cache is affected by scan operations |
|
| 544 | - * |
|
| 545 | - * @param boolean $active The active state of the cache |
|
| 546 | - */ |
|
| 547 | - public function setCacheActive($active) { |
|
| 548 | - $this->cacheActive = $active; |
|
| 549 | - } |
|
| 57 | + /** |
|
| 58 | + * @var \OC\Files\Storage\Storage $storage |
|
| 59 | + */ |
|
| 60 | + protected $storage; |
|
| 61 | + |
|
| 62 | + /** |
|
| 63 | + * @var string $storageId |
|
| 64 | + */ |
|
| 65 | + protected $storageId; |
|
| 66 | + |
|
| 67 | + /** |
|
| 68 | + * @var \OC\Files\Cache\Cache $cache |
|
| 69 | + */ |
|
| 70 | + protected $cache; |
|
| 71 | + |
|
| 72 | + /** |
|
| 73 | + * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache |
|
| 74 | + */ |
|
| 75 | + protected $cacheActive; |
|
| 76 | + |
|
| 77 | + /** |
|
| 78 | + * @var bool $useTransactions whether to use transactions |
|
| 79 | + */ |
|
| 80 | + protected $useTransactions = true; |
|
| 81 | + |
|
| 82 | + /** |
|
| 83 | + * @var \OCP\Lock\ILockingProvider |
|
| 84 | + */ |
|
| 85 | + protected $lockingProvider; |
|
| 86 | + |
|
| 87 | + public function __construct(\OC\Files\Storage\Storage $storage) { |
|
| 88 | + $this->storage = $storage; |
|
| 89 | + $this->storageId = $this->storage->getId(); |
|
| 90 | + $this->cache = $storage->getCache(); |
|
| 91 | + $this->cacheActive = !\OC::$server->getConfig()->getSystemValue('filesystem_cache_readonly', false); |
|
| 92 | + $this->lockingProvider = \OC::$server->getLockingProvider(); |
|
| 93 | + } |
|
| 94 | + |
|
| 95 | + /** |
|
| 96 | + * Whether to wrap the scanning of a folder in a database transaction |
|
| 97 | + * On default transactions are used |
|
| 98 | + * |
|
| 99 | + * @param bool $useTransactions |
|
| 100 | + */ |
|
| 101 | + public function setUseTransactions($useTransactions) { |
|
| 102 | + $this->useTransactions = $useTransactions; |
|
| 103 | + } |
|
| 104 | + |
|
| 105 | + /** |
|
| 106 | + * get all the metadata of a file or folder |
|
| 107 | + * * |
|
| 108 | + * |
|
| 109 | + * @param string $path |
|
| 110 | + * @return array an array of metadata of the file |
|
| 111 | + */ |
|
| 112 | + protected function getData($path) { |
|
| 113 | + $data = $this->storage->getMetaData($path); |
|
| 114 | + if (is_null($data)) { |
|
| 115 | + \OCP\Util::writeLog('OC\Files\Cache\Scanner', "!!! Path '$path' is not accessible or present !!!", \OCP\Util::DEBUG); |
|
| 116 | + } |
|
| 117 | + return $data; |
|
| 118 | + } |
|
| 119 | + |
|
| 120 | + /** |
|
| 121 | + * scan a single file and store it in the cache |
|
| 122 | + * |
|
| 123 | + * @param string $file |
|
| 124 | + * @param int $reuseExisting |
|
| 125 | + * @param int $parentId |
|
| 126 | + * @param array | null $cacheData existing data in the cache for the file to be scanned |
|
| 127 | + * @param bool $lock set to false to disable getting an additional read lock during scanning |
|
| 128 | + * @return array an array of metadata of the scanned file |
|
| 129 | + * @throws \OC\ServerNotAvailableException |
|
| 130 | + * @throws \OCP\Lock\LockedException |
|
| 131 | + */ |
|
| 132 | + public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) { |
|
| 133 | + if ($file !== '') { |
|
| 134 | + try { |
|
| 135 | + $this->storage->verifyPath(dirname($file), basename($file)); |
|
| 136 | + } catch (\Exception $e) { |
|
| 137 | + return null; |
|
| 138 | + } |
|
| 139 | + } |
|
| 140 | + // only proceed if $file is not a partial file nor a blacklisted file |
|
| 141 | + if (!self::isPartialFile($file) and !Filesystem::isFileBlacklisted($file)) { |
|
| 142 | + |
|
| 143 | + //acquire a lock |
|
| 144 | + if ($lock) { |
|
| 145 | + if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 146 | + $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 147 | + } |
|
| 148 | + } |
|
| 149 | + |
|
| 150 | + try { |
|
| 151 | + $data = $this->getData($file); |
|
| 152 | + } catch (ForbiddenException $e) { |
|
| 153 | + if ($lock) { |
|
| 154 | + if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 155 | + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 156 | + } |
|
| 157 | + } |
|
| 158 | + |
|
| 159 | + return null; |
|
| 160 | + } |
|
| 161 | + |
|
| 162 | + try { |
|
| 163 | + if ($data) { |
|
| 164 | + |
|
| 165 | + // pre-emit only if it was a file. By that we avoid counting/treating folders as files |
|
| 166 | + if ($data['mimetype'] !== 'httpd/unix-directory') { |
|
| 167 | + $this->emit('\OC\Files\Cache\Scanner', 'scanFile', array($file, $this->storageId)); |
|
| 168 | + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', array('path' => $file, 'storage' => $this->storageId)); |
|
| 169 | + } |
|
| 170 | + |
|
| 171 | + $parent = dirname($file); |
|
| 172 | + if ($parent === '.' or $parent === '/') { |
|
| 173 | + $parent = ''; |
|
| 174 | + } |
|
| 175 | + if ($parentId === -1) { |
|
| 176 | + $parentId = $this->cache->getParentId($file); |
|
| 177 | + } |
|
| 178 | + |
|
| 179 | + // scan the parent if it's not in the cache (id -1) and the current file is not the root folder |
|
| 180 | + if ($file and $parentId === -1) { |
|
| 181 | + $parentData = $this->scanFile($parent); |
|
| 182 | + if (!$parentData) { |
|
| 183 | + return null; |
|
| 184 | + } |
|
| 185 | + $parentId = $parentData['fileid']; |
|
| 186 | + } |
|
| 187 | + if ($parent) { |
|
| 188 | + $data['parent'] = $parentId; |
|
| 189 | + } |
|
| 190 | + if (is_null($cacheData)) { |
|
| 191 | + /** @var CacheEntry $cacheData */ |
|
| 192 | + $cacheData = $this->cache->get($file); |
|
| 193 | + } |
|
| 194 | + if ($cacheData and $reuseExisting and isset($cacheData['fileid'])) { |
|
| 195 | + // prevent empty etag |
|
| 196 | + if (empty($cacheData['etag'])) { |
|
| 197 | + $etag = $data['etag']; |
|
| 198 | + } else { |
|
| 199 | + $etag = $cacheData['etag']; |
|
| 200 | + } |
|
| 201 | + $fileId = $cacheData['fileid']; |
|
| 202 | + $data['fileid'] = $fileId; |
|
| 203 | + // only reuse data if the file hasn't explicitly changed |
|
| 204 | + if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) { |
|
| 205 | + $data['mtime'] = $cacheData['mtime']; |
|
| 206 | + if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { |
|
| 207 | + $data['size'] = $cacheData['size']; |
|
| 208 | + } |
|
| 209 | + if ($reuseExisting & self::REUSE_ETAG) { |
|
| 210 | + $data['etag'] = $etag; |
|
| 211 | + } |
|
| 212 | + } |
|
| 213 | + // Only update metadata that has changed |
|
| 214 | + $newData = array_diff_assoc($data, $cacheData->getData()); |
|
| 215 | + } else { |
|
| 216 | + $newData = $data; |
|
| 217 | + $fileId = -1; |
|
| 218 | + } |
|
| 219 | + if (!empty($newData)) { |
|
| 220 | + // Reset the checksum if the data has changed |
|
| 221 | + $newData['checksum'] = ''; |
|
| 222 | + $data['fileid'] = $this->addToCache($file, $newData, $fileId); |
|
| 223 | + } |
|
| 224 | + if (isset($cacheData['size'])) { |
|
| 225 | + $data['oldSize'] = $cacheData['size']; |
|
| 226 | + } else { |
|
| 227 | + $data['oldSize'] = 0; |
|
| 228 | + } |
|
| 229 | + |
|
| 230 | + if (isset($cacheData['encrypted'])) { |
|
| 231 | + $data['encrypted'] = $cacheData['encrypted']; |
|
| 232 | + } |
|
| 233 | + |
|
| 234 | + // post-emit only if it was a file. By that we avoid counting/treating folders as files |
|
| 235 | + if ($data['mimetype'] !== 'httpd/unix-directory') { |
|
| 236 | + $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId)); |
|
| 237 | + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId)); |
|
| 238 | + } |
|
| 239 | + |
|
| 240 | + } else { |
|
| 241 | + $this->removeFromCache($file); |
|
| 242 | + } |
|
| 243 | + } catch (\Exception $e) { |
|
| 244 | + if ($lock) { |
|
| 245 | + if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 246 | + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 247 | + } |
|
| 248 | + } |
|
| 249 | + throw $e; |
|
| 250 | + } |
|
| 251 | + |
|
| 252 | + //release the acquired lock |
|
| 253 | + if ($lock) { |
|
| 254 | + if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 255 | + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 256 | + } |
|
| 257 | + } |
|
| 258 | + |
|
| 259 | + if ($data && !isset($data['encrypted'])) { |
|
| 260 | + $data['encrypted'] = false; |
|
| 261 | + } |
|
| 262 | + return $data; |
|
| 263 | + } |
|
| 264 | + |
|
| 265 | + return null; |
|
| 266 | + } |
|
| 267 | + |
|
| 268 | + protected function removeFromCache($path) { |
|
| 269 | + \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $path)); |
|
| 270 | + $this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', array($path)); |
|
| 271 | + if ($this->cacheActive) { |
|
| 272 | + $this->cache->remove($path); |
|
| 273 | + } |
|
| 274 | + } |
|
| 275 | + |
|
| 276 | + /** |
|
| 277 | + * @param string $path |
|
| 278 | + * @param array $data |
|
| 279 | + * @param int $fileId |
|
| 280 | + * @return int the id of the added file |
|
| 281 | + */ |
|
| 282 | + protected function addToCache($path, $data, $fileId = -1) { |
|
| 283 | + if (isset($data['scan_permissions'])) { |
|
| 284 | + $data['permissions'] = $data['scan_permissions']; |
|
| 285 | + } |
|
| 286 | + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data)); |
|
| 287 | + $this->emit('\OC\Files\Cache\Scanner', 'addToCache', array($path, $this->storageId, $data)); |
|
| 288 | + if ($this->cacheActive) { |
|
| 289 | + if ($fileId !== -1) { |
|
| 290 | + $this->cache->update($fileId, $data); |
|
| 291 | + return $fileId; |
|
| 292 | + } else { |
|
| 293 | + return $this->cache->put($path, $data); |
|
| 294 | + } |
|
| 295 | + } else { |
|
| 296 | + return -1; |
|
| 297 | + } |
|
| 298 | + } |
|
| 299 | + |
|
| 300 | + /** |
|
| 301 | + * @param string $path |
|
| 302 | + * @param array $data |
|
| 303 | + * @param int $fileId |
|
| 304 | + */ |
|
| 305 | + protected function updateCache($path, $data, $fileId = -1) { |
|
| 306 | + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data)); |
|
| 307 | + $this->emit('\OC\Files\Cache\Scanner', 'updateCache', array($path, $this->storageId, $data)); |
|
| 308 | + if ($this->cacheActive) { |
|
| 309 | + if ($fileId !== -1) { |
|
| 310 | + $this->cache->update($fileId, $data); |
|
| 311 | + } else { |
|
| 312 | + $this->cache->put($path, $data); |
|
| 313 | + } |
|
| 314 | + } |
|
| 315 | + } |
|
| 316 | + |
|
| 317 | + /** |
|
| 318 | + * scan a folder and all it's children |
|
| 319 | + * |
|
| 320 | + * @param string $path |
|
| 321 | + * @param bool $recursive |
|
| 322 | + * @param int $reuse |
|
| 323 | + * @param bool $lock set to false to disable getting an additional read lock during scanning |
|
| 324 | + * @return array an array of the meta data of the scanned file or folder |
|
| 325 | + */ |
|
| 326 | + public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { |
|
| 327 | + if ($reuse === -1) { |
|
| 328 | + $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; |
|
| 329 | + } |
|
| 330 | + if ($lock) { |
|
| 331 | + if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 332 | + $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); |
|
| 333 | + $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 334 | + } |
|
| 335 | + } |
|
| 336 | + $data = $this->scanFile($path, $reuse, -1, null, $lock); |
|
| 337 | + if ($data and $data['mimetype'] === 'httpd/unix-directory') { |
|
| 338 | + $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock); |
|
| 339 | + $data['size'] = $size; |
|
| 340 | + } |
|
| 341 | + if ($lock) { |
|
| 342 | + if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
|
| 343 | + $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
|
| 344 | + $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); |
|
| 345 | + } |
|
| 346 | + } |
|
| 347 | + return $data; |
|
| 348 | + } |
|
| 349 | + |
|
| 350 | + /** |
|
| 351 | + * Get the children currently in the cache |
|
| 352 | + * |
|
| 353 | + * @param int $folderId |
|
| 354 | + * @return array[] |
|
| 355 | + */ |
|
| 356 | + protected function getExistingChildren($folderId) { |
|
| 357 | + $existingChildren = array(); |
|
| 358 | + $children = $this->cache->getFolderContentsById($folderId); |
|
| 359 | + foreach ($children as $child) { |
|
| 360 | + $existingChildren[$child['name']] = $child; |
|
| 361 | + } |
|
| 362 | + return $existingChildren; |
|
| 363 | + } |
|
| 364 | + |
|
| 365 | + /** |
|
| 366 | + * Get the children from the storage |
|
| 367 | + * |
|
| 368 | + * @param string $folder |
|
| 369 | + * @return string[] |
|
| 370 | + */ |
|
| 371 | + protected function getNewChildren($folder) { |
|
| 372 | + $children = array(); |
|
| 373 | + if ($dh = $this->storage->opendir($folder)) { |
|
| 374 | + if (is_resource($dh)) { |
|
| 375 | + while (($file = readdir($dh)) !== false) { |
|
| 376 | + if (!Filesystem::isIgnoredDir($file)) { |
|
| 377 | + $children[] = trim(\OC\Files\Filesystem::normalizePath($file), '/'); |
|
| 378 | + } |
|
| 379 | + } |
|
| 380 | + } |
|
| 381 | + } |
|
| 382 | + return $children; |
|
| 383 | + } |
|
| 384 | + |
|
| 385 | + /** |
|
| 386 | + * scan all the files and folders in a folder |
|
| 387 | + * |
|
| 388 | + * @param string $path |
|
| 389 | + * @param bool $recursive |
|
| 390 | + * @param int $reuse |
|
| 391 | + * @param int $folderId id for the folder to be scanned |
|
| 392 | + * @param bool $lock set to false to disable getting an additional read lock during scanning |
|
| 393 | + * @return int the size of the scanned folder or -1 if the size is unknown at this stage |
|
| 394 | + */ |
|
| 395 | + protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) { |
|
| 396 | + if ($reuse === -1) { |
|
| 397 | + $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; |
|
| 398 | + } |
|
| 399 | + $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', array($path, $this->storageId)); |
|
| 400 | + $size = 0; |
|
| 401 | + if (!is_null($folderId)) { |
|
| 402 | + $folderId = $this->cache->getId($path); |
|
| 403 | + } |
|
| 404 | + $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size); |
|
| 405 | + |
|
| 406 | + foreach ($childQueue as $child => $childId) { |
|
| 407 | + $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock); |
|
| 408 | + if ($childSize === -1) { |
|
| 409 | + $size = -1; |
|
| 410 | + } else if ($size !== -1) { |
|
| 411 | + $size += $childSize; |
|
| 412 | + } |
|
| 413 | + } |
|
| 414 | + if ($this->cacheActive) { |
|
| 415 | + $this->cache->update($folderId, array('size' => $size)); |
|
| 416 | + } |
|
| 417 | + $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', array($path, $this->storageId)); |
|
| 418 | + return $size; |
|
| 419 | + } |
|
| 420 | + |
|
| 421 | + private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) { |
|
| 422 | + // we put this in it's own function so it cleans up the memory before we start recursing |
|
| 423 | + $existingChildren = $this->getExistingChildren($folderId); |
|
| 424 | + $newChildren = $this->getNewChildren($path); |
|
| 425 | + |
|
| 426 | + if ($this->useTransactions) { |
|
| 427 | + \OC::$server->getDatabaseConnection()->beginTransaction(); |
|
| 428 | + } |
|
| 429 | + |
|
| 430 | + $exceptionOccurred = false; |
|
| 431 | + $childQueue = []; |
|
| 432 | + foreach ($newChildren as $file) { |
|
| 433 | + $child = ($path) ? $path . '/' . $file : $file; |
|
| 434 | + try { |
|
| 435 | + $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : null; |
|
| 436 | + $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock); |
|
| 437 | + if ($data) { |
|
| 438 | + if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) { |
|
| 439 | + $childQueue[$child] = $data['fileid']; |
|
| 440 | + } else if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE_INCOMPLETE and $data['size'] === -1) { |
|
| 441 | + // only recurse into folders which aren't fully scanned |
|
| 442 | + $childQueue[$child] = $data['fileid']; |
|
| 443 | + } else if ($data['size'] === -1) { |
|
| 444 | + $size = -1; |
|
| 445 | + } else if ($size !== -1) { |
|
| 446 | + $size += $data['size']; |
|
| 447 | + } |
|
| 448 | + } |
|
| 449 | + } catch (\Doctrine\DBAL\DBALException $ex) { |
|
| 450 | + // might happen if inserting duplicate while a scanning |
|
| 451 | + // process is running in parallel |
|
| 452 | + // log and ignore |
|
| 453 | + if ($this->useTransactions) { |
|
| 454 | + \OC::$server->getDatabaseConnection()->rollback(); |
|
| 455 | + \OC::$server->getDatabaseConnection()->beginTransaction(); |
|
| 456 | + } |
|
| 457 | + \OCP\Util::writeLog('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OCP\Util::DEBUG); |
|
| 458 | + $exceptionOccurred = true; |
|
| 459 | + } catch (\OCP\Lock\LockedException $e) { |
|
| 460 | + if ($this->useTransactions) { |
|
| 461 | + \OC::$server->getDatabaseConnection()->rollback(); |
|
| 462 | + } |
|
| 463 | + throw $e; |
|
| 464 | + } |
|
| 465 | + } |
|
| 466 | + $removedChildren = \array_diff(array_keys($existingChildren), $newChildren); |
|
| 467 | + foreach ($removedChildren as $childName) { |
|
| 468 | + $child = ($path) ? $path . '/' . $childName : $childName; |
|
| 469 | + $this->removeFromCache($child); |
|
| 470 | + } |
|
| 471 | + if ($this->useTransactions) { |
|
| 472 | + \OC::$server->getDatabaseConnection()->commit(); |
|
| 473 | + } |
|
| 474 | + if ($exceptionOccurred) { |
|
| 475 | + // It might happen that the parallel scan process has already |
|
| 476 | + // inserted mimetypes but those weren't available yet inside the transaction |
|
| 477 | + // To make sure to have the updated mime types in such cases, |
|
| 478 | + // we reload them here |
|
| 479 | + \OC::$server->getMimeTypeLoader()->reset(); |
|
| 480 | + } |
|
| 481 | + return $childQueue; |
|
| 482 | + } |
|
| 483 | + |
|
| 484 | + /** |
|
| 485 | + * check if the file should be ignored when scanning |
|
| 486 | + * NOTE: files with a '.part' extension are ignored as well! |
|
| 487 | + * prevents unfinished put requests to be scanned |
|
| 488 | + * |
|
| 489 | + * @param string $file |
|
| 490 | + * @return boolean |
|
| 491 | + */ |
|
| 492 | + public static function isPartialFile($file) { |
|
| 493 | + if (pathinfo($file, PATHINFO_EXTENSION) === 'part') { |
|
| 494 | + return true; |
|
| 495 | + } |
|
| 496 | + if (strpos($file, '.part/') !== false) { |
|
| 497 | + return true; |
|
| 498 | + } |
|
| 499 | + |
|
| 500 | + return false; |
|
| 501 | + } |
|
| 502 | + |
|
| 503 | + /** |
|
| 504 | + * walk over any folders that are not fully scanned yet and scan them |
|
| 505 | + */ |
|
| 506 | + public function backgroundScan() { |
|
| 507 | + if (!$this->cache->inCache('')) { |
|
| 508 | + $this->runBackgroundScanJob(function () { |
|
| 509 | + $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG); |
|
| 510 | + }, ''); |
|
| 511 | + } else { |
|
| 512 | + $lastPath = null; |
|
| 513 | + while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { |
|
| 514 | + $this->runBackgroundScanJob(function () use ($path) { |
|
| 515 | + $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE); |
|
| 516 | + }, $path); |
|
| 517 | + // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() |
|
| 518 | + // to make this possible |
|
| 519 | + $lastPath = $path; |
|
| 520 | + } |
|
| 521 | + } |
|
| 522 | + } |
|
| 523 | + |
|
| 524 | + private function runBackgroundScanJob(callable $callback, $path) { |
|
| 525 | + try { |
|
| 526 | + $callback(); |
|
| 527 | + \OC_Hook::emit('Scanner', 'correctFolderSize', array('path' => $path)); |
|
| 528 | + if ($this->cacheActive && $this->cache instanceof Cache) { |
|
| 529 | + $this->cache->correctFolderSize($path); |
|
| 530 | + } |
|
| 531 | + } catch (\OCP\Files\StorageInvalidException $e) { |
|
| 532 | + // skip unavailable storages |
|
| 533 | + } catch (\OCP\Files\StorageNotAvailableException $e) { |
|
| 534 | + // skip unavailable storages |
|
| 535 | + } catch (\OCP\Files\ForbiddenException $e) { |
|
| 536 | + // skip forbidden storages |
|
| 537 | + } catch (\OCP\Lock\LockedException $e) { |
|
| 538 | + // skip unavailable storages |
|
| 539 | + } |
|
| 540 | + } |
|
| 541 | + |
|
| 542 | + /** |
|
| 543 | + * Set whether the cache is affected by scan operations |
|
| 544 | + * |
|
| 545 | + * @param boolean $active The active state of the cache |
|
| 546 | + */ |
|
| 547 | + public function setCacheActive($active) { |
|
| 548 | + $this->cacheActive = $active; |
|
| 549 | + } |
|
| 550 | 550 | } |
@@ -329,7 +329,7 @@ discard block |
||
| 329 | 329 | } |
| 330 | 330 | if ($lock) { |
| 331 | 331 | if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
| 332 | - $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); |
|
| 332 | + $this->storage->acquireLock('scanner::'.$path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); |
|
| 333 | 333 | $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
| 334 | 334 | } |
| 335 | 335 | } |
@@ -341,7 +341,7 @@ discard block |
||
| 341 | 341 | if ($lock) { |
| 342 | 342 | if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { |
| 343 | 343 | $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); |
| 344 | - $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); |
|
| 344 | + $this->storage->releaseLock('scanner::'.$path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); |
|
| 345 | 345 | } |
| 346 | 346 | } |
| 347 | 347 | return $data; |
@@ -430,7 +430,7 @@ discard block |
||
| 430 | 430 | $exceptionOccurred = false; |
| 431 | 431 | $childQueue = []; |
| 432 | 432 | foreach ($newChildren as $file) { |
| 433 | - $child = ($path) ? $path . '/' . $file : $file; |
|
| 433 | + $child = ($path) ? $path.'/'.$file : $file; |
|
| 434 | 434 | try { |
| 435 | 435 | $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : null; |
| 436 | 436 | $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock); |
@@ -454,7 +454,7 @@ discard block |
||
| 454 | 454 | \OC::$server->getDatabaseConnection()->rollback(); |
| 455 | 455 | \OC::$server->getDatabaseConnection()->beginTransaction(); |
| 456 | 456 | } |
| 457 | - \OCP\Util::writeLog('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OCP\Util::DEBUG); |
|
| 457 | + \OCP\Util::writeLog('core', 'Exception while scanning file "'.$child.'": '.$ex->getMessage(), \OCP\Util::DEBUG); |
|
| 458 | 458 | $exceptionOccurred = true; |
| 459 | 459 | } catch (\OCP\Lock\LockedException $e) { |
| 460 | 460 | if ($this->useTransactions) { |
@@ -465,7 +465,7 @@ discard block |
||
| 465 | 465 | } |
| 466 | 466 | $removedChildren = \array_diff(array_keys($existingChildren), $newChildren); |
| 467 | 467 | foreach ($removedChildren as $childName) { |
| 468 | - $child = ($path) ? $path . '/' . $childName : $childName; |
|
| 468 | + $child = ($path) ? $path.'/'.$childName : $childName; |
|
| 469 | 469 | $this->removeFromCache($child); |
| 470 | 470 | } |
| 471 | 471 | if ($this->useTransactions) { |
@@ -505,13 +505,13 @@ discard block |
||
| 505 | 505 | */ |
| 506 | 506 | public function backgroundScan() { |
| 507 | 507 | if (!$this->cache->inCache('')) { |
| 508 | - $this->runBackgroundScanJob(function () { |
|
| 508 | + $this->runBackgroundScanJob(function() { |
|
| 509 | 509 | $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG); |
| 510 | 510 | }, ''); |
| 511 | 511 | } else { |
| 512 | 512 | $lastPath = null; |
| 513 | 513 | while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { |
| 514 | - $this->runBackgroundScanJob(function () use ($path) { |
|
| 514 | + $this->runBackgroundScanJob(function() use ($path) { |
|
| 515 | 515 | $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE); |
| 516 | 516 | }, $path); |
| 517 | 517 | // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() |
