nextcloud / server

apps/files_sharing/lib/Scanner.php

Indentation +44 added lines, -44 removed lines

@@ -31,52 +31,52 @@
  * Scanner for SharedStorage
  */
 class Scanner extends \OC\Files\Cache\Scanner {
-	/**
-	 * @var \OCA\Files_Sharing\SharedStorage $storage
-	 */
-	protected $storage;
+    /**
+     * @var \OCA\Files_Sharing\SharedStorage $storage
+     */
+    protected $storage;
 
-	private $sourceScanner;
+    private $sourceScanner;
 
-	/**
-	 * Returns metadata from the shared storage, but
-	 * with permissions from the source storage.
-	 *
-	 * @param string $path path of the file for which to retrieve metadata
-	 *
-	 * @return array|null an array of metadata of the file
-	 */
-	public function getData($path) {
-		$data = parent::getData($path);
-		if ($data === null) {
-			return null;
-		}
-		$internalPath = $this->storage->getUnjailedPath($path);
-		$data['permissions'] = $this->storage->getSourceStorage()->getPermissions($internalPath);
-		return $data;
-	}
+    /**
+     * Returns metadata from the shared storage, but
+     * with permissions from the source storage.
+     *
+     * @param string $path path of the file for which to retrieve metadata
+     *
+     * @return array|null an array of metadata of the file
+     */
+    public function getData($path) {
+        $data = parent::getData($path);
+        if ($data === null) {
+            return null;
+        }
+        $internalPath = $this->storage->getUnjailedPath($path);
+        $data['permissions'] = $this->storage->getSourceStorage()->getPermissions($internalPath);
+        return $data;
+    }
 
-	private function getSourceScanner() {
-		if ($this->sourceScanner) {
-			return $this->sourceScanner;
-		}
-		if ($this->storage->instanceOfStorage('\OCA\Files_Sharing\SharedStorage')) {
-			/** @var \OC\Files\Storage\Storage $storage */
-			[$storage] = $this->storage->resolvePath('');
-			$this->sourceScanner = $storage->getScanner();
-			return $this->sourceScanner;
-		} else {
-			return null;
-		}
-	}
+    private function getSourceScanner() {
+        if ($this->sourceScanner) {
+            return $this->sourceScanner;
+        }
+        if ($this->storage->instanceOfStorage('\OCA\Files_Sharing\SharedStorage')) {
+            /** @var \OC\Files\Storage\Storage $storage */
+            [$storage] = $this->storage->resolvePath('');
+            $this->sourceScanner = $storage->getScanner();
+            return $this->sourceScanner;
+        } else {
+            return null;
+        }
+    }
 
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
-		$sourceScanner = $this->getSourceScanner();
-		if ($sourceScanner instanceof ObjectStoreScanner) {
-			// ObjectStoreScanner doesn't scan
-			return [];
-		} else {
-			return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock);
-		}
-	}
+    public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
+        $sourceScanner = $this->getSourceScanner();
+        if ($sourceScanner instanceof ObjectStoreScanner) {
+            // ObjectStoreScanner doesn't scan
+            return [];
+        } else {
+            return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock);
+        }
+    }
 }

lib/private/Files/ObjectStore/ObjectStoreScanner.php

Indentation +57 added lines, -57 removed lines

@@ -31,68 +31,68 @@
 use OCP\Files\FileInfo;
 
 class ObjectStoreScanner extends Scanner {
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
-		return [];
-	}
+    public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
+        return [];
+    }
 
-	public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) {
-		return [];
-	}
+    public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) {
+        return [];
+    }
 
-	protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true, array $data = []) {
-		return 0;
-	}
+    protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true, array $data = []) {
+        return 0;
+    }
 
-	public function backgroundScan() {
-		$lastPath = null;
-		// find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck)
-		// we sort by path DESC to ensure that contents of a folder are handled before the parent folder
-		while (($path = $this->getIncomplete()) !== false && $path !== $lastPath) {
-			$this->runBackgroundScanJob(function () use ($path) {
-				$item = $this->cache->get($path);
-				if ($item && $item->getMimeType() !== FileInfo::MIMETYPE_FOLDER) {
-					$fh = $this->storage->fopen($path, 'r');
-					if ($fh) {
-						$stat = fstat($fh);
-						if ($stat['size']) {
-							$this->cache->update($item->getId(), ['size' => $stat['size']]);
-						}
-					}
-				}
-			}, $path);
-			// FIXME: this won't proceed with the next item, needs revamping of getIncomplete()
-			// to make this possible
-			$lastPath = $path;
-		}
-	}
+    public function backgroundScan() {
+        $lastPath = null;
+        // find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck)
+        // we sort by path DESC to ensure that contents of a folder are handled before the parent folder
+        while (($path = $this->getIncomplete()) !== false && $path !== $lastPath) {
+            $this->runBackgroundScanJob(function () use ($path) {
+                $item = $this->cache->get($path);
+                if ($item && $item->getMimeType() !== FileInfo::MIMETYPE_FOLDER) {
+                    $fh = $this->storage->fopen($path, 'r');
+                    if ($fh) {
+                        $stat = fstat($fh);
+                        if ($stat['size']) {
+                            $this->cache->update($item->getId(), ['size' => $stat['size']]);
+                        }
+                    }
+                }
+            }, $path);
+            // FIXME: this won't proceed with the next item, needs revamping of getIncomplete()
+            // to make this possible
+            $lastPath = $path;
+        }
+    }
 
-	/**
-	 * Unlike the default Cache::getIncomplete this one sorts by path.
-	 *
-	 * This is needed since self::backgroundScan doesn't fix child entries when running on a parent folder.
-	 * By sorting by path we ensure that we encounter the child entries first.
-	 *
-	 * @return false|string
-	 * @throws \OCP\DB\Exception
-	 */
-	private function getIncomplete() {
-		$query = $this->connection->getQueryBuilder();
-		$query->select('path')
-			->from('filecache')
-			->where($query->expr()->eq('storage', $query->createNamedParameter($this->cache->getNumericStorageId(), IQueryBuilder::PARAM_INT)))
-			->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
-			->orderBy('path', 'DESC')
-			->setMaxResults(1);
+    /**
+     * Unlike the default Cache::getIncomplete this one sorts by path.
+     *
+     * This is needed since self::backgroundScan doesn't fix child entries when running on a parent folder.
+     * By sorting by path we ensure that we encounter the child entries first.
+     *
+     * @return false|string
+     * @throws \OCP\DB\Exception
+     */
+    private function getIncomplete() {
+        $query = $this->connection->getQueryBuilder();
+        $query->select('path')
+            ->from('filecache')
+            ->where($query->expr()->eq('storage', $query->createNamedParameter($this->cache->getNumericStorageId(), IQueryBuilder::PARAM_INT)))
+            ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
+            ->orderBy('path', 'DESC')
+            ->setMaxResults(1);
 
-		$result = $query->executeQuery();
-		$path = $result->fetchOne();
-		$result->closeCursor();
+        $result = $query->executeQuery();
+        $path = $result->fetchOne();
+        $result->closeCursor();
 
-		if ($path === false) {
-			return false;
-		}
+        if ($path === false) {
+            return false;
+        }
 
-		// Make sure Oracle does not continue with null for empty strings
-		return (string)$path;
-	}
+        // Make sure Oracle does not continue with null for empty strings
+        return (string)$path;
+    }
 }

Spacing +2 added lines, -2 removed lines

@@ -48,7 +48,7 @@ 
 		// find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck)
 		// we sort by path DESC to ensure that contents of a folder are handled before the parent folder
 		while (($path = $this->getIncomplete()) !== false && $path !== $lastPath) {
-			$this->runBackgroundScanJob(function () use ($path) {
+			$this->runBackgroundScanJob(function() use ($path) {
 				$item = $this->cache->get($path);
 				if ($item && $item->getMimeType() !== FileInfo::MIMETYPE_FOLDER) {
 					$fh = $this->storage->fopen($path, 'r');
@@ -93,6 +93,6 @@ 
 		}
 
 		// Make sure Oracle does not continue with null for empty strings
-		return (string)$path;
+		return (string) $path;
 	}
 }

lib/private/Files/ObjectStore/ObjectStoreStorage.php

Indentation +665 added lines, -665 removed lines

@@ -48,672 +48,672 @@
 use OCP\Files\Storage\IStorage;
 
 class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite {
-	use CopyDirectory;
-
-	/**
-	 * @var \OCP\Files\ObjectStore\IObjectStore $objectStore
-	 */
-	protected $objectStore;
-	/**
-	 * @var string $id
-	 */
-	protected $id;
-	/**
-	 * @var \OC\User\User $user
-	 */
-	protected $user;
-
-	private $objectPrefix = 'urn:oid:';
-
-	private $logger;
-
-	/** @var bool */
-	protected $validateWrites = true;
-
-	public function __construct($params) {
-		if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) {
-			$this->objectStore = $params['objectstore'];
-		} else {
-			throw new \Exception('missing IObjectStore instance');
-		}
-		if (isset($params['storageid'])) {
-			$this->id = 'object::store:' . $params['storageid'];
-		} else {
-			$this->id = 'object::store:' . $this->objectStore->getStorageId();
-		}
-		if (isset($params['objectPrefix'])) {
-			$this->objectPrefix = $params['objectPrefix'];
-		}
-		if (isset($params['validateWrites'])) {
-			$this->validateWrites = (bool)$params['validateWrites'];
-		}
-		//initialize cache with root directory in cache
-		if (!$this->is_dir('/')) {
-			$this->mkdir('/');
-		}
-
-		$this->logger = \OC::$server->getLogger();
-	}
-
-	public function mkdir($path) {
-		$path = $this->normalizePath($path);
-		if ($this->file_exists($path)) {
-			$this->logger->warning("Tried to create an object store folder that already exists: $path");
-			return false;
-		}
-
-		$mTime = time();
-		$data = [
-			'mimetype' => 'httpd/unix-directory',
-			'size' => 0,
-			'mtime' => $mTime,
-			'storage_mtime' => $mTime,
-			'permissions' => \OCP\Constants::PERMISSION_ALL,
-		];
-		if ($path === '') {
-			//create root on the fly
-			$data['etag'] = $this->getETag('');
-			$this->getCache()->put('', $data);
-			return true;
-		} else {
-			// if parent does not exist, create it
-			$parent = $this->normalizePath(dirname($path));
-			$parentType = $this->filetype($parent);
-			if ($parentType === false) {
-				if (!$this->mkdir($parent)) {
-					// something went wrong
-					$this->logger->warning("Parent folder ($parent) doesn't exist and couldn't be created");
-					return false;
-				}
-			} elseif ($parentType === 'file') {
-				// parent is a file
-				$this->logger->warning("Parent ($parent) is a file");
-				return false;
-			}
-			// finally create the new dir
-			$mTime = time(); // update mtime
-			$data['mtime'] = $mTime;
-			$data['storage_mtime'] = $mTime;
-			$data['etag'] = $this->getETag($path);
-			$this->getCache()->put($path, $data);
-			return true;
-		}
-	}
-
-	/**
-	 * @param string $path
-	 * @return string
-	 */
-	private function normalizePath($path) {
-		$path = trim($path, '/');
-		//FIXME why do we sometimes get a path like 'files//username'?
-		$path = str_replace('//', '/', $path);
-
-		// dirname('/folder') returns '.' but internally (in the cache) we store the root as ''
-		if (!$path || $path === '.') {
-			$path = '';
-		}
-
-		return $path;
-	}
-
-	/**
-	 * Object Stores use a NoopScanner because metadata is directly stored in
-	 * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere.
-	 *
-	 * @param string $path
-	 * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner
-	 * @return \OC\Files\ObjectStore\ObjectStoreScanner
-	 */
-	public function getScanner($path = '', $storage = null) {
-		if (!$storage) {
-			$storage = $this;
-		}
-		if (!isset($this->scanner)) {
-			$this->scanner = new ObjectStoreScanner($storage);
-		}
-		return $this->scanner;
-	}
-
-	public function getId() {
-		return $this->id;
-	}
-
-	public function rmdir($path) {
-		$path = $this->normalizePath($path);
-		$entry = $this->getCache()->get($path);
-
-		if (!$entry || $entry->getMimeType() !== ICacheEntry::DIRECTORY_MIMETYPE) {
-			return false;
-		}
-
-		return $this->rmObjects($entry);
-	}
-
-	private function rmObjects(ICacheEntry $entry): bool {
-		$children = $this->getCache()->getFolderContentsById($entry->getId());
-		foreach ($children as $child) {
-			if ($child->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
-				if (!$this->rmObjects($child)) {
-					return false;
-				}
-			} else {
-				if (!$this->rmObject($child)) {
-					return false;
-				}
-			}
-		}
-
-		$this->getCache()->remove($entry->getPath());
-
-		return true;
-	}
-
-	public function unlink($path) {
-		$path = $this->normalizePath($path);
-		$entry = $this->getCache()->get($path);
-
-		if ($entry instanceof ICacheEntry) {
-			if ($entry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
-				return $this->rmObjects($entry);
-			} else {
-				return $this->rmObject($entry);
-			}
-		}
-		return false;
-	}
-
-	public function rmObject(ICacheEntry $entry): bool {
-		try {
-			$this->objectStore->deleteObject($this->getURN($entry->getId()));
-		} catch (\Exception $ex) {
-			if ($ex->getCode() !== 404) {
-				$this->logger->logException($ex, [
-					'app' => 'objectstore',
-					'message' => 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(),
-				]);
-				return false;
-			}
-			//removing from cache is ok as it does not exist in the objectstore anyway
-		}
-		$this->getCache()->remove($entry->getPath());
-		return true;
-	}
-
-	public function stat($path) {
-		$path = $this->normalizePath($path);
-		$cacheEntry = $this->getCache()->get($path);
-		if ($cacheEntry instanceof CacheEntry) {
-			return $cacheEntry->getData();
-		} else {
-			return false;
-		}
-	}
-
-	public function getPermissions($path) {
-		$stat = $this->stat($path);
-
-		if (is_array($stat) && isset($stat['permissions'])) {
-			return $stat['permissions'];
-		}
-
-		return parent::getPermissions($path);
-	}
-
-	/**
-	 * Override this method if you need a different unique resource identifier for your object storage implementation.
-	 * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users.
-	 * You may need a mapping table to store your URN if it cannot be generated from the fileid.
-	 *
-	 * @param int $fileId the fileid
-	 * @return null|string the unified resource name used to identify the object
-	 */
-	public function getURN($fileId) {
-		if (is_numeric($fileId)) {
-			return $this->objectPrefix . $fileId;
-		}
-		return null;
-	}
-
-	public function opendir($path) {
-		$path = $this->normalizePath($path);
-
-		try {
-			$files = [];
-			$folderContents = $this->getCache()->getFolderContents($path);
-			foreach ($folderContents as $file) {
-				$files[] = $file['name'];
-			}
-
-			return IteratorDirectory::wrap($files);
-		} catch (\Exception $e) {
-			$this->logger->logException($e);
-			return false;
-		}
-	}
-
-	public function filetype($path) {
-		$path = $this->normalizePath($path);
-		$stat = $this->stat($path);
-		if ($stat) {
-			if ($stat['mimetype'] === 'httpd/unix-directory') {
-				return 'dir';
-			}
-			return 'file';
-		} else {
-			return false;
-		}
-	}
-
-	public function fopen($path, $mode) {
-		$path = $this->normalizePath($path);
-
-		if (strrpos($path, '.') !== false) {
-			$ext = substr($path, strrpos($path, '.'));
-		} else {
-			$ext = '';
-		}
-
-		switch ($mode) {
-			case 'r':
-			case 'rb':
-				$stat = $this->stat($path);
-				if (is_array($stat)) {
-					$filesize = $stat['size'] ?? 0;
-					// Reading 0 sized files is a waste of time
-					if ($filesize === 0) {
-						return fopen('php://memory', $mode);
-					}
-
-					try {
-						$handle = $this->objectStore->readObject($this->getURN($stat['fileid']));
-						if ($handle === false) {
-							return false; // keep backward compatibility
-						}
-						$streamStat = fstat($handle);
-						$actualSize = $streamStat['size'] ?? -1;
-						if ($actualSize > -1 && $actualSize !== $filesize) {
-							$this->getCache()->update((int)$stat['fileid'], ['size' => $actualSize]);
-						}
-						return $handle;
-					} catch (NotFoundException $e) {
-						$this->logger->logException($e, [
-							'app' => 'objectstore',
-							'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path,
-						]);
-						throw $e;
-					} catch (\Exception $ex) {
-						$this->logger->logException($ex, [
-							'app' => 'objectstore',
-							'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path,
-						]);
-						return false;
-					}
-				} else {
-					return false;
-				}
-				// no break
-			case 'w':
-			case 'wb':
-			case 'w+':
-			case 'wb+':
-				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
-				$handle = fopen($tmpFile, $mode);
-				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
-					$this->writeBack($tmpFile, $path);
-					unlink($tmpFile);
-				});
-			case 'a':
-			case 'ab':
-			case 'r+':
-			case 'a+':
-			case 'x':
-			case 'x+':
-			case 'c':
-			case 'c+':
-				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
-				if ($this->file_exists($path)) {
-					$source = $this->fopen($path, 'r');
-					file_put_contents($tmpFile, $source);
-				}
-				$handle = fopen($tmpFile, $mode);
-				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
-					$this->writeBack($tmpFile, $path);
-					unlink($tmpFile);
-				});
-		}
-		return false;
-	}
-
-	public function file_exists($path) {
-		$path = $this->normalizePath($path);
-		return (bool)$this->stat($path);
-	}
-
-	public function rename($source, $target) {
-		$source = $this->normalizePath($source);
-		$target = $this->normalizePath($target);
-		$this->remove($target);
-		$this->getCache()->move($source, $target);
-		$this->touch(dirname($target));
-		return true;
-	}
-
-	public function getMimeType($path) {
-		$path = $this->normalizePath($path);
-		return parent::getMimeType($path);
-	}
-
-	public function touch($path, $mtime = null) {
-		if (is_null($mtime)) {
-			$mtime = time();
-		}
-
-		$path = $this->normalizePath($path);
-		$dirName = dirname($path);
-		$parentExists = $this->is_dir($dirName);
-		if (!$parentExists) {
-			return false;
-		}
-
-		$stat = $this->stat($path);
-		if (is_array($stat)) {
-			// update existing mtime in db
-			$stat['mtime'] = $mtime;
-			$this->getCache()->update($stat['fileid'], $stat);
-		} else {
-			try {
-				//create a empty file, need to have at least on char to make it
-				// work with all object storage implementations
-				$this->file_put_contents($path, ' ');
-				$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
-				$stat = [
-					'etag' => $this->getETag($path),
-					'mimetype' => $mimeType,
-					'size' => 0,
-					'mtime' => $mtime,
-					'storage_mtime' => $mtime,
-					'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
-				];
-				$this->getCache()->put($path, $stat);
-			} catch (\Exception $ex) {
-				$this->logger->logException($ex, [
-					'app' => 'objectstore',
-					'message' => 'Could not create object for ' . $path,
-				]);
-				throw $ex;
-			}
-		}
-		return true;
-	}
-
-	public function writeBack($tmpFile, $path) {
-		$size = filesize($tmpFile);
-		$this->writeStream($path, fopen($tmpFile, 'r'), $size);
-	}
-
-	/**
-	 * external changes are not supported, exclusive access to the object storage is assumed
-	 *
-	 * @param string $path
-	 * @param int $time
-	 * @return false
-	 */
-	public function hasUpdated($path, $time) {
-		return false;
-	}
-
-	public function needsPartFile() {
-		return false;
-	}
-
-	public function file_put_contents($path, $data) {
-		$handle = $this->fopen($path, 'w+');
-		$result = fwrite($handle, $data);
-		fclose($handle);
-		return $result;
-	}
-
-	public function writeStream(string $path, $stream, int $size = null): int {
-		$stat = $this->stat($path);
-		if (empty($stat)) {
-			// create new file
-			$stat = [
-				'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
-			];
-		}
-		// update stat with new data
-		$mTime = time();
-		$stat['size'] = (int)$size;
-		$stat['mtime'] = $mTime;
-		$stat['storage_mtime'] = $mTime;
-
-		$mimetypeDetector = \OC::$server->getMimeTypeDetector();
-		$mimetype = $mimetypeDetector->detectPath($path);
-
-		$stat['mimetype'] = $mimetype;
-		$stat['etag'] = $this->getETag($path);
-		$stat['checksum'] = '';
-
-		$exists = $this->getCache()->inCache($path);
-		$uploadPath = $exists ? $path : $path . '.part';
-
-		if ($exists) {
-			$fileId = $stat['fileid'];
-		} else {
-			$fileId = $this->getCache()->put($uploadPath, $stat);
-		}
-
-		$urn = $this->getURN($fileId);
-		try {
-			//upload to object storage
-			if ($size === null) {
-				$countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) {
-					$this->getCache()->update($fileId, [
-						'size' => $writtenSize,
-					]);
-					$size = $writtenSize;
-				});
-				$this->objectStore->writeObject($urn, $countStream, $mimetype);
-				if (is_resource($countStream)) {
-					fclose($countStream);
-				}
-				$stat['size'] = $size;
-			} else {
-				$this->objectStore->writeObject($urn, $stream, $mimetype);
-				if (is_resource($stream)) {
-					fclose($stream);
-				}
-			}
-		} catch (\Exception $ex) {
-			if (!$exists) {
-				/*
+    use CopyDirectory;
+
+    /**
+     * @var \OCP\Files\ObjectStore\IObjectStore $objectStore
+     */
+    protected $objectStore;
+    /**
+     * @var string $id
+     */
+    protected $id;
+    /**
+     * @var \OC\User\User $user
+     */
+    protected $user;
+
+    private $objectPrefix = 'urn:oid:';
+
+    private $logger;
+
+    /** @var bool */
+    protected $validateWrites = true;
+
+    public function __construct($params) {
+        if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) {
+            $this->objectStore = $params['objectstore'];
+        } else {
+            throw new \Exception('missing IObjectStore instance');
+        }
+        if (isset($params['storageid'])) {
+            $this->id = 'object::store:' . $params['storageid'];
+        } else {
+            $this->id = 'object::store:' . $this->objectStore->getStorageId();
+        }
+        if (isset($params['objectPrefix'])) {
+            $this->objectPrefix = $params['objectPrefix'];
+        }
+        if (isset($params['validateWrites'])) {
+            $this->validateWrites = (bool)$params['validateWrites'];
+        }
+        //initialize cache with root directory in cache
+        if (!$this->is_dir('/')) {
+            $this->mkdir('/');
+        }
+
+        $this->logger = \OC::$server->getLogger();
+    }
+
+    public function mkdir($path) {
+        $path = $this->normalizePath($path);
+        if ($this->file_exists($path)) {
+            $this->logger->warning("Tried to create an object store folder that already exists: $path");
+            return false;
+        }
+
+        $mTime = time();
+        $data = [
+            'mimetype' => 'httpd/unix-directory',
+            'size' => 0,
+            'mtime' => $mTime,
+            'storage_mtime' => $mTime,
+            'permissions' => \OCP\Constants::PERMISSION_ALL,
+        ];
+        if ($path === '') {
+            //create root on the fly
+            $data['etag'] = $this->getETag('');
+            $this->getCache()->put('', $data);
+            return true;
+        } else {
+            // if parent does not exist, create it
+            $parent = $this->normalizePath(dirname($path));
+            $parentType = $this->filetype($parent);
+            if ($parentType === false) {
+                if (!$this->mkdir($parent)) {
+                    // something went wrong
+                    $this->logger->warning("Parent folder ($parent) doesn't exist and couldn't be created");
+                    return false;
+                }
+            } elseif ($parentType === 'file') {
+                // parent is a file
+                $this->logger->warning("Parent ($parent) is a file");
+                return false;
+            }
+            // finally create the new dir
+            $mTime = time(); // update mtime
+            $data['mtime'] = $mTime;
+            $data['storage_mtime'] = $mTime;
+            $data['etag'] = $this->getETag($path);
+            $this->getCache()->put($path, $data);
+            return true;
+        }
+    }
+
+    /**
+     * @param string $path
+     * @return string
+     */
+    private function normalizePath($path) {
+        $path = trim($path, '/');
+        //FIXME why do we sometimes get a path like 'files//username'?
+        $path = str_replace('//', '/', $path);
+
+        // dirname('/folder') returns '.' but internally (in the cache) we store the root as ''
+        if (!$path || $path === '.') {
+            $path = '';
+        }
+
+        return $path;
+    }
+
+    /**
+     * Object Stores use a NoopScanner because metadata is directly stored in
+     * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere.
+     *
+     * @param string $path
+     * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner
+     * @return \OC\Files\ObjectStore\ObjectStoreScanner
+     */
+    public function getScanner($path = '', $storage = null) {
+        if (!$storage) {
+            $storage = $this;
+        }
+        if (!isset($this->scanner)) {
+            $this->scanner = new ObjectStoreScanner($storage);
+        }
+        return $this->scanner;
+    }
+
+    public function getId() {
+        return $this->id;
+    }
+
+    public function rmdir($path) {
+        $path = $this->normalizePath($path);
+        $entry = $this->getCache()->get($path);
+
+        if (!$entry || $entry->getMimeType() !== ICacheEntry::DIRECTORY_MIMETYPE) {
+            return false;
+        }
+
+        return $this->rmObjects($entry);
+    }
+
+    private function rmObjects(ICacheEntry $entry): bool {
+        $children = $this->getCache()->getFolderContentsById($entry->getId());
+        foreach ($children as $child) {
+            if ($child->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
+                if (!$this->rmObjects($child)) {
+                    return false;
+                }
+            } else {
+                if (!$this->rmObject($child)) {
+                    return false;
+                }
+            }
+        }
+
+        $this->getCache()->remove($entry->getPath());
+
+        return true;
+    }
+
+    public function unlink($path) {
+        $path = $this->normalizePath($path);
+        $entry = $this->getCache()->get($path);
+
+        if ($entry instanceof ICacheEntry) {
+            if ($entry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
+                return $this->rmObjects($entry);
+            } else {
+                return $this->rmObject($entry);
+            }
+        }
+        return false;
+    }
+
+    public function rmObject(ICacheEntry $entry): bool {
+        try {
+            $this->objectStore->deleteObject($this->getURN($entry->getId()));
+        } catch (\Exception $ex) {
+            if ($ex->getCode() !== 404) {
+                $this->logger->logException($ex, [
+                    'app' => 'objectstore',
+                    'message' => 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(),
+                ]);
+                return false;
+            }
+            //removing from cache is ok as it does not exist in the objectstore anyway
+        }
+        $this->getCache()->remove($entry->getPath());
+        return true;
+    }
+
+    public function stat($path) {
+        $path = $this->normalizePath($path);
+        $cacheEntry = $this->getCache()->get($path);
+        if ($cacheEntry instanceof CacheEntry) {
+            return $cacheEntry->getData();
+        } else {
+            return false;
+        }
+    }
+
+    public function getPermissions($path) {
+        $stat = $this->stat($path);
+
+        if (is_array($stat) && isset($stat['permissions'])) {
+            return $stat['permissions'];
+        }
+
+        return parent::getPermissions($path);
+    }
+
+    /**
+     * Override this method if you need a different unique resource identifier for your object storage implementation.
+     * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users.
+     * You may need a mapping table to store your URN if it cannot be generated from the fileid.
+     *
+     * @param int $fileId the fileid
+     * @return null|string the unified resource name used to identify the object
+     */
+    public function getURN($fileId) {
+        if (is_numeric($fileId)) {
+            return $this->objectPrefix . $fileId;
+        }
+        return null;
+    }
+
+    public function opendir($path) {
+        $path = $this->normalizePath($path);
+
+        try {
+            $files = [];
+            $folderContents = $this->getCache()->getFolderContents($path);
+            foreach ($folderContents as $file) {
+                $files[] = $file['name'];
+            }
+
+            return IteratorDirectory::wrap($files);
+        } catch (\Exception $e) {
+            $this->logger->logException($e);
+            return false;
+        }
+    }
+
+    public function filetype($path) {
+        $path = $this->normalizePath($path);
+        $stat = $this->stat($path);
+        if ($stat) {
+            if ($stat['mimetype'] === 'httpd/unix-directory') {
+                return 'dir';
+            }
+            return 'file';
+        } else {
+            return false;
+        }
+    }
+
+    public function fopen($path, $mode) {
+        $path = $this->normalizePath($path);
+
+        if (strrpos($path, '.') !== false) {
+            $ext = substr($path, strrpos($path, '.'));
+        } else {
+            $ext = '';
+        }
+
+        switch ($mode) {
+            case 'r':
+            case 'rb':
+                $stat = $this->stat($path);
+                if (is_array($stat)) {
+                    $filesize = $stat['size'] ?? 0;
+                    // Reading 0 sized files is a waste of time
+                    if ($filesize === 0) {
+                        return fopen('php://memory', $mode);
+                    }
+
+                    try {
+                        $handle = $this->objectStore->readObject($this->getURN($stat['fileid']));
+                        if ($handle === false) {
+                            return false; // keep backward compatibility
+                        }
+                        $streamStat = fstat($handle);
+                        $actualSize = $streamStat['size'] ?? -1;
+                        if ($actualSize > -1 && $actualSize !== $filesize) {
+                            $this->getCache()->update((int)$stat['fileid'], ['size' => $actualSize]);
+                        }
+                        return $handle;
+                    } catch (NotFoundException $e) {
+                        $this->logger->logException($e, [
+                            'app' => 'objectstore',
+                            'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path,
+                        ]);
+                        throw $e;
+                    } catch (\Exception $ex) {
+                        $this->logger->logException($ex, [
+                            'app' => 'objectstore',
+                            'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path,
+                        ]);
+                        return false;
+                    }
+                } else {
+                    return false;
+                }
+                // no break
+            case 'w':
+            case 'wb':
+            case 'w+':
+            case 'wb+':
+                $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
+                $handle = fopen($tmpFile, $mode);
+                return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
+                    $this->writeBack($tmpFile, $path);
+                    unlink($tmpFile);
+                });
+            case 'a':
+            case 'ab':
+            case 'r+':
+            case 'a+':
+            case 'x':
+            case 'x+':
+            case 'c':
+            case 'c+':
+                $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
+                if ($this->file_exists($path)) {
+                    $source = $this->fopen($path, 'r');
+                    file_put_contents($tmpFile, $source);
+                }
+                $handle = fopen($tmpFile, $mode);
+                return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
+                    $this->writeBack($tmpFile, $path);
+                    unlink($tmpFile);
+                });
+        }
+        return false;
+    }
+
+    public function file_exists($path) {
+        $path = $this->normalizePath($path);
+        return (bool)$this->stat($path);
+    }
+
+    public function rename($source, $target) {
+        $source = $this->normalizePath($source);
+        $target = $this->normalizePath($target);
+        $this->remove($target);
+        $this->getCache()->move($source, $target);
+        $this->touch(dirname($target));
+        return true;
+    }
+
+    public function getMimeType($path) {
+        $path = $this->normalizePath($path);
+        return parent::getMimeType($path);
+    }
+
+    public function touch($path, $mtime = null) {
+        if (is_null($mtime)) {
+            $mtime = time();
+        }
+
+        $path = $this->normalizePath($path);
+        $dirName = dirname($path);
+        $parentExists = $this->is_dir($dirName);
+        if (!$parentExists) {
+            return false;
+        }
+
+        $stat = $this->stat($path);
+        if (is_array($stat)) {
+            // update existing mtime in db
+            $stat['mtime'] = $mtime;
+            $this->getCache()->update($stat['fileid'], $stat);
+        } else {
+            try {
+                //create a empty file, need to have at least on char to make it
+                // work with all object storage implementations
+                $this->file_put_contents($path, ' ');
+                $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
+                $stat = [
+                    'etag' => $this->getETag($path),
+                    'mimetype' => $mimeType,
+                    'size' => 0,
+                    'mtime' => $mtime,
+                    'storage_mtime' => $mtime,
+                    'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
+                ];
+                $this->getCache()->put($path, $stat);
+            } catch (\Exception $ex) {
+                $this->logger->logException($ex, [
+                    'app' => 'objectstore',
+                    'message' => 'Could not create object for ' . $path,
+                ]);
+                throw $ex;
+            }
+        }
+        return true;
+    }
+
+    public function writeBack($tmpFile, $path) {
+        $size = filesize($tmpFile);
+        $this->writeStream($path, fopen($tmpFile, 'r'), $size);
+    }
+
+    /**
+     * external changes are not supported, exclusive access to the object storage is assumed
+     *
+     * @param string $path
+     * @param int $time
+     * @return false
+     */
+    public function hasUpdated($path, $time) {
+        return false;
+    }
+
+    public function needsPartFile() {
+        return false;
+    }
+
+    public function file_put_contents($path, $data) {
+        $handle = $this->fopen($path, 'w+');
+        $result = fwrite($handle, $data);
+        fclose($handle);
+        return $result;
+    }
+
+    public function writeStream(string $path, $stream, int $size = null): int {
+        $stat = $this->stat($path);
+        if (empty($stat)) {
+            // create new file
+            $stat = [
+                'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
+            ];
+        }
+        // update stat with new data
+        $mTime = time();
+        $stat['size'] = (int)$size;
+        $stat['mtime'] = $mTime;
+        $stat['storage_mtime'] = $mTime;
+
+        $mimetypeDetector = \OC::$server->getMimeTypeDetector();
+        $mimetype = $mimetypeDetector->detectPath($path);
+
+        $stat['mimetype'] = $mimetype;
+        $stat['etag'] = $this->getETag($path);
+        $stat['checksum'] = '';
+
+        $exists = $this->getCache()->inCache($path);
+        $uploadPath = $exists ? $path : $path . '.part';
+
+        if ($exists) {
+            $fileId = $stat['fileid'];
+        } else {
+            $fileId = $this->getCache()->put($uploadPath, $stat);
+        }
+
+        $urn = $this->getURN($fileId);
+        try {
+            //upload to object storage
+            if ($size === null) {
+                $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) {
+                    $this->getCache()->update($fileId, [
+                        'size' => $writtenSize,
+                    ]);
+                    $size = $writtenSize;
+                });
+                $this->objectStore->writeObject($urn, $countStream, $mimetype);
+                if (is_resource($countStream)) {
+                    fclose($countStream);
+                }
+                $stat['size'] = $size;
+            } else {
+                $this->objectStore->writeObject($urn, $stream, $mimetype);
+                if (is_resource($stream)) {
+                    fclose($stream);
+                }
+            }
+        } catch (\Exception $ex) {
+            if (!$exists) {
+                /*
 				 * Only remove the entry if we are dealing with a new file.
 				 * Else people lose access to existing files
 				 */
-				$this->getCache()->remove($uploadPath);
-				$this->logger->logException($ex, [
-					'app' => 'objectstore',
-					'message' => 'Could not create object ' . $urn . ' for ' . $path,
-				]);
-			} else {
-				$this->logger->logException($ex, [
-					'app' => 'objectstore',
-					'message' => 'Could not update object ' . $urn . ' for ' . $path,
-				]);
-			}
-			throw $ex; // make this bubble up
-		}
-
-		if ($exists) {
-			$this->getCache()->update($fileId, $stat);
-		} else {
-			if (!$this->validateWrites || $this->objectStore->objectExists($urn)) {
-				$this->getCache()->move($uploadPath, $path);
-			} else {
-				$this->getCache()->remove($uploadPath);
-				throw new \Exception("Object not found after writing (urn: $urn, path: $path)", 404);
-			}
-		}
-
-		return $size;
-	}
-
-	public function getObjectStore(): IObjectStore {
-		return $this->objectStore;
-	}
-
-	public function copyFromStorage(
-		IStorage $sourceStorage,
-		$sourceInternalPath,
-		$targetInternalPath,
-		$preserveMtime = false
-	) {
-		if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
-			/** @var ObjectStoreStorage $sourceStorage */
-			if ($sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) {
-				/** @var CacheEntry $sourceEntry */
-				$sourceEntry = $sourceStorage->getCache()->get($sourceInternalPath);
-				$sourceEntryData = $sourceEntry->getData();
-				// $sourceEntry['permissions'] here is the permissions from the jailed storage for the current
-				// user. Instead we use $sourceEntryData['scan_permissions'] that are the permissions from the
-				// unjailed storage.
-				if (is_array($sourceEntryData) && array_key_exists('scan_permissions', $sourceEntryData)) {
-					$sourceEntry['permissions'] = $sourceEntryData['scan_permissions'];
-				}
-				$this->copyInner($sourceEntry, $targetInternalPath);
-				return true;
-			}
-		}
-
-		return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
-	}
-
-	public function copy($source, $target) {
-		$source = $this->normalizePath($source);
-		$target = $this->normalizePath($target);
-
-		$cache = $this->getCache();
-		$sourceEntry = $cache->get($source);
-		if (!$sourceEntry) {
-			throw new NotFoundException('Source object not found');
-		}
-
-		$this->copyInner($sourceEntry, $target);
-
-		return true;
-	}
-
-	private function copyInner(ICacheEntry $sourceEntry, string $to) {
-		$cache = $this->getCache();
-
-		if ($sourceEntry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
-			if ($cache->inCache($to)) {
-				$cache->remove($to);
-			}
-			$this->mkdir($to);
-
-			foreach ($cache->getFolderContentsById($sourceEntry->getId()) as $child) {
-				$this->copyInner($child, $to . '/' . $child->getName());
-			}
-		} else {
-			$this->copyFile($sourceEntry, $to);
-		}
-	}
-
-	private function copyFile(ICacheEntry $sourceEntry, string $to) {
-		$cache = $this->getCache();
-
-		$sourceUrn = $this->getURN($sourceEntry->getId());
-
-		if (!$cache instanceof Cache) {
-			throw new \Exception("Invalid source cache for object store copy");
-		}
-
-		$targetId = $cache->copyFromCache($cache, $sourceEntry, $to);
-
-		$targetUrn = $this->getURN($targetId);
-
-		try {
-			$this->objectStore->copyObject($sourceUrn, $targetUrn);
-		} catch (\Exception $e) {
-			$cache->remove($to);
-
-			throw $e;
-		}
-	}
-
-	public function startChunkedWrite(string $targetPath): string {
-		if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
-			throw new GenericFileException('Object store does not support multipart upload');
-		}
-		$cacheEntry = $this->getCache()->get($targetPath);
-		$urn = $this->getURN($cacheEntry->getId());
-		return $this->objectStore->initiateMultipartUpload($urn);
-	}
-
-	/**
-	 *
-	 * @throws GenericFileException
-	 */
-	public function putChunkedWritePart(
-		string $targetPath,
-		string $writeToken,
-		string $chunkId,
-		$data,
-		$size = null
-	): ?array {
-		if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
-			throw new GenericFileException('Object store does not support multipart upload');
-		}
-		$cacheEntry = $this->getCache()->get($targetPath);
-		$urn = $this->getURN($cacheEntry->getId());
-
-		$result = $this->objectStore->uploadMultipartPart($urn, $writeToken, (int)$chunkId, $data, $size);
-
-		$parts[$chunkId] = [
-			'PartNumber' => $chunkId,
-			'ETag' => trim($result->get('ETag'), '"'),
-		];
-		return $parts[$chunkId];
-	}
-
-	public function completeChunkedWrite(string $targetPath, string $writeToken): int {
-		if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
-			throw new GenericFileException('Object store does not support multipart upload');
-		}
-		$cacheEntry = $this->getCache()->get($targetPath);
-		$urn = $this->getURN($cacheEntry->getId());
-		$parts = $this->objectStore->getMultipartUploads($urn, $writeToken);
-		$sortedParts = array_values($parts);
-		sort($sortedParts);
-		try {
-			$size = $this->objectStore->completeMultipartUpload($urn, $writeToken, $sortedParts);
-			$stat = $this->stat($targetPath);
-			$mtime = time();
-			if (is_array($stat)) {
-				$stat['size'] = $size;
-				$stat['mtime'] = $mtime;
-				$stat['mimetype'] = $this->getMimeType($targetPath);
-				$this->getCache()->update($stat['fileid'], $stat);
-			}
-		} catch (S3MultipartUploadException|S3Exception $e) {
-			$this->objectStore->abortMultipartUpload($urn, $writeToken);
-			$this->logger->logException($e, [
-				'app' => 'objectstore',
-				'message' => 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken,
-			]);
-			throw new GenericFileException('Could not write chunked file');
-		}
-		return $size;
-	}
-
-	public function cancelChunkedWrite(string $targetPath, string $writeToken): void {
-		if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
-			throw new GenericFileException('Object store does not support multipart upload');
-		}
-		$cacheEntry = $this->getCache()->get($targetPath);
-		$urn = $this->getURN($cacheEntry->getId());
-		$this->objectStore->abortMultipartUpload($urn, $writeToken);
-	}
+                $this->getCache()->remove($uploadPath);
+                $this->logger->logException($ex, [
+                    'app' => 'objectstore',
+                    'message' => 'Could not create object ' . $urn . ' for ' . $path,
+                ]);
+            } else {
+                $this->logger->logException($ex, [
+                    'app' => 'objectstore',
+                    'message' => 'Could not update object ' . $urn . ' for ' . $path,
+                ]);
+            }
+            throw $ex; // make this bubble up
+        }
+
+        if ($exists) {
+            $this->getCache()->update($fileId, $stat);
+        } else {
+            if (!$this->validateWrites || $this->objectStore->objectExists($urn)) {
+                $this->getCache()->move($uploadPath, $path);
+            } else {
+                $this->getCache()->remove($uploadPath);
+                throw new \Exception("Object not found after writing (urn: $urn, path: $path)", 404);
+            }
+        }
+
+        return $size;
+    }
+
+    public function getObjectStore(): IObjectStore {
+        return $this->objectStore;
+    }
+
+    public function copyFromStorage(
+        IStorage $sourceStorage,
+        $sourceInternalPath,
+        $targetInternalPath,
+        $preserveMtime = false
+    ) {
+        if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
+            /** @var ObjectStoreStorage $sourceStorage */
+            if ($sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) {
+                /** @var CacheEntry $sourceEntry */
+                $sourceEntry = $sourceStorage->getCache()->get($sourceInternalPath);
+                $sourceEntryData = $sourceEntry->getData();
+                // $sourceEntry['permissions'] here is the permissions from the jailed storage for the current
+                // user. Instead we use $sourceEntryData['scan_permissions'] that are the permissions from the
+                // unjailed storage.
+                if (is_array($sourceEntryData) && array_key_exists('scan_permissions', $sourceEntryData)) {
+                    $sourceEntry['permissions'] = $sourceEntryData['scan_permissions'];
+                }
+                $this->copyInner($sourceEntry, $targetInternalPath);
+                return true;
+            }
+        }
+
+        return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
+    }
+
+    public function copy($source, $target) {
+        $source = $this->normalizePath($source);
+        $target = $this->normalizePath($target);
+
+        $cache = $this->getCache();
+        $sourceEntry = $cache->get($source);
+        if (!$sourceEntry) {
+            throw new NotFoundException('Source object not found');
+        }
+
+        $this->copyInner($sourceEntry, $target);
+
+        return true;
+    }
+
+    private function copyInner(ICacheEntry $sourceEntry, string $to) {
+        $cache = $this->getCache();
+
+        if ($sourceEntry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
+            if ($cache->inCache($to)) {
+                $cache->remove($to);
+            }
+            $this->mkdir($to);
+
+            foreach ($cache->getFolderContentsById($sourceEntry->getId()) as $child) {
+                $this->copyInner($child, $to . '/' . $child->getName());
+            }
+        } else {
+            $this->copyFile($sourceEntry, $to);
+        }
+    }
+
+    private function copyFile(ICacheEntry $sourceEntry, string $to) {
+        $cache = $this->getCache();
+
+        $sourceUrn = $this->getURN($sourceEntry->getId());
+
+        if (!$cache instanceof Cache) {
+            throw new \Exception("Invalid source cache for object store copy");
+        }
+
+        $targetId = $cache->copyFromCache($cache, $sourceEntry, $to);
+
+        $targetUrn = $this->getURN($targetId);
+
+        try {
+            $this->objectStore->copyObject($sourceUrn, $targetUrn);
+        } catch (\Exception $e) {
+            $cache->remove($to);
+
+            throw $e;
+        }
+    }
+
+    public function startChunkedWrite(string $targetPath): string {
+        if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
+            throw new GenericFileException('Object store does not support multipart upload');
+        }
+        $cacheEntry = $this->getCache()->get($targetPath);
+        $urn = $this->getURN($cacheEntry->getId());
+        return $this->objectStore->initiateMultipartUpload($urn);
+    }
+
+    /**
+     *
+     * @throws GenericFileException
+     */
+    public function putChunkedWritePart(
+        string $targetPath,
+        string $writeToken,
+        string $chunkId,
+        $data,
+        $size = null
+    ): ?array {
+        if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
+            throw new GenericFileException('Object store does not support multipart upload');
+        }
+        $cacheEntry = $this->getCache()->get($targetPath);
+        $urn = $this->getURN($cacheEntry->getId());
+
+        $result = $this->objectStore->uploadMultipartPart($urn, $writeToken, (int)$chunkId, $data, $size);
+
+        $parts[$chunkId] = [
+            'PartNumber' => $chunkId,
+            'ETag' => trim($result->get('ETag'), '"'),
+        ];
+        return $parts[$chunkId];
+    }
+
+    public function completeChunkedWrite(string $targetPath, string $writeToken): int {
+        if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
+            throw new GenericFileException('Object store does not support multipart upload');
+        }
+        $cacheEntry = $this->getCache()->get($targetPath);
+        $urn = $this->getURN($cacheEntry->getId());
+        $parts = $this->objectStore->getMultipartUploads($urn, $writeToken);
+        $sortedParts = array_values($parts);
+        sort($sortedParts);
+        try {
+            $size = $this->objectStore->completeMultipartUpload($urn, $writeToken, $sortedParts);
+            $stat = $this->stat($targetPath);
+            $mtime = time();
+            if (is_array($stat)) {
+                $stat['size'] = $size;
+                $stat['mtime'] = $mtime;
+                $stat['mimetype'] = $this->getMimeType($targetPath);
+                $this->getCache()->update($stat['fileid'], $stat);
+            }
+        } catch (S3MultipartUploadException|S3Exception $e) {
+            $this->objectStore->abortMultipartUpload($urn, $writeToken);
+            $this->logger->logException($e, [
+                'app' => 'objectstore',
+                'message' => 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken,
+            ]);
+            throw new GenericFileException('Could not write chunked file');
+        }
+        return $size;
+    }
+
+    public function cancelChunkedWrite(string $targetPath, string $writeToken): void {
+        if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
+            throw new GenericFileException('Object store does not support multipart upload');
+        }
+        $cacheEntry = $this->getCache()->get($targetPath);
+        $urn = $this->getURN($cacheEntry->getId());
+        $this->objectStore->abortMultipartUpload($urn, $writeToken);
+    }
 }

lib/private/Files/Cache/Scanner.php

Indentation +528 added lines, -528 removed lines

@@ -59,532 +59,532 @@
  * @package OC\Files\Cache
  */
 class Scanner extends BasicEmitter implements IScanner {
-	/**
-	 * @var \OC\Files\Storage\Storage $storage
-	 */
-	protected $storage;
-
-	/**
-	 * @var string $storageId
-	 */
-	protected $storageId;
-
-	/**
-	 * @var \OC\Files\Cache\Cache $cache
-	 */
-	protected $cache;
-
-	/**
-	 * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache
-	 */
-	protected $cacheActive;
-
-	/**
-	 * @var bool $useTransactions whether to use transactions
-	 */
-	protected $useTransactions = true;
-
-	/**
-	 * @var \OCP\Lock\ILockingProvider
-	 */
-	protected $lockingProvider;
-
-	protected IDBConnection $connection;
-
-	public function __construct(\OC\Files\Storage\Storage $storage) {
-		$this->storage = $storage;
-		$this->storageId = $this->storage->getId();
-		$this->cache = $storage->getCache();
-		$this->cacheActive = !\OC::$server->getConfig()->getSystemValueBool('filesystem_cache_readonly', false);
-		$this->lockingProvider = \OC::$server->getLockingProvider();
-		$this->connection = \OC::$server->get(IDBConnection::class);
-	}
-
-	/**
-	 * Whether to wrap the scanning of a folder in a database transaction
-	 * On default transactions are used
-	 *
-	 * @param bool $useTransactions
-	 */
-	public function setUseTransactions($useTransactions) {
-		$this->useTransactions = $useTransactions;
-	}
-
-	/**
-	 * get all the metadata of a file or folder
-	 * *
-	 *
-	 * @param string $path
-	 * @return array|null an array of metadata of the file
-	 */
-	protected function getData($path) {
-		$data = $this->storage->getMetaData($path);
-		if (is_null($data)) {
-			\OC::$server->get(LoggerInterface::class)->debug("!!! Path '$path' is not accessible or present !!!", ['app' => 'core']);
-		}
-		return $data;
-	}
-
-	/**
-	 * scan a single file and store it in the cache
-	 *
-	 * @param string $file
-	 * @param int $reuseExisting
-	 * @param int $parentId
-	 * @param array|null|false $cacheData existing data in the cache for the file to be scanned
-	 * @param bool $lock set to false to disable getting an additional read lock during scanning
-	 * @param null $data the metadata for the file, as returned by the storage
-	 * @return array|null an array of metadata of the scanned file
-	 * @throws \OCP\Lock\LockedException
-	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
-		if ($file !== '') {
-			try {
-				$this->storage->verifyPath(dirname($file), basename($file));
-			} catch (\Exception $e) {
-				return null;
-			}
-		}
-		// only proceed if $file is not a partial file, blacklist is handled by the storage
-		if (!self::isPartialFile($file)) {
-			// acquire a lock
-			if ($lock) {
-				if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-					$this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-				}
-			}
-
-			try {
-				$data = $data ?? $this->getData($file);
-			} catch (ForbiddenException $e) {
-				if ($lock) {
-					if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-						$this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-					}
-				}
-
-				return null;
-			}
-
-			try {
-				if ($data) {
-					// pre-emit only if it was a file. By that we avoid counting/treating folders as files
-					if ($data['mimetype'] !== 'httpd/unix-directory') {
-						$this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]);
-						\OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]);
-					}
-
-					$parent = dirname($file);
-					if ($parent === '.' || $parent === '/') {
-						$parent = '';
-					}
-					if ($parentId === -1) {
-						$parentId = $this->cache->getParentId($file);
-					}
-
-					// scan the parent if it's not in the cache (id -1) and the current file is not the root folder
-					if ($file && $parentId === -1) {
-						$parentData = $this->scanFile($parent);
-						if (!$parentData) {
-							return null;
-						}
-						$parentId = $parentData['fileid'];
-					}
-					if ($parent) {
-						$data['parent'] = $parentId;
-					}
-					if (is_null($cacheData)) {
-						/** @var CacheEntry $cacheData */
-						$cacheData = $this->cache->get($file);
-					}
-					if ($cacheData && $reuseExisting && isset($cacheData['fileid'])) {
-						// prevent empty etag
-						$etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag'];
-						$fileId = $cacheData['fileid'];
-						$data['fileid'] = $fileId;
-						// only reuse data if the file hasn't explicitly changed
-						if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) {
-							$data['mtime'] = $cacheData['mtime'];
-							if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) {
-								$data['size'] = $cacheData['size'];
-							}
-							if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) {
-								$data['etag'] = $etag;
-							}
-						}
-
-						// we only updated unencrypted_size if it's already set
-						if ($cacheData['unencrypted_size'] === 0) {
-							unset($data['unencrypted_size']);
-						}
-
-						// Only update metadata that has changed
-						$newData = array_diff_assoc($data, $cacheData->getData());
-					} else {
-						// we only updated unencrypted_size if it's already set
-						unset($data['unencrypted_size']);
-						$newData = $data;
-						$fileId = -1;
-					}
-					if (!empty($newData)) {
-						// Reset the checksum if the data has changed
-						$newData['checksum'] = '';
-						$newData['parent'] = $parentId;
-						$data['fileid'] = $this->addToCache($file, $newData, $fileId);
-					}
-
-					$data['oldSize'] = ($cacheData && isset($cacheData['size'])) ? $cacheData['size'] : 0;
-
-					if ($cacheData && isset($cacheData['encrypted'])) {
-						$data['encrypted'] = $cacheData['encrypted'];
-					}
-
-					// post-emit only if it was a file. By that we avoid counting/treating folders as files
-					if ($data['mimetype'] !== 'httpd/unix-directory') {
-						$this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]);
-						\OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]);
-					}
-				} else {
-					$this->removeFromCache($file);
-				}
-			} catch (\Exception $e) {
-				if ($lock) {
-					if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-						$this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-					}
-				}
-				throw $e;
-			}
-
-			// release the acquired lock
-			if ($lock) {
-				if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-					$this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-				}
-			}
-
-			if ($data && !isset($data['encrypted'])) {
-				$data['encrypted'] = false;
-			}
-			return $data;
-		}
-
-		return null;
-	}
-
-	protected function removeFromCache($path) {
-		\OC_Hook::emit('Scanner', 'removeFromCache', ['file' => $path]);
-		$this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', [$path]);
-		if ($this->cacheActive) {
-			$this->cache->remove($path);
-		}
-	}
-
-	/**
-	 * @param string $path
-	 * @param array $data
-	 * @param int $fileId
-	 * @return int the id of the added file
-	 */
-	protected function addToCache($path, $data, $fileId = -1) {
-		if (isset($data['scan_permissions'])) {
-			$data['permissions'] = $data['scan_permissions'];
-		}
-		\OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]);
-		$this->emit('\OC\Files\Cache\Scanner', 'addToCache', [$path, $this->storageId, $data]);
-		if ($this->cacheActive) {
-			if ($fileId !== -1) {
-				$this->cache->update($fileId, $data);
-				return $fileId;
-			} else {
-				return $this->cache->insert($path, $data);
-			}
-		} else {
-			return -1;
-		}
-	}
-
-	/**
-	 * @param string $path
-	 * @param array $data
-	 * @param int $fileId
-	 */
-	protected function updateCache($path, $data, $fileId = -1) {
-		\OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]);
-		$this->emit('\OC\Files\Cache\Scanner', 'updateCache', [$path, $this->storageId, $data]);
-		if ($this->cacheActive) {
-			if ($fileId !== -1) {
-				$this->cache->update($fileId, $data);
-			} else {
-				$this->cache->put($path, $data);
-			}
-		}
-	}
-
-	/**
-	 * scan a folder and all it's children
-	 *
-	 * @param string $path
-	 * @param bool $recursive
-	 * @param int $reuse
-	 * @param bool $lock set to false to disable getting an additional read lock during scanning
-	 * @return array|null an array of the meta data of the scanned file or folder
-	 */
-	public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) {
-		if ($reuse === -1) {
-			$reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
-		}
-		if ($lock) {
-			if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-				$this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
-				$this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-			}
-		}
-		try {
-			try {
-				$data = $this->scanFile($path, $reuse, -1, null, $lock);
-				if ($data && $data['mimetype'] === 'httpd/unix-directory') {
-					$size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data);
-					$data['size'] = $size;
-				}
-			} catch (NotFoundException $e) {
-				$this->removeFromCache($path);
-				return null;
-			}
-		} finally {
-			if ($lock) {
-				if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-					$this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-					$this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
-				}
-			}
-		}
-		return $data;
-	}
-
-	/**
-	 * Get the children currently in the cache
-	 *
-	 * @param int $folderId
-	 * @return array[]
-	 */
-	protected function getExistingChildren($folderId) {
-		$existingChildren = [];
-		$children = $this->cache->getFolderContentsById($folderId);
-		foreach ($children as $child) {
-			$existingChildren[$child['name']] = $child;
-		}
-		return $existingChildren;
-	}
-
-	/**
-	 * scan all the files and folders in a folder
-	 *
-	 * @param string $path
-	 * @param bool $recursive
-	 * @param int $reuse
-	 * @param int $folderId id for the folder to be scanned
-	 * @param bool $lock set to false to disable getting an additional read lock during scanning
-	 * @param array $data the data of the folder before (re)scanning the children
-	 * @return int the size of the scanned folder or -1 if the size is unknown at this stage
-	 */
-	protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true, array $data = []) {
-		if ($reuse === -1) {
-			$reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
-		}
-		$this->emit('\OC\Files\Cache\Scanner', 'scanFolder', [$path, $this->storageId]);
-		$size = 0;
-		if (!is_null($folderId)) {
-			$folderId = $this->cache->getId($path);
-		}
-		$childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size);
-
-		foreach ($childQueue as $child => $childId) {
-			$childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock);
-			if ($childSize === -1) {
-				$size = -1;
-			} elseif ($size !== -1) {
-				$size += $childSize;
-			}
-		}
-		$oldSize = $data['size'] ?? null;
-
-		// for encrypted storages, we trigger a regular folder size calculation instead of using the calculated size
-		// to make sure we also updated the unencrypted-size where applicable
-		if ($this->storage->instanceOfStorage(Encryption::class)) {
-			$this->cache->calculateFolderSize($path);
-		} else {
-			if ($this->cacheActive && $oldSize !== $size) {
-				$this->cache->update($folderId, ['size' => $size]);
-			}
-		}
-		$this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]);
-		return $size;
-	}
-
-	private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
-		// we put this in it's own function so it cleans up the memory before we start recursing
-		$existingChildren = $this->getExistingChildren($folderId);
-		$newChildren = iterator_to_array($this->storage->getDirectoryContent($path));
-
-		if (count($existingChildren) === 0 && count($newChildren) === 0) {
-			// no need to do a transaction
-			return [];
-		}
-
-		if ($this->useTransactions) {
-			$this->connection->beginTransaction();
-		}
-
-		$exceptionOccurred = false;
-		$childQueue = [];
-		$newChildNames = [];
-		foreach ($newChildren as $fileMeta) {
-			$permissions = isset($fileMeta['scan_permissions']) ? $fileMeta['scan_permissions'] : $fileMeta['permissions'];
-			if ($permissions === 0) {
-				continue;
-			}
-			$originalFile = $fileMeta['name'];
-			$file = trim(\OC\Files\Filesystem::normalizePath($originalFile), '/');
-			if (trim($originalFile, '/') !== $file) {
-				// encoding mismatch, might require compatibility wrapper
-				\OC::$server->get(LoggerInterface::class)->debug('Scanner: Skipping non-normalized file name "'. $originalFile . '" in path "' . $path . '".', ['app' => 'core']);
-				$this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]);
-				// skip this entry
-				continue;
-			}
-
-			$newChildNames[] = $file;
-			$child = $path ? $path . '/' . $file : $file;
-			try {
-				$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
-				$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
-				if ($data) {
-					if ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE) {
-						$childQueue[$child] = $data['fileid'];
-					} elseif ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE_INCOMPLETE && $data['size'] === -1) {
-						// only recurse into folders which aren't fully scanned
-						$childQueue[$child] = $data['fileid'];
-					} elseif ($data['size'] === -1) {
-						$size = -1;
-					} elseif ($size !== -1) {
-						$size += $data['size'];
-					}
-				}
-			} catch (Exception $ex) {
-				// might happen if inserting duplicate while a scanning
-				// process is running in parallel
-				// log and ignore
-				if ($this->useTransactions) {
-					$this->connection->rollback();
-					$this->connection->beginTransaction();
-				}
-				\OC::$server->get(LoggerInterface::class)->debug('Exception while scanning file "' . $child . '"', [
-					'app' => 'core',
-					'exception' => $ex,
-				]);
-				$exceptionOccurred = true;
-			} catch (\OCP\Lock\LockedException $e) {
-				if ($this->useTransactions) {
-					$this->connection->rollback();
-				}
-				throw $e;
-			}
-		}
-		$removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
-		foreach ($removedChildren as $childName) {
-			$child = $path ? $path . '/' . $childName : $childName;
-			$this->removeFromCache($child);
-		}
-		if ($this->useTransactions) {
-			$this->connection->commit();
-		}
-		if ($exceptionOccurred) {
-			// It might happen that the parallel scan process has already
-			// inserted mimetypes but those weren't available yet inside the transaction
-			// To make sure to have the updated mime types in such cases,
-			// we reload them here
-			\OC::$server->getMimeTypeLoader()->reset();
-		}
-		return $childQueue;
-	}
-
-	/**
-	 * check if the file should be ignored when scanning
-	 * NOTE: files with a '.part' extension are ignored as well!
-	 *       prevents unfinished put requests to be scanned
-	 *
-	 * @param string $file
-	 * @return boolean
-	 */
-	public static function isPartialFile($file) {
-		if (pathinfo($file, PATHINFO_EXTENSION) === 'part') {
-			return true;
-		}
-		if (strpos($file, '.part/') !== false) {
-			return true;
-		}
-
-		return false;
-	}
-
-	/**
-	 * walk over any folders that are not fully scanned yet and scan them
-	 */
-	public function backgroundScan() {
-		if ($this->storage->instanceOfStorage(Jail::class)) {
-			// for jail storage wrappers (shares, groupfolders) we run the background scan on the source storage
-			// this is mainly done because the jail wrapper doesn't implement `getIncomplete` (because it would be inefficient).
-			//
-			// Running the scan on the source storage might scan more than "needed", but the unscanned files outside the jail will
-			// have to be scanned at some point anyway.
-			$unJailedScanner = $this->storage->getUnjailedStorage()->getScanner();
-			$unJailedScanner->backgroundScan();
-		} else {
-			if (!$this->cache->inCache('')) {
-				// if the storage isn't in the cache yet, just scan the root completely
-				$this->runBackgroundScanJob(function () {
-					$this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG);
-				}, '');
-			} else {
-				$lastPath = null;
-				// find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck)
-				while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) {
-					$this->runBackgroundScanJob(function () use ($path) {
-						$this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE);
-					}, $path);
-					// FIXME: this won't proceed with the next item, needs revamping of getIncomplete()
-					// to make this possible
-					$lastPath = $path;
-				}
-			}
-		}
-	}
-
-	protected function runBackgroundScanJob(callable $callback, $path) {
-		try {
-			$callback();
-			\OC_Hook::emit('Scanner', 'correctFolderSize', ['path' => $path]);
-			if ($this->cacheActive && $this->cache instanceof Cache) {
-				$this->cache->correctFolderSize($path, null, true);
-			}
-		} catch (\OCP\Files\StorageInvalidException $e) {
-			// skip unavailable storages
-		} catch (\OCP\Files\StorageNotAvailableException $e) {
-			// skip unavailable storages
-		} catch (\OCP\Files\ForbiddenException $e) {
-			// skip forbidden storages
-		} catch (\OCP\Lock\LockedException $e) {
-			// skip unavailable storages
-		}
-	}
-
-	/**
-	 * Set whether the cache is affected by scan operations
-	 *
-	 * @param boolean $active The active state of the cache
-	 */
-	public function setCacheActive($active) {
-		$this->cacheActive = $active;
-	}
+    /**
+     * @var \OC\Files\Storage\Storage $storage
+     */
+    protected $storage;
+
+    /**
+     * @var string $storageId
+     */
+    protected $storageId;
+
+    /**
+     * @var \OC\Files\Cache\Cache $cache
+     */
+    protected $cache;
+
+    /**
+     * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache
+     */
+    protected $cacheActive;
+
+    /**
+     * @var bool $useTransactions whether to use transactions
+     */
+    protected $useTransactions = true;
+
+    /**
+     * @var \OCP\Lock\ILockingProvider
+     */
+    protected $lockingProvider;
+
+    protected IDBConnection $connection;
+
+    public function __construct(\OC\Files\Storage\Storage $storage) {
+        $this->storage = $storage;
+        $this->storageId = $this->storage->getId();
+        $this->cache = $storage->getCache();
+        $this->cacheActive = !\OC::$server->getConfig()->getSystemValueBool('filesystem_cache_readonly', false);
+        $this->lockingProvider = \OC::$server->getLockingProvider();
+        $this->connection = \OC::$server->get(IDBConnection::class);
+    }
+
+    /**
+     * Whether to wrap the scanning of a folder in a database transaction
+     * On default transactions are used
+     *
+     * @param bool $useTransactions
+     */
+    public function setUseTransactions($useTransactions) {
+        $this->useTransactions = $useTransactions;
+    }
+
+    /**
+     * get all the metadata of a file or folder
+     * *
+     *
+     * @param string $path
+     * @return array|null an array of metadata of the file
+     */
+    protected function getData($path) {
+        $data = $this->storage->getMetaData($path);
+        if (is_null($data)) {
+            \OC::$server->get(LoggerInterface::class)->debug("!!! Path '$path' is not accessible or present !!!", ['app' => 'core']);
+        }
+        return $data;
+    }
+
+    /**
+     * scan a single file and store it in the cache
+     *
+     * @param string $file
+     * @param int $reuseExisting
+     * @param int $parentId
+     * @param array|null|false $cacheData existing data in the cache for the file to be scanned
+     * @param bool $lock set to false to disable getting an additional read lock during scanning
+     * @param null $data the metadata for the file, as returned by the storage
+     * @return array|null an array of metadata of the scanned file
+     * @throws \OCP\Lock\LockedException
+     */
+    public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
+        if ($file !== '') {
+            try {
+                $this->storage->verifyPath(dirname($file), basename($file));
+            } catch (\Exception $e) {
+                return null;
+            }
+        }
+        // only proceed if $file is not a partial file, blacklist is handled by the storage
+        if (!self::isPartialFile($file)) {
+            // acquire a lock
+            if ($lock) {
+                if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                    $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+                }
+            }
+
+            try {
+                $data = $data ?? $this->getData($file);
+            } catch (ForbiddenException $e) {
+                if ($lock) {
+                    if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                        $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+                    }
+                }
+
+                return null;
+            }
+
+            try {
+                if ($data) {
+                    // pre-emit only if it was a file. By that we avoid counting/treating folders as files
+                    if ($data['mimetype'] !== 'httpd/unix-directory') {
+                        $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]);
+                        \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]);
+                    }
+
+                    $parent = dirname($file);
+                    if ($parent === '.' || $parent === '/') {
+                        $parent = '';
+                    }
+                    if ($parentId === -1) {
+                        $parentId = $this->cache->getParentId($file);
+                    }
+
+                    // scan the parent if it's not in the cache (id -1) and the current file is not the root folder
+                    if ($file && $parentId === -1) {
+                        $parentData = $this->scanFile($parent);
+                        if (!$parentData) {
+                            return null;
+                        }
+                        $parentId = $parentData['fileid'];
+                    }
+                    if ($parent) {
+                        $data['parent'] = $parentId;
+                    }
+                    if (is_null($cacheData)) {
+                        /** @var CacheEntry $cacheData */
+                        $cacheData = $this->cache->get($file);
+                    }
+                    if ($cacheData && $reuseExisting && isset($cacheData['fileid'])) {
+                        // prevent empty etag
+                        $etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag'];
+                        $fileId = $cacheData['fileid'];
+                        $data['fileid'] = $fileId;
+                        // only reuse data if the file hasn't explicitly changed
+                        if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) {
+                            $data['mtime'] = $cacheData['mtime'];
+                            if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) {
+                                $data['size'] = $cacheData['size'];
+                            }
+                            if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) {
+                                $data['etag'] = $etag;
+                            }
+                        }
+
+                        // we only updated unencrypted_size if it's already set
+                        if ($cacheData['unencrypted_size'] === 0) {
+                            unset($data['unencrypted_size']);
+                        }
+
+                        // Only update metadata that has changed
+                        $newData = array_diff_assoc($data, $cacheData->getData());
+                    } else {
+                        // we only updated unencrypted_size if it's already set
+                        unset($data['unencrypted_size']);
+                        $newData = $data;
+                        $fileId = -1;
+                    }
+                    if (!empty($newData)) {
+                        // Reset the checksum if the data has changed
+                        $newData['checksum'] = '';
+                        $newData['parent'] = $parentId;
+                        $data['fileid'] = $this->addToCache($file, $newData, $fileId);
+                    }
+
+                    $data['oldSize'] = ($cacheData && isset($cacheData['size'])) ? $cacheData['size'] : 0;
+
+                    if ($cacheData && isset($cacheData['encrypted'])) {
+                        $data['encrypted'] = $cacheData['encrypted'];
+                    }
+
+                    // post-emit only if it was a file. By that we avoid counting/treating folders as files
+                    if ($data['mimetype'] !== 'httpd/unix-directory') {
+                        $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]);
+                        \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]);
+                    }
+                } else {
+                    $this->removeFromCache($file);
+                }
+            } catch (\Exception $e) {
+                if ($lock) {
+                    if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                        $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+                    }
+                }
+                throw $e;
+            }
+
+            // release the acquired lock
+            if ($lock) {
+                if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                    $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+                }
+            }
+
+            if ($data && !isset($data['encrypted'])) {
+                $data['encrypted'] = false;
+            }
+            return $data;
+        }
+
+        return null;
+    }
+
+    protected function removeFromCache($path) {
+        \OC_Hook::emit('Scanner', 'removeFromCache', ['file' => $path]);
+        $this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', [$path]);
+        if ($this->cacheActive) {
+            $this->cache->remove($path);
+        }
+    }
+
+    /**
+     * @param string $path
+     * @param array $data
+     * @param int $fileId
+     * @return int the id of the added file
+     */
+    protected function addToCache($path, $data, $fileId = -1) {
+        if (isset($data['scan_permissions'])) {
+            $data['permissions'] = $data['scan_permissions'];
+        }
+        \OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]);
+        $this->emit('\OC\Files\Cache\Scanner', 'addToCache', [$path, $this->storageId, $data]);
+        if ($this->cacheActive) {
+            if ($fileId !== -1) {
+                $this->cache->update($fileId, $data);
+                return $fileId;
+            } else {
+                return $this->cache->insert($path, $data);
+            }
+        } else {
+            return -1;
+        }
+    }
+
+    /**
+     * @param string $path
+     * @param array $data
+     * @param int $fileId
+     */
+    protected function updateCache($path, $data, $fileId = -1) {
+        \OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]);
+        $this->emit('\OC\Files\Cache\Scanner', 'updateCache', [$path, $this->storageId, $data]);
+        if ($this->cacheActive) {
+            if ($fileId !== -1) {
+                $this->cache->update($fileId, $data);
+            } else {
+                $this->cache->put($path, $data);
+            }
+        }
+    }
+
+    /**
+     * scan a folder and all it's children
+     *
+     * @param string $path
+     * @param bool $recursive
+     * @param int $reuse
+     * @param bool $lock set to false to disable getting an additional read lock during scanning
+     * @return array|null an array of the meta data of the scanned file or folder
+     */
+    public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) {
+        if ($reuse === -1) {
+            $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
+        }
+        if ($lock) {
+            if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+                $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+            }
+        }
+        try {
+            try {
+                $data = $this->scanFile($path, $reuse, -1, null, $lock);
+                if ($data && $data['mimetype'] === 'httpd/unix-directory') {
+                    $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data);
+                    $data['size'] = $size;
+                }
+            } catch (NotFoundException $e) {
+                $this->removeFromCache($path);
+                return null;
+            }
+        } finally {
+            if ($lock) {
+                if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                    $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+                    $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+                }
+            }
+        }
+        return $data;
+    }
+
+    /**
+     * Get the children currently in the cache
+     *
+     * @param int $folderId
+     * @return array[]
+     */
+    protected function getExistingChildren($folderId) {
+        $existingChildren = [];
+        $children = $this->cache->getFolderContentsById($folderId);
+        foreach ($children as $child) {
+            $existingChildren[$child['name']] = $child;
+        }
+        return $existingChildren;
+    }
+
+    /**
+     * scan all the files and folders in a folder
+     *
+     * @param string $path
+     * @param bool $recursive
+     * @param int $reuse
+     * @param int $folderId id for the folder to be scanned
+     * @param bool $lock set to false to disable getting an additional read lock during scanning
+     * @param array $data the data of the folder before (re)scanning the children
+     * @return int the size of the scanned folder or -1 if the size is unknown at this stage
+     */
+    protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true, array $data = []) {
+        if ($reuse === -1) {
+            $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
+        }
+        $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', [$path, $this->storageId]);
+        $size = 0;
+        if (!is_null($folderId)) {
+            $folderId = $this->cache->getId($path);
+        }
+        $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size);
+
+        foreach ($childQueue as $child => $childId) {
+            $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock);
+            if ($childSize === -1) {
+                $size = -1;
+            } elseif ($size !== -1) {
+                $size += $childSize;
+            }
+        }
+        $oldSize = $data['size'] ?? null;
+
+        // for encrypted storages, we trigger a regular folder size calculation instead of using the calculated size
+        // to make sure we also updated the unencrypted-size where applicable
+        if ($this->storage->instanceOfStorage(Encryption::class)) {
+            $this->cache->calculateFolderSize($path);
+        } else {
+            if ($this->cacheActive && $oldSize !== $size) {
+                $this->cache->update($folderId, ['size' => $size]);
+            }
+        }
+        $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]);
+        return $size;
+    }
+
+    private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
+        // we put this in it's own function so it cleans up the memory before we start recursing
+        $existingChildren = $this->getExistingChildren($folderId);
+        $newChildren = iterator_to_array($this->storage->getDirectoryContent($path));
+
+        if (count($existingChildren) === 0 && count($newChildren) === 0) {
+            // no need to do a transaction
+            return [];
+        }
+
+        if ($this->useTransactions) {
+            $this->connection->beginTransaction();
+        }
+
+        $exceptionOccurred = false;
+        $childQueue = [];
+        $newChildNames = [];
+        foreach ($newChildren as $fileMeta) {
+            $permissions = isset($fileMeta['scan_permissions']) ? $fileMeta['scan_permissions'] : $fileMeta['permissions'];
+            if ($permissions === 0) {
+                continue;
+            }
+            $originalFile = $fileMeta['name'];
+            $file = trim(\OC\Files\Filesystem::normalizePath($originalFile), '/');
+            if (trim($originalFile, '/') !== $file) {
+                // encoding mismatch, might require compatibility wrapper
+                \OC::$server->get(LoggerInterface::class)->debug('Scanner: Skipping non-normalized file name "'. $originalFile . '" in path "' . $path . '".', ['app' => 'core']);
+                $this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]);
+                // skip this entry
+                continue;
+            }
+
+            $newChildNames[] = $file;
+            $child = $path ? $path . '/' . $file : $file;
+            try {
+                $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
+                $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
+                if ($data) {
+                    if ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE) {
+                        $childQueue[$child] = $data['fileid'];
+                    } elseif ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE_INCOMPLETE && $data['size'] === -1) {
+                        // only recurse into folders which aren't fully scanned
+                        $childQueue[$child] = $data['fileid'];
+                    } elseif ($data['size'] === -1) {
+                        $size = -1;
+                    } elseif ($size !== -1) {
+                        $size += $data['size'];
+                    }
+                }
+            } catch (Exception $ex) {
+                // might happen if inserting duplicate while a scanning
+                // process is running in parallel
+                // log and ignore
+                if ($this->useTransactions) {
+                    $this->connection->rollback();
+                    $this->connection->beginTransaction();
+                }
+                \OC::$server->get(LoggerInterface::class)->debug('Exception while scanning file "' . $child . '"', [
+                    'app' => 'core',
+                    'exception' => $ex,
+                ]);
+                $exceptionOccurred = true;
+            } catch (\OCP\Lock\LockedException $e) {
+                if ($this->useTransactions) {
+                    $this->connection->rollback();
+                }
+                throw $e;
+            }
+        }
+        $removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
+        foreach ($removedChildren as $childName) {
+            $child = $path ? $path . '/' . $childName : $childName;
+            $this->removeFromCache($child);
+        }
+        if ($this->useTransactions) {
+            $this->connection->commit();
+        }
+        if ($exceptionOccurred) {
+            // It might happen that the parallel scan process has already
+            // inserted mimetypes but those weren't available yet inside the transaction
+            // To make sure to have the updated mime types in such cases,
+            // we reload them here
+            \OC::$server->getMimeTypeLoader()->reset();
+        }
+        return $childQueue;
+    }
+
+    /**
+     * check if the file should be ignored when scanning
+     * NOTE: files with a '.part' extension are ignored as well!
+     *       prevents unfinished put requests to be scanned
+     *
+     * @param string $file
+     * @return boolean
+     */
+    public static function isPartialFile($file) {
+        if (pathinfo($file, PATHINFO_EXTENSION) === 'part') {
+            return true;
+        }
+        if (strpos($file, '.part/') !== false) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * walk over any folders that are not fully scanned yet and scan them
+     */
+    public function backgroundScan() {
+        if ($this->storage->instanceOfStorage(Jail::class)) {
+            // for jail storage wrappers (shares, groupfolders) we run the background scan on the source storage
+            // this is mainly done because the jail wrapper doesn't implement `getIncomplete` (because it would be inefficient).
+            //
+            // Running the scan on the source storage might scan more than "needed", but the unscanned files outside the jail will
+            // have to be scanned at some point anyway.
+            $unJailedScanner = $this->storage->getUnjailedStorage()->getScanner();
+            $unJailedScanner->backgroundScan();
+        } else {
+            if (!$this->cache->inCache('')) {
+                // if the storage isn't in the cache yet, just scan the root completely
+                $this->runBackgroundScanJob(function () {
+                    $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG);
+                }, '');
+            } else {
+                $lastPath = null;
+                // find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck)
+                while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) {
+                    $this->runBackgroundScanJob(function () use ($path) {
+                        $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE);
+                    }, $path);
+                    // FIXME: this won't proceed with the next item, needs revamping of getIncomplete()
+                    // to make this possible
+                    $lastPath = $path;
+                }
+            }
+        }
+    }
+
+    protected function runBackgroundScanJob(callable $callback, $path) {
+        try {
+            $callback();
+            \OC_Hook::emit('Scanner', 'correctFolderSize', ['path' => $path]);
+            if ($this->cacheActive && $this->cache instanceof Cache) {
+                $this->cache->correctFolderSize($path, null, true);
+            }
+        } catch (\OCP\Files\StorageInvalidException $e) {
+            // skip unavailable storages
+        } catch (\OCP\Files\StorageNotAvailableException $e) {
+            // skip unavailable storages
+        } catch (\OCP\Files\ForbiddenException $e) {
+            // skip forbidden storages
+        } catch (\OCP\Lock\LockedException $e) {
+            // skip unavailable storages
+        }
+    }
+
+    /**
+     * Set whether the cache is affected by scan operations
+     *
+     * @param boolean $active The active state of the cache
+     */
+    public function setCacheActive($active) {
+        $this->cacheActive = $active;
+    }
 }

Spacing +9 added lines, -9 removed lines

@@ -336,7 +336,7 @@ 
 		}
 		if ($lock) {
 			if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-				$this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+				$this->storage->acquireLock('scanner::'.$path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
 				$this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
 			}
 		}
@@ -355,7 +355,7 @@ 
 			if ($lock) {
 				if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
 					$this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-					$this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+					$this->storage->releaseLock('scanner::'.$path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
 				}
 			}
 		}
@@ -448,14 +448,14 @@ 
 			$file = trim(\OC\Files\Filesystem::normalizePath($originalFile), '/');
 			if (trim($originalFile, '/') !== $file) {
 				// encoding mismatch, might require compatibility wrapper
-				\OC::$server->get(LoggerInterface::class)->debug('Scanner: Skipping non-normalized file name "'. $originalFile . '" in path "' . $path . '".', ['app' => 'core']);
-				$this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]);
+				\OC::$server->get(LoggerInterface::class)->debug('Scanner: Skipping non-normalized file name "'.$originalFile.'" in path "'.$path.'".', ['app' => 'core']);
+				$this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path.'/'.$originalFile : $originalFile]);
 				// skip this entry
 				continue;
 			}
 
 			$newChildNames[] = $file;
-			$child = $path ? $path . '/' . $file : $file;
+			$child = $path ? $path.'/'.$file : $file;
 			try {
 				$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
 				$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
@@ -479,7 +479,7 @@ 
 					$this->connection->rollback();
 					$this->connection->beginTransaction();
 				}
-				\OC::$server->get(LoggerInterface::class)->debug('Exception while scanning file "' . $child . '"', [
+				\OC::$server->get(LoggerInterface::class)->debug('Exception while scanning file "'.$child.'"', [
 					'app' => 'core',
 					'exception' => $ex,
 				]);
@@ -493,7 +493,7 @@ 
 		}
 		$removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
 		foreach ($removedChildren as $childName) {
-			$child = $path ? $path . '/' . $childName : $childName;
+			$child = $path ? $path.'/'.$childName : $childName;
 			$this->removeFromCache($child);
 		}
 		if ($this->useTransactions) {
@@ -543,14 +543,14 @@ 
 		} else {
 			if (!$this->cache->inCache('')) {
 				// if the storage isn't in the cache yet, just scan the root completely
-				$this->runBackgroundScanJob(function () {
+				$this->runBackgroundScanJob(function() {
 					$this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG);
 				}, '');
 			} else {
 				$lastPath = null;
 				// find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck)
 				while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) {
-					$this->runBackgroundScanJob(function () use ($path) {
+					$this->runBackgroundScanJob(function() use ($path) {
 						$this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE);
 					}, $path);
 					// FIXME: this won't proceed with the next item, needs revamping of getIncomplete()