nextcloud / server

apps/files_sharing/lib/External/Storage.php

Indentation +383 added lines, -383 removed lines

@@ -50,387 +50,387 @@
 use OCP\Http\Client\IClientService;
 
 class Storage extends DAV implements ISharedStorage, IDisableEncryptionStorage, IReliableEtagStorage {
-	/** @var ICloudId */
-	private $cloudId;
-	/** @var string */
-	private $mountPoint;
-	/** @var string */
-	private $token;
-	/** @var \OCP\ICacheFactory */
-	private $memcacheFactory;
-	/** @var \OCP\Http\Client\IClientService */
-	private $httpClient;
-	/** @var bool */
-	private $updateChecked = false;
-
-	/** @var ExternalShareManager */
-	private $manager;
-
-	/**
-	 * @param array{HttpClientService: IClientService, manager: ExternalShareManager, cloudId: ICloudId, mountpoint: string, token: string, password: ?string}|array $options
-	 */
-	public function __construct($options) {
-		$this->memcacheFactory = \OC::$server->getMemCacheFactory();
-		$this->httpClient = $options['HttpClientService'];
-
-		$this->manager = $options['manager'];
-		$this->cloudId = $options['cloudId'];
-		$discoveryService = \OC::$server->query(\OCP\OCS\IDiscoveryService::class);
-
-		[$protocol, $remote] = explode('://', $this->cloudId->getRemote());
-		if (strpos($remote, '/')) {
-			[$host, $root] = explode('/', $remote, 2);
-		} else {
-			$host = $remote;
-			$root = '';
-		}
-		$secure = $protocol === 'https';
-		$federatedSharingEndpoints = $discoveryService->discover($this->cloudId->getRemote(), 'FEDERATED_SHARING');
-		$webDavEndpoint = isset($federatedSharingEndpoints['webdav']) ? $federatedSharingEndpoints['webdav'] : '/public.php/webdav';
-		$root = rtrim($root, '/') . $webDavEndpoint;
-		$this->mountPoint = $options['mountpoint'];
-		$this->token = $options['token'];
-
-		parent::__construct([
-			'secure' => $secure,
-			'host' => $host,
-			'root' => $root,
-			'user' => $options['token'],
-			'password' => (string)$options['password']
-		]);
-	}
-
-	public function getWatcher($path = '', $storage = null) {
-		if (!$storage) {
-			$storage = $this;
-		}
-		if (!isset($this->watcher)) {
-			$this->watcher = new Watcher($storage);
-			$this->watcher->setPolicy(\OC\Files\Cache\Watcher::CHECK_ONCE);
-		}
-		return $this->watcher;
-	}
-
-	public function getRemoteUser(): string {
-		return $this->cloudId->getUser();
-	}
-
-	public function getRemote(): string {
-		return $this->cloudId->getRemote();
-	}
-
-	public function getMountPoint(): string {
-		return $this->mountPoint;
-	}
-
-	public function getToken(): string {
-		return $this->token;
-	}
-
-	public function getPassword(): ?string {
-		return $this->password;
-	}
-
-	/**
-	 * Get id of the mount point.
-	 * @return string
-	 */
-	public function getId() {
-		return 'shared::' . md5($this->token . '@' . $this->getRemote());
-	}
-
-	public function getCache($path = '', $storage = null) {
-		if (is_null($this->cache)) {
-			$this->cache = new Cache($this, $this->cloudId);
-		}
-		return $this->cache;
-	}
-
-	/**
-	 * @param string $path
-	 * @param \OC\Files\Storage\Storage $storage
-	 * @return \OCA\Files_Sharing\External\Scanner
-	 */
-	public function getScanner($path = '', $storage = null) {
-		if (!$storage) {
-			$storage = $this;
-		}
-		if (!isset($this->scanner)) {
-			$this->scanner = new Scanner($storage);
-		}
-		return $this->scanner;
-	}
-
-	/**
-	 * Check if a file or folder has been updated since $time
-	 *
-	 * @param string $path
-	 * @param int $time
-	 * @throws \OCP\Files\StorageNotAvailableException
-	 * @throws \OCP\Files\StorageInvalidException
-	 * @return bool
-	 */
-	public function hasUpdated($path, $time) {
-		// since for owncloud webdav servers we can rely on etag propagation we only need to check the root of the storage
-		// because of that we only do one check for the entire storage per request
-		if ($this->updateChecked) {
-			return false;
-		}
-		$this->updateChecked = true;
-		try {
-			return parent::hasUpdated('', $time);
-		} catch (StorageInvalidException $e) {
-			// check if it needs to be removed
-			$this->checkStorageAvailability();
-			throw $e;
-		} catch (StorageNotAvailableException $e) {
-			// check if it needs to be removed or just temp unavailable
-			$this->checkStorageAvailability();
-			throw $e;
-		}
-	}
-
-	public function test() {
-		try {
-			return parent::test();
-		} catch (StorageInvalidException $e) {
-			// check if it needs to be removed
-			$this->checkStorageAvailability();
-			throw $e;
-		} catch (StorageNotAvailableException $e) {
-			// check if it needs to be removed or just temp unavailable
-			$this->checkStorageAvailability();
-			throw $e;
-		}
-	}
-
-	/**
-	 * Check whether this storage is permanently or temporarily
-	 * unavailable
-	 *
-	 * @throws \OCP\Files\StorageNotAvailableException
-	 * @throws \OCP\Files\StorageInvalidException
-	 */
-	public function checkStorageAvailability() {
-		// see if we can find out why the share is unavailable
-		try {
-			$this->getShareInfo();
-		} catch (NotFoundException $e) {
-			// a 404 can either mean that the share no longer exists or there is no Nextcloud on the remote
-			if ($this->testRemote()) {
-				// valid Nextcloud instance means that the public share no longer exists
-				// since this is permanent (re-sharing the file will create a new token)
-				// we remove the invalid storage
-				$this->manager->removeShare($this->mountPoint);
-				$this->manager->getMountManager()->removeMount($this->mountPoint);
-				throw new StorageInvalidException("Remote share not found", 0, $e);
-			} else {
-				// Nextcloud instance is gone, likely to be a temporary server configuration error
-				throw new StorageNotAvailableException("No nextcloud instance found at remote", 0, $e);
-			}
-		} catch (ForbiddenException $e) {
-			// auth error, remove share for now (provide a dialog in the future)
-			$this->manager->removeShare($this->mountPoint);
-			$this->manager->getMountManager()->removeMount($this->mountPoint);
-			throw new StorageInvalidException("Auth error when getting remote share");
-		} catch (\GuzzleHttp\Exception\ConnectException $e) {
-			throw new StorageNotAvailableException("Failed to connect to remote instance", 0, $e);
-		} catch (\GuzzleHttp\Exception\RequestException $e) {
-			throw new StorageNotAvailableException("Error while sending request to remote instance", 0, $e);
-		}
-	}
-
-	public function file_exists($path) {
-		if ($path === '') {
-			return true;
-		} else {
-			return parent::file_exists($path);
-		}
-	}
-
-	/**
-	 * Check if the configured remote is a valid federated share provider
-	 *
-	 * @return bool
-	 */
-	protected function testRemote(): bool {
-		try {
-			return $this->testRemoteUrl($this->getRemote() . '/ocs-provider/index.php')
-				|| $this->testRemoteUrl($this->getRemote() . '/ocs-provider/')
-				|| $this->testRemoteUrl($this->getRemote() . '/status.php');
-		} catch (\Exception $e) {
-			return false;
-		}
-	}
-
-	private function testRemoteUrl(string $url): bool {
-		$cache = $this->memcacheFactory->createDistributed('files_sharing_remote_url');
-		if ($cache->hasKey($url)) {
-			return (bool)$cache->get($url);
-		}
-
-		$client = $this->httpClient->newClient();
-		try {
-			$result = $client->get($url, [
-				'timeout' => 10,
-				'connect_timeout' => 10,
-			])->getBody();
-			$data = json_decode($result);
-			$returnValue = (is_object($data) && !empty($data->version));
-		} catch (ConnectException $e) {
-			$returnValue = false;
-		} catch (ClientException $e) {
-			$returnValue = false;
-		} catch (RequestException $e) {
-			$returnValue = false;
-		}
-
-		$cache->set($url, $returnValue, 60 * 60 * 24);
-		return $returnValue;
-	}
-
-	/**
-	 * Check whether the remote is an ownCloud/Nextcloud. This is needed since some sharing
-	 * features are not standardized.
-	 *
-	 * @throws LocalServerException
-	 */
-	public function remoteIsOwnCloud(): bool {
-		if (defined('PHPUNIT_RUN') || !$this->testRemoteUrl($this->getRemote() . '/status.php')) {
-			return false;
-		}
-		return true;
-	}
-
-	/**
-	 * @return mixed
-	 * @throws ForbiddenException
-	 * @throws NotFoundException
-	 * @throws \Exception
-	 */
-	public function getShareInfo() {
-		$remote = $this->getRemote();
-		$token = $this->getToken();
-		$password = $this->getPassword();
-
-		try {
-			// If remote is not an ownCloud do not try to get any share info
-			if (!$this->remoteIsOwnCloud()) {
-				return ['status' => 'unsupported'];
-			}
-		} catch (LocalServerException $e) {
-			// throw this to be on the safe side: the share will still be visible
-			// in the UI in case the failure is intermittent, and the user will
-			// be able to decide whether to remove it if it's really gone
-			throw new StorageNotAvailableException();
-		}
-
-		$url = rtrim($remote, '/') . '/index.php/apps/files_sharing/shareinfo?t=' . $token;
-
-		// TODO: DI
-		$client = \OC::$server->getHTTPClientService()->newClient();
-		try {
-			$response = $client->post($url, [
-				'body' => ['password' => $password],
-				'timeout' => 10,
-				'connect_timeout' => 10,
-			]);
-		} catch (\GuzzleHttp\Exception\RequestException $e) {
-			if ($e->getCode() === Http::STATUS_UNAUTHORIZED || $e->getCode() === Http::STATUS_FORBIDDEN) {
-				throw new ForbiddenException();
-			}
-			if ($e->getCode() === Http::STATUS_NOT_FOUND) {
-				throw new NotFoundException();
-			}
-			// throw this to be on the safe side: the share will still be visible
-			// in the UI in case the failure is intermittent, and the user will
-			// be able to decide whether to remove it if it's really gone
-			throw new StorageNotAvailableException();
-		}
-
-		return json_decode($response->getBody(), true);
-	}
-
-	public function getOwner($path) {
-		return $this->cloudId->getDisplayId();
-	}
-
-	public function isSharable($path) {
-		if (\OCP\Util::isSharingDisabledForUser() || !\OC\Share\Share::isResharingAllowed()) {
-			return false;
-		}
-		return ($this->getPermissions($path) & Constants::PERMISSION_SHARE);
-	}
-
-	public function getPermissions($path) {
-		$response = $this->propfind($path);
-		// old federated sharing permissions
-		if (isset($response['{http://open-collaboration-services.org/ns}share-permissions'])) {
-			$permissions = $response['{http://open-collaboration-services.org/ns}share-permissions'];
-		} elseif (isset($response['{http://open-cloud-mesh.org/ns}share-permissions'])) {
-			// permissions provided by the OCM API
-			$permissions = $this->ocmPermissions2ncPermissions($response['{http://open-collaboration-services.org/ns}share-permissions'], $path);
-		} elseif (isset($response['{http://owncloud.org/ns}permissions'])) {
-			return $this->parsePermissions($response['{http://owncloud.org/ns}permissions']);
-		} else {
-			// use default permission if remote server doesn't provide the share permissions
-			$permissions = $this->getDefaultPermissions($path);
-		}
-
-		return $permissions;
-	}
-
-	public function needsPartFile() {
-		return false;
-	}
-
-	/**
-	 * Translate OCM Permissions to Nextcloud permissions
-	 *
-	 * @param string $ocmPermissions json encoded OCM permissions
-	 * @param string $path path to file
-	 * @return int
-	 */
-	protected function ocmPermissions2ncPermissions(string $ocmPermissions, string $path): int {
-		try {
-			$ocmPermissions = json_decode($ocmPermissions);
-			$ncPermissions = 0;
-			foreach ($ocmPermissions as $permission) {
-				switch (strtolower($permission)) {
-					case 'read':
-						$ncPermissions += Constants::PERMISSION_READ;
-						break;
-					case 'write':
-						$ncPermissions += Constants::PERMISSION_CREATE + Constants::PERMISSION_UPDATE;
-						break;
-					case 'share':
-						$ncPermissions += Constants::PERMISSION_SHARE;
-						break;
-					default:
-						throw new \Exception();
-				}
-			}
-		} catch (\Exception $e) {
-			$ncPermissions = $this->getDefaultPermissions($path);
-		}
-
-		return $ncPermissions;
-	}
-
-	/**
-	 * Calculate the default permissions in case no permissions are provided
-	 */
-	protected function getDefaultPermissions(string $path): int {
-		if ($this->is_dir($path)) {
-			$permissions = Constants::PERMISSION_ALL;
-		} else {
-			$permissions = Constants::PERMISSION_ALL & ~Constants::PERMISSION_CREATE;
-		}
-
-		return $permissions;
-	}
-
-	public function free_space($path) {
-		return parent::free_space("");
-	}
+    /** @var ICloudId */
+    private $cloudId;
+    /** @var string */
+    private $mountPoint;
+    /** @var string */
+    private $token;
+    /** @var \OCP\ICacheFactory */
+    private $memcacheFactory;
+    /** @var \OCP\Http\Client\IClientService */
+    private $httpClient;
+    /** @var bool */
+    private $updateChecked = false;
+
+    /** @var ExternalShareManager */
+    private $manager;
+
+    /**
+     * @param array{HttpClientService: IClientService, manager: ExternalShareManager, cloudId: ICloudId, mountpoint: string, token: string, password: ?string}|array $options
+     */
+    public function __construct($options) {
+        $this->memcacheFactory = \OC::$server->getMemCacheFactory();
+        $this->httpClient = $options['HttpClientService'];
+
+        $this->manager = $options['manager'];
+        $this->cloudId = $options['cloudId'];
+        $discoveryService = \OC::$server->query(\OCP\OCS\IDiscoveryService::class);
+
+        [$protocol, $remote] = explode('://', $this->cloudId->getRemote());
+        if (strpos($remote, '/')) {
+            [$host, $root] = explode('/', $remote, 2);
+        } else {
+            $host = $remote;
+            $root = '';
+        }
+        $secure = $protocol === 'https';
+        $federatedSharingEndpoints = $discoveryService->discover($this->cloudId->getRemote(), 'FEDERATED_SHARING');
+        $webDavEndpoint = isset($federatedSharingEndpoints['webdav']) ? $federatedSharingEndpoints['webdav'] : '/public.php/webdav';
+        $root = rtrim($root, '/') . $webDavEndpoint;
+        $this->mountPoint = $options['mountpoint'];
+        $this->token = $options['token'];
+
+        parent::__construct([
+            'secure' => $secure,
+            'host' => $host,
+            'root' => $root,
+            'user' => $options['token'],
+            'password' => (string)$options['password']
+        ]);
+    }
+
+    public function getWatcher($path = '', $storage = null) {
+        if (!$storage) {
+            $storage = $this;
+        }
+        if (!isset($this->watcher)) {
+            $this->watcher = new Watcher($storage);
+            $this->watcher->setPolicy(\OC\Files\Cache\Watcher::CHECK_ONCE);
+        }
+        return $this->watcher;
+    }
+
+    public function getRemoteUser(): string {
+        return $this->cloudId->getUser();
+    }
+
+    public function getRemote(): string {
+        return $this->cloudId->getRemote();
+    }
+
+    public function getMountPoint(): string {
+        return $this->mountPoint;
+    }
+
+    public function getToken(): string {
+        return $this->token;
+    }
+
+    public function getPassword(): ?string {
+        return $this->password;
+    }
+
+    /**
+     * Get id of the mount point.
+     * @return string
+     */
+    public function getId() {
+        return 'shared::' . md5($this->token . '@' . $this->getRemote());
+    }
+
+    public function getCache($path = '', $storage = null) {
+        if (is_null($this->cache)) {
+            $this->cache = new Cache($this, $this->cloudId);
+        }
+        return $this->cache;
+    }
+
+    /**
+     * @param string $path
+     * @param \OC\Files\Storage\Storage $storage
+     * @return \OCA\Files_Sharing\External\Scanner
+     */
+    public function getScanner($path = '', $storage = null) {
+        if (!$storage) {
+            $storage = $this;
+        }
+        if (!isset($this->scanner)) {
+            $this->scanner = new Scanner($storage);
+        }
+        return $this->scanner;
+    }
+
+    /**
+     * Check if a file or folder has been updated since $time
+     *
+     * @param string $path
+     * @param int $time
+     * @throws \OCP\Files\StorageNotAvailableException
+     * @throws \OCP\Files\StorageInvalidException
+     * @return bool
+     */
+    public function hasUpdated($path, $time) {
+        // since for owncloud webdav servers we can rely on etag propagation we only need to check the root of the storage
+        // because of that we only do one check for the entire storage per request
+        if ($this->updateChecked) {
+            return false;
+        }
+        $this->updateChecked = true;
+        try {
+            return parent::hasUpdated('', $time);
+        } catch (StorageInvalidException $e) {
+            // check if it needs to be removed
+            $this->checkStorageAvailability();
+            throw $e;
+        } catch (StorageNotAvailableException $e) {
+            // check if it needs to be removed or just temp unavailable
+            $this->checkStorageAvailability();
+            throw $e;
+        }
+    }
+
+    public function test() {
+        try {
+            return parent::test();
+        } catch (StorageInvalidException $e) {
+            // check if it needs to be removed
+            $this->checkStorageAvailability();
+            throw $e;
+        } catch (StorageNotAvailableException $e) {
+            // check if it needs to be removed or just temp unavailable
+            $this->checkStorageAvailability();
+            throw $e;
+        }
+    }
+
+    /**
+     * Check whether this storage is permanently or temporarily
+     * unavailable
+     *
+     * @throws \OCP\Files\StorageNotAvailableException
+     * @throws \OCP\Files\StorageInvalidException
+     */
+    public function checkStorageAvailability() {
+        // see if we can find out why the share is unavailable
+        try {
+            $this->getShareInfo();
+        } catch (NotFoundException $e) {
+            // a 404 can either mean that the share no longer exists or there is no Nextcloud on the remote
+            if ($this->testRemote()) {
+                // valid Nextcloud instance means that the public share no longer exists
+                // since this is permanent (re-sharing the file will create a new token)
+                // we remove the invalid storage
+                $this->manager->removeShare($this->mountPoint);
+                $this->manager->getMountManager()->removeMount($this->mountPoint);
+                throw new StorageInvalidException("Remote share not found", 0, $e);
+            } else {
+                // Nextcloud instance is gone, likely to be a temporary server configuration error
+                throw new StorageNotAvailableException("No nextcloud instance found at remote", 0, $e);
+            }
+        } catch (ForbiddenException $e) {
+            // auth error, remove share for now (provide a dialog in the future)
+            $this->manager->removeShare($this->mountPoint);
+            $this->manager->getMountManager()->removeMount($this->mountPoint);
+            throw new StorageInvalidException("Auth error when getting remote share");
+        } catch (\GuzzleHttp\Exception\ConnectException $e) {
+            throw new StorageNotAvailableException("Failed to connect to remote instance", 0, $e);
+        } catch (\GuzzleHttp\Exception\RequestException $e) {
+            throw new StorageNotAvailableException("Error while sending request to remote instance", 0, $e);
+        }
+    }
+
+    public function file_exists($path) {
+        if ($path === '') {
+            return true;
+        } else {
+            return parent::file_exists($path);
+        }
+    }
+
+    /**
+     * Check if the configured remote is a valid federated share provider
+     *
+     * @return bool
+     */
+    protected function testRemote(): bool {
+        try {
+            return $this->testRemoteUrl($this->getRemote() . '/ocs-provider/index.php')
+                || $this->testRemoteUrl($this->getRemote() . '/ocs-provider/')
+                || $this->testRemoteUrl($this->getRemote() . '/status.php');
+        } catch (\Exception $e) {
+            return false;
+        }
+    }
+
+    private function testRemoteUrl(string $url): bool {
+        $cache = $this->memcacheFactory->createDistributed('files_sharing_remote_url');
+        if ($cache->hasKey($url)) {
+            return (bool)$cache->get($url);
+        }
+
+        $client = $this->httpClient->newClient();
+        try {
+            $result = $client->get($url, [
+                'timeout' => 10,
+                'connect_timeout' => 10,
+            ])->getBody();
+            $data = json_decode($result);
+            $returnValue = (is_object($data) && !empty($data->version));
+        } catch (ConnectException $e) {
+            $returnValue = false;
+        } catch (ClientException $e) {
+            $returnValue = false;
+        } catch (RequestException $e) {
+            $returnValue = false;
+        }
+
+        $cache->set($url, $returnValue, 60 * 60 * 24);
+        return $returnValue;
+    }
+
+    /**
+     * Check whether the remote is an ownCloud/Nextcloud. This is needed since some sharing
+     * features are not standardized.
+     *
+     * @throws LocalServerException
+     */
+    public function remoteIsOwnCloud(): bool {
+        if (defined('PHPUNIT_RUN') || !$this->testRemoteUrl($this->getRemote() . '/status.php')) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * @return mixed
+     * @throws ForbiddenException
+     * @throws NotFoundException
+     * @throws \Exception
+     */
+    public function getShareInfo() {
+        $remote = $this->getRemote();
+        $token = $this->getToken();
+        $password = $this->getPassword();
+
+        try {
+            // If remote is not an ownCloud do not try to get any share info
+            if (!$this->remoteIsOwnCloud()) {
+                return ['status' => 'unsupported'];
+            }
+        } catch (LocalServerException $e) {
+            // throw this to be on the safe side: the share will still be visible
+            // in the UI in case the failure is intermittent, and the user will
+            // be able to decide whether to remove it if it's really gone
+            throw new StorageNotAvailableException();
+        }
+
+        $url = rtrim($remote, '/') . '/index.php/apps/files_sharing/shareinfo?t=' . $token;
+
+        // TODO: DI
+        $client = \OC::$server->getHTTPClientService()->newClient();
+        try {
+            $response = $client->post($url, [
+                'body' => ['password' => $password],
+                'timeout' => 10,
+                'connect_timeout' => 10,
+            ]);
+        } catch (\GuzzleHttp\Exception\RequestException $e) {
+            if ($e->getCode() === Http::STATUS_UNAUTHORIZED || $e->getCode() === Http::STATUS_FORBIDDEN) {
+                throw new ForbiddenException();
+            }
+            if ($e->getCode() === Http::STATUS_NOT_FOUND) {
+                throw new NotFoundException();
+            }
+            // throw this to be on the safe side: the share will still be visible
+            // in the UI in case the failure is intermittent, and the user will
+            // be able to decide whether to remove it if it's really gone
+            throw new StorageNotAvailableException();
+        }
+
+        return json_decode($response->getBody(), true);
+    }
+
+    public function getOwner($path) {
+        return $this->cloudId->getDisplayId();
+    }
+
+    public function isSharable($path) {
+        if (\OCP\Util::isSharingDisabledForUser() || !\OC\Share\Share::isResharingAllowed()) {
+            return false;
+        }
+        return ($this->getPermissions($path) & Constants::PERMISSION_SHARE);
+    }
+
+    public function getPermissions($path) {
+        $response = $this->propfind($path);
+        // old federated sharing permissions
+        if (isset($response['{http://open-collaboration-services.org/ns}share-permissions'])) {
+            $permissions = $response['{http://open-collaboration-services.org/ns}share-permissions'];
+        } elseif (isset($response['{http://open-cloud-mesh.org/ns}share-permissions'])) {
+            // permissions provided by the OCM API
+            $permissions = $this->ocmPermissions2ncPermissions($response['{http://open-collaboration-services.org/ns}share-permissions'], $path);
+        } elseif (isset($response['{http://owncloud.org/ns}permissions'])) {
+            return $this->parsePermissions($response['{http://owncloud.org/ns}permissions']);
+        } else {
+            // use default permission if remote server doesn't provide the share permissions
+            $permissions = $this->getDefaultPermissions($path);
+        }
+
+        return $permissions;
+    }
+
+    public function needsPartFile() {
+        return false;
+    }
+
+    /**
+     * Translate OCM Permissions to Nextcloud permissions
+     *
+     * @param string $ocmPermissions json encoded OCM permissions
+     * @param string $path path to file
+     * @return int
+     */
+    protected function ocmPermissions2ncPermissions(string $ocmPermissions, string $path): int {
+        try {
+            $ocmPermissions = json_decode($ocmPermissions);
+            $ncPermissions = 0;
+            foreach ($ocmPermissions as $permission) {
+                switch (strtolower($permission)) {
+                    case 'read':
+                        $ncPermissions += Constants::PERMISSION_READ;
+                        break;
+                    case 'write':
+                        $ncPermissions += Constants::PERMISSION_CREATE + Constants::PERMISSION_UPDATE;
+                        break;
+                    case 'share':
+                        $ncPermissions += Constants::PERMISSION_SHARE;
+                        break;
+                    default:
+                        throw new \Exception();
+                }
+            }
+        } catch (\Exception $e) {
+            $ncPermissions = $this->getDefaultPermissions($path);
+        }
+
+        return $ncPermissions;
+    }
+
+    /**
+     * Calculate the default permissions in case no permissions are provided
+     */
+    protected function getDefaultPermissions(string $path): int {
+        if ($this->is_dir($path)) {
+            $permissions = Constants::PERMISSION_ALL;
+        } else {
+            $permissions = Constants::PERMISSION_ALL & ~Constants::PERMISSION_CREATE;
+        }
+
+        return $permissions;
+    }
+
+    public function free_space($path) {
+        return parent::free_space("");
+    }
 }

lib/private/Files/Storage/DAV.php

Indentation +809 added lines, -809 removed lines

@@ -64,813 +64,813 @@
  * @package OC\Files\Storage
  */
 class DAV extends Common {
-	/** @var string */
-	protected $password;
-	/** @var string */
-	protected $user;
-	/** @var string|null */
-	protected $authType;
-	/** @var string */
-	protected $host;
-	/** @var bool */
-	protected $secure;
-	/** @var string */
-	protected $root;
-	/** @var string */
-	protected $certPath;
-	/** @var bool */
-	protected $ready;
-	/** @var Client */
-	protected $client;
-	/** @var ArrayCache */
-	protected $statCache;
-	/** @var IClientService */
-	protected $httpClientService;
-	/** @var ICertificateManager */
-	protected $certManager;
-
-	/**
-	 * @param array $params
-	 * @throws \Exception
-	 */
-	public function __construct($params) {
-		$this->statCache = new ArrayCache();
-		$this->httpClientService = \OC::$server->getHTTPClientService();
-		if (isset($params['host']) && isset($params['user']) && isset($params['password'])) {
-			$host = $params['host'];
-			//remove leading http[s], will be generated in createBaseUri()
-			if (substr($host, 0, 8) == "https://") {
-				$host = substr($host, 8);
-			} elseif (substr($host, 0, 7) == "http://") {
-				$host = substr($host, 7);
-			}
-			$this->host = $host;
-			$this->user = $params['user'];
-			$this->password = $params['password'];
-			if (isset($params['authType'])) {
-				$this->authType = $params['authType'];
-			}
-			if (isset($params['secure'])) {
-				if (is_string($params['secure'])) {
-					$this->secure = ($params['secure'] === 'true');
-				} else {
-					$this->secure = (bool)$params['secure'];
-				}
-			} else {
-				$this->secure = false;
-			}
-			if ($this->secure === true) {
-				// inject mock for testing
-				$this->certManager = \OC::$server->getCertificateManager();
-			}
-			$this->root = $params['root'] ?? '/';
-			$this->root = '/' . ltrim($this->root, '/');
-			$this->root = rtrim($this->root, '/') . '/';
-		} else {
-			throw new \Exception('Invalid webdav storage configuration');
-		}
-	}
-
-	protected function init() {
-		if ($this->ready) {
-			return;
-		}
-		$this->ready = true;
-
-		$settings = [
-			'baseUri' => $this->createBaseUri(),
-			'userName' => $this->user,
-			'password' => $this->password,
-		];
-		if ($this->authType !== null) {
-			$settings['authType'] = $this->authType;
-		}
-
-		$proxy = \OC::$server->getConfig()->getSystemValue('proxy', '');
-		if ($proxy !== '') {
-			$settings['proxy'] = $proxy;
-		}
-
-		$this->client = new Client($settings);
-		$this->client->setThrowExceptions(true);
-
-		if ($this->secure === true) {
-			$certPath = $this->certManager->getAbsoluteBundlePath();
-			if (file_exists($certPath)) {
-				$this->certPath = $certPath;
-			}
-			if ($this->certPath) {
-				$this->client->addCurlSetting(CURLOPT_CAINFO, $this->certPath);
-			}
-		}
-	}
-
-	/**
-	 * Clear the stat cache
-	 */
-	public function clearStatCache() {
-		$this->statCache->clear();
-	}
-
-	/** {@inheritdoc} */
-	public function getId() {
-		return 'webdav::' . $this->user . '@' . $this->host . '/' . $this->root;
-	}
-
-	/** {@inheritdoc} */
-	public function createBaseUri() {
-		$baseUri = 'http';
-		if ($this->secure) {
-			$baseUri .= 's';
-		}
-		$baseUri .= '://' . $this->host . $this->root;
-		return $baseUri;
-	}
-
-	/** {@inheritdoc} */
-	public function mkdir($path) {
-		$this->init();
-		$path = $this->cleanPath($path);
-		$result = $this->simpleResponse('MKCOL', $path, null, 201);
-		if ($result) {
-			$this->statCache->set($path, true);
-		}
-		return $result;
-	}
-
-	/** {@inheritdoc} */
-	public function rmdir($path) {
-		$this->init();
-		$path = $this->cleanPath($path);
-		// FIXME: some WebDAV impl return 403 when trying to DELETE
-		// a non-empty folder
-		$result = $this->simpleResponse('DELETE', $path . '/', null, 204);
-		$this->statCache->clear($path . '/');
-		$this->statCache->remove($path);
-		return $result;
-	}
-
-	/** {@inheritdoc} */
-	public function opendir($path) {
-		$this->init();
-		$path = $this->cleanPath($path);
-		try {
-			$response = $this->client->propFind(
-				$this->encodePath($path),
-				['{DAV:}getetag'],
-				1
-			);
-			if ($response === false) {
-				return false;
-			}
-			$content = [];
-			$files = array_keys($response);
-			array_shift($files); //the first entry is the current directory
-
-			if (!$this->statCache->hasKey($path)) {
-				$this->statCache->set($path, true);
-			}
-			foreach ($files as $file) {
-				$file = urldecode($file);
-				// do not store the real entry, we might not have all properties
-				if (!$this->statCache->hasKey($path)) {
-					$this->statCache->set($file, true);
-				}
-				$file = basename($file);
-				$content[] = $file;
-			}
-			return IteratorDirectory::wrap($content);
-		} catch (\Exception $e) {
-			$this->convertException($e, $path);
-		}
-		return false;
-	}
-
-	/**
-	 * Propfind call with cache handling.
-	 *
-	 * First checks if information is cached.
-	 * If not, request it from the server then store to cache.
-	 *
-	 * @param string $path path to propfind
-	 *
-	 * @return array|boolean propfind response or false if the entry was not found
-	 *
-	 * @throws ClientHttpException
-	 */
-	protected function propfind($path) {
-		$path = $this->cleanPath($path);
-		$cachedResponse = $this->statCache->get($path);
-		// we either don't know it, or we know it exists but need more details
-		if (is_null($cachedResponse) || $cachedResponse === true) {
-			$this->init();
-			try {
-				$response = $this->client->propFind(
-					$this->encodePath($path),
-					[
-						'{DAV:}getlastmodified',
-						'{DAV:}getcontentlength',
-						'{DAV:}getcontenttype',
-						'{http://owncloud.org/ns}permissions',
-						'{http://open-collaboration-services.org/ns}share-permissions',
-						'{DAV:}resourcetype',
-						'{DAV:}getetag',
-						'{DAV:}quota-available-bytes',
-					]
-				);
-				$this->statCache->set($path, $response);
-			} catch (ClientHttpException $e) {
-				if ($e->getHttpStatus() === 404 || $e->getHttpStatus() === 405) {
-					$this->statCache->clear($path . '/');
-					$this->statCache->set($path, false);
-					return false;
-				}
-				$this->convertException($e, $path);
-			} catch (\Exception $e) {
-				$this->convertException($e, $path);
-			}
-		} else {
-			$response = $cachedResponse;
-		}
-		return $response;
-	}
-
-	/** {@inheritdoc} */
-	public function filetype($path) {
-		try {
-			$response = $this->propfind($path);
-			if ($response === false) {
-				return false;
-			}
-			$responseType = [];
-			if (isset($response["{DAV:}resourcetype"])) {
-				/** @var ResourceType[] $response */
-				$responseType = $response["{DAV:}resourcetype"]->getValue();
-			}
-			return (count($responseType) > 0 and $responseType[0] == "{DAV:}collection") ? 'dir' : 'file';
-		} catch (\Exception $e) {
-			$this->convertException($e, $path);
-		}
-		return false;
-	}
-
-	/** {@inheritdoc} */
-	public function file_exists($path) {
-		try {
-			$path = $this->cleanPath($path);
-			$cachedState = $this->statCache->get($path);
-			if ($cachedState === false) {
-				// we know the file doesn't exist
-				return false;
-			} elseif (!is_null($cachedState)) {
-				return true;
-			}
-			// need to get from server
-			return ($this->propfind($path) !== false);
-		} catch (\Exception $e) {
-			$this->convertException($e, $path);
-		}
-		return false;
-	}
-
-	/** {@inheritdoc} */
-	public function unlink($path) {
-		$this->init();
-		$path = $this->cleanPath($path);
-		$result = $this->simpleResponse('DELETE', $path, null, 204);
-		$this->statCache->clear($path . '/');
-		$this->statCache->remove($path);
-		return $result;
-	}
-
-	/** {@inheritdoc} */
-	public function fopen($path, $mode) {
-		$this->init();
-		$path = $this->cleanPath($path);
-		switch ($mode) {
-			case 'r':
-			case 'rb':
-				try {
-					$response = $this->httpClientService
-						->newClient()
-						->get($this->createBaseUri() . $this->encodePath($path), [
-							'auth' => [$this->user, $this->password],
-							'stream' => true
-						]);
-				} catch (\GuzzleHttp\Exception\ClientException $e) {
-					if ($e->getResponse() instanceof ResponseInterface
-						&& $e->getResponse()->getStatusCode() === 404) {
-						return false;
-					} else {
-						throw $e;
-					}
-				}
-
-				if ($response->getStatusCode() !== Http::STATUS_OK) {
-					if ($response->getStatusCode() === Http::STATUS_LOCKED) {
-						throw new \OCP\Lock\LockedException($path);
-					} else {
-						Util::writeLog("webdav client", 'Guzzle get returned status code ' . $response->getStatusCode(), ILogger::ERROR);
-					}
-				}
-
-				return $response->getBody();
-			case 'w':
-			case 'wb':
-			case 'a':
-			case 'ab':
-			case 'r+':
-			case 'w+':
-			case 'wb+':
-			case 'a+':
-			case 'x':
-			case 'x+':
-			case 'c':
-			case 'c+':
-				//emulate these
-				$tempManager = \OC::$server->getTempManager();
-				if (strrpos($path, '.') !== false) {
-					$ext = substr($path, strrpos($path, '.'));
-				} else {
-					$ext = '';
-				}
-				if ($this->file_exists($path)) {
-					if (!$this->isUpdatable($path)) {
-						return false;
-					}
-					if ($mode === 'w' or $mode === 'w+') {
-						$tmpFile = $tempManager->getTemporaryFile($ext);
-					} else {
-						$tmpFile = $this->getCachedFile($path);
-					}
-				} else {
-					if (!$this->isCreatable(dirname($path))) {
-						return false;
-					}
-					$tmpFile = $tempManager->getTemporaryFile($ext);
-				}
-				$handle = fopen($tmpFile, $mode);
-				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
-					$this->writeBack($tmpFile, $path);
-				});
-		}
-	}
-
-	/**
-	 * @param string $tmpFile
-	 */
-	public function writeBack($tmpFile, $path) {
-		$this->uploadFile($tmpFile, $path);
-		unlink($tmpFile);
-	}
-
-	/** {@inheritdoc} */
-	public function free_space($path) {
-		$this->init();
-		$path = $this->cleanPath($path);
-		try {
-			$response = $this->propfind($path);
-			if ($response === false) {
-				return FileInfo::SPACE_UNKNOWN;
-			}
-			if (isset($response['{DAV:}quota-available-bytes'])) {
-				return (int)$response['{DAV:}quota-available-bytes'];
-			} else {
-				return FileInfo::SPACE_UNKNOWN;
-			}
-		} catch (\Exception $e) {
-			return FileInfo::SPACE_UNKNOWN;
-		}
-	}
-
-	/** {@inheritdoc} */
-	public function touch($path, $mtime = null) {
-		$this->init();
-		if (is_null($mtime)) {
-			$mtime = time();
-		}
-		$path = $this->cleanPath($path);
-
-		// if file exists, update the mtime, else create a new empty file
-		if ($this->file_exists($path)) {
-			try {
-				$this->statCache->remove($path);
-				$this->client->proppatch($this->encodePath($path), ['{DAV:}lastmodified' => $mtime]);
-				// non-owncloud clients might not have accepted the property, need to recheck it
-				$response = $this->client->propfind($this->encodePath($path), ['{DAV:}getlastmodified'], 0);
-				if ($response === false) {
-					return false;
-				}
-				if (isset($response['{DAV:}getlastmodified'])) {
-					$remoteMtime = strtotime($response['{DAV:}getlastmodified']);
-					if ($remoteMtime !== $mtime) {
-						// server has not accepted the mtime
-						return false;
-					}
-				}
-			} catch (ClientHttpException $e) {
-				if ($e->getHttpStatus() === 501) {
-					return false;
-				}
-				$this->convertException($e, $path);
-				return false;
-			} catch (\Exception $e) {
-				$this->convertException($e, $path);
-				return false;
-			}
-		} else {
-			$this->file_put_contents($path, '');
-		}
-		return true;
-	}
-
-	/**
-	 * @param string $path
-	 * @param mixed $data
-	 * @return int|false
-	 */
-	public function file_put_contents($path, $data) {
-		$path = $this->cleanPath($path);
-		$result = parent::file_put_contents($path, $data);
-		$this->statCache->remove($path);
-		return $result;
-	}
-
-	/**
-	 * @param string $path
-	 * @param string $target
-	 */
-	protected function uploadFile($path, $target) {
-		$this->init();
-
-		// invalidate
-		$target = $this->cleanPath($target);
-		$this->statCache->remove($target);
-		$source = fopen($path, 'r');
-
-		$this->httpClientService
-			->newClient()
-			->put($this->createBaseUri() . $this->encodePath($target), [
-				'body' => $source,
-				'auth' => [$this->user, $this->password]
-			]);
-
-		$this->removeCachedFile($target);
-	}
-
-	/** {@inheritdoc} */
-	public function rename($path1, $path2) {
-		$this->init();
-		$path1 = $this->cleanPath($path1);
-		$path2 = $this->cleanPath($path2);
-		try {
-			// overwrite directory ?
-			if ($this->is_dir($path2)) {
-				// needs trailing slash in destination
-				$path2 = rtrim($path2, '/') . '/';
-			}
-			$this->client->request(
-				'MOVE',
-				$this->encodePath($path1),
-				null,
-				[
-					'Destination' => $this->createBaseUri() . $this->encodePath($path2),
-				]
-			);
-			$this->statCache->clear($path1 . '/');
-			$this->statCache->clear($path2 . '/');
-			$this->statCache->set($path1, false);
-			$this->statCache->set($path2, true);
-			$this->removeCachedFile($path1);
-			$this->removeCachedFile($path2);
-			return true;
-		} catch (\Exception $e) {
-			$this->convertException($e);
-		}
-		return false;
-	}
-
-	/** {@inheritdoc} */
-	public function copy($path1, $path2) {
-		$this->init();
-		$path1 = $this->cleanPath($path1);
-		$path2 = $this->cleanPath($path2);
-		try {
-			// overwrite directory ?
-			if ($this->is_dir($path2)) {
-				// needs trailing slash in destination
-				$path2 = rtrim($path2, '/') . '/';
-			}
-			$this->client->request(
-				'COPY',
-				$this->encodePath($path1),
-				null,
-				[
-					'Destination' => $this->createBaseUri() . $this->encodePath($path2),
-				]
-			);
-			$this->statCache->clear($path2 . '/');
-			$this->statCache->set($path2, true);
-			$this->removeCachedFile($path2);
-			return true;
-		} catch (\Exception $e) {
-			$this->convertException($e);
-		}
-		return false;
-	}
-
-	/** {@inheritdoc} */
-	public function stat($path) {
-		try {
-			$response = $this->propfind($path);
-			if (!$response) {
-				return false;
-			}
-			return [
-				'mtime' => isset($response['{DAV:}getlastmodified']) ? strtotime($response['{DAV:}getlastmodified']) : null,
-				'size' => (int)($response['{DAV:}getcontentlength'] ?? 0),
-			];
-		} catch (\Exception $e) {
-			$this->convertException($e, $path);
-		}
-		return [];
-	}
-
-	/** {@inheritdoc} */
-	public function getMimeType($path) {
-		$remoteMimetype = $this->getMimeTypeFromRemote($path);
-		if ($remoteMimetype === 'application/octet-stream') {
-			return \OC::$server->getMimeTypeDetector()->detectPath($path);
-		} else {
-			return $remoteMimetype;
-		}
-	}
-
-	public function getMimeTypeFromRemote($path) {
-		try {
-			$response = $this->propfind($path);
-			if ($response === false) {
-				return false;
-			}
-			$responseType = [];
-			if (isset($response["{DAV:}resourcetype"])) {
-				/** @var ResourceType[] $response */
-				$responseType = $response["{DAV:}resourcetype"]->getValue();
-			}
-			$type = (count($responseType) > 0 and $responseType[0] == "{DAV:}collection") ? 'dir' : 'file';
-			if ($type == 'dir') {
-				return 'httpd/unix-directory';
-			} elseif (isset($response['{DAV:}getcontenttype'])) {
-				return $response['{DAV:}getcontenttype'];
-			} else {
-				return 'application/octet-stream';
-			}
-		} catch (\Exception $e) {
-			return false;
-		}
-	}
-
-	/**
-	 * @param string $path
-	 * @return string
-	 */
-	public function cleanPath($path) {
-		if ($path === '') {
-			return $path;
-		}
-		$path = Filesystem::normalizePath($path);
-		// remove leading slash
-		return substr($path, 1);
-	}
-
-	/**
-	 * URL encodes the given path but keeps the slashes
-	 *
-	 * @param string $path to encode
-	 * @return string encoded path
-	 */
-	protected function encodePath($path) {
-		// slashes need to stay
-		return str_replace('%2F', '/', rawurlencode($path));
-	}
-
-	/**
-	 * @param string $method
-	 * @param string $path
-	 * @param string|resource|null $body
-	 * @param int $expected
-	 * @return bool
-	 * @throws StorageInvalidException
-	 * @throws StorageNotAvailableException
-	 */
-	protected function simpleResponse($method, $path, $body, $expected) {
-		$path = $this->cleanPath($path);
-		try {
-			$response = $this->client->request($method, $this->encodePath($path), $body);
-			return $response['statusCode'] == $expected;
-		} catch (ClientHttpException $e) {
-			if ($e->getHttpStatus() === 404 && $method === 'DELETE') {
-				$this->statCache->clear($path . '/');
-				$this->statCache->set($path, false);
-				return false;
-			}
-
-			$this->convertException($e, $path);
-		} catch (\Exception $e) {
-			$this->convertException($e, $path);
-		}
-		return false;
-	}
-
-	/**
-	 * check if curl is installed
-	 */
-	public static function checkDependencies() {
-		return true;
-	}
-
-	/** {@inheritdoc} */
-	public function isUpdatable($path) {
-		return (bool)($this->getPermissions($path) & Constants::PERMISSION_UPDATE);
-	}
-
-	/** {@inheritdoc} */
-	public function isCreatable($path) {
-		return (bool)($this->getPermissions($path) & Constants::PERMISSION_CREATE);
-	}
-
-	/** {@inheritdoc} */
-	public function isSharable($path) {
-		return (bool)($this->getPermissions($path) & Constants::PERMISSION_SHARE);
-	}
-
-	/** {@inheritdoc} */
-	public function isDeletable($path) {
-		return (bool)($this->getPermissions($path) & Constants::PERMISSION_DELETE);
-	}
-
-	/** {@inheritdoc} */
-	public function getPermissions($path) {
-		$this->init();
-		$path = $this->cleanPath($path);
-		$response = $this->propfind($path);
-		if ($response === false) {
-			return 0;
-		}
-		if (isset($response['{http://owncloud.org/ns}permissions'])) {
-			return $this->parsePermissions($response['{http://owncloud.org/ns}permissions']);
-		} elseif ($this->is_dir($path)) {
-			return Constants::PERMISSION_ALL;
-		} elseif ($this->file_exists($path)) {
-			return Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
-		} else {
-			return 0;
-		}
-	}
-
-	/** {@inheritdoc} */
-	public function getETag($path) {
-		$this->init();
-		$path = $this->cleanPath($path);
-		$response = $this->propfind($path);
-		if ($response === false) {
-			return null;
-		}
-		if (isset($response['{DAV:}getetag'])) {
-			$etag = trim($response['{DAV:}getetag'], '"');
-			if (strlen($etag) > 40) {
-				$etag = md5($etag);
-			}
-			return $etag;
-		}
-		return parent::getEtag($path);
-	}
-
-	/**
-	 * @param string $permissionsString
-	 * @return int
-	 */
-	protected function parsePermissions($permissionsString) {
-		$permissions = Constants::PERMISSION_READ;
-		if (strpos($permissionsString, 'R') !== false) {
-			$permissions |= Constants::PERMISSION_SHARE;
-		}
-		if (strpos($permissionsString, 'D') !== false) {
-			$permissions |= Constants::PERMISSION_DELETE;
-		}
-		if (strpos($permissionsString, 'W') !== false) {
-			$permissions |= Constants::PERMISSION_UPDATE;
-		}
-		if (strpos($permissionsString, 'CK') !== false) {
-			$permissions |= Constants::PERMISSION_CREATE;
-			$permissions |= Constants::PERMISSION_UPDATE;
-		}
-		return $permissions;
-	}
-
-	/**
-	 * check if a file or folder has been updated since $time
-	 *
-	 * @param string $path
-	 * @param int $time
-	 * @throws \OCP\Files\StorageNotAvailableException
-	 * @return bool
-	 */
-	public function hasUpdated($path, $time) {
-		$this->init();
-		$path = $this->cleanPath($path);
-		try {
-			// force refresh for $path
-			$this->statCache->remove($path);
-			$response = $this->propfind($path);
-			if ($response === false) {
-				if ($path === '') {
-					// if root is gone it means the storage is not available
-					throw new StorageNotAvailableException('root is gone');
-				}
-				return false;
-			}
-			if (isset($response['{DAV:}getetag'])) {
-				$cachedData = $this->getCache()->get($path);
-				$etag = trim($response['{DAV:}getetag'], '"');
-				if (($cachedData === false) || (!empty($etag) && ($cachedData['etag'] !== $etag))) {
-					return true;
-				} elseif (isset($response['{http://open-collaboration-services.org/ns}share-permissions'])) {
-					$sharePermissions = (int)$response['{http://open-collaboration-services.org/ns}share-permissions'];
-					return $sharePermissions !== $cachedData['permissions'];
-				} elseif (isset($response['{http://owncloud.org/ns}permissions'])) {
-					$permissions = $this->parsePermissions($response['{http://owncloud.org/ns}permissions']);
-					return $permissions !== $cachedData['permissions'];
-				} else {
-					return false;
-				}
-			} elseif (isset($response['{DAV:}getlastmodified'])) {
-				$remoteMtime = strtotime($response['{DAV:}getlastmodified']);
-				return $remoteMtime > $time;
-			} else {
-				// neither `getetag` nor `getlastmodified` is set
-				return false;
-			}
-		} catch (ClientHttpException $e) {
-			if ($e->getHttpStatus() === 405) {
-				if ($path === '') {
-					// if root is gone it means the storage is not available
-					throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage());
-				}
-				return false;
-			}
-			$this->convertException($e, $path);
-			return false;
-		} catch (\Exception $e) {
-			$this->convertException($e, $path);
-			return false;
-		}
-	}
-
-	/**
-	 * Interpret the given exception and decide whether it is due to an
-	 * unavailable storage, invalid storage or other.
-	 * This will either throw StorageInvalidException, StorageNotAvailableException
-	 * or do nothing.
-	 *
-	 * @param Exception $e sabre exception
-	 * @param string $path optional path from the operation
-	 *
-	 * @throws StorageInvalidException if the storage is invalid, for example
-	 * when the authentication expired or is invalid
-	 * @throws StorageNotAvailableException if the storage is not available,
-	 * which might be temporary
-	 * @throws ForbiddenException if the action is not allowed
-	 */
-	protected function convertException(Exception $e, $path = '') {
-		\OC::$server->getLogger()->logException($e, ['app' => 'files_external', 'level' => ILogger::DEBUG]);
-		if ($e instanceof ClientHttpException) {
-			if ($e->getHttpStatus() === Http::STATUS_LOCKED) {
-				throw new \OCP\Lock\LockedException($path);
-			}
-			if ($e->getHttpStatus() === Http::STATUS_UNAUTHORIZED) {
-				// either password was changed or was invalid all along
-				throw new StorageInvalidException(get_class($e) . ': ' . $e->getMessage());
-			} elseif ($e->getHttpStatus() === Http::STATUS_METHOD_NOT_ALLOWED) {
-				// ignore exception for MethodNotAllowed, false will be returned
-				return;
-			} elseif ($e->getHttpStatus() === Http::STATUS_FORBIDDEN) {
-				// The operation is forbidden. Fail somewhat gracefully
-				throw new ForbiddenException(get_class($e) . ':' . $e->getMessage(), false);
-			}
-			throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage());
-		} elseif ($e instanceof ClientException) {
-			// connection timeout or refused, server could be temporarily down
-			throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage());
-		} elseif ($e instanceof \InvalidArgumentException) {
-			// parse error because the server returned HTML instead of XML,
-			// possibly temporarily down
-			throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage());
-		} elseif (($e instanceof StorageNotAvailableException) || ($e instanceof StorageInvalidException)) {
-			// rethrow
-			throw $e;
-		}
-
-		// TODO: only log for now, but in the future need to wrap/rethrow exception
-	}
+    /** @var string */
+    protected $password;
+    /** @var string */
+    protected $user;
+    /** @var string|null */
+    protected $authType;
+    /** @var string */
+    protected $host;
+    /** @var bool */
+    protected $secure;
+    /** @var string */
+    protected $root;
+    /** @var string */
+    protected $certPath;
+    /** @var bool */
+    protected $ready;
+    /** @var Client */
+    protected $client;
+    /** @var ArrayCache */
+    protected $statCache;
+    /** @var IClientService */
+    protected $httpClientService;
+    /** @var ICertificateManager */
+    protected $certManager;
+
+    /**
+     * @param array $params
+     * @throws \Exception
+     */
+    public function __construct($params) {
+        $this->statCache = new ArrayCache();
+        $this->httpClientService = \OC::$server->getHTTPClientService();
+        if (isset($params['host']) && isset($params['user']) && isset($params['password'])) {
+            $host = $params['host'];
+            //remove leading http[s], will be generated in createBaseUri()
+            if (substr($host, 0, 8) == "https://") {
+                $host = substr($host, 8);
+            } elseif (substr($host, 0, 7) == "http://") {
+                $host = substr($host, 7);
+            }
+            $this->host = $host;
+            $this->user = $params['user'];
+            $this->password = $params['password'];
+            if (isset($params['authType'])) {
+                $this->authType = $params['authType'];
+            }
+            if (isset($params['secure'])) {
+                if (is_string($params['secure'])) {
+                    $this->secure = ($params['secure'] === 'true');
+                } else {
+                    $this->secure = (bool)$params['secure'];
+                }
+            } else {
+                $this->secure = false;
+            }
+            if ($this->secure === true) {
+                // inject mock for testing
+                $this->certManager = \OC::$server->getCertificateManager();
+            }
+            $this->root = $params['root'] ?? '/';
+            $this->root = '/' . ltrim($this->root, '/');
+            $this->root = rtrim($this->root, '/') . '/';
+        } else {
+            throw new \Exception('Invalid webdav storage configuration');
+        }
+    }
+
+    protected function init() {
+        if ($this->ready) {
+            return;
+        }
+        $this->ready = true;
+
+        $settings = [
+            'baseUri' => $this->createBaseUri(),
+            'userName' => $this->user,
+            'password' => $this->password,
+        ];
+        if ($this->authType !== null) {
+            $settings['authType'] = $this->authType;
+        }
+
+        $proxy = \OC::$server->getConfig()->getSystemValue('proxy', '');
+        if ($proxy !== '') {
+            $settings['proxy'] = $proxy;
+        }
+
+        $this->client = new Client($settings);
+        $this->client->setThrowExceptions(true);
+
+        if ($this->secure === true) {
+            $certPath = $this->certManager->getAbsoluteBundlePath();
+            if (file_exists($certPath)) {
+                $this->certPath = $certPath;
+            }
+            if ($this->certPath) {
+                $this->client->addCurlSetting(CURLOPT_CAINFO, $this->certPath);
+            }
+        }
+    }
+
+    /**
+     * Clear the stat cache
+     */
+    public function clearStatCache() {
+        $this->statCache->clear();
+    }
+
+    /** {@inheritdoc} */
+    public function getId() {
+        return 'webdav::' . $this->user . '@' . $this->host . '/' . $this->root;
+    }
+
+    /** {@inheritdoc} */
+    public function createBaseUri() {
+        $baseUri = 'http';
+        if ($this->secure) {
+            $baseUri .= 's';
+        }
+        $baseUri .= '://' . $this->host . $this->root;
+        return $baseUri;
+    }
+
+    /** {@inheritdoc} */
+    public function mkdir($path) {
+        $this->init();
+        $path = $this->cleanPath($path);
+        $result = $this->simpleResponse('MKCOL', $path, null, 201);
+        if ($result) {
+            $this->statCache->set($path, true);
+        }
+        return $result;
+    }
+
+    /** {@inheritdoc} */
+    public function rmdir($path) {
+        $this->init();
+        $path = $this->cleanPath($path);
+        // FIXME: some WebDAV impl return 403 when trying to DELETE
+        // a non-empty folder
+        $result = $this->simpleResponse('DELETE', $path . '/', null, 204);
+        $this->statCache->clear($path . '/');
+        $this->statCache->remove($path);
+        return $result;
+    }
+
+    /** {@inheritdoc} */
+    public function opendir($path) {
+        $this->init();
+        $path = $this->cleanPath($path);
+        try {
+            $response = $this->client->propFind(
+                $this->encodePath($path),
+                ['{DAV:}getetag'],
+                1
+            );
+            if ($response === false) {
+                return false;
+            }
+            $content = [];
+            $files = array_keys($response);
+            array_shift($files); //the first entry is the current directory
+
+            if (!$this->statCache->hasKey($path)) {
+                $this->statCache->set($path, true);
+            }
+            foreach ($files as $file) {
+                $file = urldecode($file);
+                // do not store the real entry, we might not have all properties
+                if (!$this->statCache->hasKey($path)) {
+                    $this->statCache->set($file, true);
+                }
+                $file = basename($file);
+                $content[] = $file;
+            }
+            return IteratorDirectory::wrap($content);
+        } catch (\Exception $e) {
+            $this->convertException($e, $path);
+        }
+        return false;
+    }
+
+    /**
+     * Propfind call with cache handling.
+     *
+     * First checks if information is cached.
+     * If not, request it from the server then store to cache.
+     *
+     * @param string $path path to propfind
+     *
+     * @return array|boolean propfind response or false if the entry was not found
+     *
+     * @throws ClientHttpException
+     */
+    protected function propfind($path) {
+        $path = $this->cleanPath($path);
+        $cachedResponse = $this->statCache->get($path);
+        // we either don't know it, or we know it exists but need more details
+        if (is_null($cachedResponse) || $cachedResponse === true) {
+            $this->init();
+            try {
+                $response = $this->client->propFind(
+                    $this->encodePath($path),
+                    [
+                        '{DAV:}getlastmodified',
+                        '{DAV:}getcontentlength',
+                        '{DAV:}getcontenttype',
+                        '{http://owncloud.org/ns}permissions',
+                        '{http://open-collaboration-services.org/ns}share-permissions',
+                        '{DAV:}resourcetype',
+                        '{DAV:}getetag',
+                        '{DAV:}quota-available-bytes',
+                    ]
+                );
+                $this->statCache->set($path, $response);
+            } catch (ClientHttpException $e) {
+                if ($e->getHttpStatus() === 404 || $e->getHttpStatus() === 405) {
+                    $this->statCache->clear($path . '/');
+                    $this->statCache->set($path, false);
+                    return false;
+                }
+                $this->convertException($e, $path);
+            } catch (\Exception $e) {
+                $this->convertException($e, $path);
+            }
+        } else {
+            $response = $cachedResponse;
+        }
+        return $response;
+    }
+
+    /** {@inheritdoc} */
+    public function filetype($path) {
+        try {
+            $response = $this->propfind($path);
+            if ($response === false) {
+                return false;
+            }
+            $responseType = [];
+            if (isset($response["{DAV:}resourcetype"])) {
+                /** @var ResourceType[] $response */
+                $responseType = $response["{DAV:}resourcetype"]->getValue();
+            }
+            return (count($responseType) > 0 and $responseType[0] == "{DAV:}collection") ? 'dir' : 'file';
+        } catch (\Exception $e) {
+            $this->convertException($e, $path);
+        }
+        return false;
+    }
+
+    /** {@inheritdoc} */
+    public function file_exists($path) {
+        try {
+            $path = $this->cleanPath($path);
+            $cachedState = $this->statCache->get($path);
+            if ($cachedState === false) {
+                // we know the file doesn't exist
+                return false;
+            } elseif (!is_null($cachedState)) {
+                return true;
+            }
+            // need to get from server
+            return ($this->propfind($path) !== false);
+        } catch (\Exception $e) {
+            $this->convertException($e, $path);
+        }
+        return false;
+    }
+
+    /** {@inheritdoc} */
+    public function unlink($path) {
+        $this->init();
+        $path = $this->cleanPath($path);
+        $result = $this->simpleResponse('DELETE', $path, null, 204);
+        $this->statCache->clear($path . '/');
+        $this->statCache->remove($path);
+        return $result;
+    }
+
+    /** {@inheritdoc} */
+    public function fopen($path, $mode) {
+        $this->init();
+        $path = $this->cleanPath($path);
+        switch ($mode) {
+            case 'r':
+            case 'rb':
+                try {
+                    $response = $this->httpClientService
+                        ->newClient()
+                        ->get($this->createBaseUri() . $this->encodePath($path), [
+                            'auth' => [$this->user, $this->password],
+                            'stream' => true
+                        ]);
+                } catch (\GuzzleHttp\Exception\ClientException $e) {
+                    if ($e->getResponse() instanceof ResponseInterface
+                        && $e->getResponse()->getStatusCode() === 404) {
+                        return false;
+                    } else {
+                        throw $e;
+                    }
+                }
+
+                if ($response->getStatusCode() !== Http::STATUS_OK) {
+                    if ($response->getStatusCode() === Http::STATUS_LOCKED) {
+                        throw new \OCP\Lock\LockedException($path);
+                    } else {
+                        Util::writeLog("webdav client", 'Guzzle get returned status code ' . $response->getStatusCode(), ILogger::ERROR);
+                    }
+                }
+
+                return $response->getBody();
+            case 'w':
+            case 'wb':
+            case 'a':
+            case 'ab':
+            case 'r+':
+            case 'w+':
+            case 'wb+':
+            case 'a+':
+            case 'x':
+            case 'x+':
+            case 'c':
+            case 'c+':
+                //emulate these
+                $tempManager = \OC::$server->getTempManager();
+                if (strrpos($path, '.') !== false) {
+                    $ext = substr($path, strrpos($path, '.'));
+                } else {
+                    $ext = '';
+                }
+                if ($this->file_exists($path)) {
+                    if (!$this->isUpdatable($path)) {
+                        return false;
+                    }
+                    if ($mode === 'w' or $mode === 'w+') {
+                        $tmpFile = $tempManager->getTemporaryFile($ext);
+                    } else {
+                        $tmpFile = $this->getCachedFile($path);
+                    }
+                } else {
+                    if (!$this->isCreatable(dirname($path))) {
+                        return false;
+                    }
+                    $tmpFile = $tempManager->getTemporaryFile($ext);
+                }
+                $handle = fopen($tmpFile, $mode);
+                return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
+                    $this->writeBack($tmpFile, $path);
+                });
+        }
+    }
+
+    /**
+     * @param string $tmpFile
+     */
+    public function writeBack($tmpFile, $path) {
+        $this->uploadFile($tmpFile, $path);
+        unlink($tmpFile);
+    }
+
+    /** {@inheritdoc} */
+    public function free_space($path) {
+        $this->init();
+        $path = $this->cleanPath($path);
+        try {
+            $response = $this->propfind($path);
+            if ($response === false) {
+                return FileInfo::SPACE_UNKNOWN;
+            }
+            if (isset($response['{DAV:}quota-available-bytes'])) {
+                return (int)$response['{DAV:}quota-available-bytes'];
+            } else {
+                return FileInfo::SPACE_UNKNOWN;
+            }
+        } catch (\Exception $e) {
+            return FileInfo::SPACE_UNKNOWN;
+        }
+    }
+
+    /** {@inheritdoc} */
+    public function touch($path, $mtime = null) {
+        $this->init();
+        if (is_null($mtime)) {
+            $mtime = time();
+        }
+        $path = $this->cleanPath($path);
+
+        // if file exists, update the mtime, else create a new empty file
+        if ($this->file_exists($path)) {
+            try {
+                $this->statCache->remove($path);
+                $this->client->proppatch($this->encodePath($path), ['{DAV:}lastmodified' => $mtime]);
+                // non-owncloud clients might not have accepted the property, need to recheck it
+                $response = $this->client->propfind($this->encodePath($path), ['{DAV:}getlastmodified'], 0);
+                if ($response === false) {
+                    return false;
+                }
+                if (isset($response['{DAV:}getlastmodified'])) {
+                    $remoteMtime = strtotime($response['{DAV:}getlastmodified']);
+                    if ($remoteMtime !== $mtime) {
+                        // server has not accepted the mtime
+                        return false;
+                    }
+                }
+            } catch (ClientHttpException $e) {
+                if ($e->getHttpStatus() === 501) {
+                    return false;
+                }
+                $this->convertException($e, $path);
+                return false;
+            } catch (\Exception $e) {
+                $this->convertException($e, $path);
+                return false;
+            }
+        } else {
+            $this->file_put_contents($path, '');
+        }
+        return true;
+    }
+
+    /**
+     * @param string $path
+     * @param mixed $data
+     * @return int|false
+     */
+    public function file_put_contents($path, $data) {
+        $path = $this->cleanPath($path);
+        $result = parent::file_put_contents($path, $data);
+        $this->statCache->remove($path);
+        return $result;
+    }
+
+    /**
+     * @param string $path
+     * @param string $target
+     */
+    protected function uploadFile($path, $target) {
+        $this->init();
+
+        // invalidate
+        $target = $this->cleanPath($target);
+        $this->statCache->remove($target);
+        $source = fopen($path, 'r');
+
+        $this->httpClientService
+            ->newClient()
+            ->put($this->createBaseUri() . $this->encodePath($target), [
+                'body' => $source,
+                'auth' => [$this->user, $this->password]
+            ]);
+
+        $this->removeCachedFile($target);
+    }
+
+    /** {@inheritdoc} */
+    public function rename($path1, $path2) {
+        $this->init();
+        $path1 = $this->cleanPath($path1);
+        $path2 = $this->cleanPath($path2);
+        try {
+            // overwrite directory ?
+            if ($this->is_dir($path2)) {
+                // needs trailing slash in destination
+                $path2 = rtrim($path2, '/') . '/';
+            }
+            $this->client->request(
+                'MOVE',
+                $this->encodePath($path1),
+                null,
+                [
+                    'Destination' => $this->createBaseUri() . $this->encodePath($path2),
+                ]
+            );
+            $this->statCache->clear($path1 . '/');
+            $this->statCache->clear($path2 . '/');
+            $this->statCache->set($path1, false);
+            $this->statCache->set($path2, true);
+            $this->removeCachedFile($path1);
+            $this->removeCachedFile($path2);
+            return true;
+        } catch (\Exception $e) {
+            $this->convertException($e);
+        }
+        return false;
+    }
+
+    /** {@inheritdoc} */
+    public function copy($path1, $path2) {
+        $this->init();
+        $path1 = $this->cleanPath($path1);
+        $path2 = $this->cleanPath($path2);
+        try {
+            // overwrite directory ?
+            if ($this->is_dir($path2)) {
+                // needs trailing slash in destination
+                $path2 = rtrim($path2, '/') . '/';
+            }
+            $this->client->request(
+                'COPY',
+                $this->encodePath($path1),
+                null,
+                [
+                    'Destination' => $this->createBaseUri() . $this->encodePath($path2),
+                ]
+            );
+            $this->statCache->clear($path2 . '/');
+            $this->statCache->set($path2, true);
+            $this->removeCachedFile($path2);
+            return true;
+        } catch (\Exception $e) {
+            $this->convertException($e);
+        }
+        return false;
+    }
+
+    /** {@inheritdoc} */
+    public function stat($path) {
+        try {
+            $response = $this->propfind($path);
+            if (!$response) {
+                return false;
+            }
+            return [
+                'mtime' => isset($response['{DAV:}getlastmodified']) ? strtotime($response['{DAV:}getlastmodified']) : null,
+                'size' => (int)($response['{DAV:}getcontentlength'] ?? 0),
+            ];
+        } catch (\Exception $e) {
+            $this->convertException($e, $path);
+        }
+        return [];
+    }
+
+    /** {@inheritdoc} */
+    public function getMimeType($path) {
+        $remoteMimetype = $this->getMimeTypeFromRemote($path);
+        if ($remoteMimetype === 'application/octet-stream') {
+            return \OC::$server->getMimeTypeDetector()->detectPath($path);
+        } else {
+            return $remoteMimetype;
+        }
+    }
+
+    public function getMimeTypeFromRemote($path) {
+        try {
+            $response = $this->propfind($path);
+            if ($response === false) {
+                return false;
+            }
+            $responseType = [];
+            if (isset($response["{DAV:}resourcetype"])) {
+                /** @var ResourceType[] $response */
+                $responseType = $response["{DAV:}resourcetype"]->getValue();
+            }
+            $type = (count($responseType) > 0 and $responseType[0] == "{DAV:}collection") ? 'dir' : 'file';
+            if ($type == 'dir') {
+                return 'httpd/unix-directory';
+            } elseif (isset($response['{DAV:}getcontenttype'])) {
+                return $response['{DAV:}getcontenttype'];
+            } else {
+                return 'application/octet-stream';
+            }
+        } catch (\Exception $e) {
+            return false;
+        }
+    }
+
+    /**
+     * @param string $path
+     * @return string
+     */
+    public function cleanPath($path) {
+        if ($path === '') {
+            return $path;
+        }
+        $path = Filesystem::normalizePath($path);
+        // remove leading slash
+        return substr($path, 1);
+    }
+
+    /**
+     * URL encodes the given path but keeps the slashes
+     *
+     * @param string $path to encode
+     * @return string encoded path
+     */
+    protected function encodePath($path) {
+        // slashes need to stay
+        return str_replace('%2F', '/', rawurlencode($path));
+    }
+
+    /**
+     * @param string $method
+     * @param string $path
+     * @param string|resource|null $body
+     * @param int $expected
+     * @return bool
+     * @throws StorageInvalidException
+     * @throws StorageNotAvailableException
+     */
+    protected function simpleResponse($method, $path, $body, $expected) {
+        $path = $this->cleanPath($path);
+        try {
+            $response = $this->client->request($method, $this->encodePath($path), $body);
+            return $response['statusCode'] == $expected;
+        } catch (ClientHttpException $e) {
+            if ($e->getHttpStatus() === 404 && $method === 'DELETE') {
+                $this->statCache->clear($path . '/');
+                $this->statCache->set($path, false);
+                return false;
+            }
+
+            $this->convertException($e, $path);
+        } catch (\Exception $e) {
+            $this->convertException($e, $path);
+        }
+        return false;
+    }
+
+    /**
+     * check if curl is installed
+     */
+    public static function checkDependencies() {
+        return true;
+    }
+
+    /** {@inheritdoc} */
+    public function isUpdatable($path) {
+        return (bool)($this->getPermissions($path) & Constants::PERMISSION_UPDATE);
+    }
+
+    /** {@inheritdoc} */
+    public function isCreatable($path) {
+        return (bool)($this->getPermissions($path) & Constants::PERMISSION_CREATE);
+    }
+
+    /** {@inheritdoc} */
+    public function isSharable($path) {
+        return (bool)($this->getPermissions($path) & Constants::PERMISSION_SHARE);
+    }
+
+    /** {@inheritdoc} */
+    public function isDeletable($path) {
+        return (bool)($this->getPermissions($path) & Constants::PERMISSION_DELETE);
+    }
+
+    /** {@inheritdoc} */
+    public function getPermissions($path) {
+        $this->init();
+        $path = $this->cleanPath($path);
+        $response = $this->propfind($path);
+        if ($response === false) {
+            return 0;
+        }
+        if (isset($response['{http://owncloud.org/ns}permissions'])) {
+            return $this->parsePermissions($response['{http://owncloud.org/ns}permissions']);
+        } elseif ($this->is_dir($path)) {
+            return Constants::PERMISSION_ALL;
+        } elseif ($this->file_exists($path)) {
+            return Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
+        } else {
+            return 0;
+        }
+    }
+
+    /** {@inheritdoc} */
+    public function getETag($path) {
+        $this->init();
+        $path = $this->cleanPath($path);
+        $response = $this->propfind($path);
+        if ($response === false) {
+            return null;
+        }
+        if (isset($response['{DAV:}getetag'])) {
+            $etag = trim($response['{DAV:}getetag'], '"');
+            if (strlen($etag) > 40) {
+                $etag = md5($etag);
+            }
+            return $etag;
+        }
+        return parent::getEtag($path);
+    }
+
+    /**
+     * @param string $permissionsString
+     * @return int
+     */
+    protected function parsePermissions($permissionsString) {
+        $permissions = Constants::PERMISSION_READ;
+        if (strpos($permissionsString, 'R') !== false) {
+            $permissions |= Constants::PERMISSION_SHARE;
+        }
+        if (strpos($permissionsString, 'D') !== false) {
+            $permissions |= Constants::PERMISSION_DELETE;
+        }
+        if (strpos($permissionsString, 'W') !== false) {
+            $permissions |= Constants::PERMISSION_UPDATE;
+        }
+        if (strpos($permissionsString, 'CK') !== false) {
+            $permissions |= Constants::PERMISSION_CREATE;
+            $permissions |= Constants::PERMISSION_UPDATE;
+        }
+        return $permissions;
+    }
+
+    /**
+     * check if a file or folder has been updated since $time
+     *
+     * @param string $path
+     * @param int $time
+     * @throws \OCP\Files\StorageNotAvailableException
+     * @return bool
+     */
+    public function hasUpdated($path, $time) {
+        $this->init();
+        $path = $this->cleanPath($path);
+        try {
+            // force refresh for $path
+            $this->statCache->remove($path);
+            $response = $this->propfind($path);
+            if ($response === false) {
+                if ($path === '') {
+                    // if root is gone it means the storage is not available
+                    throw new StorageNotAvailableException('root is gone');
+                }
+                return false;
+            }
+            if (isset($response['{DAV:}getetag'])) {
+                $cachedData = $this->getCache()->get($path);
+                $etag = trim($response['{DAV:}getetag'], '"');
+                if (($cachedData === false) || (!empty($etag) && ($cachedData['etag'] !== $etag))) {
+                    return true;
+                } elseif (isset($response['{http://open-collaboration-services.org/ns}share-permissions'])) {
+                    $sharePermissions = (int)$response['{http://open-collaboration-services.org/ns}share-permissions'];
+                    return $sharePermissions !== $cachedData['permissions'];
+                } elseif (isset($response['{http://owncloud.org/ns}permissions'])) {
+                    $permissions = $this->parsePermissions($response['{http://owncloud.org/ns}permissions']);
+                    return $permissions !== $cachedData['permissions'];
+                } else {
+                    return false;
+                }
+            } elseif (isset($response['{DAV:}getlastmodified'])) {
+                $remoteMtime = strtotime($response['{DAV:}getlastmodified']);
+                return $remoteMtime > $time;
+            } else {
+                // neither `getetag` nor `getlastmodified` is set
+                return false;
+            }
+        } catch (ClientHttpException $e) {
+            if ($e->getHttpStatus() === 405) {
+                if ($path === '') {
+                    // if root is gone it means the storage is not available
+                    throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage());
+                }
+                return false;
+            }
+            $this->convertException($e, $path);
+            return false;
+        } catch (\Exception $e) {
+            $this->convertException($e, $path);
+            return false;
+        }
+    }
+
+    /**
+     * Interpret the given exception and decide whether it is due to an
+     * unavailable storage, invalid storage or other.
+     * This will either throw StorageInvalidException, StorageNotAvailableException
+     * or do nothing.
+     *
+     * @param Exception $e sabre exception
+     * @param string $path optional path from the operation
+     *
+     * @throws StorageInvalidException if the storage is invalid, for example
+     * when the authentication expired or is invalid
+     * @throws StorageNotAvailableException if the storage is not available,
+     * which might be temporary
+     * @throws ForbiddenException if the action is not allowed
+     */
+    protected function convertException(Exception $e, $path = '') {
+        \OC::$server->getLogger()->logException($e, ['app' => 'files_external', 'level' => ILogger::DEBUG]);
+        if ($e instanceof ClientHttpException) {
+            if ($e->getHttpStatus() === Http::STATUS_LOCKED) {
+                throw new \OCP\Lock\LockedException($path);
+            }
+            if ($e->getHttpStatus() === Http::STATUS_UNAUTHORIZED) {
+                // either password was changed or was invalid all along
+                throw new StorageInvalidException(get_class($e) . ': ' . $e->getMessage());
+            } elseif ($e->getHttpStatus() === Http::STATUS_METHOD_NOT_ALLOWED) {
+                // ignore exception for MethodNotAllowed, false will be returned
+                return;
+            } elseif ($e->getHttpStatus() === Http::STATUS_FORBIDDEN) {
+                // The operation is forbidden. Fail somewhat gracefully
+                throw new ForbiddenException(get_class($e) . ':' . $e->getMessage(), false);
+            }
+            throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage());
+        } elseif ($e instanceof ClientException) {
+            // connection timeout or refused, server could be temporarily down
+            throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage());
+        } elseif ($e instanceof \InvalidArgumentException) {
+            // parse error because the server returned HTML instead of XML,
+            // possibly temporarily down
+            throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage());
+        } elseif (($e instanceof StorageNotAvailableException) || ($e instanceof StorageInvalidException)) {
+            // rethrow
+            throw $e;
+        }
+
+        // TODO: only log for now, but in the future need to wrap/rethrow exception
+    }
 }

lib/private/Files/Cache/Scanner.php

Indentation +508 added lines, -508 removed lines

@@ -57,512 +57,512 @@
  * @package OC\Files\Cache
  */
 class Scanner extends BasicEmitter implements IScanner {
-	/**
-	 * @var \OC\Files\Storage\Storage $storage
-	 */
-	protected $storage;
-
-	/**
-	 * @var string $storageId
-	 */
-	protected $storageId;
-
-	/**
-	 * @var \OC\Files\Cache\Cache $cache
-	 */
-	protected $cache;
-
-	/**
-	 * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache
-	 */
-	protected $cacheActive;
-
-	/**
-	 * @var bool $useTransactions whether to use transactions
-	 */
-	protected $useTransactions = true;
-
-	/**
-	 * @var \OCP\Lock\ILockingProvider
-	 */
-	protected $lockingProvider;
-
-	public function __construct(\OC\Files\Storage\Storage $storage) {
-		$this->storage = $storage;
-		$this->storageId = $this->storage->getId();
-		$this->cache = $storage->getCache();
-		$this->cacheActive = !\OC::$server->getConfig()->getSystemValue('filesystem_cache_readonly', false);
-		$this->lockingProvider = \OC::$server->getLockingProvider();
-	}
-
-	/**
-	 * Whether to wrap the scanning of a folder in a database transaction
-	 * On default transactions are used
-	 *
-	 * @param bool $useTransactions
-	 */
-	public function setUseTransactions($useTransactions) {
-		$this->useTransactions = $useTransactions;
-	}
-
-	/**
-	 * get all the metadata of a file or folder
-	 * *
-	 *
-	 * @param string $path
-	 * @return array|null an array of metadata of the file
-	 */
-	protected function getData($path) {
-		$data = $this->storage->getMetaData($path);
-		if (is_null($data)) {
-			\OCP\Util::writeLog(Scanner::class, "!!! Path '$path' is not accessible or present !!!", ILogger::DEBUG);
-		}
-		return $data;
-	}
-
-	/**
-	 * scan a single file and store it in the cache
-	 *
-	 * @param string $file
-	 * @param int $reuseExisting
-	 * @param int $parentId
-	 * @param array|null|false $cacheData existing data in the cache for the file to be scanned
-	 * @param bool $lock set to false to disable getting an additional read lock during scanning
-	 * @param null $data the metadata for the file, as returned by the storage
-	 * @return array|null an array of metadata of the scanned file
-	 * @throws \OCP\Lock\LockedException
-	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
-		if ($file !== '') {
-			try {
-				$this->storage->verifyPath(dirname($file), basename($file));
-			} catch (\Exception $e) {
-				return null;
-			}
-		}
-		// only proceed if $file is not a partial file, blacklist is handled by the storage
-		if (!self::isPartialFile($file)) {
-
-			//acquire a lock
-			if ($lock) {
-				if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-					$this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-				}
-			}
-
-			try {
-				$data = $data ?? $this->getData($file);
-			} catch (ForbiddenException $e) {
-				if ($lock) {
-					if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-						$this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-					}
-				}
-
-				return null;
-			}
-
-			try {
-				if ($data) {
-
-					// pre-emit only if it was a file. By that we avoid counting/treating folders as files
-					if ($data['mimetype'] !== 'httpd/unix-directory') {
-						$this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]);
-						\OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]);
-					}
-
-					$parent = dirname($file);
-					if ($parent === '.' or $parent === '/') {
-						$parent = '';
-					}
-					if ($parentId === -1) {
-						$parentId = $this->cache->getParentId($file);
-					}
-
-					// scan the parent if it's not in the cache (id -1) and the current file is not the root folder
-					if ($file and $parentId === -1) {
-						$parentData = $this->scanFile($parent);
-						if (!$parentData) {
-							return null;
-						}
-						$parentId = $parentData['fileid'];
-					}
-					if ($parent) {
-						$data['parent'] = $parentId;
-					}
-					if (is_null($cacheData)) {
-						/** @var CacheEntry $cacheData */
-						$cacheData = $this->cache->get($file);
-					}
-					if ($cacheData and $reuseExisting and isset($cacheData['fileid'])) {
-						// prevent empty etag
-						if (empty($cacheData['etag'])) {
-							$etag = $data['etag'];
-						} else {
-							$etag = $cacheData['etag'];
-						}
-						$fileId = $cacheData['fileid'];
-						$data['fileid'] = $fileId;
-						// only reuse data if the file hasn't explicitly changed
-						if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) {
-							$data['mtime'] = $cacheData['mtime'];
-							if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) {
-								$data['size'] = $cacheData['size'];
-							}
-							if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) {
-								$data['etag'] = $etag;
-							}
-						}
-						// Only update metadata that has changed
-						$newData = array_diff_assoc($data, $cacheData->getData());
-					} else {
-						$newData = $data;
-						$fileId = -1;
-					}
-					if (!empty($newData)) {
-						// Reset the checksum if the data has changed
-						$newData['checksum'] = '';
-						$newData['parent'] = $parentId;
-						$data['fileid'] = $this->addToCache($file, $newData, $fileId);
-					}
-					if ($cacheData && isset($cacheData['size'])) {
-						$data['oldSize'] = $cacheData['size'];
-					} else {
-						$data['oldSize'] = 0;
-					}
-
-					if ($cacheData && isset($cacheData['encrypted'])) {
-						$data['encrypted'] = $cacheData['encrypted'];
-					}
-
-					// post-emit only if it was a file. By that we avoid counting/treating folders as files
-					if ($data['mimetype'] !== 'httpd/unix-directory') {
-						$this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]);
-						\OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]);
-					}
-				} else {
-					$this->removeFromCache($file);
-				}
-			} catch (\Exception $e) {
-				if ($lock) {
-					if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-						$this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-					}
-				}
-				throw $e;
-			}
-
-			//release the acquired lock
-			if ($lock) {
-				if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-					$this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-				}
-			}
-
-			if ($data && !isset($data['encrypted'])) {
-				$data['encrypted'] = false;
-			}
-			return $data;
-		}
-
-		return null;
-	}
-
-	protected function removeFromCache($path) {
-		\OC_Hook::emit('Scanner', 'removeFromCache', ['file' => $path]);
-		$this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', [$path]);
-		if ($this->cacheActive) {
-			$this->cache->remove($path);
-		}
-	}
-
-	/**
-	 * @param string $path
-	 * @param array $data
-	 * @param int $fileId
-	 * @return int the id of the added file
-	 */
-	protected function addToCache($path, $data, $fileId = -1) {
-		if (isset($data['scan_permissions'])) {
-			$data['permissions'] = $data['scan_permissions'];
-		}
-		\OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]);
-		$this->emit('\OC\Files\Cache\Scanner', 'addToCache', [$path, $this->storageId, $data]);
-		if ($this->cacheActive) {
-			if ($fileId !== -1) {
-				$this->cache->update($fileId, $data);
-				return $fileId;
-			} else {
-				return $this->cache->insert($path, $data);
-			}
-		} else {
-			return -1;
-		}
-	}
-
-	/**
-	 * @param string $path
-	 * @param array $data
-	 * @param int $fileId
-	 */
-	protected function updateCache($path, $data, $fileId = -1) {
-		\OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]);
-		$this->emit('\OC\Files\Cache\Scanner', 'updateCache', [$path, $this->storageId, $data]);
-		if ($this->cacheActive) {
-			if ($fileId !== -1) {
-				$this->cache->update($fileId, $data);
-			} else {
-				$this->cache->put($path, $data);
-			}
-		}
-	}
-
-	/**
-	 * scan a folder and all it's children
-	 *
-	 * @param string $path
-	 * @param bool $recursive
-	 * @param int $reuse
-	 * @param bool $lock set to false to disable getting an additional read lock during scanning
-	 * @return array|null an array of the meta data of the scanned file or folder
-	 */
-	public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) {
-		if ($reuse === -1) {
-			$reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
-		}
-		if ($lock) {
-			if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-				$this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
-				$this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-			}
-		}
-		try {
-			$data = $this->scanFile($path, $reuse, -1, null, $lock);
-			if ($data and $data['mimetype'] === 'httpd/unix-directory') {
-				$size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock);
-				$data['size'] = $size;
-			}
-		} finally {
-			if ($lock) {
-				if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
-					$this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
-					$this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
-				}
-			}
-		}
-		return $data;
-	}
-
-	/**
-	 * Get the children currently in the cache
-	 *
-	 * @param int $folderId
-	 * @return array[]
-	 */
-	protected function getExistingChildren($folderId) {
-		$existingChildren = [];
-		$children = $this->cache->getFolderContentsById($folderId);
-		foreach ($children as $child) {
-			$existingChildren[$child['name']] = $child;
-		}
-		return $existingChildren;
-	}
-
-	/**
-	 * scan all the files and folders in a folder
-	 *
-	 * @param string $path
-	 * @param bool $recursive
-	 * @param int $reuse
-	 * @param int $folderId id for the folder to be scanned
-	 * @param bool $lock set to false to disable getting an additional read lock during scanning
-	 * @return int the size of the scanned folder or -1 if the size is unknown at this stage
-	 */
-	protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) {
-		if ($reuse === -1) {
-			$reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
-		}
-		$this->emit('\OC\Files\Cache\Scanner', 'scanFolder', [$path, $this->storageId]);
-		$size = 0;
-		if (!is_null($folderId)) {
-			$folderId = $this->cache->getId($path);
-		}
-		$childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size);
-
-		foreach ($childQueue as $child => $childId) {
-			$childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock);
-			if ($childSize === -1) {
-				$size = -1;
-			} elseif ($size !== -1) {
-				$size += $childSize;
-			}
-		}
-		if ($this->cacheActive) {
-			$this->cache->update($folderId, ['size' => $size]);
-		}
-		$this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]);
-		return $size;
-	}
-
-	private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
-		// we put this in it's own function so it cleans up the memory before we start recursing
-		$existingChildren = $this->getExistingChildren($folderId);
-		$newChildren = iterator_to_array($this->storage->getDirectoryContent($path));
-
-		if ($this->useTransactions) {
-			\OC::$server->getDatabaseConnection()->beginTransaction();
-		}
-
-		$exceptionOccurred = false;
-		$childQueue = [];
-		$newChildNames = [];
-		foreach ($newChildren as $fileMeta) {
-			$permissions = isset($fileMeta['scan_permissions']) ? $fileMeta['scan_permissions'] : $fileMeta['permissions'];
-			if ($permissions === 0) {
-				continue;
-			}
-			$originalFile = $fileMeta['name'];
-			$file = trim(\OC\Files\Filesystem::normalizePath($originalFile), '/');
-			if (trim($originalFile, '/') !== $file) {
-				// encoding mismatch, might require compatibility wrapper
-				\OC::$server->getLogger()->debug('Scanner: Skipping non-normalized file name "'. $originalFile . '" in path "' . $path . '".', ['app' => 'core']);
-				$this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]);
-				// skip this entry
-				continue;
-			}
-
-			$newChildNames[] = $file;
-			$child = $path ? $path . '/' . $file : $file;
-			try {
-				$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
-				$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
-				if ($data) {
-					if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
-						$childQueue[$child] = $data['fileid'];
-					} elseif ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE_INCOMPLETE and $data['size'] === -1) {
-						// only recurse into folders which aren't fully scanned
-						$childQueue[$child] = $data['fileid'];
-					} elseif ($data['size'] === -1) {
-						$size = -1;
-					} elseif ($size !== -1) {
-						$size += $data['size'];
-					}
-				}
-			} catch (Exception $ex) {
-				// might happen if inserting duplicate while a scanning
-				// process is running in parallel
-				// log and ignore
-				if ($this->useTransactions) {
-					\OC::$server->getDatabaseConnection()->rollback();
-					\OC::$server->getDatabaseConnection()->beginTransaction();
-				}
-				\OC::$server->getLogger()->logException($ex, [
-					'message' => 'Exception while scanning file "' . $child . '"',
-					'level' => ILogger::DEBUG,
-					'app' => 'core',
-				]);
-				$exceptionOccurred = true;
-			} catch (\OCP\Lock\LockedException $e) {
-				if ($this->useTransactions) {
-					\OC::$server->getDatabaseConnection()->rollback();
-				}
-				throw $e;
-			}
-		}
-		$removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
-		foreach ($removedChildren as $childName) {
-			$child = $path ? $path . '/' . $childName : $childName;
-			$this->removeFromCache($child);
-		}
-		if ($this->useTransactions) {
-			\OC::$server->getDatabaseConnection()->commit();
-		}
-		if ($exceptionOccurred) {
-			// It might happen that the parallel scan process has already
-			// inserted mimetypes but those weren't available yet inside the transaction
-			// To make sure to have the updated mime types in such cases,
-			// we reload them here
-			\OC::$server->getMimeTypeLoader()->reset();
-		}
-		return $childQueue;
-	}
-
-	/**
-	 * check if the file should be ignored when scanning
-	 * NOTE: files with a '.part' extension are ignored as well!
-	 *       prevents unfinished put requests to be scanned
-	 *
-	 * @param string $file
-	 * @return boolean
-	 */
-	public static function isPartialFile($file) {
-		if (pathinfo($file, PATHINFO_EXTENSION) === 'part') {
-			return true;
-		}
-		if (strpos($file, '.part/') !== false) {
-			return true;
-		}
-
-		return false;
-	}
-
-	/**
-	 * walk over any folders that are not fully scanned yet and scan them
-	 */
-	public function backgroundScan() {
-		if ($this->storage->instanceOfStorage(Jail::class)) {
-			// for jail storage wrappers (shares, groupfolders) we run the background scan on the source storage
-			// this is mainly done because the jail wrapper doesn't implement `getIncomplete` (because it would be inefficient).
-			//
-			// Running the scan on the source storage might scan more than "needed", but the unscanned files outside the jail will
-			// have to be scanned at some point anyway.
-			$unJailedScanner = $this->storage->getUnjailedStorage()->getScanner();
-			$unJailedScanner->backgroundScan();
-		} else {
-			if (!$this->cache->inCache('')) {
-				// if the storage isn't in the cache yet, just scan the root completely
-				$this->runBackgroundScanJob(function () {
-					$this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG);
-				}, '');
-			} else {
-				$lastPath = null;
-				// find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck)
-				while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) {
-					$this->runBackgroundScanJob(function () use ($path) {
-						$this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE);
-					}, $path);
-					// FIXME: this won't proceed with the next item, needs revamping of getIncomplete()
-					// to make this possible
-					$lastPath = $path;
-				}
-			}
-		}
-	}
-
-	private function runBackgroundScanJob(callable $callback, $path) {
-		try {
-			$callback();
-			\OC_Hook::emit('Scanner', 'correctFolderSize', ['path' => $path]);
-			if ($this->cacheActive && $this->cache instanceof Cache) {
-				$this->cache->correctFolderSize($path, null, true);
-			}
-		} catch (\OCP\Files\StorageInvalidException $e) {
-			// skip unavailable storages
-		} catch (\OCP\Files\StorageNotAvailableException $e) {
-			// skip unavailable storages
-		} catch (\OCP\Files\ForbiddenException $e) {
-			// skip forbidden storages
-		} catch (\OCP\Lock\LockedException $e) {
-			// skip unavailable storages
-		}
-	}
-
-	/**
-	 * Set whether the cache is affected by scan operations
-	 *
-	 * @param boolean $active The active state of the cache
-	 */
-	public function setCacheActive($active) {
-		$this->cacheActive = $active;
-	}
+    /**
+     * @var \OC\Files\Storage\Storage $storage
+     */
+    protected $storage;
+
+    /**
+     * @var string $storageId
+     */
+    protected $storageId;
+
+    /**
+     * @var \OC\Files\Cache\Cache $cache
+     */
+    protected $cache;
+
+    /**
+     * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache
+     */
+    protected $cacheActive;
+
+    /**
+     * @var bool $useTransactions whether to use transactions
+     */
+    protected $useTransactions = true;
+
+    /**
+     * @var \OCP\Lock\ILockingProvider
+     */
+    protected $lockingProvider;
+
+    public function __construct(\OC\Files\Storage\Storage $storage) {
+        $this->storage = $storage;
+        $this->storageId = $this->storage->getId();
+        $this->cache = $storage->getCache();
+        $this->cacheActive = !\OC::$server->getConfig()->getSystemValue('filesystem_cache_readonly', false);
+        $this->lockingProvider = \OC::$server->getLockingProvider();
+    }
+
+    /**
+     * Whether to wrap the scanning of a folder in a database transaction
+     * On default transactions are used
+     *
+     * @param bool $useTransactions
+     */
+    public function setUseTransactions($useTransactions) {
+        $this->useTransactions = $useTransactions;
+    }
+
+    /**
+     * get all the metadata of a file or folder
+     * *
+     *
+     * @param string $path
+     * @return array|null an array of metadata of the file
+     */
+    protected function getData($path) {
+        $data = $this->storage->getMetaData($path);
+        if (is_null($data)) {
+            \OCP\Util::writeLog(Scanner::class, "!!! Path '$path' is not accessible or present !!!", ILogger::DEBUG);
+        }
+        return $data;
+    }
+
+    /**
+     * scan a single file and store it in the cache
+     *
+     * @param string $file
+     * @param int $reuseExisting
+     * @param int $parentId
+     * @param array|null|false $cacheData existing data in the cache for the file to be scanned
+     * @param bool $lock set to false to disable getting an additional read lock during scanning
+     * @param null $data the metadata for the file, as returned by the storage
+     * @return array|null an array of metadata of the scanned file
+     * @throws \OCP\Lock\LockedException
+     */
+    public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
+        if ($file !== '') {
+            try {
+                $this->storage->verifyPath(dirname($file), basename($file));
+            } catch (\Exception $e) {
+                return null;
+            }
+        }
+        // only proceed if $file is not a partial file, blacklist is handled by the storage
+        if (!self::isPartialFile($file)) {
+
+            //acquire a lock
+            if ($lock) {
+                if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                    $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+                }
+            }
+
+            try {
+                $data = $data ?? $this->getData($file);
+            } catch (ForbiddenException $e) {
+                if ($lock) {
+                    if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                        $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+                    }
+                }
+
+                return null;
+            }
+
+            try {
+                if ($data) {
+
+                    // pre-emit only if it was a file. By that we avoid counting/treating folders as files
+                    if ($data['mimetype'] !== 'httpd/unix-directory') {
+                        $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]);
+                        \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]);
+                    }
+
+                    $parent = dirname($file);
+                    if ($parent === '.' or $parent === '/') {
+                        $parent = '';
+                    }
+                    if ($parentId === -1) {
+                        $parentId = $this->cache->getParentId($file);
+                    }
+
+                    // scan the parent if it's not in the cache (id -1) and the current file is not the root folder
+                    if ($file and $parentId === -1) {
+                        $parentData = $this->scanFile($parent);
+                        if (!$parentData) {
+                            return null;
+                        }
+                        $parentId = $parentData['fileid'];
+                    }
+                    if ($parent) {
+                        $data['parent'] = $parentId;
+                    }
+                    if (is_null($cacheData)) {
+                        /** @var CacheEntry $cacheData */
+                        $cacheData = $this->cache->get($file);
+                    }
+                    if ($cacheData and $reuseExisting and isset($cacheData['fileid'])) {
+                        // prevent empty etag
+                        if (empty($cacheData['etag'])) {
+                            $etag = $data['etag'];
+                        } else {
+                            $etag = $cacheData['etag'];
+                        }
+                        $fileId = $cacheData['fileid'];
+                        $data['fileid'] = $fileId;
+                        // only reuse data if the file hasn't explicitly changed
+                        if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) {
+                            $data['mtime'] = $cacheData['mtime'];
+                            if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) {
+                                $data['size'] = $cacheData['size'];
+                            }
+                            if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) {
+                                $data['etag'] = $etag;
+                            }
+                        }
+                        // Only update metadata that has changed
+                        $newData = array_diff_assoc($data, $cacheData->getData());
+                    } else {
+                        $newData = $data;
+                        $fileId = -1;
+                    }
+                    if (!empty($newData)) {
+                        // Reset the checksum if the data has changed
+                        $newData['checksum'] = '';
+                        $newData['parent'] = $parentId;
+                        $data['fileid'] = $this->addToCache($file, $newData, $fileId);
+                    }
+                    if ($cacheData && isset($cacheData['size'])) {
+                        $data['oldSize'] = $cacheData['size'];
+                    } else {
+                        $data['oldSize'] = 0;
+                    }
+
+                    if ($cacheData && isset($cacheData['encrypted'])) {
+                        $data['encrypted'] = $cacheData['encrypted'];
+                    }
+
+                    // post-emit only if it was a file. By that we avoid counting/treating folders as files
+                    if ($data['mimetype'] !== 'httpd/unix-directory') {
+                        $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]);
+                        \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]);
+                    }
+                } else {
+                    $this->removeFromCache($file);
+                }
+            } catch (\Exception $e) {
+                if ($lock) {
+                    if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                        $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+                    }
+                }
+                throw $e;
+            }
+
+            //release the acquired lock
+            if ($lock) {
+                if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                    $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+                }
+            }
+
+            if ($data && !isset($data['encrypted'])) {
+                $data['encrypted'] = false;
+            }
+            return $data;
+        }
+
+        return null;
+    }
+
+    protected function removeFromCache($path) {
+        \OC_Hook::emit('Scanner', 'removeFromCache', ['file' => $path]);
+        $this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', [$path]);
+        if ($this->cacheActive) {
+            $this->cache->remove($path);
+        }
+    }
+
+    /**
+     * @param string $path
+     * @param array $data
+     * @param int $fileId
+     * @return int the id of the added file
+     */
+    protected function addToCache($path, $data, $fileId = -1) {
+        if (isset($data['scan_permissions'])) {
+            $data['permissions'] = $data['scan_permissions'];
+        }
+        \OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]);
+        $this->emit('\OC\Files\Cache\Scanner', 'addToCache', [$path, $this->storageId, $data]);
+        if ($this->cacheActive) {
+            if ($fileId !== -1) {
+                $this->cache->update($fileId, $data);
+                return $fileId;
+            } else {
+                return $this->cache->insert($path, $data);
+            }
+        } else {
+            return -1;
+        }
+    }
+
+    /**
+     * @param string $path
+     * @param array $data
+     * @param int $fileId
+     */
+    protected function updateCache($path, $data, $fileId = -1) {
+        \OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]);
+        $this->emit('\OC\Files\Cache\Scanner', 'updateCache', [$path, $this->storageId, $data]);
+        if ($this->cacheActive) {
+            if ($fileId !== -1) {
+                $this->cache->update($fileId, $data);
+            } else {
+                $this->cache->put($path, $data);
+            }
+        }
+    }
+
+    /**
+     * scan a folder and all it's children
+     *
+     * @param string $path
+     * @param bool $recursive
+     * @param int $reuse
+     * @param bool $lock set to false to disable getting an additional read lock during scanning
+     * @return array|null an array of the meta data of the scanned file or folder
+     */
+    public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) {
+        if ($reuse === -1) {
+            $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
+        }
+        if ($lock) {
+            if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+                $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+            }
+        }
+        try {
+            $data = $this->scanFile($path, $reuse, -1, null, $lock);
+            if ($data and $data['mimetype'] === 'httpd/unix-directory') {
+                $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock);
+                $data['size'] = $size;
+            }
+        } finally {
+            if ($lock) {
+                if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
+                    $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider);
+                    $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+                }
+            }
+        }
+        return $data;
+    }
+
+    /**
+     * Get the children currently in the cache
+     *
+     * @param int $folderId
+     * @return array[]
+     */
+    protected function getExistingChildren($folderId) {
+        $existingChildren = [];
+        $children = $this->cache->getFolderContentsById($folderId);
+        foreach ($children as $child) {
+            $existingChildren[$child['name']] = $child;
+        }
+        return $existingChildren;
+    }
+
+    /**
+     * scan all the files and folders in a folder
+     *
+     * @param string $path
+     * @param bool $recursive
+     * @param int $reuse
+     * @param int $folderId id for the folder to be scanned
+     * @param bool $lock set to false to disable getting an additional read lock during scanning
+     * @return int the size of the scanned folder or -1 if the size is unknown at this stage
+     */
+    protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) {
+        if ($reuse === -1) {
+            $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
+        }
+        $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', [$path, $this->storageId]);
+        $size = 0;
+        if (!is_null($folderId)) {
+            $folderId = $this->cache->getId($path);
+        }
+        $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size);
+
+        foreach ($childQueue as $child => $childId) {
+            $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock);
+            if ($childSize === -1) {
+                $size = -1;
+            } elseif ($size !== -1) {
+                $size += $childSize;
+            }
+        }
+        if ($this->cacheActive) {
+            $this->cache->update($folderId, ['size' => $size]);
+        }
+        $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]);
+        return $size;
+    }
+
+    private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
+        // we put this in it's own function so it cleans up the memory before we start recursing
+        $existingChildren = $this->getExistingChildren($folderId);
+        $newChildren = iterator_to_array($this->storage->getDirectoryContent($path));
+
+        if ($this->useTransactions) {
+            \OC::$server->getDatabaseConnection()->beginTransaction();
+        }
+
+        $exceptionOccurred = false;
+        $childQueue = [];
+        $newChildNames = [];
+        foreach ($newChildren as $fileMeta) {
+            $permissions = isset($fileMeta['scan_permissions']) ? $fileMeta['scan_permissions'] : $fileMeta['permissions'];
+            if ($permissions === 0) {
+                continue;
+            }
+            $originalFile = $fileMeta['name'];
+            $file = trim(\OC\Files\Filesystem::normalizePath($originalFile), '/');
+            if (trim($originalFile, '/') !== $file) {
+                // encoding mismatch, might require compatibility wrapper
+                \OC::$server->getLogger()->debug('Scanner: Skipping non-normalized file name "'. $originalFile . '" in path "' . $path . '".', ['app' => 'core']);
+                $this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]);
+                // skip this entry
+                continue;
+            }
+
+            $newChildNames[] = $file;
+            $child = $path ? $path . '/' . $file : $file;
+            try {
+                $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
+                $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
+                if ($data) {
+                    if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
+                        $childQueue[$child] = $data['fileid'];
+                    } elseif ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE_INCOMPLETE and $data['size'] === -1) {
+                        // only recurse into folders which aren't fully scanned
+                        $childQueue[$child] = $data['fileid'];
+                    } elseif ($data['size'] === -1) {
+                        $size = -1;
+                    } elseif ($size !== -1) {
+                        $size += $data['size'];
+                    }
+                }
+            } catch (Exception $ex) {
+                // might happen if inserting duplicate while a scanning
+                // process is running in parallel
+                // log and ignore
+                if ($this->useTransactions) {
+                    \OC::$server->getDatabaseConnection()->rollback();
+                    \OC::$server->getDatabaseConnection()->beginTransaction();
+                }
+                \OC::$server->getLogger()->logException($ex, [
+                    'message' => 'Exception while scanning file "' . $child . '"',
+                    'level' => ILogger::DEBUG,
+                    'app' => 'core',
+                ]);
+                $exceptionOccurred = true;
+            } catch (\OCP\Lock\LockedException $e) {
+                if ($this->useTransactions) {
+                    \OC::$server->getDatabaseConnection()->rollback();
+                }
+                throw $e;
+            }
+        }
+        $removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
+        foreach ($removedChildren as $childName) {
+            $child = $path ? $path . '/' . $childName : $childName;
+            $this->removeFromCache($child);
+        }
+        if ($this->useTransactions) {
+            \OC::$server->getDatabaseConnection()->commit();
+        }
+        if ($exceptionOccurred) {
+            // It might happen that the parallel scan process has already
+            // inserted mimetypes but those weren't available yet inside the transaction
+            // To make sure to have the updated mime types in such cases,
+            // we reload them here
+            \OC::$server->getMimeTypeLoader()->reset();
+        }
+        return $childQueue;
+    }
+
+    /**
+     * check if the file should be ignored when scanning
+     * NOTE: files with a '.part' extension are ignored as well!
+     *       prevents unfinished put requests to be scanned
+     *
+     * @param string $file
+     * @return boolean
+     */
+    public static function isPartialFile($file) {
+        if (pathinfo($file, PATHINFO_EXTENSION) === 'part') {
+            return true;
+        }
+        if (strpos($file, '.part/') !== false) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * walk over any folders that are not fully scanned yet and scan them
+     */
+    public function backgroundScan() {
+        if ($this->storage->instanceOfStorage(Jail::class)) {
+            // for jail storage wrappers (shares, groupfolders) we run the background scan on the source storage
+            // this is mainly done because the jail wrapper doesn't implement `getIncomplete` (because it would be inefficient).
+            //
+            // Running the scan on the source storage might scan more than "needed", but the unscanned files outside the jail will
+            // have to be scanned at some point anyway.
+            $unJailedScanner = $this->storage->getUnjailedStorage()->getScanner();
+            $unJailedScanner->backgroundScan();
+        } else {
+            if (!$this->cache->inCache('')) {
+                // if the storage isn't in the cache yet, just scan the root completely
+                $this->runBackgroundScanJob(function () {
+                    $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG);
+                }, '');
+            } else {
+                $lastPath = null;
+                // find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck)
+                while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) {
+                    $this->runBackgroundScanJob(function () use ($path) {
+                        $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE);
+                    }, $path);
+                    // FIXME: this won't proceed with the next item, needs revamping of getIncomplete()
+                    // to make this possible
+                    $lastPath = $path;
+                }
+            }
+        }
+    }
+
+    private function runBackgroundScanJob(callable $callback, $path) {
+        try {
+            $callback();
+            \OC_Hook::emit('Scanner', 'correctFolderSize', ['path' => $path]);
+            if ($this->cacheActive && $this->cache instanceof Cache) {
+                $this->cache->correctFolderSize($path, null, true);
+            }
+        } catch (\OCP\Files\StorageInvalidException $e) {
+            // skip unavailable storages
+        } catch (\OCP\Files\StorageNotAvailableException $e) {
+            // skip unavailable storages
+        } catch (\OCP\Files\ForbiddenException $e) {
+            // skip forbidden storages
+        } catch (\OCP\Lock\LockedException $e) {
+            // skip unavailable storages
+        }
+    }
+
+    /**
+     * Set whether the cache is affected by scan operations
+     *
+     * @param boolean $active The active state of the cache
+     */
+    public function setCacheActive($active) {
+        $this->cacheActive = $active;
+    }
 }

lib/private/Files/Cache/Propagator.php

Indentation +167 added lines, -167 removed lines

@@ -33,171 +33,171 @@
  * Propagate etags and mtimes within the storage
  */
 class Propagator implements IPropagator {
-	private $inBatch = false;
-
-	private $batch = [];
-
-	/**
-	 * @var \OC\Files\Storage\Storage
-	 */
-	protected $storage;
-
-	/**
-	 * @var IDBConnection
-	 */
-	private $connection;
-
-	/**
-	 * @var array
-	 */
-	private $ignore = [];
-
-	public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection, array $ignore = []) {
-		$this->storage = $storage;
-		$this->connection = $connection;
-		$this->ignore = $ignore;
-	}
-
-
-	/**
-	 * @param string $internalPath
-	 * @param int $time
-	 * @param int $sizeDifference number of bytes the file has grown
-	 */
-	public function propagateChange($internalPath, $time, $sizeDifference = 0) {
-		// Do not propogate changes in ignored paths
-		foreach ($this->ignore as $ignore) {
-			if (strpos($internalPath, $ignore) === 0) {
-				return;
-			}
-		}
-
-		$storageId = (int)$this->storage->getStorageCache()->getNumericId();
-
-		$parents = $this->getParents($internalPath);
-
-		if ($this->inBatch) {
-			foreach ($parents as $parent) {
-				$this->addToBatch($parent, $time, $sizeDifference);
-			}
-			return;
-		}
-
-		$parentHashes = array_map('md5', $parents);
-		$etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag
-
-		$builder = $this->connection->getQueryBuilder();
-		$hashParams = array_map(function ($hash) use ($builder) {
-			return $builder->expr()->literal($hash);
-		}, $parentHashes);
-
-		$builder->update('filecache')
-			->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int)$time, IQueryBuilder::PARAM_INT)))
-			->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
-			->andWhere($builder->expr()->in('path_hash', $hashParams));
-		if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) {
-			$builder->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR));
-		}
-
-		$builder->execute();
-
-		if ($sizeDifference !== 0) {
-			// we need to do size separably so we can ignore entries with uncalculated size
-			$builder = $this->connection->getQueryBuilder();
-			$builder->update('filecache')
-				->set('size', $builder->func()->greatest(
-					$builder->func()->add('size', $builder->createNamedParameter($sizeDifference)),
-					$builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT)
-				))
-				->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
-				->andWhere($builder->expr()->in('path_hash', $hashParams))
-				->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT)));
-
-			$builder->execute();
-		}
-	}
-
-	protected function getParents($path) {
-		$parts = explode('/', $path);
-		$parent = '';
-		$parents = [];
-		foreach ($parts as $part) {
-			$parents[] = $parent;
-			$parent = trim($parent . '/' . $part, '/');
-		}
-		return $parents;
-	}
-
-	/**
-	 * Mark the beginning of a propagation batch
-	 *
-	 * Note that not all cache setups support propagation in which case this will be a noop
-	 *
-	 * Batching for cache setups that do support it has to be explicit since the cache state is not fully consistent
-	 * before the batch is committed.
-	 */
-	public function beginBatch() {
-		$this->inBatch = true;
-	}
-
-	private function addToBatch($internalPath, $time, $sizeDifference) {
-		if (!isset($this->batch[$internalPath])) {
-			$this->batch[$internalPath] = [
-				'hash' => md5($internalPath),
-				'time' => $time,
-				'size' => $sizeDifference,
-			];
-		} else {
-			$this->batch[$internalPath]['size'] += $sizeDifference;
-			if ($time > $this->batch[$internalPath]['time']) {
-				$this->batch[$internalPath]['time'] = $time;
-			}
-		}
-	}
-
-	/**
-	 * Commit the active propagation batch
-	 */
-	public function commitBatch() {
-		if (!$this->inBatch) {
-			throw new \BadMethodCallException('Not in batch');
-		}
-		$this->inBatch = false;
-
-		$this->connection->beginTransaction();
-
-		$query = $this->connection->getQueryBuilder();
-		$storageId = (int)$this->storage->getStorageCache()->getNumericId();
-
-		$query->update('filecache')
-			->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
-			->set('etag', $query->expr()->literal(uniqid()))
-			->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT)))
-			->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')));
-
-		$sizeQuery = $this->connection->getQueryBuilder();
-		$sizeQuery->update('filecache')
-			->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size')))
-			->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT)))
-			->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')))
-			->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->expr()->literal(-1, IQueryBuilder::PARAM_INT)));
-
-		foreach ($this->batch as $item) {
-			$query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT);
-			$query->setParameter('hash', $item['hash']);
-
-			$query->execute();
-
-			if ($item['size']) {
-				$sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT);
-				$sizeQuery->setParameter('hash', $item['hash']);
-
-				$sizeQuery->execute();
-			}
-		}
-
-		$this->batch = [];
-
-		$this->connection->commit();
-	}
+    private $inBatch = false;
+
+    private $batch = [];
+
+    /**
+     * @var \OC\Files\Storage\Storage
+     */
+    protected $storage;
+
+    /**
+     * @var IDBConnection
+     */
+    private $connection;
+
+    /**
+     * @var array
+     */
+    private $ignore = [];
+
+    public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection, array $ignore = []) {
+        $this->storage = $storage;
+        $this->connection = $connection;
+        $this->ignore = $ignore;
+    }
+
+
+    /**
+     * @param string $internalPath
+     * @param int $time
+     * @param int $sizeDifference number of bytes the file has grown
+     */
+    public function propagateChange($internalPath, $time, $sizeDifference = 0) {
+        // Do not propogate changes in ignored paths
+        foreach ($this->ignore as $ignore) {
+            if (strpos($internalPath, $ignore) === 0) {
+                return;
+            }
+        }
+
+        $storageId = (int)$this->storage->getStorageCache()->getNumericId();
+
+        $parents = $this->getParents($internalPath);
+
+        if ($this->inBatch) {
+            foreach ($parents as $parent) {
+                $this->addToBatch($parent, $time, $sizeDifference);
+            }
+            return;
+        }
+
+        $parentHashes = array_map('md5', $parents);
+        $etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag
+
+        $builder = $this->connection->getQueryBuilder();
+        $hashParams = array_map(function ($hash) use ($builder) {
+            return $builder->expr()->literal($hash);
+        }, $parentHashes);
+
+        $builder->update('filecache')
+            ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int)$time, IQueryBuilder::PARAM_INT)))
+            ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
+            ->andWhere($builder->expr()->in('path_hash', $hashParams));
+        if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) {
+            $builder->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR));
+        }
+
+        $builder->execute();
+
+        if ($sizeDifference !== 0) {
+            // we need to do size separably so we can ignore entries with uncalculated size
+            $builder = $this->connection->getQueryBuilder();
+            $builder->update('filecache')
+                ->set('size', $builder->func()->greatest(
+                    $builder->func()->add('size', $builder->createNamedParameter($sizeDifference)),
+                    $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT)
+                ))
+                ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
+                ->andWhere($builder->expr()->in('path_hash', $hashParams))
+                ->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT)));
+
+            $builder->execute();
+        }
+    }
+
+    protected function getParents($path) {
+        $parts = explode('/', $path);
+        $parent = '';
+        $parents = [];
+        foreach ($parts as $part) {
+            $parents[] = $parent;
+            $parent = trim($parent . '/' . $part, '/');
+        }
+        return $parents;
+    }
+
+    /**
+     * Mark the beginning of a propagation batch
+     *
+     * Note that not all cache setups support propagation in which case this will be a noop
+     *
+     * Batching for cache setups that do support it has to be explicit since the cache state is not fully consistent
+     * before the batch is committed.
+     */
+    public function beginBatch() {
+        $this->inBatch = true;
+    }
+
+    private function addToBatch($internalPath, $time, $sizeDifference) {
+        if (!isset($this->batch[$internalPath])) {
+            $this->batch[$internalPath] = [
+                'hash' => md5($internalPath),
+                'time' => $time,
+                'size' => $sizeDifference,
+            ];
+        } else {
+            $this->batch[$internalPath]['size'] += $sizeDifference;
+            if ($time > $this->batch[$internalPath]['time']) {
+                $this->batch[$internalPath]['time'] = $time;
+            }
+        }
+    }
+
+    /**
+     * Commit the active propagation batch
+     */
+    public function commitBatch() {
+        if (!$this->inBatch) {
+            throw new \BadMethodCallException('Not in batch');
+        }
+        $this->inBatch = false;
+
+        $this->connection->beginTransaction();
+
+        $query = $this->connection->getQueryBuilder();
+        $storageId = (int)$this->storage->getStorageCache()->getNumericId();
+
+        $query->update('filecache')
+            ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
+            ->set('etag', $query->expr()->literal(uniqid()))
+            ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT)))
+            ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')));
+
+        $sizeQuery = $this->connection->getQueryBuilder();
+        $sizeQuery->update('filecache')
+            ->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size')))
+            ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT)))
+            ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')))
+            ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->expr()->literal(-1, IQueryBuilder::PARAM_INT)));
+
+        foreach ($this->batch as $item) {
+            $query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT);
+            $query->setParameter('hash', $item['hash']);
+
+            $query->execute();
+
+            if ($item['size']) {
+                $sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT);
+                $sizeQuery->setParameter('hash', $item['hash']);
+
+                $sizeQuery->execute();
+            }
+        }
+
+        $this->batch = [];
+
+        $this->connection->commit();
+    }
 }

Spacing +5 added lines, -5 removed lines

@@ -72,7 +72,7 @@ 
 			}
 		}
 
-		$storageId = (int)$this->storage->getStorageCache()->getNumericId();
+		$storageId = (int) $this->storage->getStorageCache()->getNumericId();
 
 		$parents = $this->getParents($internalPath);
 
@@ -87,12 +87,12 @@ 
 		$etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag
 
 		$builder = $this->connection->getQueryBuilder();
-		$hashParams = array_map(function ($hash) use ($builder) {
+		$hashParams = array_map(function($hash) use ($builder) {
 			return $builder->expr()->literal($hash);
 		}, $parentHashes);
 
 		$builder->update('filecache')
-			->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int)$time, IQueryBuilder::PARAM_INT)))
+			->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int) $time, IQueryBuilder::PARAM_INT)))
 			->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
 			->andWhere($builder->expr()->in('path_hash', $hashParams));
 		if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) {
@@ -123,7 +123,7 @@ 
 		$parents = [];
 		foreach ($parts as $part) {
 			$parents[] = $parent;
-			$parent = trim($parent . '/' . $part, '/');
+			$parent = trim($parent.'/'.$part, '/');
 		}
 		return $parents;
 	}
@@ -167,7 +167,7 @@ 
 		$this->connection->beginTransaction();
 
 		$query = $this->connection->getQueryBuilder();
-		$storageId = (int)$this->storage->getStorageCache()->getNumericId();
+		$storageId = (int) $this->storage->getStorageCache()->getNumericId();
 
 		$query->update('filecache')
 			->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))