AmazonS3::clearCache() - Code Metrics - Inspection of "Merge pull request #14913 from nextcloud/bugfix/69..." - nextcloud/server - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 1cb1b1...e38718 )

by Roeland

created 2019-09-26 09:39 UTC

AmazonS3::clearCache() A

↳ Parent: AmazonS3

Complexity

Conditions	1
Paths	1

Size

Total Lines	4
Code Lines	3

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	1
eloc	3
c	1
b	0
f	0
nc	1
nop	0
dl	0
loc	4
rs	10

<?php
/**
 * @copyright Copyright (c) 2016, ownCloud, Inc.
 *
 * @author André Gaul <[email protected]>
 * @author Arthur Schiwon <[email protected]>
 * @author Christian Berendt <[email protected]>
 * @author Christopher T. Johnson <[email protected]>
 * @author Johan Björk <[email protected]>
 * @author Jörn Friedrich Dreyer <[email protected]>
 * @author Martin Mattel <[email protected]>
 * @author Michael Gapczynski <[email protected]>
 * @author Morris Jobke <[email protected]>
 * @author Philipp Kapfer <[email protected]>
 * @author Robin Appelman <[email protected]>
 * @author Robin McCorkell <[email protected]>
 * @author Thomas Müller <[email protected]>
 * @author Vincent Petry <[email protected]>
 *
 * @license AGPL-3.0
 *
 * This code is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License, version 3,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License, version 3,
 * along with this program.  If not, see <http://www.gnu.org/licenses/>
 *
 */

namespace OCA\Files_External\Lib\Storage;

use Aws\Result;
use Aws\S3\S3Client;
use Aws\S3\Exception\S3Exception;
use Icewind\Streams\CallbackWrapper;
use Icewind\Streams\IteratorDirectory;
use OC\Cache\CappedMemoryCache;
use OC\Files\ObjectStore\S3ConnectionTrait;
use OC\Files\ObjectStore\S3ObjectTrait;
use OCP\Constants;

class AmazonS3 extends \OC\Files\Storage\Common {
	use S3ConnectionTrait;
	use S3ObjectTrait;

	public function needsPartFile() {
		return false;
	}

	/**
	 * @var int in seconds
	 */
	private $rescanDelay = 10;

	/** @var CappedMemoryCache|Result[] */
	private $objectCache;

	/** @var CappedMemoryCache|bool[] */
	private $directoryCache;

	/** @var CappedMemoryCache|array */
	private $filesCache;

	public function __construct($parameters) {
		parent::__construct($parameters);
		$this->parseParams($parameters);
		$this->objectCache = new CappedMemoryCache();
		$this->directoryCache = new CappedMemoryCache();
		$this->filesCache = new CappedMemoryCache();
	}

	/**
	 * @param string $path
	 * @return string correctly encoded path
	 */
	private function normalizePath($path) {
		$path = trim($path, '/');

		if (!$path) {
			$path = '.';
		}

		return $path;
	}

	private function isRoot($path) {
		return $path === '.';
	}

	private function cleanKey($path) {
		if ($this->isRoot($path)) {
			return '/';
		}
		return $path;
	}

	private function clearCache() {
		$this->objectCache = new CappedMemoryCache();
		$this->directoryCache = new CappedMemoryCache();
		$this->filesCache = new CappedMemoryCache();
	}

	private function invalidateCache($key) {
		unset($this->objectCache[$key]);
		$keys = array_keys($this->objectCache->getData());
		$keyLength = strlen($key);
		foreach ($keys as $existingKey) {
			if (substr($existingKey, 0, $keyLength) === $key) {
				unset($this->objectCache[$existingKey]);
			}
		}
		unset($this->directoryCache[$key], $this->filesCache[$key]);
	}

	/**
	 * @param $key
	 * @return Result|boolean
	 */
	private function headObject($key) {
		if (!isset($this->objectCache[$key])) {
			try {
				$this->objectCache[$key] = $this->getConnection()->headObject(array(
					'Bucket' => $this->bucket,
					'Key' => $key
				));
			} catch (S3Exception $e) {
				if ($e->getStatusCode() >= 500) {
					throw $e;
				}
				$this->objectCache[$key] = false;
			}
		}

		return $this->objectCache[$key];
	}

	/**
	 * Return true if directory exists
	 *
	 * There are no folders in s3. A folder like structure could be archived
	 * by prefixing files with the folder name.
	 *
	 * Implementation from flysystem-aws-s3-v3:
	 * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
	 *
	 * @param $path
	 * @return bool
	 * @throws \Exception
	 */
	private function doesDirectoryExist($path) {
		if (!isset($this->directoryCache[$path])) {
			// Maybe this isn't an actual key, but a prefix.
			// Do a prefix listing of objects to determine.
			try {
				$result = $this->getConnection()->listObjects([
					'Bucket' => $this->bucket,
					'Prefix' => rtrim($path, '/') . '/',
					'MaxKeys' => 1,
				]);
				$this->directoryCache[$path] = $result['Contents'] || $result['CommonPrefixes'];
			} catch (S3Exception $e) {
				if ($e->getStatusCode() === 403) {
					$this->directoryCache[$path] = false;
				}
				throw $e;
			}
		}

		return $this->directoryCache[$path];
	}

	/**
	 * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
	 * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
	 *
	 * @param array $params
	 */
	public function updateLegacyId(array $params) {
		$oldId = 'amazon::' . $params['key'] . md5($params['secret']);

		// find by old id or bucket
		$stmt = \OC::$server->getDatabaseConnection()->prepare(
			'SELECT `numeric_id`, `id` FROM `*PREFIX*storages` WHERE `id` IN (?, ?)'
		);
		$stmt->execute(array($oldId, $this->id));
		while ($row = $stmt->fetch()) {
			$storages[$row['id']] = $row['numeric_id'];
		}

		if (isset($storages[$this->id]) && isset($storages[$oldId])) {
			// if both ids exist, delete the old storage and corresponding filecache entries
			\OC\Files\Cache\Storage::remove($oldId);
		} else if (isset($storages[$oldId])) {
			// if only the old id exists do an update
			$stmt = \OC::$server->getDatabaseConnection()->prepare(
				'UPDATE `*PREFIX*storages` SET `id` = ? WHERE `id` = ?'
			);
			$stmt->execute(array($this->id, $oldId));
		}
		// only the bucket based id may exist, do nothing
	}

	/**
	 * Remove a file or folder
	 *
	 * @param string $path
	 * @return bool
	 */
	protected function remove($path) {
		// remember fileType to reduce http calls
		$fileType = $this->filetype($path);
		if ($fileType === 'dir') {
			return $this->rmdir($path);
		} else if ($fileType === 'file') {
			return $this->unlink($path);
		} else {
			return false;
		}
	}

	public function mkdir($path) {
		$path = $this->normalizePath($path);

		if ($this->is_dir($path)) {
			return false;
		}

		try {
			$this->getConnection()->putObject(array(
				'Bucket' => $this->bucket,
				'Key' => $path . '/',
				'Body' => '',
				'ContentType' => 'httpd/unix-directory'
			));
			$this->testTimeout();
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}

		$this->invalidateCache($path);

		return true;
	}

	public function file_exists($path) {
		return $this->filetype($path) !== false;
	}


	public function rmdir($path) {
		$path = $this->normalizePath($path);

		if ($this->isRoot($path)) {
			return $this->clearBucket();
		}

		if (!$this->file_exists($path)) {
			return false;
		}

		$this->invalidateCache($path);
		return $this->batchDelete($path);
	}

	protected function clearBucket() {
		$this->clearCache();
		try {
			$this->getConnection()->clearBucket($this->bucket);
			return true;
			// clearBucket() is not working with Ceph, so if it fails we try the slower approach
		} catch (\Exception $e) {
			return $this->batchDelete();
		}
	}

	private function batchDelete($path = null) {
		$params = array(
			'Bucket' => $this->bucket
		);
		if ($path !== null) {
			$params['Prefix'] = $path . '/';
		}
		try {
			$connection = $this->getConnection();
			// Since there are no real directories on S3, we need
			// to delete all objects prefixed with the path.
			do {
				// instead of the iterator, manually loop over the list ...
				$objects = $connection->listObjects($params);
				// ... so we can delete the files in batches
				if (isset($objects['Contents'])) {
					$connection->deleteObjects([
						'Bucket' => $this->bucket,
						'Delete' => [
							'Objects' => $objects['Contents']
						]
					]);
					$this->testTimeout();
				}
				// we reached the end when the list is no longer truncated
			} while ($objects['IsTruncated']);
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}
		return true;
	}

	public function opendir($path) {
		$path = $this->normalizePath($path);

		if ($this->isRoot($path)) {
			$path = '';
		} else {
			$path .= '/';
		}

		try {
			$files = array();
			$results = $this->getConnection()->getPaginator('ListObjects', [
				'Bucket' => $this->bucket,
				'Delimiter' => '/',
				'Prefix' => $path,
			]);

			foreach ($results as $result) {
				// sub folders
				if (is_array($result['CommonPrefixes'])) {
					foreach ($result['CommonPrefixes'] as $prefix) {
						$directoryName = trim($prefix['Prefix'], '/');
						$files[] = substr($directoryName, strlen($path));
						$this->directoryCache[$directoryName] = true;
					}
				}
				if (is_array($result['Contents'])) {
					foreach ($result['Contents'] as $object) {
						if (isset($object['Key']) && $object['Key'] === $path) {
							// it's the directory itself, skip
							continue;
						}
						$file = basename(
							isset($object['Key']) ? $object['Key'] : $object['Prefix']
						);
						$files[] = $file;

						// store this information for later usage
						$this->filesCache[$path . $file] = [
							'ContentLength' => $object['Size'],
							'LastModified' => (string)$object['LastModified'],
						];
					}
				}
			}

			return IteratorDirectory::wrap($files);
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}
	}

	public function stat($path) {
		$path = $this->normalizePath($path);

		try {
			$stat = [];
			if ($this->is_dir($path)) {
				//folders don't really exist
				$stat['size'] = -1; //unknown
				$stat['mtime'] = time() - $this->rescanDelay * 1000;
			} else {
				$stat['size'] = $this->getContentLength($path);
				$stat['mtime'] = strtotime($this->getLastModified($path));
			}
			$stat['atime'] = time();

			return $stat;
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}
	}

	/**
	 * Return content length for object
	 *
	 * When the information is already present (e.g. opendir has been called before)
	 * this value is return. Otherwise a headObject is emitted.
	 *
	 * @param $path
	 * @return int|mixed
	 */
	private function getContentLength($path) {
		if (isset($this->filesCache[$path])) {
			return $this->filesCache[$path]['ContentLength'];
		}

		$result = $this->headObject($path);
		if (isset($result['ContentLength'])) {
			return $result['ContentLength'];
		}

		return 0;
	}

	/**
	 * Return last modified for object
	 *
	 * When the information is already present (e.g. opendir has been called before)
	 * this value is return. Otherwise a headObject is emitted.
	 *
	 * @param $path
	 * @return mixed|string
	 */
	private function getLastModified($path) {
		if (isset($this->filesCache[$path])) {
			return $this->filesCache[$path]['LastModified'];
		}

		$result = $this->headObject($path);
		if (isset($result['LastModified'])) {
			return $result['LastModified'];
		}

		return 'now';
	}

	public function is_dir($path) {
		$path = $this->normalizePath($path);

		if (isset($this->filesCache[$path])) {
			return false;
		}

		try {
			return $this->isRoot($path) || $this->doesDirectoryExist($path);
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}
	}

	public function filetype($path) {
		$path = $this->normalizePath($path);

		if ($this->isRoot($path)) {
			return 'dir';
		}

		try {
			if (isset($this->filesCache[$path]) || $this->headObject($path)) {
				return 'file';
			}
			if ($this->doesDirectoryExist($path)) {
				return 'dir';
			}
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}

		return false;
	}

	public function getPermissions($path) {
		$type = $this->filetype($path);
		if (!$type) {
			return 0;
		}
		return $type === 'dir' ? Constants::PERMISSION_ALL : Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
	}

	public function unlink($path) {
		$path = $this->normalizePath($path);

		if ($this->is_dir($path)) {
			return $this->rmdir($path);
		}

		try {
			$this->deleteObject($path);
			$this->invalidateCache($path);
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}

		return true;
	}

	public function fopen($path, $mode) {
		$path = $this->normalizePath($path);

		switch ($mode) {
			case 'r':
			case 'rb':
				try {
					return $this->readObject($path);
				} catch (S3Exception $e) {
					\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
					return false;
				}
			case 'w':
			case 'wb':
				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile();

				$handle = fopen($tmpFile, 'w');
				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
					$this->writeBack($tmpFile, $path);
				});
			case 'a':
			case 'ab':
			case 'r+':
			case 'w+':
			case 'wb+':
			case 'a+':
			case 'x':
			case 'x+':
			case 'c':
			case 'c+':
				if (strrpos($path, '.') !== false) {
					$ext = substr($path, strrpos($path, '.'));
				} else {
					$ext = '';
				}
				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
				if ($this->file_exists($path)) {
					$source = $this->readObject($path);
					file_put_contents($tmpFile, $source);
				}

				$handle = fopen($tmpFile, $mode);
				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
					$this->writeBack($tmpFile, $path);
				});
		}
		return false;
	}

	public function touch($path, $mtime = null) {
		$path = $this->normalizePath($path);

		$metadata = array();

		if (is_null($mtime)) {
			$mtime = time();
		}
		$metadata = [
			'lastmodified' => gmdate(\DateTime::RFC1123, $mtime)
		];

		$fileType = $this->filetype($path);
		try {
			if ($fileType !== false) {
				if ($fileType === 'dir' && !$this->isRoot($path)) {
					$path .= '/';
				}
				$this->getConnection()->copyObject([
					'Bucket' => $this->bucket,
					'Key' => $this->cleanKey($path),
					'Metadata' => $metadata,
					'CopySource' => $this->bucket . '/' . $path,
					'MetadataDirective' => 'REPLACE',
				]);
				$this->testTimeout();
			} else {
				$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
				$this->getConnection()->putObject([
					'Bucket' => $this->bucket,
					'Key' => $this->cleanKey($path),
					'Metadata' => $metadata,
					'Body' => '',
					'ContentType' => $mimeType,
					'MetadataDirective' => 'REPLACE',
				]);
				$this->testTimeout();
			}
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}

		$this->invalidateCache($path);
		return true;
	}

	public function copy($path1, $path2) {
		$path1 = $this->normalizePath($path1);
		$path2 = $this->normalizePath($path2);

		if ($this->is_file($path1)) {
			try {
				$this->getConnection()->copyObject(array(
					'Bucket' => $this->bucket,
					'Key' => $this->cleanKey($path2),
					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
				));
				$this->testTimeout();
			} catch (S3Exception $e) {
				\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
				return false;
			}
		} else {
			$this->remove($path2);

			try {
				$this->getConnection()->copyObject(array(
					'Bucket' => $this->bucket,
					'Key' => $path2 . '/',
					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
				));
				$this->testTimeout();
			} catch (S3Exception $e) {
				\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
				return false;
			}

			$dh = $this->opendir($path1);
			if (is_resource($dh)) {
				while (($file = readdir($dh)) !== false) {
					if (\OC\Files\Filesystem::isIgnoredDir($file)) {
						continue;
					}

					$source = $path1 . '/' . $file;
					$target = $path2 . '/' . $file;
					$this->copy($source, $target);
				}
			}
		}

		$this->invalidateCache($path2);

		return true;
	}

	public function rename($path1, $path2) {
		$path1 = $this->normalizePath($path1);
		$path2 = $this->normalizePath($path2);

		if ($this->is_file($path1)) {

			if ($this->copy($path1, $path2) === false) {
				return false;
			}

			if ($this->unlink($path1) === false) {
				$this->unlink($path2);
				return false;
			}
		} else {

			if ($this->copy($path1, $path2) === false) {
				return false;
			}

			if ($this->rmdir($path1) === false) {
				$this->rmdir($path2);
				return false;
			}
		}

		return true;
	}

	public function test() {
		$this->getConnection()->headBucket([
			'Bucket' => $this->bucket
		]);
		return true;
	}

	public function getId() {
		return $this->id;
	}

	public function writeBack($tmpFile, $path) {
		try {
			$source = fopen($tmpFile, 'r');
			$this->writeObject($path, $source);
			$this->invalidateCache($path);
			fclose($source);

			unlink($tmpFile);
			return true;
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}
	}

	/**
	 * check if curl is installed
	 */
	public static function checkDependencies() {
		return true;
	}

}


1			<?php
2			/**
3			* @copyright Copyright (c) 2016, ownCloud, Inc.
4			*
5			* @author André Gaul <[email protected]>
6			* @author Arthur Schiwon <[email protected]>
7			* @author Christian Berendt <[email protected]>
8			* @author Christopher T. Johnson <[email protected]>
9			* @author Johan Björk <[email protected]>
10			* @author Jörn Friedrich Dreyer <[email protected]>
11			* @author Martin Mattel <[email protected]>
12			* @author Michael Gapczynski <[email protected]>
13			* @author Morris Jobke <[email protected]>
14			* @author Philipp Kapfer <[email protected]>
15			* @author Robin Appelman <[email protected]>
16			* @author Robin McCorkell <[email protected]>
17			* @author Thomas Müller <[email protected]>
18			* @author Vincent Petry <[email protected]>
19			*
20			* @license AGPL-3.0
21			*
22			* This code is free software: you can redistribute it and/or modify
23			* it under the terms of the GNU Affero General Public License, version 3,
24			* as published by the Free Software Foundation.
25			*
26			* This program is distributed in the hope that it will be useful,
27			* but WITHOUT ANY WARRANTY; without even the implied warranty of
28			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
29			* GNU Affero General Public License for more details.
30			*
31			* You should have received a copy of the GNU Affero General Public License, version 3,
32			* along with this program. If not, see <http://www.gnu.org/licenses/>
33			*
34			*/
35
36			namespace OCA\Files_External\Lib\Storage;
37
38			use Aws\Result;
39			use Aws\S3\S3Client;
40			use Aws\S3\Exception\S3Exception;
41			use Icewind\Streams\CallbackWrapper;
42			use Icewind\Streams\IteratorDirectory;
43			use OC\Cache\CappedMemoryCache;
44			use OC\Files\ObjectStore\S3ConnectionTrait;
45			use OC\Files\ObjectStore\S3ObjectTrait;
46			use OCP\Constants;
47
48			class AmazonS3 extends \OC\Files\Storage\Common {
49			use S3ConnectionTrait;
50			use S3ObjectTrait;
51
52			public function needsPartFile() {
53			return false;
54			}
55
56			/**
57			* @var int in seconds
58			*/
59			private $rescanDelay = 10;
60
61			/** @var CappedMemoryCache\|Result[] */
62			private $objectCache;
63
64			/** @var CappedMemoryCache\|bool[] */
65			private $directoryCache;
66
67			/** @var CappedMemoryCache\|array */
68			private $filesCache;
69
70			public function __construct($parameters) {
71			parent::__construct($parameters);
72			$this->parseParams($parameters);
73			$this->objectCache = new CappedMemoryCache();
74			$this->directoryCache = new CappedMemoryCache();
75			$this->filesCache = new CappedMemoryCache();
76			}
77
78			/**
79			* @param string $path
80			* @return string correctly encoded path
81			*/
82			private function normalizePath($path) {
83			$path = trim($path, '/');
84
85			if (!$path) {
86			$path = '.';
87			}
88
89			return $path;
90			}
91
92			private function isRoot($path) {
93			return $path === '.';
94			}
95
96			private function cleanKey($path) {
97			if ($this->isRoot($path)) {
98			return '/';
99			}
100			return $path;
101			}
102
103			private function clearCache() {
104			$this->objectCache = new CappedMemoryCache();
105			$this->directoryCache = new CappedMemoryCache();
106			$this->filesCache = new CappedMemoryCache();
107			}
108
109			private function invalidateCache($key) {
110			unset($this->objectCache[$key]);
111			$keys = array_keys($this->objectCache->getData());
112			$keyLength = strlen($key);
113			foreach ($keys as $existingKey) {
114			if (substr($existingKey, 0, $keyLength) === $key) {
115			unset($this->objectCache[$existingKey]);
116			}
117			}
118			unset($this->directoryCache[$key], $this->filesCache[$key]);
119			}
120
121			/**
122			* @param $key
123			* @return Result\|boolean
124			*/
125			private function headObject($key) {
126			if (!isset($this->objectCache[$key])) {
127			try {
128			$this->objectCache[$key] = $this->getConnection()->headObject(array(
129			'Bucket' => $this->bucket,
130			'Key' => $key
131			));
132			} catch (S3Exception $e) {
133			if ($e->getStatusCode() >= 500) {
134			throw $e;
135			}
136			$this->objectCache[$key] = false;
137			}
138			}
139
140			return $this->objectCache[$key];
141			}
142
143			/**
144			* Return true if directory exists
145			*
146			* There are no folders in s3. A folder like structure could be archived
147			* by prefixing files with the folder name.
148			*
149			* Implementation from flysystem-aws-s3-v3:
150			* https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
151			*
152			* @param $path
153			* @return bool
154			* @throws \Exception
155			*/
156			private function doesDirectoryExist($path) {
157			if (!isset($this->directoryCache[$path])) {
158			// Maybe this isn't an actual key, but a prefix.
159			// Do a prefix listing of objects to determine.
160			try {
161			$result = $this->getConnection()->listObjects([
162			'Bucket' => $this->bucket,
163			'Prefix' => rtrim($path, '/') . '/',
164			'MaxKeys' => 1,
165			]);
166			$this->directoryCache[$path] = $result['Contents'] \|\| $result['CommonPrefixes'];
167			} catch (S3Exception $e) {
168			if ($e->getStatusCode() === 403) {
169			$this->directoryCache[$path] = false;
170			}
171			throw $e;
172			}
173			}
174
175			return $this->directoryCache[$path];
176			}
177
178			/**
179			* Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
180			* TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
181			*
182			* @param array $params
183			*/
184			public function updateLegacyId(array $params) {
185			$oldId = 'amazon::' . $params['key'] . md5($params['secret']);
186
187			// find by old id or bucket
188			$stmt = \OC::$server->getDatabaseConnection()->prepare(
189			'SELECT `numeric_id`, `id` FROM `PREFIXstorages` WHERE `id` IN (?, ?)'
190			);
191			$stmt->execute(array($oldId, $this->id));
192			while ($row = $stmt->fetch()) {
193			$storages[$row['id']] = $row['numeric_id'];
194			}
195
196			if (isset($storages[$this->id]) && isset($storages[$oldId])) {
197			// if both ids exist, delete the old storage and corresponding filecache entries
198			\OC\Files\Cache\Storage::remove($oldId);
199			} else if (isset($storages[$oldId])) {
200			// if only the old id exists do an update
201			$stmt = \OC::$server->getDatabaseConnection()->prepare(
202			'UPDATE `PREFIXstorages` SET `id` = ? WHERE `id` = ?'
203			);
204			$stmt->execute(array($this->id, $oldId));
205			}
206			// only the bucket based id may exist, do nothing
207			}
208
209			/**
210			* Remove a file or folder
211			*
212			* @param string $path
213			* @return bool
214			*/
215			protected function remove($path) {
216			// remember fileType to reduce http calls
217			$fileType = $this->filetype($path);
218			if ($fileType === 'dir') {
219			return $this->rmdir($path);
220			} else if ($fileType === 'file') {
221			return $this->unlink($path);
222			} else {
223			return false;
224			}
225			}
226
227			public function mkdir($path) {
228			$path = $this->normalizePath($path);
229
230			if ($this->is_dir($path)) {
231			return false;
232			}
233
234			try {
235			$this->getConnection()->putObject(array(
236			'Bucket' => $this->bucket,
237			'Key' => $path . '/',
238			'Body' => '',
239			'ContentType' => 'httpd/unix-directory'
240			));
241			$this->testTimeout();
242			} catch (S3Exception $e) {
243			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
244			return false;
245			}
246
247			$this->invalidateCache($path);
248
249			return true;
250			}
251
252			public function file_exists($path) {
253			return $this->filetype($path) !== false;
254			}
255
256
257			public function rmdir($path) {
258			$path = $this->normalizePath($path);
259
260			if ($this->isRoot($path)) {
261			return $this->clearBucket();
262			}
263
264			if (!$this->file_exists($path)) {
265			return false;
266			}
267
268			$this->invalidateCache($path);
269			return $this->batchDelete($path);
270			}
271
272			protected function clearBucket() {
273			$this->clearCache();
274			try {
275			$this->getConnection()->clearBucket($this->bucket);
276			return true;
277			// clearBucket() is not working with Ceph, so if it fails we try the slower approach
278			} catch (\Exception $e) {
279			return $this->batchDelete();
280			}
281			}
282
283			private function batchDelete($path = null) {
284			$params = array(
285			'Bucket' => $this->bucket
286			);
287			if ($path !== null) {
288			$params['Prefix'] = $path . '/';
289			}
290			try {
291			$connection = $this->getConnection();
292			// Since there are no real directories on S3, we need
293			// to delete all objects prefixed with the path.
294			do {
295			// instead of the iterator, manually loop over the list ...
296			$objects = $connection->listObjects($params);
297			// ... so we can delete the files in batches
298			if (isset($objects['Contents'])) {
299			$connection->deleteObjects([
300			'Bucket' => $this->bucket,
301			'Delete' => [
302			'Objects' => $objects['Contents']
303			]
304			]);
305			$this->testTimeout();
306			}
307			// we reached the end when the list is no longer truncated
308			} while ($objects['IsTruncated']);
309			} catch (S3Exception $e) {
310			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
311			return false;
312			}
313			return true;
314			}
315
316			public function opendir($path) {
317			$path = $this->normalizePath($path);
318
319			if ($this->isRoot($path)) {
320			$path = '';
321			} else {
322			$path .= '/';
323			}
324
325			try {
326			$files = array();
327			$results = $this->getConnection()->getPaginator('ListObjects', [
328			'Bucket' => $this->bucket,
329			'Delimiter' => '/',
330			'Prefix' => $path,
331			]);
332
333			foreach ($results as $result) {
334			// sub folders
335			if (is_array($result['CommonPrefixes'])) {
336			foreach ($result['CommonPrefixes'] as $prefix) {
337			$directoryName = trim($prefix['Prefix'], '/');
338			$files[] = substr($directoryName, strlen($path));
339			$this->directoryCache[$directoryName] = true;
340			}
341			}
342			if (is_array($result['Contents'])) {
343			foreach ($result['Contents'] as $object) {
344			if (isset($object['Key']) && $object['Key'] === $path) {
345			// it's the directory itself, skip
346			continue;
347			}
348			$file = basename(
349			isset($object['Key']) ? $object['Key'] : $object['Prefix']
350			);
351			$files[] = $file;
352
353			// store this information for later usage
354			$this->filesCache[$path . $file] = [
355			'ContentLength' => $object['Size'],
356			'LastModified' => (string)$object['LastModified'],
357			];
358			}
359			}
360			}
361
362			return IteratorDirectory::wrap($files);
363			} catch (S3Exception $e) {
364			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
365			return false;
366			}
367			}
368
369			public function stat($path) {
370			$path = $this->normalizePath($path);
371
372			try {
373			$stat = [];
374			if ($this->is_dir($path)) {
375			//folders don't really exist
376			$stat['size'] = -1; //unknown
377			$stat['mtime'] = time() - $this->rescanDelay * 1000;
378			} else {
379			$stat['size'] = $this->getContentLength($path);
380			$stat['mtime'] = strtotime($this->getLastModified($path));
381			}
382			$stat['atime'] = time();
383
384			return $stat;
385			} catch (S3Exception $e) {
386			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
387			return false;
388			}
389			}
390
391			/**
392			* Return content length for object
393			*
394			* When the information is already present (e.g. opendir has been called before)
395			* this value is return. Otherwise a headObject is emitted.
396			*
397			* @param $path
398			* @return int\|mixed
399			*/
400			private function getContentLength($path) {
401			if (isset($this->filesCache[$path])) {
402			return $this->filesCache[$path]['ContentLength'];
403			}
404
405			$result = $this->headObject($path);
406			if (isset($result['ContentLength'])) {
407			return $result['ContentLength'];
408			}
409
410			return 0;
411			}
412
413			/**
414			* Return last modified for object
415			*
416			* When the information is already present (e.g. opendir has been called before)
417			* this value is return. Otherwise a headObject is emitted.
418			*
419			* @param $path
420			* @return mixed\|string
421			*/
422			private function getLastModified($path) {
423			if (isset($this->filesCache[$path])) {
424			return $this->filesCache[$path]['LastModified'];
425			}
426
427			$result = $this->headObject($path);
428			if (isset($result['LastModified'])) {
429			return $result['LastModified'];
430			}
431
432			return 'now';
433			}
434
435			public function is_dir($path) {
436			$path = $this->normalizePath($path);
437
438			if (isset($this->filesCache[$path])) {
439			return false;
440			}
441
442			try {
443			return $this->isRoot($path) \|\| $this->doesDirectoryExist($path);
444			} catch (S3Exception $e) {
445			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
446			return false;
447			}
448			}
449
450			public function filetype($path) {
451			$path = $this->normalizePath($path);
452
453			if ($this->isRoot($path)) {
454			return 'dir';
455			}
456
457			try {
458			if (isset($this->filesCache[$path]) \|\| $this->headObject($path)) {
459			return 'file';
460			}
461			if ($this->doesDirectoryExist($path)) {
462			return 'dir';
463			}
464			} catch (S3Exception $e) {
465			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
466			return false;
467			}
468
469			return false;
470			}
471
472			public function getPermissions($path) {
473			$type = $this->filetype($path);
474			if (!$type) {
475			return 0;
476			}
477			return $type === 'dir' ? Constants::PERMISSION_ALL : Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
478			}
479
480			public function unlink($path) {
481			$path = $this->normalizePath($path);
482
483			if ($this->is_dir($path)) {
484			return $this->rmdir($path);
485			}
486
487			try {
488			$this->deleteObject($path);
489			$this->invalidateCache($path);
490			} catch (S3Exception $e) {
491			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
492			return false;
493			}
494
495			return true;
496			}
497
498			public function fopen($path, $mode) {
499			$path = $this->normalizePath($path);
500
501			switch ($mode) {
502			case 'r':
503			case 'rb':
504			try {
505			return $this->readObject($path);
506			} catch (S3Exception $e) {
507			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
508			return false;
509			}
510			case 'w':
511			case 'wb':
512			$tmpFile = \OC::$server->getTempManager()->getTemporaryFile();
513
514			$handle = fopen($tmpFile, 'w');
515			return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
516			$this->writeBack($tmpFile, $path);
517			});
518			case 'a':
519			case 'ab':
520			case 'r+':
521			case 'w+':
522			case 'wb+':
523			case 'a+':
524			case 'x':
525			case 'x+':
526			case 'c':
527			case 'c+':
528			if (strrpos($path, '.') !== false) {
529			$ext = substr($path, strrpos($path, '.'));
530			} else {
531			$ext = '';
532			}
533			$tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
534			if ($this->file_exists($path)) {
535			$source = $this->readObject($path);
536			file_put_contents($tmpFile, $source);
537			}
538
539			$handle = fopen($tmpFile, $mode);
540			return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
541			$this->writeBack($tmpFile, $path);
542			});
543			}
544			return false;
545			}
546
547			public function touch($path, $mtime = null) {
548			$path = $this->normalizePath($path);
549
550			$metadata = array();
			0 ignored issues – show Unused Code introduced 2018-10-24 12:40 UTC by Report Bug Copy Issue Report The assignment to `$metadata` is dead and can be removed. Loading history...
551			if (is_null($mtime)) {
552			$mtime = time();
553			}
554			$metadata = [
555			'lastmodified' => gmdate(\DateTime::RFC1123, $mtime)
556			];
557
558			$fileType = $this->filetype($path);
559			try {
560			if ($fileType !== false) {
561			if ($fileType === 'dir' && !$this->isRoot($path)) {
562			$path .= '/';
563			}
564			$this->getConnection()->copyObject([
565			'Bucket' => $this->bucket,
566			'Key' => $this->cleanKey($path),
567			'Metadata' => $metadata,
568			'CopySource' => $this->bucket . '/' . $path,
569			'MetadataDirective' => 'REPLACE',
570			]);
571			$this->testTimeout();
572			} else {
573			$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
574			$this->getConnection()->putObject([
575			'Bucket' => $this->bucket,
576			'Key' => $this->cleanKey($path),
577			'Metadata' => $metadata,
578			'Body' => '',
579			'ContentType' => $mimeType,
580			'MetadataDirective' => 'REPLACE',
581			]);
582			$this->testTimeout();
583			}
584			} catch (S3Exception $e) {
585			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
586			return false;
587			}
588
589			$this->invalidateCache($path);
590			return true;
591			}
592
593			public function copy($path1, $path2) {
594			$path1 = $this->normalizePath($path1);
595			$path2 = $this->normalizePath($path2);
596
597			if ($this->is_file($path1)) {
598			try {
599			$this->getConnection()->copyObject(array(
600			'Bucket' => $this->bucket,
601			'Key' => $this->cleanKey($path2),
602			'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
603			));
604			$this->testTimeout();
605			} catch (S3Exception $e) {
606			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
607			return false;
608			}
609			} else {
610			$this->remove($path2);
611
612			try {
613			$this->getConnection()->copyObject(array(
614			'Bucket' => $this->bucket,
615			'Key' => $path2 . '/',
616			'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
617			));
618			$this->testTimeout();
619			} catch (S3Exception $e) {
620			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
621			return false;
622			}
623
624			$dh = $this->opendir($path1);
625			if (is_resource($dh)) {
626			while (($file = readdir($dh)) !== false) {
627			if (\OC\Files\Filesystem::isIgnoredDir($file)) {
628			continue;
629			}
630
631			$source = $path1 . '/' . $file;
632			$target = $path2 . '/' . $file;
633			$this->copy($source, $target);
634			}
635			}
636			}
637
638			$this->invalidateCache($path2);
639
640			return true;
641			}
642
643			public function rename($path1, $path2) {
644			$path1 = $this->normalizePath($path1);
645			$path2 = $this->normalizePath($path2);
646
647			if ($this->is_file($path1)) {
648
649			if ($this->copy($path1, $path2) === false) {
650			return false;
651			}
652
653			if ($this->unlink($path1) === false) {
654			$this->unlink($path2);
655			return false;
656			}
657			} else {
658
659			if ($this->copy($path1, $path2) === false) {
660			return false;
661			}
662
663			if ($this->rmdir($path1) === false) {
664			$this->rmdir($path2);
665			return false;
666			}
667			}
668
669			return true;
670			}
671
672			public function test() {
673			$this->getConnection()->headBucket([
674			'Bucket' => $this->bucket
675			]);
676			return true;
677			}
678
679			public function getId() {
680			return $this->id;
681			}
682
683			public function writeBack($tmpFile, $path) {
684			try {
685			$source = fopen($tmpFile, 'r');
686			$this->writeObject($path, $source);
687			$this->invalidateCache($path);
688			fclose($source);
689
690			unlink($tmpFile);
691			return true;
692			} catch (S3Exception $e) {
693			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
694			return false;
695			}
696			}
697
698			/**
699			* check if curl is installed
700			*/
701			public static function checkDependencies() {
702			return true;
703			}
704
705			}
706

nextcloud / server

Push — master ( 1cb1b1...e38718 )

AmazonS3::clearCache() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like