AmazonS3::needsPartFile() - Code Metrics - Inspection of "Merge pull request #23645 from nextcloud/fix/fe/s3..." - nextcloud/server - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( d38a7c...25f347 )

by Roeland

created 2020-10-27 10:55 UTC

AmazonS3::needsPartFile() A

↳ Parent: AmazonS3

Complexity

Conditions	1
Paths	1

Size

Total Lines	2
Code Lines	1

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	1
eloc	1
nc	1
nop	0
dl	0
loc	2
rs	10
c	0
b	0
f	0

<?php
/**
 * @copyright Copyright (c) 2016, ownCloud, Inc.
 *
 * @author André Gaul <[email protected]>
 * @author Arthur Schiwon <[email protected]>
 * @author Christian Berendt <[email protected]>
 * @author Christopher T. Johnson <[email protected]>
 * @author Christoph Wurst <[email protected]>
 * @author Daniel Kesselberg <[email protected]>
 * @author enoch <[email protected]>
 * @author Johan Björk <[email protected]>
 * @author Jörn Friedrich Dreyer <[email protected]>
 * @author Julius Härtl <[email protected]>
 * @author Martin Mattel <[email protected]>
 * @author Michael Gapczynski <[email protected]>
 * @author Morris Jobke <[email protected]>
 * @author Philipp Kapfer <[email protected]>
 * @author Robin Appelman <[email protected]>
 * @author Robin McCorkell <[email protected]>
 * @author Roeland Jago Douma <[email protected]>
 * @author Thomas Müller <[email protected]>
 * @author Vincent Petry <[email protected]>
 *
 * @license AGPL-3.0
 *
 * This code is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License, version 3,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License, version 3,
 * along with this program. If not, see <http://www.gnu.org/licenses/>
 *
 */

namespace OCA\Files_External\Lib\Storage;

use Aws\Result;
use Aws\S3\Exception\S3Exception;
use Aws\S3\S3Client;
use Icewind\Streams\CallbackWrapper;
use Icewind\Streams\IteratorDirectory;
use OC\Cache\CappedMemoryCache;
use OC\Files\Cache\CacheEntry;
use OC\Files\ObjectStore\S3ConnectionTrait;
use OC\Files\ObjectStore\S3ObjectTrait;
use OCP\Constants;

class AmazonS3 extends \OC\Files\Storage\Common {
	use S3ConnectionTrait;
	use S3ObjectTrait;

	public function needsPartFile() {
		return false;
	}

	/** @var CappedMemoryCache|Result[] */
	private $objectCache;

	/** @var CappedMemoryCache|bool[] */
	private $directoryCache;

	/** @var CappedMemoryCache|array */
	private $filesCache;

	public function __construct($parameters) {
		parent::__construct($parameters);
		$this->parseParams($parameters);
		$this->objectCache = new CappedMemoryCache();
		$this->directoryCache = new CappedMemoryCache();
		$this->filesCache = new CappedMemoryCache();
	}

	/**
	 * @param string $path
	 * @return string correctly encoded path
	 */
	private function normalizePath($path) {
		$path = trim($path, '/');

		if (!$path) {
			$path = '.';
		}

		return $path;
	}

	private function isRoot($path) {
		return $path === '.';
	}

	private function cleanKey($path) {
		if ($this->isRoot($path)) {
			return '/';
		}
		return $path;
	}

	private function clearCache() {
		$this->objectCache = new CappedMemoryCache();
		$this->directoryCache = new CappedMemoryCache();
		$this->filesCache = new CappedMemoryCache();
	}

	private function invalidateCache($key) {
		unset($this->objectCache[$key]);
		$keys = array_keys($this->objectCache->getData());
		$keyLength = strlen($key);
		foreach ($keys as $existingKey) {
			if (substr($existingKey, 0, $keyLength) === $key) {
				unset($this->objectCache[$existingKey]);
			}
		}
		unset($this->directoryCache[$key], $this->filesCache[$key]);
	}

	/**
	 * @param $key
	 * @return Result|boolean
	 */
	private function headObject($key) {
		if (!isset($this->objectCache[$key])) {
			try {
				$this->objectCache[$key] = $this->getConnection()->headObject([
					'Bucket' => $this->bucket,
					'Key' => $key
				]);
			} catch (S3Exception $e) {
				if ($e->getStatusCode() >= 500) {
					throw $e;
				}
				$this->objectCache[$key] = false;
			}
		}

		return $this->objectCache[$key];
	}

	/**
	 * Return true if directory exists
	 *
	 * There are no folders in s3. A folder like structure could be archived
	 * by prefixing files with the folder name.
	 *
	 * Implementation from flysystem-aws-s3-v3:
	 * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
	 *
	 * @param $path
	 * @return bool
	 * @throws \Exception
	 */
	private function doesDirectoryExist($path) {
		if (!isset($this->directoryCache[$path])) {
			// Maybe this isn't an actual key, but a prefix.
			// Do a prefix listing of objects to determine.
			try {
				$result = $this->getConnection()->listObjects([
					'Bucket' => $this->bucket,
					'Prefix' => rtrim($path, '/'),
					'MaxKeys' => 1,
					'Delimiter' => '/',
				]);

				if ((isset($result['Contents'][0]['Key']) && $result['Contents'][0]['Key'] === rtrim($path, '/') . '/')
					 || isset($result['CommonPrefixes'])) {
					$this->directoryCache[$path] = true;
				} else {
					$this->directoryCache[$path] = false;
				}
			} catch (S3Exception $e) {
				if ($e->getStatusCode() === 403) {
					$this->directoryCache[$path] = false;
				}
				throw $e;
			}
		}

		return $this->directoryCache[$path];
	}

	/**
	 * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
	 * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
	 *
	 * @param array $params
	 */
	public function updateLegacyId(array $params) {
		$oldId = 'amazon::' . $params['key'] . md5($params['secret']);

		// find by old id or bucket
		$stmt = \OC::$server->getDatabaseConnection()->prepare(
			'SELECT `numeric_id`, `id` FROM `*PREFIX*storages` WHERE `id` IN (?, ?)'
		);
		$stmt->execute([$oldId, $this->id]);
		while ($row = $stmt->fetch()) {
			$storages[$row['id']] = $row['numeric_id'];
		}

		if (isset($storages[$this->id]) && isset($storages[$oldId])) {
			// if both ids exist, delete the old storage and corresponding filecache entries
			\OC\Files\Cache\Storage::remove($oldId);
		} elseif (isset($storages[$oldId])) {
			// if only the old id exists do an update
			$stmt = \OC::$server->getDatabaseConnection()->prepare(
				'UPDATE `*PREFIX*storages` SET `id` = ? WHERE `id` = ?'
			);
			$stmt->execute([$this->id, $oldId]);
		}
		// only the bucket based id may exist, do nothing
	}

	/**
	 * Remove a file or folder
	 *
	 * @param string $path
	 * @return bool
	 */
	protected function remove($path) {
		// remember fileType to reduce http calls
		$fileType = $this->filetype($path);
		if ($fileType === 'dir') {
			return $this->rmdir($path);
		} elseif ($fileType === 'file') {
			return $this->unlink($path);
		} else {
			return false;
		}
	}

	public function mkdir($path) {
		$path = $this->normalizePath($path);

		if ($this->is_dir($path)) {
			return false;
		}

		try {
			$this->getConnection()->putObject([
				'Bucket' => $this->bucket,
				'Key' => $path . '/',
				'Body' => '',
				'ContentType' => 'httpd/unix-directory'
			]);
			$this->testTimeout();
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}

		$this->invalidateCache($path);

		return true;
	}

	public function file_exists($path) {
		return $this->filetype($path) !== false;
	}


	public function rmdir($path) {
		$path = $this->normalizePath($path);

		if ($this->isRoot($path)) {
			return $this->clearBucket();
		}

		if (!$this->file_exists($path)) {
			return false;
		}

		$this->invalidateCache($path);
		return $this->batchDelete($path);
	}

	protected function clearBucket() {
		$this->clearCache();
		try {
			$this->getConnection()->clearBucket($this->bucket);
			return true;
			// clearBucket() is not working with Ceph, so if it fails we try the slower approach
		} catch (\Exception $e) {
			return $this->batchDelete();
		}
	}

	private function batchDelete($path = null) {
		$params = [
			'Bucket' => $this->bucket
		];
		if ($path !== null) {
			$params['Prefix'] = $path . '/';
		}
		try {
			$connection = $this->getConnection();
			// Since there are no real directories on S3, we need
			// to delete all objects prefixed with the path.
			do {
				// instead of the iterator, manually loop over the list ...
				$objects = $connection->listObjects($params);
				// ... so we can delete the files in batches
				if (isset($objects['Contents'])) {
					$connection->deleteObjects([
						'Bucket' => $this->bucket,
						'Delete' => [
							'Objects' => $objects['Contents']
						]
					]);
					$this->testTimeout();
				}
				// we reached the end when the list is no longer truncated
			} while ($objects['IsTruncated']);
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}
		return true;
	}

	public function opendir($path) {
		$path = $this->normalizePath($path);

		if ($this->isRoot($path)) {
			$path = '';
		} else {
			$path .= '/';
		}

		try {
			$files = [];
			$results = $this->getConnection()->getPaginator('ListObjects', [
				'Bucket' => $this->bucket,
				'Delimiter' => '/',
				'Prefix' => $path,
			]);

			foreach ($results as $result) {
				// sub folders
				if (is_array($result['CommonPrefixes'])) {
					foreach ($result['CommonPrefixes'] as $prefix) {
						$directoryName = trim($prefix['Prefix'], '/');
						$files[] = substr($directoryName, strlen($path));
						$this->directoryCache[$directoryName] = true;
					}
				}
				if (is_array($result['Contents'])) {
					foreach ($result['Contents'] as $object) {
						if (isset($object['Key']) && $object['Key'] === $path) {
							// it's the directory itself, skip
							continue;
						}
						$file = basename(
							isset($object['Key']) ? $object['Key'] : $object['Prefix']
						);
						$files[] = $file;

						// store this information for later usage
						$this->filesCache[$path . $file] = [
							'ContentLength' => $object['Size'],
							'LastModified' => (string)$object['LastModified'],
						];
					}
				}
			}

			return IteratorDirectory::wrap($files);
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}
	}

	public function stat($path) {
		$path = $this->normalizePath($path);

		try {
			$stat = [];
			if ($this->is_dir($path)) {
				//folders don't really exist
				$stat['size'] = -1; //unknown
				$stat['mtime'] = time();
				$cacheEntry = $this->getCache()->get($path);
				if ($cacheEntry instanceof CacheEntry && $this->getMountOption('filesystem_check_changes', 1) !== 1) {
					$stat['size'] = $cacheEntry->getSize();
					$stat['mtime'] = $cacheEntry->getMTime();
				}
			} else {
				$stat['size'] = $this->getContentLength($path);
				$stat['mtime'] = strtotime($this->getLastModified($path));
			}
			$stat['atime'] = time();

			return $stat;
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}
	}

	/**
	 * Return content length for object
	 *
	 * When the information is already present (e.g. opendir has been called before)
	 * this value is return. Otherwise a headObject is emitted.
	 *
	 * @param $path
	 * @return int|mixed
	 */
	private function getContentLength($path) {
		if (isset($this->filesCache[$path])) {
			return (int)$this->filesCache[$path]['ContentLength'];
		}

		$result = $this->headObject($path);
		if (isset($result['ContentLength'])) {
			return (int)$result['ContentLength'];
		}

		return 0;
	}

	/**
	 * Return last modified for object
	 *
	 * When the information is already present (e.g. opendir has been called before)
	 * this value is return. Otherwise a headObject is emitted.
	 *
	 * @param $path
	 * @return mixed|string
	 */
	private function getLastModified($path) {
		if (isset($this->filesCache[$path])) {
			return $this->filesCache[$path]['LastModified'];
		}

		$result = $this->headObject($path);
		if (isset($result['LastModified'])) {
			return $result['LastModified'];
		}

		return 'now';
	}

	public function is_dir($path) {
		$path = $this->normalizePath($path);

		if (isset($this->filesCache[$path])) {
			return false;
		}

		try {
			return $this->isRoot($path) || $this->doesDirectoryExist($path);
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}
	}

	public function filetype($path) {
		$path = $this->normalizePath($path);

		if ($this->isRoot($path)) {
			return 'dir';
		}

		try {
			if (isset($this->filesCache[$path]) || $this->headObject($path)) {
				return 'file';
			}
			if ($this->doesDirectoryExist($path)) {
				return 'dir';
			}
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}

		return false;
	}

	public function getPermissions($path) {
		$type = $this->filetype($path);
		if (!$type) {
			return 0;
		}
		return $type === 'dir' ? Constants::PERMISSION_ALL : Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
	}

	public function unlink($path) {
		$path = $this->normalizePath($path);

		if ($this->is_dir($path)) {
			return $this->rmdir($path);
		}

		try {
			$this->deleteObject($path);
			$this->invalidateCache($path);
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}

		return true;
	}

	public function fopen($path, $mode) {
		$path = $this->normalizePath($path);

		switch ($mode) {
			case 'r':
			case 'rb':
				// Don't try to fetch empty files
				$stat = $this->stat($path);
				if (is_array($stat) && isset($stat['size']) && $stat['size'] === 0) {
					return fopen('php://memory', $mode);
				}

				try {
					return $this->readObject($path);
				} catch (S3Exception $e) {
					\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
					return false;
				}
			case 'w':
			case 'wb':
				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile();

				$handle = fopen($tmpFile, 'w');
				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
					$this->writeBack($tmpFile, $path);
				});
			case 'a':
			case 'ab':
			case 'r+':
			case 'w+':
			case 'wb+':
			case 'a+':
			case 'x':
			case 'x+':
			case 'c':
			case 'c+':
				if (strrpos($path, '.') !== false) {
					$ext = substr($path, strrpos($path, '.'));
				} else {
					$ext = '';
				}
				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
				if ($this->file_exists($path)) {
					$source = $this->readObject($path);
					file_put_contents($tmpFile, $source);
				}

				$handle = fopen($tmpFile, $mode);
				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
					$this->writeBack($tmpFile, $path);
				});
		}
		return false;
	}

	public function touch($path, $mtime = null) {
		if (is_null($mtime)) {
			$mtime = time();
		}
		$metadata = [
			'lastmodified' => gmdate(\DateTime::RFC1123, $mtime)
		];

		try {
			if (!$this->file_exists($path)) {
				$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
				$this->getConnection()->putObject([
					'Bucket' => $this->bucket,
					'Key' => $this->cleanKey($path),
					'Metadata' => $metadata,
					'Body' => '',
					'ContentType' => $mimeType,
					'MetadataDirective' => 'REPLACE',
				]);
				$this->testTimeout();
			}
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}

		$this->invalidateCache($path);
		return true;
	}

	public function copy($path1, $path2) {
		$path1 = $this->normalizePath($path1);
		$path2 = $this->normalizePath($path2);

		if ($this->is_file($path1)) {
			try {
				$this->getConnection()->copyObject([
					'Bucket' => $this->bucket,
					'Key' => $this->cleanKey($path2),
					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
				]);
				$this->testTimeout();
			} catch (S3Exception $e) {
				\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
				return false;
			}
		} else {
			$this->remove($path2);

			try {
				$this->getConnection()->copyObject([
					'Bucket' => $this->bucket,
					'Key' => $path2 . '/',
					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
				]);
				$this->testTimeout();
			} catch (S3Exception $e) {
				\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
				return false;
			}

			$dh = $this->opendir($path1);
			if (is_resource($dh)) {
				while (($file = readdir($dh)) !== false) {
					if (\OC\Files\Filesystem::isIgnoredDir($file)) {
						continue;
					}

					$source = $path1 . '/' . $file;
					$target = $path2 . '/' . $file;
					$this->copy($source, $target);
				}
			}
		}

		$this->invalidateCache($path2);

		return true;
	}

	public function rename($path1, $path2) {
		$path1 = $this->normalizePath($path1);
		$path2 = $this->normalizePath($path2);

		if ($this->is_file($path1)) {
			if ($this->copy($path1, $path2) === false) {
				return false;
			}

			if ($this->unlink($path1) === false) {
				$this->unlink($path2);
				return false;
			}
		} else {
			if ($this->copy($path1, $path2) === false) {
				return false;
			}

			if ($this->rmdir($path1) === false) {
				$this->rmdir($path2);
				return false;
			}
		}

		return true;
	}

	public function test() {
		$this->getConnection()->headBucket([
			'Bucket' => $this->bucket
		]);
		return true;
	}

	public function getId() {
		return $this->id;
	}

	public function writeBack($tmpFile, $path) {
		try {
			$source = fopen($tmpFile, 'r');
			$this->writeObject($path, $source);
			$this->invalidateCache($path);

			unlink($tmpFile);
			return true;
		} catch (S3Exception $e) {
			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
			return false;
		}
	}

	/**
	 * check if curl is installed
	 */
	public static function checkDependencies() {
		return true;
	}
}


1			<?php
2			/**
3			* @copyright Copyright (c) 2016, ownCloud, Inc.
4			*
5			* @author André Gaul <[email protected]>
6			* @author Arthur Schiwon <[email protected]>
7			* @author Christian Berendt <[email protected]>
8			* @author Christopher T. Johnson <[email protected]>
9			* @author Christoph Wurst <[email protected]>
10			* @author Daniel Kesselberg <[email protected]>
11			* @author enoch <[email protected]>
12			* @author Johan Björk <[email protected]>
13			* @author Jörn Friedrich Dreyer <[email protected]>
14			* @author Julius Härtl <[email protected]>
15			* @author Martin Mattel <[email protected]>
16			* @author Michael Gapczynski <[email protected]>
17			* @author Morris Jobke <[email protected]>
18			* @author Philipp Kapfer <[email protected]>
19			* @author Robin Appelman <[email protected]>
20			* @author Robin McCorkell <[email protected]>
21			* @author Roeland Jago Douma <[email protected]>
22			* @author Thomas Müller <[email protected]>
23			* @author Vincent Petry <[email protected]>
24			*
25			* @license AGPL-3.0
26			*
27			* This code is free software: you can redistribute it and/or modify
28			* it under the terms of the GNU Affero General Public License, version 3,
29			* as published by the Free Software Foundation.
30			*
31			* This program is distributed in the hope that it will be useful,
32			* but WITHOUT ANY WARRANTY; without even the implied warranty of
33			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
34			* GNU Affero General Public License for more details.
35			*
36			* You should have received a copy of the GNU Affero General Public License, version 3,
37			* along with this program. If not, see <http://www.gnu.org/licenses/>
38			*
39			*/
40
41			namespace OCA\Files_External\Lib\Storage;
42
43			use Aws\Result;
44			use Aws\S3\Exception\S3Exception;
45			use Aws\S3\S3Client;
46			use Icewind\Streams\CallbackWrapper;
47			use Icewind\Streams\IteratorDirectory;
48			use OC\Cache\CappedMemoryCache;
49			use OC\Files\Cache\CacheEntry;
50			use OC\Files\ObjectStore\S3ConnectionTrait;
51			use OC\Files\ObjectStore\S3ObjectTrait;
52			use OCP\Constants;
53
54			class AmazonS3 extends \OC\Files\Storage\Common {
55			use S3ConnectionTrait;
56			use S3ObjectTrait;
57
58			public function needsPartFile() {
59			return false;
60			}
61
62			/** @var CappedMemoryCache\|Result[] */
63			private $objectCache;
64
65			/** @var CappedMemoryCache\|bool[] */
66			private $directoryCache;
67
68			/** @var CappedMemoryCache\|array */
69			private $filesCache;
70
71			public function __construct($parameters) {
72			parent::__construct($parameters);
73			$this->parseParams($parameters);
74			$this->objectCache = new CappedMemoryCache();
75			$this->directoryCache = new CappedMemoryCache();
76			$this->filesCache = new CappedMemoryCache();
77			}
78
79			/**
80			* @param string $path
81			* @return string correctly encoded path
82			*/
83			private function normalizePath($path) {
84			$path = trim($path, '/');
85
86			if (!$path) {
87			$path = '.';
88			}
89
90			return $path;
91			}
92
93			private function isRoot($path) {
94			return $path === '.';
95			}
96
97			private function cleanKey($path) {
98			if ($this->isRoot($path)) {
99			return '/';
100			}
101			return $path;
102			}
103
104			private function clearCache() {
105			$this->objectCache = new CappedMemoryCache();
106			$this->directoryCache = new CappedMemoryCache();
107			$this->filesCache = new CappedMemoryCache();
108			}
109
110			private function invalidateCache($key) {
111			unset($this->objectCache[$key]);
112			$keys = array_keys($this->objectCache->getData());
113			$keyLength = strlen($key);
114			foreach ($keys as $existingKey) {
115			if (substr($existingKey, 0, $keyLength) === $key) {
116			unset($this->objectCache[$existingKey]);
117			}
118			}
119			unset($this->directoryCache[$key], $this->filesCache[$key]);
120			}
121
122			/**
123			* @param $key
124			* @return Result\|boolean
125			*/
126			private function headObject($key) {
127			if (!isset($this->objectCache[$key])) {
128			try {
129			$this->objectCache[$key] = $this->getConnection()->headObject([
130			'Bucket' => $this->bucket,
131			'Key' => $key
132			]);
133			} catch (S3Exception $e) {
134			if ($e->getStatusCode() >= 500) {
135			throw $e;
136			}
137			$this->objectCache[$key] = false;
138			}
139			}
140
141			return $this->objectCache[$key];
142			}
143
144			/**
145			* Return true if directory exists
146			*
147			* There are no folders in s3. A folder like structure could be archived
148			* by prefixing files with the folder name.
149			*
150			* Implementation from flysystem-aws-s3-v3:
151			* https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
152			*
153			* @param $path
154			* @return bool
155			* @throws \Exception
156			*/
157			private function doesDirectoryExist($path) {
158			if (!isset($this->directoryCache[$path])) {
159			// Maybe this isn't an actual key, but a prefix.
160			// Do a prefix listing of objects to determine.
161			try {
162			$result = $this->getConnection()->listObjects([
163			'Bucket' => $this->bucket,
164			'Prefix' => rtrim($path, '/'),
165			'MaxKeys' => 1,
166			'Delimiter' => '/',
167			]);
168
169			if ((isset($result['Contents'][0]['Key']) && $result['Contents'][0]['Key'] === rtrim($path, '/') . '/')
170			\|\| isset($result['CommonPrefixes'])) {
171			$this->directoryCache[$path] = true;
172			} else {
173			$this->directoryCache[$path] = false;
174			}
175			} catch (S3Exception $e) {
176			if ($e->getStatusCode() === 403) {
177			$this->directoryCache[$path] = false;
178			}
179			throw $e;
180			}
181			}
182
183			return $this->directoryCache[$path];
184			}
185
186			/**
187			* Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
188			* TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
189			*
190			* @param array $params
191			*/
192			public function updateLegacyId(array $params) {
193			$oldId = 'amazon::' . $params['key'] . md5($params['secret']);
194
195			// find by old id or bucket
196			$stmt = \OC::$server->getDatabaseConnection()->prepare(
197			'SELECT `numeric_id`, `id` FROM `PREFIXstorages` WHERE `id` IN (?, ?)'
198			);
199			$stmt->execute([$oldId, $this->id]);
200			while ($row = $stmt->fetch()) {
201			$storages[$row['id']] = $row['numeric_id'];
202			}
203
204			if (isset($storages[$this->id]) && isset($storages[$oldId])) {
205			// if both ids exist, delete the old storage and corresponding filecache entries
206			\OC\Files\Cache\Storage::remove($oldId);
207			} elseif (isset($storages[$oldId])) {
208			// if only the old id exists do an update
209			$stmt = \OC::$server->getDatabaseConnection()->prepare(
210			'UPDATE `PREFIXstorages` SET `id` = ? WHERE `id` = ?'
211			);
212			$stmt->execute([$this->id, $oldId]);
213			}
214			// only the bucket based id may exist, do nothing
215			}
216
217			/**
218			* Remove a file or folder
219			*
220			* @param string $path
221			* @return bool
222			*/
223			protected function remove($path) {
224			// remember fileType to reduce http calls
225			$fileType = $this->filetype($path);
226			if ($fileType === 'dir') {
227			return $this->rmdir($path);
228			} elseif ($fileType === 'file') {
229			return $this->unlink($path);
230			} else {
231			return false;
232			}
233			}
234
235			public function mkdir($path) {
236			$path = $this->normalizePath($path);
237
238			if ($this->is_dir($path)) {
239			return false;
240			}
241
242			try {
243			$this->getConnection()->putObject([
244			'Bucket' => $this->bucket,
245			'Key' => $path . '/',
246			'Body' => '',
247			'ContentType' => 'httpd/unix-directory'
248			]);
249			$this->testTimeout();
250			} catch (S3Exception $e) {
251			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
252			return false;
253			}
254
255			$this->invalidateCache($path);
256
257			return true;
258			}
259
260			public function file_exists($path) {
261			return $this->filetype($path) !== false;
262			}
263
264
265			public function rmdir($path) {
266			$path = $this->normalizePath($path);
267
268			if ($this->isRoot($path)) {
269			return $this->clearBucket();
270			}
271
272			if (!$this->file_exists($path)) {
273			return false;
274			}
275
276			$this->invalidateCache($path);
277			return $this->batchDelete($path);
278			}
279
280			protected function clearBucket() {
281			$this->clearCache();
282			try {
283			$this->getConnection()->clearBucket($this->bucket);
284			return true;
285			// clearBucket() is not working with Ceph, so if it fails we try the slower approach
286			} catch (\Exception $e) {
287			return $this->batchDelete();
288			}
289			}
290
291			private function batchDelete($path = null) {
292			$params = [
293			'Bucket' => $this->bucket
294			];
295			if ($path !== null) {
296			$params['Prefix'] = $path . '/';
297			}
298			try {
299			$connection = $this->getConnection();
300			// Since there are no real directories on S3, we need
301			// to delete all objects prefixed with the path.
302			do {
303			// instead of the iterator, manually loop over the list ...
304			$objects = $connection->listObjects($params);
305			// ... so we can delete the files in batches
306			if (isset($objects['Contents'])) {
307			$connection->deleteObjects([
308			'Bucket' => $this->bucket,
309			'Delete' => [
310			'Objects' => $objects['Contents']
311			]
312			]);
313			$this->testTimeout();
314			}
315			// we reached the end when the list is no longer truncated
316			} while ($objects['IsTruncated']);
317			} catch (S3Exception $e) {
318			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
319			return false;
320			}
321			return true;
322			}
323
324			public function opendir($path) {
325			$path = $this->normalizePath($path);
326
327			if ($this->isRoot($path)) {
328			$path = '';
329			} else {
330			$path .= '/';
331			}
332
333			try {
334			$files = [];
335			$results = $this->getConnection()->getPaginator('ListObjects', [
336			'Bucket' => $this->bucket,
337			'Delimiter' => '/',
338			'Prefix' => $path,
339			]);
340
341			foreach ($results as $result) {
342			// sub folders
343			if (is_array($result['CommonPrefixes'])) {
344			foreach ($result['CommonPrefixes'] as $prefix) {
345			$directoryName = trim($prefix['Prefix'], '/');
346			$files[] = substr($directoryName, strlen($path));
347			$this->directoryCache[$directoryName] = true;
348			}
349			}
350			if (is_array($result['Contents'])) {
351			foreach ($result['Contents'] as $object) {
352			if (isset($object['Key']) && $object['Key'] === $path) {
353			// it's the directory itself, skip
354			continue;
355			}
356			$file = basename(
357			isset($object['Key']) ? $object['Key'] : $object['Prefix']
358			);
359			$files[] = $file;
360
361			// store this information for later usage
362			$this->filesCache[$path . $file] = [
363			'ContentLength' => $object['Size'],
364			'LastModified' => (string)$object['LastModified'],
365			];
366			}
367			}
368			}
369
370			return IteratorDirectory::wrap($files);
371			} catch (S3Exception $e) {
372			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
373			return false;
374			}
375			}
376
377			public function stat($path) {
378			$path = $this->normalizePath($path);
379
380			try {
381			$stat = [];
382			if ($this->is_dir($path)) {
383			//folders don't really exist
384			$stat['size'] = -1; //unknown
385			$stat['mtime'] = time();
386			$cacheEntry = $this->getCache()->get($path);
387			if ($cacheEntry instanceof CacheEntry && $this->getMountOption('filesystem_check_changes', 1) !== 1) {
388			$stat['size'] = $cacheEntry->getSize();
389			$stat['mtime'] = $cacheEntry->getMTime();
390			}
391			} else {
392			$stat['size'] = $this->getContentLength($path);
393			$stat['mtime'] = strtotime($this->getLastModified($path));
394			}
395			$stat['atime'] = time();
396
397			return $stat;
398			} catch (S3Exception $e) {
399			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
400			return false;
401			}
402			}
403
404			/**
405			* Return content length for object
406			*
407			* When the information is already present (e.g. opendir has been called before)
408			* this value is return. Otherwise a headObject is emitted.
409			*
410			* @param $path
411			* @return int\|mixed
412			*/
413			private function getContentLength($path) {
414			if (isset($this->filesCache[$path])) {
415			return (int)$this->filesCache[$path]['ContentLength'];
416			}
417
418			$result = $this->headObject($path);
419			if (isset($result['ContentLength'])) {
420			return (int)$result['ContentLength'];
421			}
422
423			return 0;
424			}
425
426			/**
427			* Return last modified for object
428			*
429			* When the information is already present (e.g. opendir has been called before)
430			* this value is return. Otherwise a headObject is emitted.
431			*
432			* @param $path
433			* @return mixed\|string
434			*/
435			private function getLastModified($path) {
436			if (isset($this->filesCache[$path])) {
437			return $this->filesCache[$path]['LastModified'];
438			}
439
440			$result = $this->headObject($path);
441			if (isset($result['LastModified'])) {
442			return $result['LastModified'];
443			}
444
445			return 'now';
446			}
447
448			public function is_dir($path) {
449			$path = $this->normalizePath($path);
450
451			if (isset($this->filesCache[$path])) {
452			return false;
453			}
454
455			try {
456			return $this->isRoot($path) \|\| $this->doesDirectoryExist($path);
457			} catch (S3Exception $e) {
458			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
459			return false;
460			}
461			}
462
463			public function filetype($path) {
464			$path = $this->normalizePath($path);
465
466			if ($this->isRoot($path)) {
467			return 'dir';
468			}
469
470			try {
471			if (isset($this->filesCache[$path]) \|\| $this->headObject($path)) {
472			return 'file';
473			}
474			if ($this->doesDirectoryExist($path)) {
475			return 'dir';
476			}
477			} catch (S3Exception $e) {
478			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
479			return false;
480			}
481
482			return false;
483			}
484
485			public function getPermissions($path) {
486			$type = $this->filetype($path);
487			if (!$type) {
488			return 0;
489			}
490			return $type === 'dir' ? Constants::PERMISSION_ALL : Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
491			}
492
493			public function unlink($path) {
494			$path = $this->normalizePath($path);
495
496			if ($this->is_dir($path)) {
497			return $this->rmdir($path);
498			}
499
500			try {
501			$this->deleteObject($path);
502			$this->invalidateCache($path);
503			} catch (S3Exception $e) {
504			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
505			return false;
506			}
507
508			return true;
509			}
510
511			public function fopen($path, $mode) {
512			$path = $this->normalizePath($path);
513
514			switch ($mode) {
515			case 'r':
516			case 'rb':
517			// Don't try to fetch empty files
518			$stat = $this->stat($path);
519			if (is_array($stat) && isset($stat['size']) && $stat['size'] === 0) {
520			return fopen('php://memory', $mode);
521			}
522
523			try {
524			return $this->readObject($path);
525			} catch (S3Exception $e) {
526			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
527			return false;
528			}
529			case 'w':
530			case 'wb':
531			$tmpFile = \OC::$server->getTempManager()->getTemporaryFile();
532
533			$handle = fopen($tmpFile, 'w');
534			return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
535			$this->writeBack($tmpFile, $path);
536			});
537			case 'a':
538			case 'ab':
539			case 'r+':
540			case 'w+':
541			case 'wb+':
542			case 'a+':
543			case 'x':
544			case 'x+':
545			case 'c':
546			case 'c+':
547			if (strrpos($path, '.') !== false) {
548			$ext = substr($path, strrpos($path, '.'));
549			} else {
550			$ext = '';
551			}
552			$tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
553			if ($this->file_exists($path)) {
554			$source = $this->readObject($path);
555			file_put_contents($tmpFile, $source);
556			}
557
558			$handle = fopen($tmpFile, $mode);
559			return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
560			$this->writeBack($tmpFile, $path);
561			});
562			}
563			return false;
564			}
565
566			public function touch($path, $mtime = null) {
567			if (is_null($mtime)) {
568			$mtime = time();
569			}
570			$metadata = [
571			'lastmodified' => gmdate(\DateTime::RFC1123, $mtime)
572			];
573
574			try {
575			if (!$this->file_exists($path)) {
576			$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
577			$this->getConnection()->putObject([
578			'Bucket' => $this->bucket,
579			'Key' => $this->cleanKey($path),
580			'Metadata' => $metadata,
581			'Body' => '',
582			'ContentType' => $mimeType,
583			'MetadataDirective' => 'REPLACE',
584			]);
585			$this->testTimeout();
586			}
587			} catch (S3Exception $e) {
588			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
589			return false;
590			}
591
592			$this->invalidateCache($path);
593			return true;
594			}
595
596			public function copy($path1, $path2) {
597			$path1 = $this->normalizePath($path1);
598			$path2 = $this->normalizePath($path2);
599
600			if ($this->is_file($path1)) {
601			try {
602			$this->getConnection()->copyObject([
603			'Bucket' => $this->bucket,
604			'Key' => $this->cleanKey($path2),
605			'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
606			]);
607			$this->testTimeout();
608			} catch (S3Exception $e) {
609			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
610			return false;
611			}
612			} else {
613			$this->remove($path2);
614
615			try {
616			$this->getConnection()->copyObject([
617			'Bucket' => $this->bucket,
618			'Key' => $path2 . '/',
619			'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
620			]);
621			$this->testTimeout();
622			} catch (S3Exception $e) {
623			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
624			return false;
625			}
626
627			$dh = $this->opendir($path1);
628			if (is_resource($dh)) {
629			while (($file = readdir($dh)) !== false) {
630			if (\OC\Files\Filesystem::isIgnoredDir($file)) {
631			continue;
632			}
633
634			$source = $path1 . '/' . $file;
635			$target = $path2 . '/' . $file;
636			$this->copy($source, $target);
637			}
638			}
639			}
640
641			$this->invalidateCache($path2);
642
643			return true;
644			}
645
646			public function rename($path1, $path2) {
647			$path1 = $this->normalizePath($path1);
648			$path2 = $this->normalizePath($path2);
649
650			if ($this->is_file($path1)) {
651			if ($this->copy($path1, $path2) === false) {
652			return false;
653			}
654
655			if ($this->unlink($path1) === false) {
656			$this->unlink($path2);
657			return false;
658			}
659			} else {
660			if ($this->copy($path1, $path2) === false) {
661			return false;
662			}
663
664			if ($this->rmdir($path1) === false) {
665			$this->rmdir($path2);
666			return false;
667			}
668			}
669
670			return true;
671			}
672
673			public function test() {
674			$this->getConnection()->headBucket([
675			'Bucket' => $this->bucket
676			]);
677			return true;
678			}
679
680			public function getId() {
681			return $this->id;
682			}
683
684			public function writeBack($tmpFile, $path) {
685			try {
686			$source = fopen($tmpFile, 'r');
687			$this->writeObject($path, $source);
688			$this->invalidateCache($path);
689
690			unlink($tmpFile);
691			return true;
692			} catch (S3Exception $e) {
693			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
694			return false;
695			}
696			}
697
698			/**
699			* check if curl is installed
700			*/
701			public static function checkDependencies() {
702			return true;
703			}
704			}
705

nextcloud / server

Push — master ( d38a7c...25f347 )

AmazonS3::needsPartFile() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like