Passed
Push — master ( 03603d...ac7398 )
by Morris
12:40 queued 10s
created
apps/files_external/lib/Lib/Storage/AmazonS3.php 2 patches
Indentation   +636 added lines, -636 removed lines patch added patch discarded remove patch
@@ -51,640 +51,640 @@
 block discarded – undo
51 51
 use OCP\Constants;
52 52
 
53 53
 class AmazonS3 extends \OC\Files\Storage\Common {
54
-	use S3ConnectionTrait;
55
-	use S3ObjectTrait;
56
-
57
-	public function needsPartFile() {
58
-		return false;
59
-	}
60
-
61
-	/** @var CappedMemoryCache|Result[] */
62
-	private $objectCache;
63
-
64
-	/** @var CappedMemoryCache|bool[] */
65
-	private $directoryCache;
66
-
67
-	/** @var CappedMemoryCache|array */
68
-	private $filesCache;
69
-
70
-	public function __construct($parameters) {
71
-		parent::__construct($parameters);
72
-		$this->parseParams($parameters);
73
-		$this->objectCache = new CappedMemoryCache();
74
-		$this->directoryCache = new CappedMemoryCache();
75
-		$this->filesCache = new CappedMemoryCache();
76
-	}
77
-
78
-	/**
79
-	 * @param string $path
80
-	 * @return string correctly encoded path
81
-	 */
82
-	private function normalizePath($path) {
83
-		$path = trim($path, '/');
84
-
85
-		if (!$path) {
86
-			$path = '.';
87
-		}
88
-
89
-		return $path;
90
-	}
91
-
92
-	private function isRoot($path) {
93
-		return $path === '.';
94
-	}
95
-
96
-	private function cleanKey($path) {
97
-		if ($this->isRoot($path)) {
98
-			return '/';
99
-		}
100
-		return $path;
101
-	}
102
-
103
-	private function clearCache() {
104
-		$this->objectCache = new CappedMemoryCache();
105
-		$this->directoryCache = new CappedMemoryCache();
106
-		$this->filesCache = new CappedMemoryCache();
107
-	}
108
-
109
-	private function invalidateCache($key) {
110
-		unset($this->objectCache[$key]);
111
-		$keys = array_keys($this->objectCache->getData());
112
-		$keyLength = strlen($key);
113
-		foreach ($keys as $existingKey) {
114
-			if (substr($existingKey, 0, $keyLength) === $key) {
115
-				unset($this->objectCache[$existingKey]);
116
-			}
117
-		}
118
-		unset($this->directoryCache[$key], $this->filesCache[$key]);
119
-	}
120
-
121
-	/**
122
-	 * @param $key
123
-	 * @return Result|boolean
124
-	 */
125
-	private function headObject($key) {
126
-		if (!isset($this->objectCache[$key])) {
127
-			try {
128
-				$this->objectCache[$key] = $this->getConnection()->headObject([
129
-					'Bucket' => $this->bucket,
130
-					'Key' => $key
131
-				]);
132
-			} catch (S3Exception $e) {
133
-				if ($e->getStatusCode() >= 500) {
134
-					throw $e;
135
-				}
136
-				$this->objectCache[$key] = false;
137
-			}
138
-		}
139
-
140
-		return $this->objectCache[$key];
141
-	}
142
-
143
-	/**
144
-	 * Return true if directory exists
145
-	 *
146
-	 * There are no folders in s3. A folder like structure could be archived
147
-	 * by prefixing files with the folder name.
148
-	 *
149
-	 * Implementation from flysystem-aws-s3-v3:
150
-	 * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
151
-	 *
152
-	 * @param $path
153
-	 * @return bool
154
-	 * @throws \Exception
155
-	 */
156
-	private function doesDirectoryExist($path) {
157
-		if (!isset($this->directoryCache[$path])) {
158
-			// Maybe this isn't an actual key, but a prefix.
159
-			// Do a prefix listing of objects to determine.
160
-			try {
161
-				$result = $this->getConnection()->listObjects([
162
-					'Bucket' => $this->bucket,
163
-					'Prefix' => rtrim($path, '/') . '/',
164
-					'MaxKeys' => 1,
165
-				]);
166
-				$this->directoryCache[$path] = $result['Contents'] || $result['CommonPrefixes'];
167
-			} catch (S3Exception $e) {
168
-				if ($e->getStatusCode() === 403) {
169
-					$this->directoryCache[$path] = false;
170
-				}
171
-				throw $e;
172
-			}
173
-		}
174
-
175
-		return $this->directoryCache[$path];
176
-	}
177
-
178
-	/**
179
-	 * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
180
-	 * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
181
-	 *
182
-	 * @param array $params
183
-	 */
184
-	public function updateLegacyId(array $params) {
185
-		$oldId = 'amazon::' . $params['key'] . md5($params['secret']);
186
-
187
-		// find by old id or bucket
188
-		$stmt = \OC::$server->getDatabaseConnection()->prepare(
189
-			'SELECT `numeric_id`, `id` FROM `*PREFIX*storages` WHERE `id` IN (?, ?)'
190
-		);
191
-		$stmt->execute([$oldId, $this->id]);
192
-		while ($row = $stmt->fetch()) {
193
-			$storages[$row['id']] = $row['numeric_id'];
194
-		}
195
-
196
-		if (isset($storages[$this->id]) && isset($storages[$oldId])) {
197
-			// if both ids exist, delete the old storage and corresponding filecache entries
198
-			\OC\Files\Cache\Storage::remove($oldId);
199
-		} elseif (isset($storages[$oldId])) {
200
-			// if only the old id exists do an update
201
-			$stmt = \OC::$server->getDatabaseConnection()->prepare(
202
-				'UPDATE `*PREFIX*storages` SET `id` = ? WHERE `id` = ?'
203
-			);
204
-			$stmt->execute([$this->id, $oldId]);
205
-		}
206
-		// only the bucket based id may exist, do nothing
207
-	}
208
-
209
-	/**
210
-	 * Remove a file or folder
211
-	 *
212
-	 * @param string $path
213
-	 * @return bool
214
-	 */
215
-	protected function remove($path) {
216
-		// remember fileType to reduce http calls
217
-		$fileType = $this->filetype($path);
218
-		if ($fileType === 'dir') {
219
-			return $this->rmdir($path);
220
-		} elseif ($fileType === 'file') {
221
-			return $this->unlink($path);
222
-		} else {
223
-			return false;
224
-		}
225
-	}
226
-
227
-	public function mkdir($path) {
228
-		$path = $this->normalizePath($path);
229
-
230
-		if ($this->is_dir($path)) {
231
-			return false;
232
-		}
233
-
234
-		try {
235
-			$this->getConnection()->putObject([
236
-				'Bucket' => $this->bucket,
237
-				'Key' => $path . '/',
238
-				'Body' => '',
239
-				'ContentType' => 'httpd/unix-directory'
240
-			]);
241
-			$this->testTimeout();
242
-		} catch (S3Exception $e) {
243
-			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
244
-			return false;
245
-		}
246
-
247
-		$this->invalidateCache($path);
248
-
249
-		return true;
250
-	}
251
-
252
-	public function file_exists($path) {
253
-		return $this->filetype($path) !== false;
254
-	}
255
-
256
-
257
-	public function rmdir($path) {
258
-		$path = $this->normalizePath($path);
259
-
260
-		if ($this->isRoot($path)) {
261
-			return $this->clearBucket();
262
-		}
263
-
264
-		if (!$this->file_exists($path)) {
265
-			return false;
266
-		}
267
-
268
-		$this->invalidateCache($path);
269
-		return $this->batchDelete($path);
270
-	}
271
-
272
-	protected function clearBucket() {
273
-		$this->clearCache();
274
-		try {
275
-			$this->getConnection()->clearBucket($this->bucket);
276
-			return true;
277
-			// clearBucket() is not working with Ceph, so if it fails we try the slower approach
278
-		} catch (\Exception $e) {
279
-			return $this->batchDelete();
280
-		}
281
-	}
282
-
283
-	private function batchDelete($path = null) {
284
-		$params = [
285
-			'Bucket' => $this->bucket
286
-		];
287
-		if ($path !== null) {
288
-			$params['Prefix'] = $path . '/';
289
-		}
290
-		try {
291
-			$connection = $this->getConnection();
292
-			// Since there are no real directories on S3, we need
293
-			// to delete all objects prefixed with the path.
294
-			do {
295
-				// instead of the iterator, manually loop over the list ...
296
-				$objects = $connection->listObjects($params);
297
-				// ... so we can delete the files in batches
298
-				if (isset($objects['Contents'])) {
299
-					$connection->deleteObjects([
300
-						'Bucket' => $this->bucket,
301
-						'Delete' => [
302
-							'Objects' => $objects['Contents']
303
-						]
304
-					]);
305
-					$this->testTimeout();
306
-				}
307
-				// we reached the end when the list is no longer truncated
308
-			} while ($objects['IsTruncated']);
309
-		} catch (S3Exception $e) {
310
-			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
311
-			return false;
312
-		}
313
-		return true;
314
-	}
315
-
316
-	public function opendir($path) {
317
-		$path = $this->normalizePath($path);
318
-
319
-		if ($this->isRoot($path)) {
320
-			$path = '';
321
-		} else {
322
-			$path .= '/';
323
-		}
324
-
325
-		try {
326
-			$files = [];
327
-			$results = $this->getConnection()->getPaginator('ListObjects', [
328
-				'Bucket' => $this->bucket,
329
-				'Delimiter' => '/',
330
-				'Prefix' => $path,
331
-			]);
332
-
333
-			foreach ($results as $result) {
334
-				// sub folders
335
-				if (is_array($result['CommonPrefixes'])) {
336
-					foreach ($result['CommonPrefixes'] as $prefix) {
337
-						$directoryName = trim($prefix['Prefix'], '/');
338
-						$files[] = substr($directoryName, strlen($path));
339
-						$this->directoryCache[$directoryName] = true;
340
-					}
341
-				}
342
-				if (is_array($result['Contents'])) {
343
-					foreach ($result['Contents'] as $object) {
344
-						if (isset($object['Key']) && $object['Key'] === $path) {
345
-							// it's the directory itself, skip
346
-							continue;
347
-						}
348
-						$file = basename(
349
-							isset($object['Key']) ? $object['Key'] : $object['Prefix']
350
-						);
351
-						$files[] = $file;
352
-
353
-						// store this information for later usage
354
-						$this->filesCache[$path . $file] = [
355
-							'ContentLength' => $object['Size'],
356
-							'LastModified' => (string)$object['LastModified'],
357
-						];
358
-					}
359
-				}
360
-			}
361
-
362
-			return IteratorDirectory::wrap($files);
363
-		} catch (S3Exception $e) {
364
-			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
365
-			return false;
366
-		}
367
-	}
368
-
369
-	public function stat($path) {
370
-		$path = $this->normalizePath($path);
371
-
372
-		try {
373
-			$stat = [];
374
-			if ($this->is_dir($path)) {
375
-				//folders don't really exist
376
-				$stat['size'] = -1; //unknown
377
-				$stat['mtime'] = time();
378
-				$cacheEntry = $this->getCache()->get($path);
379
-				if ($cacheEntry instanceof CacheEntry && $this->getMountOption('filesystem_check_changes', 1) !== 1) {
380
-					$stat['size'] = $cacheEntry->getSize();
381
-					$stat['mtime'] = $cacheEntry->getMTime();
382
-				}
383
-			} else {
384
-				$stat['size'] = $this->getContentLength($path);
385
-				$stat['mtime'] = strtotime($this->getLastModified($path));
386
-			}
387
-			$stat['atime'] = time();
388
-
389
-			return $stat;
390
-		} catch (S3Exception $e) {
391
-			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
392
-			return false;
393
-		}
394
-	}
395
-
396
-	/**
397
-	 * Return content length for object
398
-	 *
399
-	 * When the information is already present (e.g. opendir has been called before)
400
-	 * this value is return. Otherwise a headObject is emitted.
401
-	 *
402
-	 * @param $path
403
-	 * @return int|mixed
404
-	 */
405
-	private function getContentLength($path) {
406
-		if (isset($this->filesCache[$path])) {
407
-			return $this->filesCache[$path]['ContentLength'];
408
-		}
409
-
410
-		$result = $this->headObject($path);
411
-		if (isset($result['ContentLength'])) {
412
-			return $result['ContentLength'];
413
-		}
414
-
415
-		return 0;
416
-	}
417
-
418
-	/**
419
-	 * Return last modified for object
420
-	 *
421
-	 * When the information is already present (e.g. opendir has been called before)
422
-	 * this value is return. Otherwise a headObject is emitted.
423
-	 *
424
-	 * @param $path
425
-	 * @return mixed|string
426
-	 */
427
-	private function getLastModified($path) {
428
-		if (isset($this->filesCache[$path])) {
429
-			return $this->filesCache[$path]['LastModified'];
430
-		}
431
-
432
-		$result = $this->headObject($path);
433
-		if (isset($result['LastModified'])) {
434
-			return $result['LastModified'];
435
-		}
436
-
437
-		return 'now';
438
-	}
439
-
440
-	public function is_dir($path) {
441
-		$path = $this->normalizePath($path);
442
-
443
-		if (isset($this->filesCache[$path])) {
444
-			return false;
445
-		}
446
-
447
-		try {
448
-			return $this->isRoot($path) || $this->doesDirectoryExist($path);
449
-		} catch (S3Exception $e) {
450
-			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
451
-			return false;
452
-		}
453
-	}
454
-
455
-	public function filetype($path) {
456
-		$path = $this->normalizePath($path);
457
-
458
-		if ($this->isRoot($path)) {
459
-			return 'dir';
460
-		}
461
-
462
-		try {
463
-			if (isset($this->filesCache[$path]) || $this->headObject($path)) {
464
-				return 'file';
465
-			}
466
-			if ($this->doesDirectoryExist($path)) {
467
-				return 'dir';
468
-			}
469
-		} catch (S3Exception $e) {
470
-			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
471
-			return false;
472
-		}
473
-
474
-		return false;
475
-	}
476
-
477
-	public function getPermissions($path) {
478
-		$type = $this->filetype($path);
479
-		if (!$type) {
480
-			return 0;
481
-		}
482
-		return $type === 'dir' ? Constants::PERMISSION_ALL : Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
483
-	}
484
-
485
-	public function unlink($path) {
486
-		$path = $this->normalizePath($path);
487
-
488
-		if ($this->is_dir($path)) {
489
-			return $this->rmdir($path);
490
-		}
491
-
492
-		try {
493
-			$this->deleteObject($path);
494
-			$this->invalidateCache($path);
495
-		} catch (S3Exception $e) {
496
-			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
497
-			return false;
498
-		}
499
-
500
-		return true;
501
-	}
502
-
503
-	public function fopen($path, $mode) {
504
-		$path = $this->normalizePath($path);
505
-
506
-		switch ($mode) {
507
-			case 'r':
508
-			case 'rb':
509
-				try {
510
-					return $this->readObject($path);
511
-				} catch (S3Exception $e) {
512
-					\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
513
-					return false;
514
-				}
515
-			case 'w':
516
-			case 'wb':
517
-				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile();
518
-
519
-				$handle = fopen($tmpFile, 'w');
520
-				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
521
-					$this->writeBack($tmpFile, $path);
522
-				});
523
-			case 'a':
524
-			case 'ab':
525
-			case 'r+':
526
-			case 'w+':
527
-			case 'wb+':
528
-			case 'a+':
529
-			case 'x':
530
-			case 'x+':
531
-			case 'c':
532
-			case 'c+':
533
-				if (strrpos($path, '.') !== false) {
534
-					$ext = substr($path, strrpos($path, '.'));
535
-				} else {
536
-					$ext = '';
537
-				}
538
-				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
539
-				if ($this->file_exists($path)) {
540
-					$source = $this->readObject($path);
541
-					file_put_contents($tmpFile, $source);
542
-				}
543
-
544
-				$handle = fopen($tmpFile, $mode);
545
-				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
546
-					$this->writeBack($tmpFile, $path);
547
-				});
548
-		}
549
-		return false;
550
-	}
551
-
552
-	public function touch($path, $mtime = null) {
553
-		if (is_null($mtime)) {
554
-			$mtime = time();
555
-		}
556
-		$metadata = [
557
-			'lastmodified' => gmdate(\DateTime::RFC1123, $mtime)
558
-		];
559
-
560
-		try {
561
-			if (!$this->file_exists($path)) {
562
-				$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
563
-				$this->getConnection()->putObject([
564
-					'Bucket' => $this->bucket,
565
-					'Key' => $this->cleanKey($path),
566
-					'Metadata' => $metadata,
567
-					'Body' => '',
568
-					'ContentType' => $mimeType,
569
-					'MetadataDirective' => 'REPLACE',
570
-				]);
571
-				$this->testTimeout();
572
-			}
573
-		} catch (S3Exception $e) {
574
-			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
575
-			return false;
576
-		}
577
-
578
-		$this->invalidateCache($path);
579
-		return true;
580
-	}
581
-
582
-	public function copy($path1, $path2) {
583
-		$path1 = $this->normalizePath($path1);
584
-		$path2 = $this->normalizePath($path2);
585
-
586
-		if ($this->is_file($path1)) {
587
-			try {
588
-				$this->getConnection()->copyObject([
589
-					'Bucket' => $this->bucket,
590
-					'Key' => $this->cleanKey($path2),
591
-					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
592
-				]);
593
-				$this->testTimeout();
594
-			} catch (S3Exception $e) {
595
-				\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
596
-				return false;
597
-			}
598
-		} else {
599
-			$this->remove($path2);
600
-
601
-			try {
602
-				$this->getConnection()->copyObject([
603
-					'Bucket' => $this->bucket,
604
-					'Key' => $path2 . '/',
605
-					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
606
-				]);
607
-				$this->testTimeout();
608
-			} catch (S3Exception $e) {
609
-				\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
610
-				return false;
611
-			}
612
-
613
-			$dh = $this->opendir($path1);
614
-			if (is_resource($dh)) {
615
-				while (($file = readdir($dh)) !== false) {
616
-					if (\OC\Files\Filesystem::isIgnoredDir($file)) {
617
-						continue;
618
-					}
619
-
620
-					$source = $path1 . '/' . $file;
621
-					$target = $path2 . '/' . $file;
622
-					$this->copy($source, $target);
623
-				}
624
-			}
625
-		}
626
-
627
-		$this->invalidateCache($path2);
628
-
629
-		return true;
630
-	}
631
-
632
-	public function rename($path1, $path2) {
633
-		$path1 = $this->normalizePath($path1);
634
-		$path2 = $this->normalizePath($path2);
635
-
636
-		if ($this->is_file($path1)) {
637
-			if ($this->copy($path1, $path2) === false) {
638
-				return false;
639
-			}
640
-
641
-			if ($this->unlink($path1) === false) {
642
-				$this->unlink($path2);
643
-				return false;
644
-			}
645
-		} else {
646
-			if ($this->copy($path1, $path2) === false) {
647
-				return false;
648
-			}
649
-
650
-			if ($this->rmdir($path1) === false) {
651
-				$this->rmdir($path2);
652
-				return false;
653
-			}
654
-		}
655
-
656
-		return true;
657
-	}
658
-
659
-	public function test() {
660
-		$this->getConnection()->headBucket([
661
-			'Bucket' => $this->bucket
662
-		]);
663
-		return true;
664
-	}
665
-
666
-	public function getId() {
667
-		return $this->id;
668
-	}
669
-
670
-	public function writeBack($tmpFile, $path) {
671
-		try {
672
-			$source = fopen($tmpFile, 'r');
673
-			$this->writeObject($path, $source);
674
-			$this->invalidateCache($path);
675
-
676
-			unlink($tmpFile);
677
-			return true;
678
-		} catch (S3Exception $e) {
679
-			\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
680
-			return false;
681
-		}
682
-	}
683
-
684
-	/**
685
-	 * check if curl is installed
686
-	 */
687
-	public static function checkDependencies() {
688
-		return true;
689
-	}
54
+    use S3ConnectionTrait;
55
+    use S3ObjectTrait;
56
+
57
+    public function needsPartFile() {
58
+        return false;
59
+    }
60
+
61
+    /** @var CappedMemoryCache|Result[] */
62
+    private $objectCache;
63
+
64
+    /** @var CappedMemoryCache|bool[] */
65
+    private $directoryCache;
66
+
67
+    /** @var CappedMemoryCache|array */
68
+    private $filesCache;
69
+
70
+    public function __construct($parameters) {
71
+        parent::__construct($parameters);
72
+        $this->parseParams($parameters);
73
+        $this->objectCache = new CappedMemoryCache();
74
+        $this->directoryCache = new CappedMemoryCache();
75
+        $this->filesCache = new CappedMemoryCache();
76
+    }
77
+
78
+    /**
79
+     * @param string $path
80
+     * @return string correctly encoded path
81
+     */
82
+    private function normalizePath($path) {
83
+        $path = trim($path, '/');
84
+
85
+        if (!$path) {
86
+            $path = '.';
87
+        }
88
+
89
+        return $path;
90
+    }
91
+
92
+    private function isRoot($path) {
93
+        return $path === '.';
94
+    }
95
+
96
+    private function cleanKey($path) {
97
+        if ($this->isRoot($path)) {
98
+            return '/';
99
+        }
100
+        return $path;
101
+    }
102
+
103
+    private function clearCache() {
104
+        $this->objectCache = new CappedMemoryCache();
105
+        $this->directoryCache = new CappedMemoryCache();
106
+        $this->filesCache = new CappedMemoryCache();
107
+    }
108
+
109
+    private function invalidateCache($key) {
110
+        unset($this->objectCache[$key]);
111
+        $keys = array_keys($this->objectCache->getData());
112
+        $keyLength = strlen($key);
113
+        foreach ($keys as $existingKey) {
114
+            if (substr($existingKey, 0, $keyLength) === $key) {
115
+                unset($this->objectCache[$existingKey]);
116
+            }
117
+        }
118
+        unset($this->directoryCache[$key], $this->filesCache[$key]);
119
+    }
120
+
121
+    /**
122
+     * @param $key
123
+     * @return Result|boolean
124
+     */
125
+    private function headObject($key) {
126
+        if (!isset($this->objectCache[$key])) {
127
+            try {
128
+                $this->objectCache[$key] = $this->getConnection()->headObject([
129
+                    'Bucket' => $this->bucket,
130
+                    'Key' => $key
131
+                ]);
132
+            } catch (S3Exception $e) {
133
+                if ($e->getStatusCode() >= 500) {
134
+                    throw $e;
135
+                }
136
+                $this->objectCache[$key] = false;
137
+            }
138
+        }
139
+
140
+        return $this->objectCache[$key];
141
+    }
142
+
143
+    /**
144
+     * Return true if directory exists
145
+     *
146
+     * There are no folders in s3. A folder like structure could be archived
147
+     * by prefixing files with the folder name.
148
+     *
149
+     * Implementation from flysystem-aws-s3-v3:
150
+     * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
151
+     *
152
+     * @param $path
153
+     * @return bool
154
+     * @throws \Exception
155
+     */
156
+    private function doesDirectoryExist($path) {
157
+        if (!isset($this->directoryCache[$path])) {
158
+            // Maybe this isn't an actual key, but a prefix.
159
+            // Do a prefix listing of objects to determine.
160
+            try {
161
+                $result = $this->getConnection()->listObjects([
162
+                    'Bucket' => $this->bucket,
163
+                    'Prefix' => rtrim($path, '/') . '/',
164
+                    'MaxKeys' => 1,
165
+                ]);
166
+                $this->directoryCache[$path] = $result['Contents'] || $result['CommonPrefixes'];
167
+            } catch (S3Exception $e) {
168
+                if ($e->getStatusCode() === 403) {
169
+                    $this->directoryCache[$path] = false;
170
+                }
171
+                throw $e;
172
+            }
173
+        }
174
+
175
+        return $this->directoryCache[$path];
176
+    }
177
+
178
+    /**
179
+     * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
180
+     * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
181
+     *
182
+     * @param array $params
183
+     */
184
+    public function updateLegacyId(array $params) {
185
+        $oldId = 'amazon::' . $params['key'] . md5($params['secret']);
186
+
187
+        // find by old id or bucket
188
+        $stmt = \OC::$server->getDatabaseConnection()->prepare(
189
+            'SELECT `numeric_id`, `id` FROM `*PREFIX*storages` WHERE `id` IN (?, ?)'
190
+        );
191
+        $stmt->execute([$oldId, $this->id]);
192
+        while ($row = $stmt->fetch()) {
193
+            $storages[$row['id']] = $row['numeric_id'];
194
+        }
195
+
196
+        if (isset($storages[$this->id]) && isset($storages[$oldId])) {
197
+            // if both ids exist, delete the old storage and corresponding filecache entries
198
+            \OC\Files\Cache\Storage::remove($oldId);
199
+        } elseif (isset($storages[$oldId])) {
200
+            // if only the old id exists do an update
201
+            $stmt = \OC::$server->getDatabaseConnection()->prepare(
202
+                'UPDATE `*PREFIX*storages` SET `id` = ? WHERE `id` = ?'
203
+            );
204
+            $stmt->execute([$this->id, $oldId]);
205
+        }
206
+        // only the bucket based id may exist, do nothing
207
+    }
208
+
209
+    /**
210
+     * Remove a file or folder
211
+     *
212
+     * @param string $path
213
+     * @return bool
214
+     */
215
+    protected function remove($path) {
216
+        // remember fileType to reduce http calls
217
+        $fileType = $this->filetype($path);
218
+        if ($fileType === 'dir') {
219
+            return $this->rmdir($path);
220
+        } elseif ($fileType === 'file') {
221
+            return $this->unlink($path);
222
+        } else {
223
+            return false;
224
+        }
225
+    }
226
+
227
+    public function mkdir($path) {
228
+        $path = $this->normalizePath($path);
229
+
230
+        if ($this->is_dir($path)) {
231
+            return false;
232
+        }
233
+
234
+        try {
235
+            $this->getConnection()->putObject([
236
+                'Bucket' => $this->bucket,
237
+                'Key' => $path . '/',
238
+                'Body' => '',
239
+                'ContentType' => 'httpd/unix-directory'
240
+            ]);
241
+            $this->testTimeout();
242
+        } catch (S3Exception $e) {
243
+            \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
244
+            return false;
245
+        }
246
+
247
+        $this->invalidateCache($path);
248
+
249
+        return true;
250
+    }
251
+
252
+    public function file_exists($path) {
253
+        return $this->filetype($path) !== false;
254
+    }
255
+
256
+
257
+    public function rmdir($path) {
258
+        $path = $this->normalizePath($path);
259
+
260
+        if ($this->isRoot($path)) {
261
+            return $this->clearBucket();
262
+        }
263
+
264
+        if (!$this->file_exists($path)) {
265
+            return false;
266
+        }
267
+
268
+        $this->invalidateCache($path);
269
+        return $this->batchDelete($path);
270
+    }
271
+
272
+    protected function clearBucket() {
273
+        $this->clearCache();
274
+        try {
275
+            $this->getConnection()->clearBucket($this->bucket);
276
+            return true;
277
+            // clearBucket() is not working with Ceph, so if it fails we try the slower approach
278
+        } catch (\Exception $e) {
279
+            return $this->batchDelete();
280
+        }
281
+    }
282
+
283
+    private function batchDelete($path = null) {
284
+        $params = [
285
+            'Bucket' => $this->bucket
286
+        ];
287
+        if ($path !== null) {
288
+            $params['Prefix'] = $path . '/';
289
+        }
290
+        try {
291
+            $connection = $this->getConnection();
292
+            // Since there are no real directories on S3, we need
293
+            // to delete all objects prefixed with the path.
294
+            do {
295
+                // instead of the iterator, manually loop over the list ...
296
+                $objects = $connection->listObjects($params);
297
+                // ... so we can delete the files in batches
298
+                if (isset($objects['Contents'])) {
299
+                    $connection->deleteObjects([
300
+                        'Bucket' => $this->bucket,
301
+                        'Delete' => [
302
+                            'Objects' => $objects['Contents']
303
+                        ]
304
+                    ]);
305
+                    $this->testTimeout();
306
+                }
307
+                // we reached the end when the list is no longer truncated
308
+            } while ($objects['IsTruncated']);
309
+        } catch (S3Exception $e) {
310
+            \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
311
+            return false;
312
+        }
313
+        return true;
314
+    }
315
+
316
+    public function opendir($path) {
317
+        $path = $this->normalizePath($path);
318
+
319
+        if ($this->isRoot($path)) {
320
+            $path = '';
321
+        } else {
322
+            $path .= '/';
323
+        }
324
+
325
+        try {
326
+            $files = [];
327
+            $results = $this->getConnection()->getPaginator('ListObjects', [
328
+                'Bucket' => $this->bucket,
329
+                'Delimiter' => '/',
330
+                'Prefix' => $path,
331
+            ]);
332
+
333
+            foreach ($results as $result) {
334
+                // sub folders
335
+                if (is_array($result['CommonPrefixes'])) {
336
+                    foreach ($result['CommonPrefixes'] as $prefix) {
337
+                        $directoryName = trim($prefix['Prefix'], '/');
338
+                        $files[] = substr($directoryName, strlen($path));
339
+                        $this->directoryCache[$directoryName] = true;
340
+                    }
341
+                }
342
+                if (is_array($result['Contents'])) {
343
+                    foreach ($result['Contents'] as $object) {
344
+                        if (isset($object['Key']) && $object['Key'] === $path) {
345
+                            // it's the directory itself, skip
346
+                            continue;
347
+                        }
348
+                        $file = basename(
349
+                            isset($object['Key']) ? $object['Key'] : $object['Prefix']
350
+                        );
351
+                        $files[] = $file;
352
+
353
+                        // store this information for later usage
354
+                        $this->filesCache[$path . $file] = [
355
+                            'ContentLength' => $object['Size'],
356
+                            'LastModified' => (string)$object['LastModified'],
357
+                        ];
358
+                    }
359
+                }
360
+            }
361
+
362
+            return IteratorDirectory::wrap($files);
363
+        } catch (S3Exception $e) {
364
+            \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
365
+            return false;
366
+        }
367
+    }
368
+
369
+    public function stat($path) {
370
+        $path = $this->normalizePath($path);
371
+
372
+        try {
373
+            $stat = [];
374
+            if ($this->is_dir($path)) {
375
+                //folders don't really exist
376
+                $stat['size'] = -1; //unknown
377
+                $stat['mtime'] = time();
378
+                $cacheEntry = $this->getCache()->get($path);
379
+                if ($cacheEntry instanceof CacheEntry && $this->getMountOption('filesystem_check_changes', 1) !== 1) {
380
+                    $stat['size'] = $cacheEntry->getSize();
381
+                    $stat['mtime'] = $cacheEntry->getMTime();
382
+                }
383
+            } else {
384
+                $stat['size'] = $this->getContentLength($path);
385
+                $stat['mtime'] = strtotime($this->getLastModified($path));
386
+            }
387
+            $stat['atime'] = time();
388
+
389
+            return $stat;
390
+        } catch (S3Exception $e) {
391
+            \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
392
+            return false;
393
+        }
394
+    }
395
+
396
+    /**
397
+     * Return content length for object
398
+     *
399
+     * When the information is already present (e.g. opendir has been called before)
400
+     * this value is return. Otherwise a headObject is emitted.
401
+     *
402
+     * @param $path
403
+     * @return int|mixed
404
+     */
405
+    private function getContentLength($path) {
406
+        if (isset($this->filesCache[$path])) {
407
+            return $this->filesCache[$path]['ContentLength'];
408
+        }
409
+
410
+        $result = $this->headObject($path);
411
+        if (isset($result['ContentLength'])) {
412
+            return $result['ContentLength'];
413
+        }
414
+
415
+        return 0;
416
+    }
417
+
418
+    /**
419
+     * Return last modified for object
420
+     *
421
+     * When the information is already present (e.g. opendir has been called before)
422
+     * this value is return. Otherwise a headObject is emitted.
423
+     *
424
+     * @param $path
425
+     * @return mixed|string
426
+     */
427
+    private function getLastModified($path) {
428
+        if (isset($this->filesCache[$path])) {
429
+            return $this->filesCache[$path]['LastModified'];
430
+        }
431
+
432
+        $result = $this->headObject($path);
433
+        if (isset($result['LastModified'])) {
434
+            return $result['LastModified'];
435
+        }
436
+
437
+        return 'now';
438
+    }
439
+
440
+    public function is_dir($path) {
441
+        $path = $this->normalizePath($path);
442
+
443
+        if (isset($this->filesCache[$path])) {
444
+            return false;
445
+        }
446
+
447
+        try {
448
+            return $this->isRoot($path) || $this->doesDirectoryExist($path);
449
+        } catch (S3Exception $e) {
450
+            \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
451
+            return false;
452
+        }
453
+    }
454
+
455
+    public function filetype($path) {
456
+        $path = $this->normalizePath($path);
457
+
458
+        if ($this->isRoot($path)) {
459
+            return 'dir';
460
+        }
461
+
462
+        try {
463
+            if (isset($this->filesCache[$path]) || $this->headObject($path)) {
464
+                return 'file';
465
+            }
466
+            if ($this->doesDirectoryExist($path)) {
467
+                return 'dir';
468
+            }
469
+        } catch (S3Exception $e) {
470
+            \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
471
+            return false;
472
+        }
473
+
474
+        return false;
475
+    }
476
+
477
+    public function getPermissions($path) {
478
+        $type = $this->filetype($path);
479
+        if (!$type) {
480
+            return 0;
481
+        }
482
+        return $type === 'dir' ? Constants::PERMISSION_ALL : Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
483
+    }
484
+
485
+    public function unlink($path) {
486
+        $path = $this->normalizePath($path);
487
+
488
+        if ($this->is_dir($path)) {
489
+            return $this->rmdir($path);
490
+        }
491
+
492
+        try {
493
+            $this->deleteObject($path);
494
+            $this->invalidateCache($path);
495
+        } catch (S3Exception $e) {
496
+            \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
497
+            return false;
498
+        }
499
+
500
+        return true;
501
+    }
502
+
503
+    public function fopen($path, $mode) {
504
+        $path = $this->normalizePath($path);
505
+
506
+        switch ($mode) {
507
+            case 'r':
508
+            case 'rb':
509
+                try {
510
+                    return $this->readObject($path);
511
+                } catch (S3Exception $e) {
512
+                    \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
513
+                    return false;
514
+                }
515
+            case 'w':
516
+            case 'wb':
517
+                $tmpFile = \OC::$server->getTempManager()->getTemporaryFile();
518
+
519
+                $handle = fopen($tmpFile, 'w');
520
+                return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
521
+                    $this->writeBack($tmpFile, $path);
522
+                });
523
+            case 'a':
524
+            case 'ab':
525
+            case 'r+':
526
+            case 'w+':
527
+            case 'wb+':
528
+            case 'a+':
529
+            case 'x':
530
+            case 'x+':
531
+            case 'c':
532
+            case 'c+':
533
+                if (strrpos($path, '.') !== false) {
534
+                    $ext = substr($path, strrpos($path, '.'));
535
+                } else {
536
+                    $ext = '';
537
+                }
538
+                $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
539
+                if ($this->file_exists($path)) {
540
+                    $source = $this->readObject($path);
541
+                    file_put_contents($tmpFile, $source);
542
+                }
543
+
544
+                $handle = fopen($tmpFile, $mode);
545
+                return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
546
+                    $this->writeBack($tmpFile, $path);
547
+                });
548
+        }
549
+        return false;
550
+    }
551
+
552
+    public function touch($path, $mtime = null) {
553
+        if (is_null($mtime)) {
554
+            $mtime = time();
555
+        }
556
+        $metadata = [
557
+            'lastmodified' => gmdate(\DateTime::RFC1123, $mtime)
558
+        ];
559
+
560
+        try {
561
+            if (!$this->file_exists($path)) {
562
+                $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
563
+                $this->getConnection()->putObject([
564
+                    'Bucket' => $this->bucket,
565
+                    'Key' => $this->cleanKey($path),
566
+                    'Metadata' => $metadata,
567
+                    'Body' => '',
568
+                    'ContentType' => $mimeType,
569
+                    'MetadataDirective' => 'REPLACE',
570
+                ]);
571
+                $this->testTimeout();
572
+            }
573
+        } catch (S3Exception $e) {
574
+            \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
575
+            return false;
576
+        }
577
+
578
+        $this->invalidateCache($path);
579
+        return true;
580
+    }
581
+
582
+    public function copy($path1, $path2) {
583
+        $path1 = $this->normalizePath($path1);
584
+        $path2 = $this->normalizePath($path2);
585
+
586
+        if ($this->is_file($path1)) {
587
+            try {
588
+                $this->getConnection()->copyObject([
589
+                    'Bucket' => $this->bucket,
590
+                    'Key' => $this->cleanKey($path2),
591
+                    'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
592
+                ]);
593
+                $this->testTimeout();
594
+            } catch (S3Exception $e) {
595
+                \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
596
+                return false;
597
+            }
598
+        } else {
599
+            $this->remove($path2);
600
+
601
+            try {
602
+                $this->getConnection()->copyObject([
603
+                    'Bucket' => $this->bucket,
604
+                    'Key' => $path2 . '/',
605
+                    'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
606
+                ]);
607
+                $this->testTimeout();
608
+            } catch (S3Exception $e) {
609
+                \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
610
+                return false;
611
+            }
612
+
613
+            $dh = $this->opendir($path1);
614
+            if (is_resource($dh)) {
615
+                while (($file = readdir($dh)) !== false) {
616
+                    if (\OC\Files\Filesystem::isIgnoredDir($file)) {
617
+                        continue;
618
+                    }
619
+
620
+                    $source = $path1 . '/' . $file;
621
+                    $target = $path2 . '/' . $file;
622
+                    $this->copy($source, $target);
623
+                }
624
+            }
625
+        }
626
+
627
+        $this->invalidateCache($path2);
628
+
629
+        return true;
630
+    }
631
+
632
+    public function rename($path1, $path2) {
633
+        $path1 = $this->normalizePath($path1);
634
+        $path2 = $this->normalizePath($path2);
635
+
636
+        if ($this->is_file($path1)) {
637
+            if ($this->copy($path1, $path2) === false) {
638
+                return false;
639
+            }
640
+
641
+            if ($this->unlink($path1) === false) {
642
+                $this->unlink($path2);
643
+                return false;
644
+            }
645
+        } else {
646
+            if ($this->copy($path1, $path2) === false) {
647
+                return false;
648
+            }
649
+
650
+            if ($this->rmdir($path1) === false) {
651
+                $this->rmdir($path2);
652
+                return false;
653
+            }
654
+        }
655
+
656
+        return true;
657
+    }
658
+
659
+    public function test() {
660
+        $this->getConnection()->headBucket([
661
+            'Bucket' => $this->bucket
662
+        ]);
663
+        return true;
664
+    }
665
+
666
+    public function getId() {
667
+        return $this->id;
668
+    }
669
+
670
+    public function writeBack($tmpFile, $path) {
671
+        try {
672
+            $source = fopen($tmpFile, 'r');
673
+            $this->writeObject($path, $source);
674
+            $this->invalidateCache($path);
675
+
676
+            unlink($tmpFile);
677
+            return true;
678
+        } catch (S3Exception $e) {
679
+            \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
680
+            return false;
681
+        }
682
+    }
683
+
684
+    /**
685
+     * check if curl is installed
686
+     */
687
+    public static function checkDependencies() {
688
+        return true;
689
+    }
690 690
 }
Please login to merge, or discard this patch.
Spacing   +13 added lines, -13 removed lines patch added patch discarded remove patch
@@ -160,7 +160,7 @@  discard block
 block discarded – undo
160 160
 			try {
161 161
 				$result = $this->getConnection()->listObjects([
162 162
 					'Bucket' => $this->bucket,
163
-					'Prefix' => rtrim($path, '/') . '/',
163
+					'Prefix' => rtrim($path, '/').'/',
164 164
 					'MaxKeys' => 1,
165 165
 				]);
166 166
 				$this->directoryCache[$path] = $result['Contents'] || $result['CommonPrefixes'];
@@ -182,7 +182,7 @@  discard block
 block discarded – undo
182 182
 	 * @param array $params
183 183
 	 */
184 184
 	public function updateLegacyId(array $params) {
185
-		$oldId = 'amazon::' . $params['key'] . md5($params['secret']);
185
+		$oldId = 'amazon::'.$params['key'].md5($params['secret']);
186 186
 
187 187
 		// find by old id or bucket
188 188
 		$stmt = \OC::$server->getDatabaseConnection()->prepare(
@@ -234,7 +234,7 @@  discard block
 block discarded – undo
234 234
 		try {
235 235
 			$this->getConnection()->putObject([
236 236
 				'Bucket' => $this->bucket,
237
-				'Key' => $path . '/',
237
+				'Key' => $path.'/',
238 238
 				'Body' => '',
239 239
 				'ContentType' => 'httpd/unix-directory'
240 240
 			]);
@@ -285,7 +285,7 @@  discard block
 block discarded – undo
285 285
 			'Bucket' => $this->bucket
286 286
 		];
287 287
 		if ($path !== null) {
288
-			$params['Prefix'] = $path . '/';
288
+			$params['Prefix'] = $path.'/';
289 289
 		}
290 290
 		try {
291 291
 			$connection = $this->getConnection();
@@ -351,9 +351,9 @@  discard block
 block discarded – undo
351 351
 						$files[] = $file;
352 352
 
353 353
 						// store this information for later usage
354
-						$this->filesCache[$path . $file] = [
354
+						$this->filesCache[$path.$file] = [
355 355
 							'ContentLength' => $object['Size'],
356
-							'LastModified' => (string)$object['LastModified'],
356
+							'LastModified' => (string) $object['LastModified'],
357 357
 						];
358 358
 					}
359 359
 				}
@@ -517,7 +517,7 @@  discard block
 block discarded – undo
517 517
 				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile();
518 518
 
519 519
 				$handle = fopen($tmpFile, 'w');
520
-				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
520
+				return CallbackWrapper::wrap($handle, null, null, function() use ($path, $tmpFile) {
521 521
 					$this->writeBack($tmpFile, $path);
522 522
 				});
523 523
 			case 'a':
@@ -542,7 +542,7 @@  discard block
 block discarded – undo
542 542
 				}
543 543
 
544 544
 				$handle = fopen($tmpFile, $mode);
545
-				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
545
+				return CallbackWrapper::wrap($handle, null, null, function() use ($path, $tmpFile) {
546 546
 					$this->writeBack($tmpFile, $path);
547 547
 				});
548 548
 		}
@@ -588,7 +588,7 @@  discard block
 block discarded – undo
588 588
 				$this->getConnection()->copyObject([
589 589
 					'Bucket' => $this->bucket,
590 590
 					'Key' => $this->cleanKey($path2),
591
-					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
591
+					'CopySource' => S3Client::encodeKey($this->bucket.'/'.$path1)
592 592
 				]);
593 593
 				$this->testTimeout();
594 594
 			} catch (S3Exception $e) {
@@ -601,8 +601,8 @@  discard block
 block discarded – undo
601 601
 			try {
602 602
 				$this->getConnection()->copyObject([
603 603
 					'Bucket' => $this->bucket,
604
-					'Key' => $path2 . '/',
605
-					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
604
+					'Key' => $path2.'/',
605
+					'CopySource' => S3Client::encodeKey($this->bucket.'/'.$path1.'/')
606 606
 				]);
607 607
 				$this->testTimeout();
608 608
 			} catch (S3Exception $e) {
@@ -617,8 +617,8 @@  discard block
 block discarded – undo
617 617
 						continue;
618 618
 					}
619 619
 
620
-					$source = $path1 . '/' . $file;
621
-					$target = $path2 . '/' . $file;
620
+					$source = $path1.'/'.$file;
621
+					$target = $path2.'/'.$file;
622 622
 					$this->copy($source, $target);
623 623
 				}
624 624
 			}
Please login to merge, or discard this patch.