@@ -44,221 +44,221 @@ |
||
44 | 44 | use Psr\Log\LoggerInterface; |
45 | 45 | |
46 | 46 | trait S3ConnectionTrait { |
47 | - /** @var array */ |
|
48 | - protected $params; |
|
49 | - |
|
50 | - /** @var S3Client */ |
|
51 | - protected $connection; |
|
52 | - |
|
53 | - /** @var string */ |
|
54 | - protected $id; |
|
55 | - |
|
56 | - /** @var string */ |
|
57 | - protected $bucket; |
|
58 | - |
|
59 | - /** @var int */ |
|
60 | - protected $timeout; |
|
61 | - |
|
62 | - /** @var string */ |
|
63 | - protected $proxy; |
|
64 | - |
|
65 | - /** @var string */ |
|
66 | - protected $storageClass; |
|
67 | - |
|
68 | - /** @var int */ |
|
69 | - protected $uploadPartSize; |
|
70 | - |
|
71 | - /** @var int */ |
|
72 | - private $putSizeLimit; |
|
73 | - |
|
74 | - protected $test; |
|
75 | - |
|
76 | - protected function parseParams($params) { |
|
77 | - if (empty($params['bucket'])) { |
|
78 | - throw new \Exception("Bucket has to be configured."); |
|
79 | - } |
|
80 | - |
|
81 | - $this->id = 'amazon::' . $params['bucket']; |
|
82 | - |
|
83 | - $this->test = isset($params['test']); |
|
84 | - $this->bucket = $params['bucket']; |
|
85 | - $this->proxy = $params['proxy'] ?? false; |
|
86 | - $this->timeout = $params['timeout'] ?? 15; |
|
87 | - $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD'; |
|
88 | - $this->uploadPartSize = $params['uploadPartSize'] ?? 524288000; |
|
89 | - $this->putSizeLimit = $params['putSizeLimit'] ?? 104857600; |
|
90 | - $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; |
|
91 | - $params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname']; |
|
92 | - if (!isset($params['port']) || $params['port'] === '') { |
|
93 | - $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; |
|
94 | - } |
|
95 | - $params['verify_bucket_exists'] = empty($params['verify_bucket_exists']) ? true : $params['verify_bucket_exists']; |
|
96 | - $this->params = $params; |
|
97 | - } |
|
98 | - |
|
99 | - public function getBucket() { |
|
100 | - return $this->bucket; |
|
101 | - } |
|
102 | - |
|
103 | - public function getProxy() { |
|
104 | - return $this->proxy; |
|
105 | - } |
|
106 | - |
|
107 | - /** |
|
108 | - * Returns the connection |
|
109 | - * |
|
110 | - * @return S3Client connected client |
|
111 | - * @throws \Exception if connection could not be made |
|
112 | - */ |
|
113 | - public function getConnection() { |
|
114 | - if (!is_null($this->connection)) { |
|
115 | - return $this->connection; |
|
116 | - } |
|
117 | - |
|
118 | - $scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https'; |
|
119 | - $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/'; |
|
120 | - |
|
121 | - // Adding explicit credential provider to the beginning chain. |
|
122 | - // Including default credential provider (skipping AWS shared config files). |
|
123 | - $provider = CredentialProvider::memoize( |
|
124 | - CredentialProvider::chain( |
|
125 | - $this->paramCredentialProvider(), |
|
126 | - CredentialProvider::defaultProvider(['use_aws_shared_config_files' => false]) |
|
127 | - ) |
|
128 | - ); |
|
129 | - |
|
130 | - $options = [ |
|
131 | - 'version' => isset($this->params['version']) ? $this->params['version'] : 'latest', |
|
132 | - 'credentials' => $provider, |
|
133 | - 'endpoint' => $base_url, |
|
134 | - 'region' => $this->params['region'], |
|
135 | - 'use_path_style_endpoint' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false, |
|
136 | - 'signature_provider' => \Aws\or_chain([self::class, 'legacySignatureProvider'], ClientResolver::_default_signature_provider()), |
|
137 | - 'csm' => false, |
|
138 | - 'use_arn_region' => false, |
|
139 | - 'http' => ['verify' => $this->getCertificateBundlePath()], |
|
140 | - 'use_aws_shared_config_files' => false, |
|
141 | - ]; |
|
142 | - if ($this->getProxy()) { |
|
143 | - $options['http']['proxy'] = $this->getProxy(); |
|
144 | - } |
|
145 | - if (isset($this->params['legacy_auth']) && $this->params['legacy_auth']) { |
|
146 | - $options['signature_version'] = 'v2'; |
|
147 | - } |
|
148 | - $this->connection = new S3Client($options); |
|
149 | - |
|
150 | - if (!$this->connection::isBucketDnsCompatible($this->bucket)) { |
|
151 | - $logger = \OC::$server->get(LoggerInterface::class); |
|
152 | - $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.', |
|
153 | - ['app' => 'objectstore']); |
|
154 | - } |
|
155 | - |
|
156 | - if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) { |
|
157 | - $logger = \OC::$server->get(LoggerInterface::class); |
|
158 | - try { |
|
159 | - $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']); |
|
160 | - if (!$this->connection::isBucketDnsCompatible($this->bucket)) { |
|
161 | - throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket); |
|
162 | - } |
|
163 | - $this->connection->createBucket(['Bucket' => $this->bucket]); |
|
164 | - $this->testTimeout(); |
|
165 | - } catch (S3Exception $e) { |
|
166 | - $logger->debug('Invalid remote storage.', [ |
|
167 | - 'exception' => $e, |
|
168 | - 'app' => 'objectstore', |
|
169 | - ]); |
|
170 | - throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage()); |
|
171 | - } |
|
172 | - } |
|
173 | - |
|
174 | - // google cloud's s3 compatibility doesn't like the EncodingType parameter |
|
175 | - if (strpos($base_url, 'storage.googleapis.com')) { |
|
176 | - $this->connection->getHandlerList()->remove('s3.auto_encode'); |
|
177 | - } |
|
178 | - |
|
179 | - return $this->connection; |
|
180 | - } |
|
181 | - |
|
182 | - /** |
|
183 | - * when running the tests wait to let the buckets catch up |
|
184 | - */ |
|
185 | - private function testTimeout() { |
|
186 | - if ($this->test) { |
|
187 | - sleep($this->timeout); |
|
188 | - } |
|
189 | - } |
|
190 | - |
|
191 | - public static function legacySignatureProvider($version, $service, $region) { |
|
192 | - switch ($version) { |
|
193 | - case 'v2': |
|
194 | - case 's3': |
|
195 | - return new S3Signature(); |
|
196 | - default: |
|
197 | - return null; |
|
198 | - } |
|
199 | - } |
|
200 | - |
|
201 | - /** |
|
202 | - * This function creates a credential provider based on user parameter file |
|
203 | - */ |
|
204 | - protected function paramCredentialProvider(): callable { |
|
205 | - return function () { |
|
206 | - $key = empty($this->params['key']) ? null : $this->params['key']; |
|
207 | - $secret = empty($this->params['secret']) ? null : $this->params['secret']; |
|
208 | - |
|
209 | - if ($key && $secret) { |
|
210 | - return Promise\promise_for( |
|
211 | - new Credentials($key, $secret) |
|
212 | - ); |
|
213 | - } |
|
214 | - |
|
215 | - $msg = 'Could not find parameters set for credentials in config file.'; |
|
216 | - return new RejectedPromise(new CredentialsException($msg)); |
|
217 | - }; |
|
218 | - } |
|
219 | - |
|
220 | - protected function getCertificateBundlePath(): ?string { |
|
221 | - if ((int)($this->params['use_nextcloud_bundle'] ?? "0")) { |
|
222 | - // since we store the certificate bundles on the primary storage, we can't get the bundle while setting up the primary storage |
|
223 | - if (!isset($this->params['primary_storage'])) { |
|
224 | - /** @var ICertificateManager $certManager */ |
|
225 | - $certManager = \OC::$server->get(ICertificateManager::class); |
|
226 | - return $certManager->getAbsoluteBundlePath(); |
|
227 | - } else { |
|
228 | - return \OC::$SERVERROOT . '/resources/config/ca-bundle.crt'; |
|
229 | - } |
|
230 | - } else { |
|
231 | - return null; |
|
232 | - } |
|
233 | - } |
|
234 | - |
|
235 | - protected function getSSECKey(): ?string { |
|
236 | - if (isset($this->params['sse_c_key'])) { |
|
237 | - return $this->params['sse_c_key']; |
|
238 | - } |
|
239 | - |
|
240 | - return null; |
|
241 | - } |
|
242 | - |
|
243 | - protected function getSSECParameters(bool $copy = false): array { |
|
244 | - $key = $this->getSSECKey(); |
|
245 | - |
|
246 | - if ($key === null) { |
|
247 | - return []; |
|
248 | - } |
|
249 | - |
|
250 | - $rawKey = base64_decode($key); |
|
251 | - if ($copy) { |
|
252 | - return [ |
|
253 | - 'CopySourceSSECustomerAlgorithm' => 'AES256', |
|
254 | - 'CopySourceSSECustomerKey' => $rawKey, |
|
255 | - 'CopySourceSSECustomerKeyMD5' => md5($rawKey, true) |
|
256 | - ]; |
|
257 | - } |
|
258 | - return [ |
|
259 | - 'SSECustomerAlgorithm' => 'AES256', |
|
260 | - 'SSECustomerKey' => $rawKey, |
|
261 | - 'SSECustomerKeyMD5' => md5($rawKey, true) |
|
262 | - ]; |
|
263 | - } |
|
47 | + /** @var array */ |
|
48 | + protected $params; |
|
49 | + |
|
50 | + /** @var S3Client */ |
|
51 | + protected $connection; |
|
52 | + |
|
53 | + /** @var string */ |
|
54 | + protected $id; |
|
55 | + |
|
56 | + /** @var string */ |
|
57 | + protected $bucket; |
|
58 | + |
|
59 | + /** @var int */ |
|
60 | + protected $timeout; |
|
61 | + |
|
62 | + /** @var string */ |
|
63 | + protected $proxy; |
|
64 | + |
|
65 | + /** @var string */ |
|
66 | + protected $storageClass; |
|
67 | + |
|
68 | + /** @var int */ |
|
69 | + protected $uploadPartSize; |
|
70 | + |
|
71 | + /** @var int */ |
|
72 | + private $putSizeLimit; |
|
73 | + |
|
74 | + protected $test; |
|
75 | + |
|
76 | + protected function parseParams($params) { |
|
77 | + if (empty($params['bucket'])) { |
|
78 | + throw new \Exception("Bucket has to be configured."); |
|
79 | + } |
|
80 | + |
|
81 | + $this->id = 'amazon::' . $params['bucket']; |
|
82 | + |
|
83 | + $this->test = isset($params['test']); |
|
84 | + $this->bucket = $params['bucket']; |
|
85 | + $this->proxy = $params['proxy'] ?? false; |
|
86 | + $this->timeout = $params['timeout'] ?? 15; |
|
87 | + $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD'; |
|
88 | + $this->uploadPartSize = $params['uploadPartSize'] ?? 524288000; |
|
89 | + $this->putSizeLimit = $params['putSizeLimit'] ?? 104857600; |
|
90 | + $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; |
|
91 | + $params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname']; |
|
92 | + if (!isset($params['port']) || $params['port'] === '') { |
|
93 | + $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; |
|
94 | + } |
|
95 | + $params['verify_bucket_exists'] = empty($params['verify_bucket_exists']) ? true : $params['verify_bucket_exists']; |
|
96 | + $this->params = $params; |
|
97 | + } |
|
98 | + |
|
99 | + public function getBucket() { |
|
100 | + return $this->bucket; |
|
101 | + } |
|
102 | + |
|
103 | + public function getProxy() { |
|
104 | + return $this->proxy; |
|
105 | + } |
|
106 | + |
|
107 | + /** |
|
108 | + * Returns the connection |
|
109 | + * |
|
110 | + * @return S3Client connected client |
|
111 | + * @throws \Exception if connection could not be made |
|
112 | + */ |
|
113 | + public function getConnection() { |
|
114 | + if (!is_null($this->connection)) { |
|
115 | + return $this->connection; |
|
116 | + } |
|
117 | + |
|
118 | + $scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https'; |
|
119 | + $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/'; |
|
120 | + |
|
121 | + // Adding explicit credential provider to the beginning chain. |
|
122 | + // Including default credential provider (skipping AWS shared config files). |
|
123 | + $provider = CredentialProvider::memoize( |
|
124 | + CredentialProvider::chain( |
|
125 | + $this->paramCredentialProvider(), |
|
126 | + CredentialProvider::defaultProvider(['use_aws_shared_config_files' => false]) |
|
127 | + ) |
|
128 | + ); |
|
129 | + |
|
130 | + $options = [ |
|
131 | + 'version' => isset($this->params['version']) ? $this->params['version'] : 'latest', |
|
132 | + 'credentials' => $provider, |
|
133 | + 'endpoint' => $base_url, |
|
134 | + 'region' => $this->params['region'], |
|
135 | + 'use_path_style_endpoint' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false, |
|
136 | + 'signature_provider' => \Aws\or_chain([self::class, 'legacySignatureProvider'], ClientResolver::_default_signature_provider()), |
|
137 | + 'csm' => false, |
|
138 | + 'use_arn_region' => false, |
|
139 | + 'http' => ['verify' => $this->getCertificateBundlePath()], |
|
140 | + 'use_aws_shared_config_files' => false, |
|
141 | + ]; |
|
142 | + if ($this->getProxy()) { |
|
143 | + $options['http']['proxy'] = $this->getProxy(); |
|
144 | + } |
|
145 | + if (isset($this->params['legacy_auth']) && $this->params['legacy_auth']) { |
|
146 | + $options['signature_version'] = 'v2'; |
|
147 | + } |
|
148 | + $this->connection = new S3Client($options); |
|
149 | + |
|
150 | + if (!$this->connection::isBucketDnsCompatible($this->bucket)) { |
|
151 | + $logger = \OC::$server->get(LoggerInterface::class); |
|
152 | + $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.', |
|
153 | + ['app' => 'objectstore']); |
|
154 | + } |
|
155 | + |
|
156 | + if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) { |
|
157 | + $logger = \OC::$server->get(LoggerInterface::class); |
|
158 | + try { |
|
159 | + $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']); |
|
160 | + if (!$this->connection::isBucketDnsCompatible($this->bucket)) { |
|
161 | + throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket); |
|
162 | + } |
|
163 | + $this->connection->createBucket(['Bucket' => $this->bucket]); |
|
164 | + $this->testTimeout(); |
|
165 | + } catch (S3Exception $e) { |
|
166 | + $logger->debug('Invalid remote storage.', [ |
|
167 | + 'exception' => $e, |
|
168 | + 'app' => 'objectstore', |
|
169 | + ]); |
|
170 | + throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage()); |
|
171 | + } |
|
172 | + } |
|
173 | + |
|
174 | + // google cloud's s3 compatibility doesn't like the EncodingType parameter |
|
175 | + if (strpos($base_url, 'storage.googleapis.com')) { |
|
176 | + $this->connection->getHandlerList()->remove('s3.auto_encode'); |
|
177 | + } |
|
178 | + |
|
179 | + return $this->connection; |
|
180 | + } |
|
181 | + |
|
182 | + /** |
|
183 | + * when running the tests wait to let the buckets catch up |
|
184 | + */ |
|
185 | + private function testTimeout() { |
|
186 | + if ($this->test) { |
|
187 | + sleep($this->timeout); |
|
188 | + } |
|
189 | + } |
|
190 | + |
|
191 | + public static function legacySignatureProvider($version, $service, $region) { |
|
192 | + switch ($version) { |
|
193 | + case 'v2': |
|
194 | + case 's3': |
|
195 | + return new S3Signature(); |
|
196 | + default: |
|
197 | + return null; |
|
198 | + } |
|
199 | + } |
|
200 | + |
|
201 | + /** |
|
202 | + * This function creates a credential provider based on user parameter file |
|
203 | + */ |
|
204 | + protected function paramCredentialProvider(): callable { |
|
205 | + return function () { |
|
206 | + $key = empty($this->params['key']) ? null : $this->params['key']; |
|
207 | + $secret = empty($this->params['secret']) ? null : $this->params['secret']; |
|
208 | + |
|
209 | + if ($key && $secret) { |
|
210 | + return Promise\promise_for( |
|
211 | + new Credentials($key, $secret) |
|
212 | + ); |
|
213 | + } |
|
214 | + |
|
215 | + $msg = 'Could not find parameters set for credentials in config file.'; |
|
216 | + return new RejectedPromise(new CredentialsException($msg)); |
|
217 | + }; |
|
218 | + } |
|
219 | + |
|
220 | + protected function getCertificateBundlePath(): ?string { |
|
221 | + if ((int)($this->params['use_nextcloud_bundle'] ?? "0")) { |
|
222 | + // since we store the certificate bundles on the primary storage, we can't get the bundle while setting up the primary storage |
|
223 | + if (!isset($this->params['primary_storage'])) { |
|
224 | + /** @var ICertificateManager $certManager */ |
|
225 | + $certManager = \OC::$server->get(ICertificateManager::class); |
|
226 | + return $certManager->getAbsoluteBundlePath(); |
|
227 | + } else { |
|
228 | + return \OC::$SERVERROOT . '/resources/config/ca-bundle.crt'; |
|
229 | + } |
|
230 | + } else { |
|
231 | + return null; |
|
232 | + } |
|
233 | + } |
|
234 | + |
|
235 | + protected function getSSECKey(): ?string { |
|
236 | + if (isset($this->params['sse_c_key'])) { |
|
237 | + return $this->params['sse_c_key']; |
|
238 | + } |
|
239 | + |
|
240 | + return null; |
|
241 | + } |
|
242 | + |
|
243 | + protected function getSSECParameters(bool $copy = false): array { |
|
244 | + $key = $this->getSSECKey(); |
|
245 | + |
|
246 | + if ($key === null) { |
|
247 | + return []; |
|
248 | + } |
|
249 | + |
|
250 | + $rawKey = base64_decode($key); |
|
251 | + if ($copy) { |
|
252 | + return [ |
|
253 | + 'CopySourceSSECustomerAlgorithm' => 'AES256', |
|
254 | + 'CopySourceSSECustomerKey' => $rawKey, |
|
255 | + 'CopySourceSSECustomerKeyMD5' => md5($rawKey, true) |
|
256 | + ]; |
|
257 | + } |
|
258 | + return [ |
|
259 | + 'SSECustomerAlgorithm' => 'AES256', |
|
260 | + 'SSECustomerKey' => $rawKey, |
|
261 | + 'SSECustomerKeyMD5' => md5($rawKey, true) |
|
262 | + ]; |
|
263 | + } |
|
264 | 264 | } |
@@ -35,159 +35,159 @@ |
||
35 | 35 | use Psr\Http\Message\StreamInterface; |
36 | 36 | |
37 | 37 | trait S3ObjectTrait { |
38 | - /** |
|
39 | - * Returns the connection |
|
40 | - * |
|
41 | - * @return S3Client connected client |
|
42 | - * @throws \Exception if connection could not be made |
|
43 | - */ |
|
44 | - abstract protected function getConnection(); |
|
45 | - |
|
46 | - abstract protected function getCertificateBundlePath(): ?string; |
|
47 | - abstract protected function getSSECParameters(bool $copy = false): array; |
|
48 | - |
|
49 | - /** |
|
50 | - * @param string $urn the unified resource name used to identify the object |
|
51 | - * |
|
52 | - * @return resource stream with the read data |
|
53 | - * @throws \Exception when something goes wrong, message will be logged |
|
54 | - * @since 7.0.0 |
|
55 | - */ |
|
56 | - public function readObject($urn) { |
|
57 | - return SeekableHttpStream::open(function ($range) use ($urn) { |
|
58 | - $command = $this->getConnection()->getCommand('GetObject', [ |
|
59 | - 'Bucket' => $this->bucket, |
|
60 | - 'Key' => $urn, |
|
61 | - 'Range' => 'bytes=' . $range, |
|
62 | - ] + $this->getSSECParameters()); |
|
63 | - $request = \Aws\serialize($command); |
|
64 | - $headers = []; |
|
65 | - foreach ($request->getHeaders() as $key => $values) { |
|
66 | - foreach ($values as $value) { |
|
67 | - $headers[] = "$key: $value"; |
|
68 | - } |
|
69 | - } |
|
70 | - $opts = [ |
|
71 | - 'http' => [ |
|
72 | - 'protocol_version' => $request->getProtocolVersion(), |
|
73 | - 'header' => $headers, |
|
74 | - ] |
|
75 | - ]; |
|
76 | - $bundle = $this->getCertificateBundlePath(); |
|
77 | - if ($bundle) { |
|
78 | - $opts['ssl'] = [ |
|
79 | - 'cafile' => $bundle |
|
80 | - ]; |
|
81 | - } |
|
82 | - |
|
83 | - if ($this->getProxy()) { |
|
84 | - $opts['http']['proxy'] = $this->getProxy(); |
|
85 | - $opts['http']['request_fulluri'] = true; |
|
86 | - } |
|
87 | - |
|
88 | - $context = stream_context_create($opts); |
|
89 | - return fopen($request->getUri(), 'r', false, $context); |
|
90 | - }); |
|
91 | - } |
|
92 | - |
|
93 | - |
|
94 | - /** |
|
95 | - * Single object put helper |
|
96 | - * |
|
97 | - * @param string $urn the unified resource name used to identify the object |
|
98 | - * @param StreamInterface $stream stream with the data to write |
|
99 | - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 |
|
100 | - * @throws \Exception when something goes wrong, message will be logged |
|
101 | - */ |
|
102 | - protected function writeSingle(string $urn, StreamInterface $stream, string $mimetype = null): void { |
|
103 | - $this->getConnection()->putObject([ |
|
104 | - 'Bucket' => $this->bucket, |
|
105 | - 'Key' => $urn, |
|
106 | - 'Body' => $stream, |
|
107 | - 'ACL' => 'private', |
|
108 | - 'ContentType' => $mimetype, |
|
109 | - 'StorageClass' => $this->storageClass, |
|
110 | - ] + $this->getSSECParameters()); |
|
111 | - } |
|
112 | - |
|
113 | - |
|
114 | - /** |
|
115 | - * Multipart upload helper that tries to avoid orphaned fragments in S3 |
|
116 | - * |
|
117 | - * @param string $urn the unified resource name used to identify the object |
|
118 | - * @param StreamInterface $stream stream with the data to write |
|
119 | - * @param string|null $mimetype the mimetype to set for the remove object |
|
120 | - * @throws \Exception when something goes wrong, message will be logged |
|
121 | - */ |
|
122 | - protected function writeMultiPart(string $urn, StreamInterface $stream, string $mimetype = null): void { |
|
123 | - $uploader = new MultipartUploader($this->getConnection(), $stream, [ |
|
124 | - 'bucket' => $this->bucket, |
|
125 | - 'key' => $urn, |
|
126 | - 'part_size' => $this->uploadPartSize, |
|
127 | - 'params' => [ |
|
128 | - 'ContentType' => $mimetype, |
|
129 | - 'StorageClass' => $this->storageClass, |
|
130 | - ] + $this->getSSECParameters(), |
|
131 | - ]); |
|
132 | - |
|
133 | - try { |
|
134 | - $uploader->upload(); |
|
135 | - } catch (S3MultipartUploadException $e) { |
|
136 | - // if anything goes wrong with multipart, make sure that you don´t poison and |
|
137 | - // slow down s3 bucket with orphaned fragments |
|
138 | - $uploadInfo = $e->getState()->getId(); |
|
139 | - if ($e->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) { |
|
140 | - $this->getConnection()->abortMultipartUpload($uploadInfo); |
|
141 | - } |
|
142 | - throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway("Error while uploading to S3 bucket", 0, $e); |
|
143 | - } |
|
144 | - } |
|
145 | - |
|
146 | - |
|
147 | - /** |
|
148 | - * @param string $urn the unified resource name used to identify the object |
|
149 | - * @param resource $stream stream with the data to write |
|
150 | - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 |
|
151 | - * @throws \Exception when something goes wrong, message will be logged |
|
152 | - * @since 7.0.0 |
|
153 | - */ |
|
154 | - public function writeObject($urn, $stream, string $mimetype = null) { |
|
155 | - $psrStream = Utils::streamFor($stream); |
|
156 | - |
|
157 | - // ($psrStream->isSeekable() && $psrStream->getSize() !== null) evaluates to true for a On-Seekable stream |
|
158 | - // so the optimisation does not apply |
|
159 | - $buffer = new Psr7\Stream(fopen("php://memory", 'rwb+')); |
|
160 | - Utils::copyToStream($psrStream, $buffer, $this->putSizeLimit); |
|
161 | - $buffer->seek(0); |
|
162 | - if ($buffer->getSize() < $this->putSizeLimit) { |
|
163 | - // buffer is fully seekable, so use it directly for the small upload |
|
164 | - $this->writeSingle($urn, $buffer, $mimetype); |
|
165 | - } else { |
|
166 | - $loadStream = new Psr7\AppendStream([$buffer, $psrStream]); |
|
167 | - $this->writeMultiPart($urn, $loadStream, $mimetype); |
|
168 | - } |
|
169 | - } |
|
170 | - |
|
171 | - /** |
|
172 | - * @param string $urn the unified resource name used to identify the object |
|
173 | - * @return void |
|
174 | - * @throws \Exception when something goes wrong, message will be logged |
|
175 | - * @since 7.0.0 |
|
176 | - */ |
|
177 | - public function deleteObject($urn) { |
|
178 | - $this->getConnection()->deleteObject([ |
|
179 | - 'Bucket' => $this->bucket, |
|
180 | - 'Key' => $urn, |
|
181 | - ]); |
|
182 | - } |
|
183 | - |
|
184 | - public function objectExists($urn) { |
|
185 | - return $this->getConnection()->doesObjectExist($this->bucket, $urn, $this->getSSECParameters()); |
|
186 | - } |
|
187 | - |
|
188 | - public function copyObject($from, $to) { |
|
189 | - $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [ |
|
190 | - 'params' => $this->getSSECParameters() + $this->getSSECParameters(true) |
|
191 | - ]); |
|
192 | - } |
|
38 | + /** |
|
39 | + * Returns the connection |
|
40 | + * |
|
41 | + * @return S3Client connected client |
|
42 | + * @throws \Exception if connection could not be made |
|
43 | + */ |
|
44 | + abstract protected function getConnection(); |
|
45 | + |
|
46 | + abstract protected function getCertificateBundlePath(): ?string; |
|
47 | + abstract protected function getSSECParameters(bool $copy = false): array; |
|
48 | + |
|
49 | + /** |
|
50 | + * @param string $urn the unified resource name used to identify the object |
|
51 | + * |
|
52 | + * @return resource stream with the read data |
|
53 | + * @throws \Exception when something goes wrong, message will be logged |
|
54 | + * @since 7.0.0 |
|
55 | + */ |
|
56 | + public function readObject($urn) { |
|
57 | + return SeekableHttpStream::open(function ($range) use ($urn) { |
|
58 | + $command = $this->getConnection()->getCommand('GetObject', [ |
|
59 | + 'Bucket' => $this->bucket, |
|
60 | + 'Key' => $urn, |
|
61 | + 'Range' => 'bytes=' . $range, |
|
62 | + ] + $this->getSSECParameters()); |
|
63 | + $request = \Aws\serialize($command); |
|
64 | + $headers = []; |
|
65 | + foreach ($request->getHeaders() as $key => $values) { |
|
66 | + foreach ($values as $value) { |
|
67 | + $headers[] = "$key: $value"; |
|
68 | + } |
|
69 | + } |
|
70 | + $opts = [ |
|
71 | + 'http' => [ |
|
72 | + 'protocol_version' => $request->getProtocolVersion(), |
|
73 | + 'header' => $headers, |
|
74 | + ] |
|
75 | + ]; |
|
76 | + $bundle = $this->getCertificateBundlePath(); |
|
77 | + if ($bundle) { |
|
78 | + $opts['ssl'] = [ |
|
79 | + 'cafile' => $bundle |
|
80 | + ]; |
|
81 | + } |
|
82 | + |
|
83 | + if ($this->getProxy()) { |
|
84 | + $opts['http']['proxy'] = $this->getProxy(); |
|
85 | + $opts['http']['request_fulluri'] = true; |
|
86 | + } |
|
87 | + |
|
88 | + $context = stream_context_create($opts); |
|
89 | + return fopen($request->getUri(), 'r', false, $context); |
|
90 | + }); |
|
91 | + } |
|
92 | + |
|
93 | + |
|
94 | + /** |
|
95 | + * Single object put helper |
|
96 | + * |
|
97 | + * @param string $urn the unified resource name used to identify the object |
|
98 | + * @param StreamInterface $stream stream with the data to write |
|
99 | + * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 |
|
100 | + * @throws \Exception when something goes wrong, message will be logged |
|
101 | + */ |
|
102 | + protected function writeSingle(string $urn, StreamInterface $stream, string $mimetype = null): void { |
|
103 | + $this->getConnection()->putObject([ |
|
104 | + 'Bucket' => $this->bucket, |
|
105 | + 'Key' => $urn, |
|
106 | + 'Body' => $stream, |
|
107 | + 'ACL' => 'private', |
|
108 | + 'ContentType' => $mimetype, |
|
109 | + 'StorageClass' => $this->storageClass, |
|
110 | + ] + $this->getSSECParameters()); |
|
111 | + } |
|
112 | + |
|
113 | + |
|
114 | + /** |
|
115 | + * Multipart upload helper that tries to avoid orphaned fragments in S3 |
|
116 | + * |
|
117 | + * @param string $urn the unified resource name used to identify the object |
|
118 | + * @param StreamInterface $stream stream with the data to write |
|
119 | + * @param string|null $mimetype the mimetype to set for the remove object |
|
120 | + * @throws \Exception when something goes wrong, message will be logged |
|
121 | + */ |
|
122 | + protected function writeMultiPart(string $urn, StreamInterface $stream, string $mimetype = null): void { |
|
123 | + $uploader = new MultipartUploader($this->getConnection(), $stream, [ |
|
124 | + 'bucket' => $this->bucket, |
|
125 | + 'key' => $urn, |
|
126 | + 'part_size' => $this->uploadPartSize, |
|
127 | + 'params' => [ |
|
128 | + 'ContentType' => $mimetype, |
|
129 | + 'StorageClass' => $this->storageClass, |
|
130 | + ] + $this->getSSECParameters(), |
|
131 | + ]); |
|
132 | + |
|
133 | + try { |
|
134 | + $uploader->upload(); |
|
135 | + } catch (S3MultipartUploadException $e) { |
|
136 | + // if anything goes wrong with multipart, make sure that you don´t poison and |
|
137 | + // slow down s3 bucket with orphaned fragments |
|
138 | + $uploadInfo = $e->getState()->getId(); |
|
139 | + if ($e->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) { |
|
140 | + $this->getConnection()->abortMultipartUpload($uploadInfo); |
|
141 | + } |
|
142 | + throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway("Error while uploading to S3 bucket", 0, $e); |
|
143 | + } |
|
144 | + } |
|
145 | + |
|
146 | + |
|
147 | + /** |
|
148 | + * @param string $urn the unified resource name used to identify the object |
|
149 | + * @param resource $stream stream with the data to write |
|
150 | + * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 |
|
151 | + * @throws \Exception when something goes wrong, message will be logged |
|
152 | + * @since 7.0.0 |
|
153 | + */ |
|
154 | + public function writeObject($urn, $stream, string $mimetype = null) { |
|
155 | + $psrStream = Utils::streamFor($stream); |
|
156 | + |
|
157 | + // ($psrStream->isSeekable() && $psrStream->getSize() !== null) evaluates to true for a On-Seekable stream |
|
158 | + // so the optimisation does not apply |
|
159 | + $buffer = new Psr7\Stream(fopen("php://memory", 'rwb+')); |
|
160 | + Utils::copyToStream($psrStream, $buffer, $this->putSizeLimit); |
|
161 | + $buffer->seek(0); |
|
162 | + if ($buffer->getSize() < $this->putSizeLimit) { |
|
163 | + // buffer is fully seekable, so use it directly for the small upload |
|
164 | + $this->writeSingle($urn, $buffer, $mimetype); |
|
165 | + } else { |
|
166 | + $loadStream = new Psr7\AppendStream([$buffer, $psrStream]); |
|
167 | + $this->writeMultiPart($urn, $loadStream, $mimetype); |
|
168 | + } |
|
169 | + } |
|
170 | + |
|
171 | + /** |
|
172 | + * @param string $urn the unified resource name used to identify the object |
|
173 | + * @return void |
|
174 | + * @throws \Exception when something goes wrong, message will be logged |
|
175 | + * @since 7.0.0 |
|
176 | + */ |
|
177 | + public function deleteObject($urn) { |
|
178 | + $this->getConnection()->deleteObject([ |
|
179 | + 'Bucket' => $this->bucket, |
|
180 | + 'Key' => $urn, |
|
181 | + ]); |
|
182 | + } |
|
183 | + |
|
184 | + public function objectExists($urn) { |
|
185 | + return $this->getConnection()->doesObjectExist($this->bucket, $urn, $this->getSSECParameters()); |
|
186 | + } |
|
187 | + |
|
188 | + public function copyObject($from, $to) { |
|
189 | + $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [ |
|
190 | + 'params' => $this->getSSECParameters() + $this->getSSECParameters(true) |
|
191 | + ]); |
|
192 | + } |
|
193 | 193 | } |
@@ -54,11 +54,11 @@ |
||
54 | 54 | * @since 7.0.0 |
55 | 55 | */ |
56 | 56 | public function readObject($urn) { |
57 | - return SeekableHttpStream::open(function ($range) use ($urn) { |
|
57 | + return SeekableHttpStream::open(function($range) use ($urn) { |
|
58 | 58 | $command = $this->getConnection()->getCommand('GetObject', [ |
59 | 59 | 'Bucket' => $this->bucket, |
60 | 60 | 'Key' => $urn, |
61 | - 'Range' => 'bytes=' . $range, |
|
61 | + 'Range' => 'bytes='.$range, |
|
62 | 62 | ] + $this->getSSECParameters()); |
63 | 63 | $request = \Aws\serialize($command); |
64 | 64 | $headers = []; |