1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
declare(strict_types=1); |
4
|
|
|
|
5
|
|
|
/** |
6
|
|
|
* balloon |
7
|
|
|
* |
8
|
|
|
* @copyright Copryright (c) 2012-2019 gyselroth GmbH (https://gyselroth.com) |
9
|
|
|
* @license GPL-3.0 https://opensource.org/licenses/GPL-3.0 |
10
|
|
|
*/ |
11
|
|
|
|
12
|
|
|
namespace Balloon\Filesystem\Storage\Adapter; |
13
|
|
|
|
14
|
|
|
use Balloon\Filesystem\Node\Collection; |
15
|
|
|
use Balloon\Filesystem\Node\File; |
16
|
|
|
use Balloon\Filesystem\Node\NodeInterface; |
17
|
|
|
use Balloon\Filesystem\Storage\Exception; |
18
|
|
|
use Balloon\Server\User; |
19
|
|
|
use MongoDB\BSON\Binary; |
20
|
|
|
use MongoDB\BSON\ObjectId; |
21
|
|
|
use MongoDB\BSON\UTCDateTime; |
22
|
|
|
use MongoDB\Database; |
23
|
|
|
use MongoDB\GridFS\Bucket; |
24
|
|
|
use Psr\Log\LoggerInterface; |
25
|
|
|
|
26
|
|
|
class Gridfs implements AdapterInterface |
27
|
|
|
{ |
28
|
|
|
/** |
29
|
|
|
* Grid chunks. |
30
|
|
|
*/ |
31
|
|
|
public const CHUNK_SIZE = 261120; |
32
|
|
|
|
33
|
|
|
/** |
34
|
|
|
* Database. |
35
|
|
|
* |
36
|
|
|
* @var Database |
37
|
|
|
*/ |
38
|
|
|
protected $db; |
39
|
|
|
|
40
|
|
|
/** |
41
|
|
|
* GridFS. |
42
|
|
|
* |
43
|
|
|
* @var Bucket |
44
|
|
|
*/ |
45
|
|
|
protected $gridfs; |
46
|
|
|
|
47
|
|
|
/** |
48
|
|
|
* Logger. |
49
|
|
|
* |
50
|
|
|
* @var LoggerInterface |
51
|
|
|
*/ |
52
|
|
|
protected $logger; |
53
|
|
|
|
54
|
|
|
/** |
55
|
|
|
* GridFS storage. |
56
|
|
|
* |
57
|
|
|
* @param Database |
58
|
|
|
*/ |
59
|
|
|
public function __construct(Database $db, LoggerInterface $logger) |
60
|
|
|
{ |
61
|
|
|
$this->db = $db; |
62
|
|
|
$this->gridfs = $db->selectGridFSBucket(); |
63
|
|
|
$this->logger = $logger; |
64
|
|
|
} |
65
|
|
|
|
66
|
|
|
/** |
67
|
|
|
* {@inheritdoc} |
68
|
|
|
*/ |
69
|
|
|
public function deleteCollection(Collection $collection): ?array |
70
|
|
|
{ |
71
|
|
|
return $collection->getAttributes()['storage']; |
72
|
|
|
} |
73
|
|
|
|
74
|
|
|
/** |
75
|
|
|
* {@inheritdoc} |
76
|
|
|
*/ |
77
|
|
|
public function forceDeleteCollection(Collection $collection): bool |
78
|
|
|
{ |
79
|
|
|
return true; |
80
|
|
|
} |
81
|
|
|
|
82
|
|
|
/** |
83
|
|
|
* {@inheritdoc} |
84
|
|
|
*/ |
85
|
|
|
public function undelete(NodeInterface $node): ?array |
86
|
|
|
{ |
87
|
|
|
return $node->getAttributes()['storage']; |
88
|
|
|
} |
89
|
|
|
|
90
|
|
|
/** |
91
|
|
|
* {@inheritdoc} |
92
|
|
|
*/ |
93
|
|
|
public function rename(NodeInterface $node, string $new_name): ?array |
94
|
|
|
{ |
95
|
|
|
return $node->getAttributes()['storage']; |
96
|
|
|
} |
97
|
|
|
|
98
|
|
|
/** |
99
|
|
|
* {@inheritdoc} |
100
|
|
|
*/ |
101
|
|
|
public function readonly(NodeInterface $node, bool $readonly = true): ?array |
102
|
|
|
{ |
103
|
|
|
return $node->getAttributes()['storage']; |
104
|
|
|
} |
105
|
|
|
|
106
|
|
|
/** |
107
|
|
|
* {@inheritdoc} |
108
|
|
|
*/ |
109
|
|
|
public function hasNode(NodeInterface $node): bool |
110
|
|
|
{ |
111
|
|
|
return null !== $this->getFileById($node->getId()); |
112
|
|
|
} |
113
|
|
|
|
114
|
|
|
/** |
115
|
|
|
* {@inheritdoc} |
116
|
|
|
*/ |
117
|
|
|
public function forceDeleteFile(File $file, ?int $version = null): bool |
118
|
|
|
{ |
119
|
|
|
try { |
120
|
|
|
$exists = $this->getFileById($this->getId($file, $version)); |
121
|
|
|
} catch (Exception\BlobNotFound $e) { |
122
|
|
|
return true; |
123
|
|
|
} |
124
|
|
|
|
125
|
|
|
if (null === $exists) { |
126
|
|
|
$this->logger->debug('gridfs blob ['.$exists['_id'].'] was not found for file reference ['.$file->getId().']', [ |
127
|
|
|
'category' => get_class($this), |
128
|
|
|
]); |
129
|
|
|
|
130
|
|
|
return false; |
131
|
|
|
} |
132
|
|
|
|
133
|
|
|
if (!isset($exists['metadata']['references'])) { |
134
|
|
|
$this->gridfs->delete($exists['_id']); |
135
|
|
|
|
136
|
|
|
return true; |
137
|
|
|
} |
138
|
|
|
|
139
|
|
|
$refs = $exists['metadata']['references']; |
140
|
|
|
if (($key = array_search($file->getId(), $refs)) !== false) { |
141
|
|
|
unset($refs[$key]); |
142
|
|
|
$refs = array_values($refs); |
143
|
|
|
} |
144
|
|
|
|
145
|
|
|
if (count($refs) >= 1) { |
146
|
|
|
$this->logger->debug('gridfs content node ['.$exists['_id'].'] still has references left, just remove the reference ['.$file->getId().']', [ |
147
|
|
|
'category' => get_class($this), |
148
|
|
|
]); |
149
|
|
|
|
150
|
|
|
$this->db->{'fs.files'}->updateOne(['_id' => $exists['_id']], [ |
151
|
|
|
'$set' => ['metadata.references' => $refs], |
152
|
|
|
]); |
153
|
|
|
} else { |
154
|
|
|
$this->logger->debug('gridfs content node ['.$exists['_id'].'] has no references left, delete node completely', [ |
155
|
|
|
'category' => get_class($this), |
156
|
|
|
]); |
157
|
|
|
|
158
|
|
|
$this->gridfs->delete($exists['_id']); |
159
|
|
|
} |
160
|
|
|
|
161
|
|
|
return true; |
162
|
|
|
} |
163
|
|
|
|
164
|
|
|
/** |
165
|
|
|
* {@inheritdoc} |
166
|
|
|
*/ |
167
|
|
|
public function deleteFile(File $file, ?int $version = null): ?array |
168
|
|
|
{ |
169
|
|
|
return $file->getAttributes()['storage']; |
170
|
|
|
} |
171
|
|
|
|
172
|
|
|
/** |
173
|
|
|
* {@inheritdoc} |
174
|
|
|
*/ |
175
|
|
|
public function openReadStream(File $file) |
176
|
|
|
{ |
177
|
|
|
return $this->gridfs->openDownloadStream($this->getId($file)); |
178
|
|
|
} |
179
|
|
|
|
180
|
|
|
/** |
181
|
|
|
* {@inheritdoc} |
182
|
|
|
*/ |
183
|
|
|
public function storeFile(File $file, ObjectId $session): array |
184
|
|
|
{ |
185
|
|
|
$this->logger->debug('finalize temporary file ['.$session.'] and add file ['.$file->getId().'] as reference', [ |
186
|
|
|
'category' => get_class($this), |
187
|
|
|
]); |
188
|
|
|
|
189
|
|
|
$md5 = $this->db->command([ |
190
|
|
|
'filemd5' => $session, |
191
|
|
|
'root' => 'fs', |
192
|
|
|
])->toArray()[0]['md5']; |
193
|
|
|
|
194
|
|
|
$blob = $this->getFileByHash($md5); |
195
|
|
|
|
196
|
|
|
if ($blob !== null) { |
197
|
|
|
$this->logger->debug('found existing file with hash ['.$md5.'], add file ['.$file->getId().'] as reference to ['.$blob['_id'].']', [ |
198
|
|
|
'category' => get_class($this), |
199
|
|
|
]); |
200
|
|
|
|
201
|
|
|
$this->db->selectCollection('fs.files')->updateOne([ |
202
|
|
|
'md5' => $blob['md5'], |
203
|
|
|
], [ |
204
|
|
|
'$addToSet' => [ |
205
|
|
|
'metadata.references' => $file->getId(), |
206
|
|
|
], |
207
|
|
|
]); |
208
|
|
|
|
209
|
|
|
$this->gridfs->delete($session); |
210
|
|
|
|
211
|
|
|
return [ |
212
|
|
|
'reference' => ['_id' => $blob['_id']], |
213
|
|
|
'size' => $blob['length'], |
214
|
|
|
'hash' => $md5, |
215
|
|
|
]; |
216
|
|
|
} |
217
|
|
|
|
218
|
|
|
$this->logger->debug('calculated hash ['.$md5.'] for temporary file ['.$session.'], remove temporary flag', [ |
219
|
|
|
'category' => get_class($this), |
220
|
|
|
]); |
221
|
|
|
|
222
|
|
|
$this->db->selectCollection('fs.files')->updateOne([ |
223
|
|
|
'_id' => $session, |
224
|
|
|
], [ |
225
|
|
|
'$set' => [ |
226
|
|
|
'md5' => $md5, |
227
|
|
|
'metadata.references' => [$file->getId()], |
228
|
|
|
], |
229
|
|
|
'$unset' => [ |
230
|
|
|
'metadata.temporary' => true, |
231
|
|
|
], |
232
|
|
|
]); |
233
|
|
|
|
234
|
|
|
$blob = $this->getFileById($session); |
235
|
|
|
|
236
|
|
|
return [ |
237
|
|
|
'reference' => ['_id' => $session], |
238
|
|
|
'size' => $blob['length'], |
239
|
|
|
'hash' => $md5, |
240
|
|
|
]; |
241
|
|
|
} |
242
|
|
|
|
243
|
|
|
/** |
244
|
|
|
* {@inheritdoc} |
245
|
|
|
*/ |
246
|
|
|
public function createCollection(Collection $parent, string $name): array |
247
|
|
|
{ |
248
|
|
|
return []; |
249
|
|
|
} |
250
|
|
|
|
251
|
|
|
/** |
252
|
|
|
* {@inheritdoc} |
253
|
|
|
*/ |
254
|
|
|
public function move(NodeInterface $node, Collection $parent): ?array |
255
|
|
|
{ |
256
|
|
|
return $node->getAttributes()['storage']; |
257
|
|
|
} |
258
|
|
|
|
259
|
|
|
/** |
260
|
|
|
* {@inheritdoc} |
261
|
|
|
*/ |
262
|
|
|
public function storeTemporaryFile($stream, User $user, ?ObjectId $session = null): ObjectId |
263
|
|
|
{ |
264
|
|
|
$exists = $session; |
265
|
|
|
|
266
|
|
|
if ($session === null) { |
267
|
|
|
$session = new ObjectId(); |
268
|
|
|
|
269
|
|
|
$this->logger->info('create new tempory storage file ['.$session.']', [ |
270
|
|
|
'category' => get_class($this), |
271
|
|
|
]); |
272
|
|
|
|
273
|
|
|
$this->db->selectCollection('fs.files')->insertOne([ |
274
|
|
|
'_id' => $session, |
275
|
|
|
'chunkSize' => self::CHUNK_SIZE, |
276
|
|
|
'length' => 0, |
277
|
|
|
'uploadDate' => new UTCDateTime(), |
278
|
|
|
'metadata' => ['temporary' => true], |
279
|
|
|
]); |
280
|
|
|
} |
281
|
|
|
|
282
|
|
|
$temp = $this->db->selectCollection('fs.files')->findOne([ |
283
|
|
|
'_id' => $session, |
284
|
|
|
]); |
285
|
|
|
|
286
|
|
|
if ($temp === null) { |
287
|
|
|
throw new Exception\SessionNotFound('temporary storage for this file is gone'); |
288
|
|
|
} |
289
|
|
|
|
290
|
|
|
$this->storeStream($stream, $user, $exists, $temp); |
|
|
|
|
291
|
|
|
|
292
|
|
|
return $session; |
293
|
|
|
} |
294
|
|
|
|
295
|
|
|
/** |
296
|
|
|
* Get file blob id. |
297
|
|
|
*/ |
298
|
|
|
protected function getId(NodeInterface $node, ?int $version = null): ObjectId |
299
|
|
|
{ |
300
|
|
|
$attributes = $node->getAttributes(); |
301
|
|
|
|
302
|
|
|
if ($version !== null) { |
303
|
|
|
$history = $node->getHistory(); |
|
|
|
|
304
|
|
|
|
305
|
|
|
$key = array_search($version, array_column($history, 'version'), true); |
306
|
|
|
$blobs = array_column($history, 'storage'); |
307
|
|
|
|
308
|
|
|
if ($key === false || !isset($blobs[$key]['_id'])) { |
309
|
|
|
throw new Exception\BlobNotFound('attributes do not contain a gridfs id storage._id'); |
310
|
|
|
} |
311
|
|
|
|
312
|
|
|
return $blobs[$key]['_id']; |
313
|
|
|
} |
314
|
|
|
|
315
|
|
|
if (!isset($attributes['storage']['_id'])) { |
316
|
|
|
throw new Exception\BlobNotFound('attributes do not contain a gridfs id storage._id'); |
317
|
|
|
} |
318
|
|
|
|
319
|
|
|
return $attributes['storage']['_id']; |
320
|
|
|
} |
321
|
|
|
|
322
|
|
|
/** |
323
|
|
|
* Get stored file. |
324
|
|
|
*/ |
325
|
|
|
protected function getFileById(ObjectId $id): ?array |
326
|
|
|
{ |
327
|
|
|
return $this->gridfs->findOne(['_id' => $id]); |
328
|
|
|
} |
329
|
|
|
|
330
|
|
|
/** |
331
|
|
|
* Get stored file. |
332
|
|
|
*/ |
333
|
|
|
protected function getFileByHash(string $hash): ?array |
334
|
|
|
{ |
335
|
|
|
return $this->gridfs->findOne(['md5' => $hash]); |
336
|
|
|
} |
337
|
|
|
|
338
|
|
|
/** |
339
|
|
|
* Store stream content. |
340
|
|
|
*/ |
341
|
|
|
protected function storeStream($stream, User $user, ?ObjectId $exists, array $temp): int |
342
|
|
|
{ |
343
|
|
|
$data = null; |
344
|
|
|
$length = $temp['length']; |
345
|
|
|
$chunks = 0; |
346
|
|
|
$left = 0; |
347
|
|
|
|
348
|
|
|
if ($exists !== null) { |
349
|
|
|
$chunks = (int) ceil($temp['length'] / $temp['chunkSize']); |
350
|
|
|
$left = (int) ($chunks * $temp['chunkSize'] - $temp['length']); |
351
|
|
|
|
352
|
|
|
if ($left < 0) { |
353
|
|
|
$left = 0; |
354
|
|
|
} else { |
355
|
|
|
$this->logger->debug('found existing chunks ['.$chunks.'] for temporary file ['.$temp['_id'].'] while the last chunk has ['.$left.'] bytes free', [ |
356
|
|
|
'category' => get_class($this), |
357
|
|
|
]); |
358
|
|
|
|
359
|
|
|
--$chunks; |
360
|
|
|
|
361
|
|
|
$last = $this->db->selectCollection('fs.chunks')->findOne([ |
362
|
|
|
'files_id' => $temp['_id'], |
363
|
|
|
'n' => $chunks, |
364
|
|
|
]); |
365
|
|
|
|
366
|
|
|
if ($last === null) { |
367
|
|
|
throw new Exception\ChunkNotFound('Chunk not found, file is corrupt'); |
368
|
|
|
} |
369
|
|
|
|
370
|
|
|
$data = $last['data']->getData(); |
371
|
|
|
} |
372
|
|
|
} |
373
|
|
|
|
374
|
|
|
while (!feof($stream)) { |
375
|
|
|
if ($data !== null) { |
376
|
|
|
$append = $this->readStream($stream, $left); |
377
|
|
|
$data .= $append; |
378
|
|
|
$chunk = new Binary($data, Binary::TYPE_GENERIC); |
379
|
|
|
$size = mb_strlen($append, '8bit'); |
380
|
|
|
$length += $size; |
381
|
|
|
|
382
|
|
|
$this->logger->debug('append data ['.$size.'] to last chunk ['.$chunks.'] in temporary file ['.$temp['_id'].']', [ |
383
|
|
|
'category' => get_class($this), |
384
|
|
|
]); |
385
|
|
|
|
386
|
|
|
$last = $this->db->selectCollection('fs.chunks')->updateOne([ |
|
|
|
|
387
|
|
|
'files_id' => $temp['_id'], |
388
|
|
|
'n' => $chunks, |
389
|
|
|
], [ |
390
|
|
|
'$set' => [ |
391
|
|
|
'data' => $chunk, |
392
|
|
|
], |
393
|
|
|
]); |
394
|
|
|
|
395
|
|
|
++$chunks; |
396
|
|
|
$data = null; |
397
|
|
|
|
398
|
|
|
continue; |
399
|
|
|
} |
400
|
|
|
|
401
|
|
|
$content = $this->readStream($stream, $temp['chunkSize']); |
402
|
|
|
$size = mb_strlen($content, '8bit'); |
403
|
|
|
|
404
|
|
|
if ($size === 0) { |
405
|
|
|
continue; |
406
|
|
|
} |
407
|
|
|
|
408
|
|
|
$length += $size; |
409
|
|
|
|
410
|
|
|
$chunk = new Binary($content, Binary::TYPE_GENERIC); |
411
|
|
|
$last = $this->db->selectCollection('fs.chunks')->insertOne([ |
412
|
|
|
'files_id' => $temp['_id'], |
413
|
|
|
'n' => $chunks, |
414
|
|
|
'data' => $chunk, |
415
|
|
|
]); |
416
|
|
|
|
417
|
|
|
$this->logger->debug('inserted new chunk ['.$last->getInsertedId().'] ['.$size.'] in temporary file ['.$temp['_id'].']', [ |
418
|
|
|
'category' => get_class($this), |
419
|
|
|
]); |
420
|
|
|
|
421
|
|
|
++$chunks; |
422
|
|
|
|
423
|
|
|
continue; |
424
|
|
|
} |
425
|
|
|
|
426
|
|
|
$this->verifyQuota($user, $length, $temp['_id']); |
427
|
|
|
|
428
|
|
|
$this->db->selectCollection('fs.files')->updateOne([ |
429
|
|
|
'_id' => $temp['_id'], |
430
|
|
|
], [ |
431
|
|
|
'$set' => [ |
432
|
|
|
'uploadDate' => new UTCDateTime(), |
433
|
|
|
'length' => $length, |
434
|
|
|
], |
435
|
|
|
]); |
436
|
|
|
|
437
|
|
|
return $length; |
438
|
|
|
} |
439
|
|
|
|
440
|
|
|
/** |
441
|
|
|
* Verify quota. |
442
|
|
|
*/ |
443
|
|
|
protected function verifyQuota(User $user, int $size, ObjectId $session): bool |
444
|
|
|
{ |
445
|
|
|
if (!$user->checkQuota($size)) { |
446
|
|
|
$this->logger->warning('stop adding chunk, user ['.$user->getId().'] quota is full, remove upload session', [ |
447
|
|
|
'category' => get_class($this), |
448
|
|
|
]); |
449
|
|
|
|
450
|
|
|
$this->gridfs->delete($session); |
451
|
|
|
|
452
|
|
|
throw new Exception\InsufficientStorage( |
453
|
|
|
'user quota is full', |
454
|
|
|
Exception\InsufficientStorage::USER_QUOTA_FULL |
455
|
|
|
); |
456
|
|
|
} |
457
|
|
|
|
458
|
|
|
return true; |
459
|
|
|
} |
460
|
|
|
|
461
|
|
|
/** |
462
|
|
|
* Read x bytes from stream. |
463
|
|
|
*/ |
464
|
|
|
protected function readStream($stream, int $bytes): string |
465
|
|
|
{ |
466
|
|
|
$length = 0; |
467
|
|
|
$data = ''; |
468
|
|
|
while (!feof($stream)) { |
469
|
|
|
if ($length + 8192 > $bytes) { |
470
|
|
|
$max = $bytes - $length; |
471
|
|
|
|
472
|
|
|
if ($max === 0) { |
473
|
|
|
return $data; |
474
|
|
|
} |
475
|
|
|
|
476
|
|
|
$length += $max; |
477
|
|
|
|
478
|
|
|
return $data .= fread($stream, $max); |
479
|
|
|
} |
480
|
|
|
|
481
|
|
|
$length += 8192; |
482
|
|
|
$data .= fread($stream, 8192); |
483
|
|
|
} |
484
|
|
|
|
485
|
|
|
return $data; |
486
|
|
|
} |
487
|
|
|
} |
488
|
|
|
|
If a method or function can return multiple different values and unless you are sure that you only can receive a single value in this context, we recommend to add an additional type check:
If this a common case that PHP Analyzer should handle natively, please let us know by opening an issue.