1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* @copyright Copyright (c) 2016, ownCloud, Inc. |
4
|
|
|
* |
5
|
|
|
* @author Jörn Friedrich Dreyer <[email protected]> |
6
|
|
|
* @author Morris Jobke <[email protected]> |
7
|
|
|
* @author Olivier Paroz <[email protected]> |
8
|
|
|
* @author Robin Appelman <[email protected]> |
9
|
|
|
* @author Roeland Jago Douma <[email protected]> |
10
|
|
|
* @author Thomas Müller <[email protected]> |
11
|
|
|
* @author Vincent Petry <[email protected]> |
12
|
|
|
* |
13
|
|
|
* @license AGPL-3.0 |
14
|
|
|
* |
15
|
|
|
* This code is free software: you can redistribute it and/or modify |
16
|
|
|
* it under the terms of the GNU Affero General Public License, version 3, |
17
|
|
|
* as published by the Free Software Foundation. |
18
|
|
|
* |
19
|
|
|
* This program is distributed in the hope that it will be useful, |
20
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
21
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
22
|
|
|
* GNU Affero General Public License for more details. |
23
|
|
|
* |
24
|
|
|
* You should have received a copy of the GNU Affero General Public License, version 3, |
25
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/> |
26
|
|
|
* |
27
|
|
|
*/ |
28
|
|
|
|
29
|
|
|
namespace OC\Files\Utils; |
30
|
|
|
|
31
|
|
|
use OC\Files\Cache\Cache; |
32
|
|
|
use OC\Files\Filesystem; |
33
|
|
|
use OC\ForbiddenException; |
34
|
|
|
use OC\Hooks\PublicEmitter; |
35
|
|
|
use OC\Lock\DBLockingProvider; |
36
|
|
|
use OCA\Files_Sharing\SharedStorage; |
37
|
|
|
use OCP\Files\NotFoundException; |
38
|
|
|
use OCP\Files\Storage\IStorage; |
39
|
|
|
use OCP\Files\StorageNotAvailableException; |
40
|
|
|
use OCP\ILogger; |
41
|
|
|
use OC\Files\Storage\FailedStorage; |
42
|
|
|
|
43
|
|
|
/** |
44
|
|
|
* Class Scanner |
45
|
|
|
* |
46
|
|
|
* Hooks available in scope \OC\Utils\Scanner |
47
|
|
|
* - scanFile(string $absolutePath) |
48
|
|
|
* - scanFolder(string $absolutePath) |
49
|
|
|
* |
50
|
|
|
* @package OC\Files\Utils |
51
|
|
|
*/ |
52
|
|
|
class Scanner extends PublicEmitter { |
53
|
|
|
const MAX_ENTRIES_TO_COMMIT = 10000; |
54
|
|
|
|
55
|
|
|
/** |
56
|
|
|
* @var string $user |
57
|
|
|
*/ |
58
|
|
|
private $user; |
59
|
|
|
|
60
|
|
|
/** |
61
|
|
|
* @var \OCP\IDBConnection |
62
|
|
|
*/ |
63
|
|
|
protected $db; |
64
|
|
|
|
65
|
|
|
/** |
66
|
|
|
* @var ILogger |
67
|
|
|
*/ |
68
|
|
|
protected $logger; |
69
|
|
|
|
70
|
|
|
/** |
71
|
|
|
* Whether to use a DB transaction |
72
|
|
|
* |
73
|
|
|
* @var bool |
74
|
|
|
*/ |
75
|
|
|
protected $useTransaction; |
76
|
|
|
|
77
|
|
|
/** |
78
|
|
|
* Number of entries scanned to commit |
79
|
|
|
* |
80
|
|
|
* @var int |
81
|
|
|
*/ |
82
|
|
|
protected $entriesToCommit; |
83
|
|
|
|
84
|
|
|
/** |
85
|
|
|
* @param string $user |
86
|
|
|
* @param \OCP\IDBConnection $db |
87
|
|
|
* @param ILogger $logger |
88
|
|
|
*/ |
89
|
|
|
public function __construct($user, $db, ILogger $logger) { |
90
|
|
|
$this->logger = $logger; |
91
|
|
|
$this->user = $user; |
92
|
|
|
$this->db = $db; |
93
|
|
|
// when DB locking is used, no DB transactions will be used |
94
|
|
|
$this->useTransaction = !(\OC::$server->getLockingProvider() instanceof DBLockingProvider); |
95
|
|
|
} |
96
|
|
|
|
97
|
|
|
/** |
98
|
|
|
* get all storages for $dir |
99
|
|
|
* |
100
|
|
|
* @param string $dir |
101
|
|
|
* @return \OC\Files\Mount\MountPoint[] |
102
|
|
|
*/ |
103
|
|
|
protected function getMounts($dir) { |
104
|
|
|
//TODO: move to the node based fileapi once that's done |
105
|
|
|
\OC_Util::tearDownFS(); |
106
|
|
|
\OC_Util::setupFS($this->user); |
107
|
|
|
|
108
|
|
|
$mountManager = Filesystem::getMountManager(); |
109
|
|
|
$mounts = $mountManager->findIn($dir); |
110
|
|
|
$mounts[] = $mountManager->find($dir); |
111
|
|
|
$mounts = array_reverse($mounts); //start with the mount of $dir |
112
|
|
|
|
113
|
|
|
return $mounts; |
114
|
|
|
} |
115
|
|
|
|
116
|
|
|
/** |
117
|
|
|
* attach listeners to the scanner |
118
|
|
|
* |
119
|
|
|
* @param \OC\Files\Mount\MountPoint $mount |
120
|
|
|
*/ |
121
|
|
|
protected function attachListener($mount) { |
122
|
|
|
$scanner = $mount->getStorage()->getScanner(); |
123
|
|
|
$emitter = $this; |
124
|
|
|
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function ($path) use ($mount, $emitter) { |
125
|
|
|
$emitter->emit('\OC\Files\Utils\Scanner', 'scanFile', array($mount->getMountPoint() . $path)); |
126
|
|
|
}); |
127
|
|
|
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function ($path) use ($mount, $emitter) { |
128
|
|
|
$emitter->emit('\OC\Files\Utils\Scanner', 'scanFolder', array($mount->getMountPoint() . $path)); |
129
|
|
|
}); |
130
|
|
|
$scanner->listen('\OC\Files\Cache\Scanner', 'postScanFile', function ($path) use ($mount, $emitter) { |
131
|
|
|
$emitter->emit('\OC\Files\Utils\Scanner', 'postScanFile', array($mount->getMountPoint() . $path)); |
132
|
|
|
}); |
133
|
|
|
$scanner->listen('\OC\Files\Cache\Scanner', 'postScanFolder', function ($path) use ($mount, $emitter) { |
134
|
|
|
$emitter->emit('\OC\Files\Utils\Scanner', 'postScanFolder', array($mount->getMountPoint() . $path)); |
135
|
|
|
}); |
136
|
|
|
} |
137
|
|
|
|
138
|
|
|
/** |
139
|
|
|
* @param string $dir |
140
|
|
|
*/ |
141
|
|
|
public function backgroundScan($dir) { |
142
|
|
|
$mounts = $this->getMounts($dir); |
143
|
|
|
foreach ($mounts as $mount) { |
144
|
|
|
$storage = $mount->getStorage(); |
145
|
|
|
if (is_null($storage)) { |
146
|
|
|
continue; |
147
|
|
|
} |
148
|
|
|
|
149
|
|
|
// don't bother scanning failed storages (shortcut for same result) |
150
|
|
|
if ($storage->instanceOfStorage(FailedStorage::class)) { |
151
|
|
|
continue; |
152
|
|
|
} |
153
|
|
|
|
154
|
|
|
// don't scan received local shares, these can be scanned when scanning the owner's storage |
155
|
|
|
if ($storage->instanceOfStorage(SharedStorage::class)) { |
156
|
|
|
continue; |
157
|
|
|
} |
158
|
|
|
$scanner = $storage->getScanner(); |
159
|
|
|
$this->attachListener($mount); |
|
|
|
|
160
|
|
|
|
161
|
|
|
$scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) { |
162
|
|
|
$this->triggerPropagator($storage, $path); |
163
|
|
|
}); |
164
|
|
|
$scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) { |
165
|
|
|
$this->triggerPropagator($storage, $path); |
166
|
|
|
}); |
167
|
|
|
$scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path) use ($storage) { |
168
|
|
|
$this->triggerPropagator($storage, $path); |
169
|
|
|
}); |
170
|
|
|
|
171
|
|
|
$propagator = $storage->getPropagator(); |
172
|
|
|
$propagator->beginBatch(); |
173
|
|
|
$scanner->backgroundScan(); |
174
|
|
|
$propagator->commitBatch(); |
175
|
|
|
} |
176
|
|
|
} |
177
|
|
|
|
178
|
|
|
/** |
179
|
|
|
* @param string $dir |
180
|
|
|
* @param $recursive |
181
|
|
|
* @param callable|null $mountFilter |
182
|
|
|
* @throws ForbiddenException |
183
|
|
|
* @throws NotFoundException |
184
|
|
|
*/ |
185
|
|
|
public function scan($dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECURSIVE, callable $mountFilter = null) { |
186
|
|
|
if (!Filesystem::isValidPath($dir)) { |
187
|
|
|
throw new \InvalidArgumentException('Invalid path to scan'); |
188
|
|
|
} |
189
|
|
|
$mounts = $this->getMounts($dir); |
190
|
|
|
foreach ($mounts as $mount) { |
191
|
|
|
if ($mountFilter && !$mountFilter($mount)) { |
192
|
|
|
continue; |
193
|
|
|
} |
194
|
|
|
$storage = $mount->getStorage(); |
195
|
|
|
if (is_null($storage)) { |
196
|
|
|
continue; |
197
|
|
|
} |
198
|
|
|
|
199
|
|
|
// don't bother scanning failed storages (shortcut for same result) |
200
|
|
|
if ($storage->instanceOfStorage(FailedStorage::class)) { |
201
|
|
|
continue; |
202
|
|
|
} |
203
|
|
|
|
204
|
|
|
// if the home storage isn't writable then the scanner is run as the wrong user |
205
|
|
|
if ($storage->instanceOfStorage('\OC\Files\Storage\Home') and |
|
|
|
|
206
|
|
|
(!$storage->isCreatable('') or !$storage->isCreatable('files')) |
|
|
|
|
207
|
|
|
) { |
208
|
|
|
if ($storage->file_exists('') or $storage->getCache()->inCache('')) { |
|
|
|
|
209
|
|
|
throw new ForbiddenException(); |
210
|
|
|
} else {// if the root exists in neither the cache nor the storage the user isn't setup yet |
211
|
|
|
break; |
212
|
|
|
} |
213
|
|
|
|
214
|
|
|
} |
215
|
|
|
|
216
|
|
|
// don't scan received local shares, these can be scanned when scanning the owner's storage |
217
|
|
|
if ($storage->instanceOfStorage(SharedStorage::class)) { |
218
|
|
|
continue; |
219
|
|
|
} |
220
|
|
|
$relativePath = $mount->getInternalPath($dir); |
221
|
|
|
$scanner = $storage->getScanner(); |
222
|
|
|
$scanner->setUseTransactions(false); |
223
|
|
|
$this->attachListener($mount); |
|
|
|
|
224
|
|
|
|
225
|
|
|
$scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) { |
226
|
|
|
$this->postProcessEntry($storage, $path); |
227
|
|
|
}); |
228
|
|
|
$scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) { |
229
|
|
|
$this->postProcessEntry($storage, $path); |
230
|
|
|
}); |
231
|
|
|
$scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path) use ($storage) { |
232
|
|
|
$this->postProcessEntry($storage, $path); |
233
|
|
|
}); |
234
|
|
|
|
235
|
|
|
if (!$storage->file_exists($relativePath)) { |
236
|
|
|
throw new NotFoundException($dir); |
237
|
|
|
} |
238
|
|
|
|
239
|
|
|
if ($this->useTransaction) { |
240
|
|
|
$this->db->beginTransaction(); |
241
|
|
|
} |
242
|
|
|
try { |
243
|
|
|
$propagator = $storage->getPropagator(); |
244
|
|
|
$propagator->beginBatch(); |
245
|
|
|
$scanner->scan($relativePath, $recursive, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); |
246
|
|
|
$cache = $storage->getCache(); |
247
|
|
|
if ($cache instanceof Cache) { |
248
|
|
|
// only re-calculate for the root folder we scanned, anything below that is taken care of by the scanner |
249
|
|
|
$cache->correctFolderSize($relativePath); |
250
|
|
|
} |
251
|
|
|
$propagator->commitBatch(); |
252
|
|
|
} catch (StorageNotAvailableException $e) { |
253
|
|
|
$this->logger->error('Storage ' . $storage->getId() . ' not available'); |
254
|
|
|
$this->logger->logException($e); |
|
|
|
|
255
|
|
|
$this->emit('\OC\Files\Utils\Scanner', 'StorageNotAvailable', [$e]); |
256
|
|
|
} |
257
|
|
|
if ($this->useTransaction) { |
258
|
|
|
$this->db->commit(); |
259
|
|
|
} |
260
|
|
|
} |
261
|
|
|
} |
262
|
|
|
|
263
|
|
|
private function triggerPropagator(IStorage $storage, $internalPath) { |
264
|
|
|
$storage->getPropagator()->propagateChange($internalPath, time()); |
265
|
|
|
} |
266
|
|
|
|
267
|
|
|
private function postProcessEntry(IStorage $storage, $internalPath) { |
268
|
|
|
$this->triggerPropagator($storage, $internalPath); |
269
|
|
|
if ($this->useTransaction) { |
270
|
|
|
$this->entriesToCommit++; |
271
|
|
|
if ($this->entriesToCommit >= self::MAX_ENTRIES_TO_COMMIT) { |
272
|
|
|
$propagator = $storage->getPropagator(); |
273
|
|
|
$this->entriesToCommit = 0; |
274
|
|
|
$this->db->commit(); |
275
|
|
|
$propagator->commitBatch(); |
276
|
|
|
$this->db->beginTransaction(); |
277
|
|
|
$propagator->beginBatch(); |
278
|
|
|
} |
279
|
|
|
} |
280
|
|
|
} |
281
|
|
|
} |
282
|
|
|
|
283
|
|
|
|
Unless you are absolutely sure that the expression can never be null because of other conditions, we strongly recommend to add an additional type check to your code: