@@ -35,343 +35,343 @@ |
||
35 | 35 | use Symfony\Component\Console\Output\OutputInterface; |
36 | 36 | |
37 | 37 | class Scan extends Base { |
38 | - protected float $execTime = 0; |
|
39 | - protected int $foldersCounter = 0; |
|
40 | - protected int $filesCounter = 0; |
|
41 | - protected int $errorsCounter = 0; |
|
42 | - protected int $newCounter = 0; |
|
43 | - protected int $updatedCounter = 0; |
|
44 | - protected int $removedCounter = 0; |
|
45 | - |
|
46 | - public function __construct( |
|
47 | - private IUserManager $userManager, |
|
48 | - private IRootFolder $rootFolder, |
|
49 | - private FilesMetadataManager $filesMetadataManager, |
|
50 | - private IEventDispatcher $eventDispatcher, |
|
51 | - private LoggerInterface $logger, |
|
52 | - ) { |
|
53 | - parent::__construct(); |
|
54 | - } |
|
55 | - |
|
56 | - protected function configure(): void { |
|
57 | - parent::configure(); |
|
58 | - |
|
59 | - $this |
|
60 | - ->setName('files:scan') |
|
61 | - ->setDescription('rescan filesystem') |
|
62 | - ->addArgument( |
|
63 | - 'user_id', |
|
64 | - InputArgument::OPTIONAL | InputArgument::IS_ARRAY, |
|
65 | - 'will rescan all files of the given user(s)' |
|
66 | - ) |
|
67 | - ->addOption( |
|
68 | - 'path', |
|
69 | - 'p', |
|
70 | - InputOption::VALUE_REQUIRED, |
|
71 | - 'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored' |
|
72 | - ) |
|
73 | - ->addOption( |
|
74 | - 'generate-metadata', |
|
75 | - null, |
|
76 | - InputOption::VALUE_OPTIONAL, |
|
77 | - 'Generate metadata for all scanned files; if specified only generate for named value', |
|
78 | - '' |
|
79 | - ) |
|
80 | - ->addOption( |
|
81 | - 'all', |
|
82 | - null, |
|
83 | - InputOption::VALUE_NONE, |
|
84 | - 'will rescan all files of all known users' |
|
85 | - )->addOption( |
|
86 | - 'unscanned', |
|
87 | - null, |
|
88 | - InputOption::VALUE_NONE, |
|
89 | - 'only scan files which are marked as not fully scanned' |
|
90 | - )->addOption( |
|
91 | - 'shallow', |
|
92 | - null, |
|
93 | - InputOption::VALUE_NONE, |
|
94 | - 'do not scan folders recursively' |
|
95 | - )->addOption( |
|
96 | - 'home-only', |
|
97 | - null, |
|
98 | - InputOption::VALUE_NONE, |
|
99 | - 'only scan the home storage, ignoring any mounted external storage or share' |
|
100 | - ); |
|
101 | - } |
|
102 | - |
|
103 | - protected function scanFiles( |
|
104 | - string $user, |
|
105 | - string $path, |
|
106 | - ?string $scanMetadata, |
|
107 | - OutputInterface $output, |
|
108 | - callable $mountFilter, |
|
109 | - bool $backgroundScan = false, |
|
110 | - bool $recursive = true, |
|
111 | - ): void { |
|
112 | - $connection = $this->reconnectToDatabase($output); |
|
113 | - $scanner = new Scanner( |
|
114 | - $user, |
|
115 | - new ConnectionAdapter($connection), |
|
116 | - Server::get(IEventDispatcher::class), |
|
117 | - Server::get(LoggerInterface::class) |
|
118 | - ); |
|
119 | - |
|
120 | - # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception |
|
121 | - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function (string $path) use ($output, $scanMetadata): void { |
|
122 | - $output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
|
123 | - ++$this->filesCounter; |
|
124 | - $this->abortIfInterrupted(); |
|
125 | - if ($scanMetadata !== null) { |
|
126 | - $node = $this->rootFolder->get($path); |
|
127 | - $this->filesMetadataManager->refreshMetadata( |
|
128 | - $node, |
|
129 | - ($scanMetadata !== '') ? IFilesMetadataManager::PROCESS_NAMED : IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND, |
|
130 | - $scanMetadata |
|
131 | - ); |
|
132 | - } |
|
133 | - }); |
|
134 | - |
|
135 | - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output): void { |
|
136 | - $output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
|
137 | - ++$this->foldersCounter; |
|
138 | - $this->abortIfInterrupted(); |
|
139 | - }); |
|
140 | - |
|
141 | - $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output): void { |
|
142 | - $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); |
|
143 | - ++$this->errorsCounter; |
|
144 | - }); |
|
145 | - |
|
146 | - $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function ($fullPath) use ($output): void { |
|
147 | - $output->writeln("\t<error>Entry \"" . $fullPath . '" will not be accessible due to incompatible encoding</error>'); |
|
148 | - ++$this->errorsCounter; |
|
149 | - }); |
|
150 | - |
|
151 | - $this->eventDispatcher->addListener(NodeAddedToCache::class, function (): void { |
|
152 | - ++$this->newCounter; |
|
153 | - }); |
|
154 | - $this->eventDispatcher->addListener(FileCacheUpdated::class, function (): void { |
|
155 | - ++$this->updatedCounter; |
|
156 | - }); |
|
157 | - $this->eventDispatcher->addListener(NodeRemovedFromCache::class, function (): void { |
|
158 | - ++$this->removedCounter; |
|
159 | - }); |
|
160 | - |
|
161 | - try { |
|
162 | - if ($backgroundScan) { |
|
163 | - $scanner->backgroundScan($path); |
|
164 | - } else { |
|
165 | - $scanner->scan($path, $recursive, $mountFilter); |
|
166 | - } |
|
167 | - } catch (ForbiddenException $e) { |
|
168 | - $output->writeln("<error>Home storage for user $user not writable or 'files' subdirectory missing</error>"); |
|
169 | - $output->writeln(' ' . $e->getMessage()); |
|
170 | - $output->writeln('Make sure you\'re running the scan command only as the user the web server runs as'); |
|
171 | - ++$this->errorsCounter; |
|
172 | - } catch (InterruptedException $e) { |
|
173 | - # exit the function if ctrl-c has been pressed |
|
174 | - $output->writeln('Interrupted by user'); |
|
175 | - } catch (NotFoundException $e) { |
|
176 | - $output->writeln('<error>Path not found: ' . $e->getMessage() . '</error>'); |
|
177 | - ++$this->errorsCounter; |
|
178 | - } catch (LockedException $e) { |
|
179 | - if (str_starts_with($e->getPath(), 'scanner::')) { |
|
180 | - $output->writeln('<error>Another process is already scanning \'' . substr($e->getPath(), strlen('scanner::')) . '\'</error>'); |
|
181 | - } else { |
|
182 | - throw $e; |
|
183 | - } |
|
184 | - } catch (\Exception $e) { |
|
185 | - $output->writeln('<error>Exception during scan: ' . $e->getMessage() . '</error>'); |
|
186 | - $output->writeln('<error>' . $e->getTraceAsString() . '</error>'); |
|
187 | - ++$this->errorsCounter; |
|
188 | - } |
|
189 | - } |
|
190 | - |
|
191 | - public function isHomeMount(IMountPoint $mountPoint): bool { |
|
192 | - // any mountpoint inside '/$user/files/' |
|
193 | - return substr_count($mountPoint->getMountPoint(), '/') <= 3; |
|
194 | - } |
|
195 | - |
|
196 | - protected function execute(InputInterface $input, OutputInterface $output): int { |
|
197 | - $inputPath = $input->getOption('path'); |
|
198 | - if ($inputPath) { |
|
199 | - $inputPath = '/' . trim($inputPath, '/'); |
|
200 | - [, $user,] = explode('/', $inputPath, 3); |
|
201 | - $users = [$user]; |
|
202 | - } elseif ($input->getOption('all')) { |
|
203 | - $users = $this->userManager->search(''); |
|
204 | - } else { |
|
205 | - $users = $input->getArgument('user_id'); |
|
206 | - } |
|
207 | - |
|
208 | - # check quantity of users to be process and show it on the command line |
|
209 | - $users_total = count($users); |
|
210 | - if ($users_total === 0) { |
|
211 | - $output->writeln('<error>Please specify the user id to scan, --all to scan for all users or --path=...</error>'); |
|
212 | - return self::FAILURE; |
|
213 | - } |
|
214 | - |
|
215 | - $this->initTools($output); |
|
216 | - |
|
217 | - // getOption() logic on VALUE_OPTIONAL |
|
218 | - $metadata = null; // null if --generate-metadata is not set, empty if option have no value, value if set |
|
219 | - if ($input->getOption('generate-metadata') !== '') { |
|
220 | - $metadata = $input->getOption('generate-metadata') ?? ''; |
|
221 | - } |
|
222 | - |
|
223 | - $homeOnly = $input->getOption('home-only'); |
|
224 | - $scannedStorages = []; |
|
225 | - $mountFilter = function (IMountPoint $mount) use ($homeOnly, &$scannedStorages) { |
|
226 | - if ($homeOnly && !$this->isHomeMount($mount)) { |
|
227 | - return false; |
|
228 | - } |
|
229 | - |
|
230 | - // when scanning multiple users, the scanner might encounter the same storage multiple times (e.g. external storages, or group folders) |
|
231 | - // we can filter out any storage we've already scanned to avoid double work |
|
232 | - $storage = $mount->getStorage(); |
|
233 | - $storageKey = $storage->getId(); |
|
234 | - while ($storage->instanceOfStorage(Jail::class)) { |
|
235 | - $storageKey .= '/' . $storage->getUnjailedPath(''); |
|
236 | - $storage = $storage->getUnjailedStorage(); |
|
237 | - } |
|
238 | - if (array_key_exists($storageKey, $scannedStorages)) { |
|
239 | - return false; |
|
240 | - } |
|
241 | - |
|
242 | - $scannedStorages[$storageKey] = true; |
|
243 | - return true; |
|
244 | - }; |
|
245 | - |
|
246 | - $user_count = 0; |
|
247 | - foreach ($users as $user) { |
|
248 | - if (is_object($user)) { |
|
249 | - $user = $user->getUID(); |
|
250 | - } |
|
251 | - $path = $inputPath ?: '/' . $user; |
|
252 | - ++$user_count; |
|
253 | - if ($this->userManager->userExists($user)) { |
|
254 | - $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); |
|
255 | - $this->scanFiles( |
|
256 | - $user, |
|
257 | - $path, |
|
258 | - $metadata, |
|
259 | - $output, |
|
260 | - $mountFilter, |
|
261 | - $input->getOption('unscanned'), |
|
262 | - !$input->getOption('shallow'), |
|
263 | - ); |
|
264 | - $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); |
|
265 | - } else { |
|
266 | - $output->writeln("<error>Unknown user $user_count $user</error>"); |
|
267 | - $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); |
|
268 | - } |
|
269 | - |
|
270 | - try { |
|
271 | - $this->abortIfInterrupted(); |
|
272 | - } catch (InterruptedException $e) { |
|
273 | - break; |
|
274 | - } |
|
275 | - } |
|
276 | - |
|
277 | - $this->presentStats($output); |
|
278 | - return self::SUCCESS; |
|
279 | - } |
|
280 | - |
|
281 | - /** |
|
282 | - * Initialises some useful tools for the Command |
|
283 | - */ |
|
284 | - protected function initTools(OutputInterface $output): void { |
|
285 | - // Start the timer |
|
286 | - $this->execTime = -microtime(true); |
|
287 | - // Convert PHP errors to exceptions |
|
288 | - set_error_handler( |
|
289 | - fn (int $severity, string $message, string $file, int $line): bool => |
|
290 | - $this->exceptionErrorHandler($output, $severity, $message, $file, $line), |
|
291 | - E_ALL |
|
292 | - ); |
|
293 | - } |
|
294 | - |
|
295 | - /** |
|
296 | - * Processes PHP errors in order to be able to show them in the output |
|
297 | - * |
|
298 | - * @see https://www.php.net/manual/en/function.set-error-handler.php |
|
299 | - * |
|
300 | - * @param int $severity the level of the error raised |
|
301 | - * @param string $message |
|
302 | - * @param string $file the filename that the error was raised in |
|
303 | - * @param int $line the line number the error was raised |
|
304 | - */ |
|
305 | - public function exceptionErrorHandler(OutputInterface $output, int $severity, string $message, string $file, int $line): bool { |
|
306 | - if (($severity === E_DEPRECATED) || ($severity === E_USER_DEPRECATED)) { |
|
307 | - // Do not show deprecation warnings |
|
308 | - return false; |
|
309 | - } |
|
310 | - $e = new \ErrorException($message, 0, $severity, $file, $line); |
|
311 | - $output->writeln('<error>Error during scan: ' . $e->getMessage() . '</error>'); |
|
312 | - $output->writeln('<error>' . $e->getTraceAsString() . '</error>', OutputInterface::VERBOSITY_VERY_VERBOSE); |
|
313 | - ++$this->errorsCounter; |
|
314 | - return true; |
|
315 | - } |
|
316 | - |
|
317 | - protected function presentStats(OutputInterface $output): void { |
|
318 | - // Stop the timer |
|
319 | - $this->execTime += microtime(true); |
|
320 | - |
|
321 | - $this->logger->info("Completed scan of {$this->filesCounter} files in {$this->foldersCounter} folder. Found {$this->newCounter} new, {$this->updatedCounter} updated and {$this->removedCounter} removed items"); |
|
322 | - |
|
323 | - $headers = [ |
|
324 | - 'Folders', |
|
325 | - 'Files', |
|
326 | - 'New', |
|
327 | - 'Updated', |
|
328 | - 'Removed', |
|
329 | - 'Errors', |
|
330 | - 'Elapsed time', |
|
331 | - ]; |
|
332 | - $niceDate = $this->formatExecTime(); |
|
333 | - $rows = [ |
|
334 | - $this->foldersCounter, |
|
335 | - $this->filesCounter, |
|
336 | - $this->newCounter, |
|
337 | - $this->updatedCounter, |
|
338 | - $this->removedCounter, |
|
339 | - $this->errorsCounter, |
|
340 | - $niceDate, |
|
341 | - ]; |
|
342 | - $table = new Table($output); |
|
343 | - $table |
|
344 | - ->setHeaders($headers) |
|
345 | - ->setRows([$rows]); |
|
346 | - $table->render(); |
|
347 | - } |
|
348 | - |
|
349 | - |
|
350 | - /** |
|
351 | - * Formats microtime into a human-readable format |
|
352 | - */ |
|
353 | - protected function formatExecTime(): string { |
|
354 | - $secs = (int)round($this->execTime); |
|
355 | - # convert seconds into HH:MM:SS form |
|
356 | - return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60); |
|
357 | - } |
|
358 | - |
|
359 | - protected function reconnectToDatabase(OutputInterface $output): Connection { |
|
360 | - /** @var Connection $connection */ |
|
361 | - $connection = Server::get(Connection::class); |
|
362 | - try { |
|
363 | - $connection->close(); |
|
364 | - } catch (\Exception $ex) { |
|
365 | - $output->writeln("<info>Error while disconnecting from database: {$ex->getMessage()}</info>"); |
|
366 | - } |
|
367 | - while (!$connection->isConnected()) { |
|
368 | - try { |
|
369 | - $connection->connect(); |
|
370 | - } catch (\Exception $ex) { |
|
371 | - $output->writeln("<info>Error while re-connecting to database: {$ex->getMessage()}</info>"); |
|
372 | - sleep(60); |
|
373 | - } |
|
374 | - } |
|
375 | - return $connection; |
|
376 | - } |
|
38 | + protected float $execTime = 0; |
|
39 | + protected int $foldersCounter = 0; |
|
40 | + protected int $filesCounter = 0; |
|
41 | + protected int $errorsCounter = 0; |
|
42 | + protected int $newCounter = 0; |
|
43 | + protected int $updatedCounter = 0; |
|
44 | + protected int $removedCounter = 0; |
|
45 | + |
|
46 | + public function __construct( |
|
47 | + private IUserManager $userManager, |
|
48 | + private IRootFolder $rootFolder, |
|
49 | + private FilesMetadataManager $filesMetadataManager, |
|
50 | + private IEventDispatcher $eventDispatcher, |
|
51 | + private LoggerInterface $logger, |
|
52 | + ) { |
|
53 | + parent::__construct(); |
|
54 | + } |
|
55 | + |
|
56 | + protected function configure(): void { |
|
57 | + parent::configure(); |
|
58 | + |
|
59 | + $this |
|
60 | + ->setName('files:scan') |
|
61 | + ->setDescription('rescan filesystem') |
|
62 | + ->addArgument( |
|
63 | + 'user_id', |
|
64 | + InputArgument::OPTIONAL | InputArgument::IS_ARRAY, |
|
65 | + 'will rescan all files of the given user(s)' |
|
66 | + ) |
|
67 | + ->addOption( |
|
68 | + 'path', |
|
69 | + 'p', |
|
70 | + InputOption::VALUE_REQUIRED, |
|
71 | + 'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored' |
|
72 | + ) |
|
73 | + ->addOption( |
|
74 | + 'generate-metadata', |
|
75 | + null, |
|
76 | + InputOption::VALUE_OPTIONAL, |
|
77 | + 'Generate metadata for all scanned files; if specified only generate for named value', |
|
78 | + '' |
|
79 | + ) |
|
80 | + ->addOption( |
|
81 | + 'all', |
|
82 | + null, |
|
83 | + InputOption::VALUE_NONE, |
|
84 | + 'will rescan all files of all known users' |
|
85 | + )->addOption( |
|
86 | + 'unscanned', |
|
87 | + null, |
|
88 | + InputOption::VALUE_NONE, |
|
89 | + 'only scan files which are marked as not fully scanned' |
|
90 | + )->addOption( |
|
91 | + 'shallow', |
|
92 | + null, |
|
93 | + InputOption::VALUE_NONE, |
|
94 | + 'do not scan folders recursively' |
|
95 | + )->addOption( |
|
96 | + 'home-only', |
|
97 | + null, |
|
98 | + InputOption::VALUE_NONE, |
|
99 | + 'only scan the home storage, ignoring any mounted external storage or share' |
|
100 | + ); |
|
101 | + } |
|
102 | + |
|
103 | + protected function scanFiles( |
|
104 | + string $user, |
|
105 | + string $path, |
|
106 | + ?string $scanMetadata, |
|
107 | + OutputInterface $output, |
|
108 | + callable $mountFilter, |
|
109 | + bool $backgroundScan = false, |
|
110 | + bool $recursive = true, |
|
111 | + ): void { |
|
112 | + $connection = $this->reconnectToDatabase($output); |
|
113 | + $scanner = new Scanner( |
|
114 | + $user, |
|
115 | + new ConnectionAdapter($connection), |
|
116 | + Server::get(IEventDispatcher::class), |
|
117 | + Server::get(LoggerInterface::class) |
|
118 | + ); |
|
119 | + |
|
120 | + # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception |
|
121 | + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function (string $path) use ($output, $scanMetadata): void { |
|
122 | + $output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
|
123 | + ++$this->filesCounter; |
|
124 | + $this->abortIfInterrupted(); |
|
125 | + if ($scanMetadata !== null) { |
|
126 | + $node = $this->rootFolder->get($path); |
|
127 | + $this->filesMetadataManager->refreshMetadata( |
|
128 | + $node, |
|
129 | + ($scanMetadata !== '') ? IFilesMetadataManager::PROCESS_NAMED : IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND, |
|
130 | + $scanMetadata |
|
131 | + ); |
|
132 | + } |
|
133 | + }); |
|
134 | + |
|
135 | + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output): void { |
|
136 | + $output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
|
137 | + ++$this->foldersCounter; |
|
138 | + $this->abortIfInterrupted(); |
|
139 | + }); |
|
140 | + |
|
141 | + $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output): void { |
|
142 | + $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); |
|
143 | + ++$this->errorsCounter; |
|
144 | + }); |
|
145 | + |
|
146 | + $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function ($fullPath) use ($output): void { |
|
147 | + $output->writeln("\t<error>Entry \"" . $fullPath . '" will not be accessible due to incompatible encoding</error>'); |
|
148 | + ++$this->errorsCounter; |
|
149 | + }); |
|
150 | + |
|
151 | + $this->eventDispatcher->addListener(NodeAddedToCache::class, function (): void { |
|
152 | + ++$this->newCounter; |
|
153 | + }); |
|
154 | + $this->eventDispatcher->addListener(FileCacheUpdated::class, function (): void { |
|
155 | + ++$this->updatedCounter; |
|
156 | + }); |
|
157 | + $this->eventDispatcher->addListener(NodeRemovedFromCache::class, function (): void { |
|
158 | + ++$this->removedCounter; |
|
159 | + }); |
|
160 | + |
|
161 | + try { |
|
162 | + if ($backgroundScan) { |
|
163 | + $scanner->backgroundScan($path); |
|
164 | + } else { |
|
165 | + $scanner->scan($path, $recursive, $mountFilter); |
|
166 | + } |
|
167 | + } catch (ForbiddenException $e) { |
|
168 | + $output->writeln("<error>Home storage for user $user not writable or 'files' subdirectory missing</error>"); |
|
169 | + $output->writeln(' ' . $e->getMessage()); |
|
170 | + $output->writeln('Make sure you\'re running the scan command only as the user the web server runs as'); |
|
171 | + ++$this->errorsCounter; |
|
172 | + } catch (InterruptedException $e) { |
|
173 | + # exit the function if ctrl-c has been pressed |
|
174 | + $output->writeln('Interrupted by user'); |
|
175 | + } catch (NotFoundException $e) { |
|
176 | + $output->writeln('<error>Path not found: ' . $e->getMessage() . '</error>'); |
|
177 | + ++$this->errorsCounter; |
|
178 | + } catch (LockedException $e) { |
|
179 | + if (str_starts_with($e->getPath(), 'scanner::')) { |
|
180 | + $output->writeln('<error>Another process is already scanning \'' . substr($e->getPath(), strlen('scanner::')) . '\'</error>'); |
|
181 | + } else { |
|
182 | + throw $e; |
|
183 | + } |
|
184 | + } catch (\Exception $e) { |
|
185 | + $output->writeln('<error>Exception during scan: ' . $e->getMessage() . '</error>'); |
|
186 | + $output->writeln('<error>' . $e->getTraceAsString() . '</error>'); |
|
187 | + ++$this->errorsCounter; |
|
188 | + } |
|
189 | + } |
|
190 | + |
|
191 | + public function isHomeMount(IMountPoint $mountPoint): bool { |
|
192 | + // any mountpoint inside '/$user/files/' |
|
193 | + return substr_count($mountPoint->getMountPoint(), '/') <= 3; |
|
194 | + } |
|
195 | + |
|
196 | + protected function execute(InputInterface $input, OutputInterface $output): int { |
|
197 | + $inputPath = $input->getOption('path'); |
|
198 | + if ($inputPath) { |
|
199 | + $inputPath = '/' . trim($inputPath, '/'); |
|
200 | + [, $user,] = explode('/', $inputPath, 3); |
|
201 | + $users = [$user]; |
|
202 | + } elseif ($input->getOption('all')) { |
|
203 | + $users = $this->userManager->search(''); |
|
204 | + } else { |
|
205 | + $users = $input->getArgument('user_id'); |
|
206 | + } |
|
207 | + |
|
208 | + # check quantity of users to be process and show it on the command line |
|
209 | + $users_total = count($users); |
|
210 | + if ($users_total === 0) { |
|
211 | + $output->writeln('<error>Please specify the user id to scan, --all to scan for all users or --path=...</error>'); |
|
212 | + return self::FAILURE; |
|
213 | + } |
|
214 | + |
|
215 | + $this->initTools($output); |
|
216 | + |
|
217 | + // getOption() logic on VALUE_OPTIONAL |
|
218 | + $metadata = null; // null if --generate-metadata is not set, empty if option have no value, value if set |
|
219 | + if ($input->getOption('generate-metadata') !== '') { |
|
220 | + $metadata = $input->getOption('generate-metadata') ?? ''; |
|
221 | + } |
|
222 | + |
|
223 | + $homeOnly = $input->getOption('home-only'); |
|
224 | + $scannedStorages = []; |
|
225 | + $mountFilter = function (IMountPoint $mount) use ($homeOnly, &$scannedStorages) { |
|
226 | + if ($homeOnly && !$this->isHomeMount($mount)) { |
|
227 | + return false; |
|
228 | + } |
|
229 | + |
|
230 | + // when scanning multiple users, the scanner might encounter the same storage multiple times (e.g. external storages, or group folders) |
|
231 | + // we can filter out any storage we've already scanned to avoid double work |
|
232 | + $storage = $mount->getStorage(); |
|
233 | + $storageKey = $storage->getId(); |
|
234 | + while ($storage->instanceOfStorage(Jail::class)) { |
|
235 | + $storageKey .= '/' . $storage->getUnjailedPath(''); |
|
236 | + $storage = $storage->getUnjailedStorage(); |
|
237 | + } |
|
238 | + if (array_key_exists($storageKey, $scannedStorages)) { |
|
239 | + return false; |
|
240 | + } |
|
241 | + |
|
242 | + $scannedStorages[$storageKey] = true; |
|
243 | + return true; |
|
244 | + }; |
|
245 | + |
|
246 | + $user_count = 0; |
|
247 | + foreach ($users as $user) { |
|
248 | + if (is_object($user)) { |
|
249 | + $user = $user->getUID(); |
|
250 | + } |
|
251 | + $path = $inputPath ?: '/' . $user; |
|
252 | + ++$user_count; |
|
253 | + if ($this->userManager->userExists($user)) { |
|
254 | + $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); |
|
255 | + $this->scanFiles( |
|
256 | + $user, |
|
257 | + $path, |
|
258 | + $metadata, |
|
259 | + $output, |
|
260 | + $mountFilter, |
|
261 | + $input->getOption('unscanned'), |
|
262 | + !$input->getOption('shallow'), |
|
263 | + ); |
|
264 | + $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); |
|
265 | + } else { |
|
266 | + $output->writeln("<error>Unknown user $user_count $user</error>"); |
|
267 | + $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); |
|
268 | + } |
|
269 | + |
|
270 | + try { |
|
271 | + $this->abortIfInterrupted(); |
|
272 | + } catch (InterruptedException $e) { |
|
273 | + break; |
|
274 | + } |
|
275 | + } |
|
276 | + |
|
277 | + $this->presentStats($output); |
|
278 | + return self::SUCCESS; |
|
279 | + } |
|
280 | + |
|
281 | + /** |
|
282 | + * Initialises some useful tools for the Command |
|
283 | + */ |
|
284 | + protected function initTools(OutputInterface $output): void { |
|
285 | + // Start the timer |
|
286 | + $this->execTime = -microtime(true); |
|
287 | + // Convert PHP errors to exceptions |
|
288 | + set_error_handler( |
|
289 | + fn (int $severity, string $message, string $file, int $line): bool => |
|
290 | + $this->exceptionErrorHandler($output, $severity, $message, $file, $line), |
|
291 | + E_ALL |
|
292 | + ); |
|
293 | + } |
|
294 | + |
|
295 | + /** |
|
296 | + * Processes PHP errors in order to be able to show them in the output |
|
297 | + * |
|
298 | + * @see https://www.php.net/manual/en/function.set-error-handler.php |
|
299 | + * |
|
300 | + * @param int $severity the level of the error raised |
|
301 | + * @param string $message |
|
302 | + * @param string $file the filename that the error was raised in |
|
303 | + * @param int $line the line number the error was raised |
|
304 | + */ |
|
305 | + public function exceptionErrorHandler(OutputInterface $output, int $severity, string $message, string $file, int $line): bool { |
|
306 | + if (($severity === E_DEPRECATED) || ($severity === E_USER_DEPRECATED)) { |
|
307 | + // Do not show deprecation warnings |
|
308 | + return false; |
|
309 | + } |
|
310 | + $e = new \ErrorException($message, 0, $severity, $file, $line); |
|
311 | + $output->writeln('<error>Error during scan: ' . $e->getMessage() . '</error>'); |
|
312 | + $output->writeln('<error>' . $e->getTraceAsString() . '</error>', OutputInterface::VERBOSITY_VERY_VERBOSE); |
|
313 | + ++$this->errorsCounter; |
|
314 | + return true; |
|
315 | + } |
|
316 | + |
|
317 | + protected function presentStats(OutputInterface $output): void { |
|
318 | + // Stop the timer |
|
319 | + $this->execTime += microtime(true); |
|
320 | + |
|
321 | + $this->logger->info("Completed scan of {$this->filesCounter} files in {$this->foldersCounter} folder. Found {$this->newCounter} new, {$this->updatedCounter} updated and {$this->removedCounter} removed items"); |
|
322 | + |
|
323 | + $headers = [ |
|
324 | + 'Folders', |
|
325 | + 'Files', |
|
326 | + 'New', |
|
327 | + 'Updated', |
|
328 | + 'Removed', |
|
329 | + 'Errors', |
|
330 | + 'Elapsed time', |
|
331 | + ]; |
|
332 | + $niceDate = $this->formatExecTime(); |
|
333 | + $rows = [ |
|
334 | + $this->foldersCounter, |
|
335 | + $this->filesCounter, |
|
336 | + $this->newCounter, |
|
337 | + $this->updatedCounter, |
|
338 | + $this->removedCounter, |
|
339 | + $this->errorsCounter, |
|
340 | + $niceDate, |
|
341 | + ]; |
|
342 | + $table = new Table($output); |
|
343 | + $table |
|
344 | + ->setHeaders($headers) |
|
345 | + ->setRows([$rows]); |
|
346 | + $table->render(); |
|
347 | + } |
|
348 | + |
|
349 | + |
|
350 | + /** |
|
351 | + * Formats microtime into a human-readable format |
|
352 | + */ |
|
353 | + protected function formatExecTime(): string { |
|
354 | + $secs = (int)round($this->execTime); |
|
355 | + # convert seconds into HH:MM:SS form |
|
356 | + return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60); |
|
357 | + } |
|
358 | + |
|
359 | + protected function reconnectToDatabase(OutputInterface $output): Connection { |
|
360 | + /** @var Connection $connection */ |
|
361 | + $connection = Server::get(Connection::class); |
|
362 | + try { |
|
363 | + $connection->close(); |
|
364 | + } catch (\Exception $ex) { |
|
365 | + $output->writeln("<info>Error while disconnecting from database: {$ex->getMessage()}</info>"); |
|
366 | + } |
|
367 | + while (!$connection->isConnected()) { |
|
368 | + try { |
|
369 | + $connection->connect(); |
|
370 | + } catch (\Exception $ex) { |
|
371 | + $output->writeln("<info>Error while re-connecting to database: {$ex->getMessage()}</info>"); |
|
372 | + sleep(60); |
|
373 | + } |
|
374 | + } |
|
375 | + return $connection; |
|
376 | + } |
|
377 | 377 | } |
@@ -118,7 +118,7 @@ discard block |
||
118 | 118 | ); |
119 | 119 | |
120 | 120 | # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception |
121 | - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function (string $path) use ($output, $scanMetadata): void { |
|
121 | + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function(string $path) use ($output, $scanMetadata): void { |
|
122 | 122 | $output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
123 | 123 | ++$this->filesCounter; |
124 | 124 | $this->abortIfInterrupted(); |
@@ -132,29 +132,29 @@ discard block |
||
132 | 132 | } |
133 | 133 | }); |
134 | 134 | |
135 | - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output): void { |
|
135 | + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function($path) use ($output): void { |
|
136 | 136 | $output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
137 | 137 | ++$this->foldersCounter; |
138 | 138 | $this->abortIfInterrupted(); |
139 | 139 | }); |
140 | 140 | |
141 | - $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output): void { |
|
142 | - $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); |
|
141 | + $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function(StorageNotAvailableException $e) use ($output): void { |
|
142 | + $output->writeln('Error while scanning, storage not available ('.$e->getMessage().')', OutputInterface::VERBOSITY_VERBOSE); |
|
143 | 143 | ++$this->errorsCounter; |
144 | 144 | }); |
145 | 145 | |
146 | - $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function ($fullPath) use ($output): void { |
|
147 | - $output->writeln("\t<error>Entry \"" . $fullPath . '" will not be accessible due to incompatible encoding</error>'); |
|
146 | + $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function($fullPath) use ($output): void { |
|
147 | + $output->writeln("\t<error>Entry \"".$fullPath.'" will not be accessible due to incompatible encoding</error>'); |
|
148 | 148 | ++$this->errorsCounter; |
149 | 149 | }); |
150 | 150 | |
151 | - $this->eventDispatcher->addListener(NodeAddedToCache::class, function (): void { |
|
151 | + $this->eventDispatcher->addListener(NodeAddedToCache::class, function(): void { |
|
152 | 152 | ++$this->newCounter; |
153 | 153 | }); |
154 | - $this->eventDispatcher->addListener(FileCacheUpdated::class, function (): void { |
|
154 | + $this->eventDispatcher->addListener(FileCacheUpdated::class, function(): void { |
|
155 | 155 | ++$this->updatedCounter; |
156 | 156 | }); |
157 | - $this->eventDispatcher->addListener(NodeRemovedFromCache::class, function (): void { |
|
157 | + $this->eventDispatcher->addListener(NodeRemovedFromCache::class, function(): void { |
|
158 | 158 | ++$this->removedCounter; |
159 | 159 | }); |
160 | 160 | |
@@ -166,24 +166,24 @@ discard block |
||
166 | 166 | } |
167 | 167 | } catch (ForbiddenException $e) { |
168 | 168 | $output->writeln("<error>Home storage for user $user not writable or 'files' subdirectory missing</error>"); |
169 | - $output->writeln(' ' . $e->getMessage()); |
|
169 | + $output->writeln(' '.$e->getMessage()); |
|
170 | 170 | $output->writeln('Make sure you\'re running the scan command only as the user the web server runs as'); |
171 | 171 | ++$this->errorsCounter; |
172 | 172 | } catch (InterruptedException $e) { |
173 | 173 | # exit the function if ctrl-c has been pressed |
174 | 174 | $output->writeln('Interrupted by user'); |
175 | 175 | } catch (NotFoundException $e) { |
176 | - $output->writeln('<error>Path not found: ' . $e->getMessage() . '</error>'); |
|
176 | + $output->writeln('<error>Path not found: '.$e->getMessage().'</error>'); |
|
177 | 177 | ++$this->errorsCounter; |
178 | 178 | } catch (LockedException $e) { |
179 | 179 | if (str_starts_with($e->getPath(), 'scanner::')) { |
180 | - $output->writeln('<error>Another process is already scanning \'' . substr($e->getPath(), strlen('scanner::')) . '\'</error>'); |
|
180 | + $output->writeln('<error>Another process is already scanning \''.substr($e->getPath(), strlen('scanner::')).'\'</error>'); |
|
181 | 181 | } else { |
182 | 182 | throw $e; |
183 | 183 | } |
184 | 184 | } catch (\Exception $e) { |
185 | - $output->writeln('<error>Exception during scan: ' . $e->getMessage() . '</error>'); |
|
186 | - $output->writeln('<error>' . $e->getTraceAsString() . '</error>'); |
|
185 | + $output->writeln('<error>Exception during scan: '.$e->getMessage().'</error>'); |
|
186 | + $output->writeln('<error>'.$e->getTraceAsString().'</error>'); |
|
187 | 187 | ++$this->errorsCounter; |
188 | 188 | } |
189 | 189 | } |
@@ -196,8 +196,8 @@ discard block |
||
196 | 196 | protected function execute(InputInterface $input, OutputInterface $output): int { |
197 | 197 | $inputPath = $input->getOption('path'); |
198 | 198 | if ($inputPath) { |
199 | - $inputPath = '/' . trim($inputPath, '/'); |
|
200 | - [, $user,] = explode('/', $inputPath, 3); |
|
199 | + $inputPath = '/'.trim($inputPath, '/'); |
|
200 | + [, $user, ] = explode('/', $inputPath, 3); |
|
201 | 201 | $users = [$user]; |
202 | 202 | } elseif ($input->getOption('all')) { |
203 | 203 | $users = $this->userManager->search(''); |
@@ -222,7 +222,7 @@ discard block |
||
222 | 222 | |
223 | 223 | $homeOnly = $input->getOption('home-only'); |
224 | 224 | $scannedStorages = []; |
225 | - $mountFilter = function (IMountPoint $mount) use ($homeOnly, &$scannedStorages) { |
|
225 | + $mountFilter = function(IMountPoint $mount) use ($homeOnly, &$scannedStorages) { |
|
226 | 226 | if ($homeOnly && !$this->isHomeMount($mount)) { |
227 | 227 | return false; |
228 | 228 | } |
@@ -232,7 +232,7 @@ discard block |
||
232 | 232 | $storage = $mount->getStorage(); |
233 | 233 | $storageKey = $storage->getId(); |
234 | 234 | while ($storage->instanceOfStorage(Jail::class)) { |
235 | - $storageKey .= '/' . $storage->getUnjailedPath(''); |
|
235 | + $storageKey .= '/'.$storage->getUnjailedPath(''); |
|
236 | 236 | $storage = $storage->getUnjailedStorage(); |
237 | 237 | } |
238 | 238 | if (array_key_exists($storageKey, $scannedStorages)) { |
@@ -248,7 +248,7 @@ discard block |
||
248 | 248 | if (is_object($user)) { |
249 | 249 | $user = $user->getUID(); |
250 | 250 | } |
251 | - $path = $inputPath ?: '/' . $user; |
|
251 | + $path = $inputPath ?: '/'.$user; |
|
252 | 252 | ++$user_count; |
253 | 253 | if ($this->userManager->userExists($user)) { |
254 | 254 | $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); |
@@ -308,8 +308,8 @@ discard block |
||
308 | 308 | return false; |
309 | 309 | } |
310 | 310 | $e = new \ErrorException($message, 0, $severity, $file, $line); |
311 | - $output->writeln('<error>Error during scan: ' . $e->getMessage() . '</error>'); |
|
312 | - $output->writeln('<error>' . $e->getTraceAsString() . '</error>', OutputInterface::VERBOSITY_VERY_VERBOSE); |
|
311 | + $output->writeln('<error>Error during scan: '.$e->getMessage().'</error>'); |
|
312 | + $output->writeln('<error>'.$e->getTraceAsString().'</error>', OutputInterface::VERBOSITY_VERY_VERBOSE); |
|
313 | 313 | ++$this->errorsCounter; |
314 | 314 | return true; |
315 | 315 | } |
@@ -351,9 +351,9 @@ discard block |
||
351 | 351 | * Formats microtime into a human-readable format |
352 | 352 | */ |
353 | 353 | protected function formatExecTime(): string { |
354 | - $secs = (int)round($this->execTime); |
|
354 | + $secs = (int) round($this->execTime); |
|
355 | 355 | # convert seconds into HH:MM:SS form |
356 | - return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60); |
|
356 | + return sprintf('%02d:%02d:%02d', (int) ($secs / 3600), ((int) ($secs / 60) % 60), $secs % 60); |
|
357 | 357 | } |
358 | 358 | |
359 | 359 | protected function reconnectToDatabase(OutputInterface $output): Connection { |