@@ -35,343 +35,343 @@ |
||
| 35 | 35 | use Symfony\Component\Console\Output\OutputInterface; |
| 36 | 36 | |
| 37 | 37 | class Scan extends Base { |
| 38 | - protected float $execTime = 0; |
|
| 39 | - protected int $foldersCounter = 0; |
|
| 40 | - protected int $filesCounter = 0; |
|
| 41 | - protected int $errorsCounter = 0; |
|
| 42 | - protected int $newCounter = 0; |
|
| 43 | - protected int $updatedCounter = 0; |
|
| 44 | - protected int $removedCounter = 0; |
|
| 45 | - |
|
| 46 | - public function __construct( |
|
| 47 | - private IUserManager $userManager, |
|
| 48 | - private IRootFolder $rootFolder, |
|
| 49 | - private FilesMetadataManager $filesMetadataManager, |
|
| 50 | - private IEventDispatcher $eventDispatcher, |
|
| 51 | - private LoggerInterface $logger, |
|
| 52 | - ) { |
|
| 53 | - parent::__construct(); |
|
| 54 | - } |
|
| 55 | - |
|
| 56 | - protected function configure(): void { |
|
| 57 | - parent::configure(); |
|
| 58 | - |
|
| 59 | - $this |
|
| 60 | - ->setName('files:scan') |
|
| 61 | - ->setDescription('rescan filesystem') |
|
| 62 | - ->addArgument( |
|
| 63 | - 'user_id', |
|
| 64 | - InputArgument::OPTIONAL | InputArgument::IS_ARRAY, |
|
| 65 | - 'will rescan all files of the given user(s)' |
|
| 66 | - ) |
|
| 67 | - ->addOption( |
|
| 68 | - 'path', |
|
| 69 | - 'p', |
|
| 70 | - InputOption::VALUE_REQUIRED, |
|
| 71 | - 'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored' |
|
| 72 | - ) |
|
| 73 | - ->addOption( |
|
| 74 | - 'generate-metadata', |
|
| 75 | - null, |
|
| 76 | - InputOption::VALUE_OPTIONAL, |
|
| 77 | - 'Generate metadata for all scanned files; if specified only generate for named value', |
|
| 78 | - '' |
|
| 79 | - ) |
|
| 80 | - ->addOption( |
|
| 81 | - 'all', |
|
| 82 | - null, |
|
| 83 | - InputOption::VALUE_NONE, |
|
| 84 | - 'will rescan all files of all known users' |
|
| 85 | - )->addOption( |
|
| 86 | - 'unscanned', |
|
| 87 | - null, |
|
| 88 | - InputOption::VALUE_NONE, |
|
| 89 | - 'only scan files which are marked as not fully scanned' |
|
| 90 | - )->addOption( |
|
| 91 | - 'shallow', |
|
| 92 | - null, |
|
| 93 | - InputOption::VALUE_NONE, |
|
| 94 | - 'do not scan folders recursively' |
|
| 95 | - )->addOption( |
|
| 96 | - 'home-only', |
|
| 97 | - null, |
|
| 98 | - InputOption::VALUE_NONE, |
|
| 99 | - 'only scan the home storage, ignoring any mounted external storage or share' |
|
| 100 | - ); |
|
| 101 | - } |
|
| 102 | - |
|
| 103 | - protected function scanFiles( |
|
| 104 | - string $user, |
|
| 105 | - string $path, |
|
| 106 | - ?string $scanMetadata, |
|
| 107 | - OutputInterface $output, |
|
| 108 | - callable $mountFilter, |
|
| 109 | - bool $backgroundScan = false, |
|
| 110 | - bool $recursive = true, |
|
| 111 | - ): void { |
|
| 112 | - $connection = $this->reconnectToDatabase($output); |
|
| 113 | - $scanner = new Scanner( |
|
| 114 | - $user, |
|
| 115 | - new ConnectionAdapter($connection), |
|
| 116 | - Server::get(IEventDispatcher::class), |
|
| 117 | - Server::get(LoggerInterface::class) |
|
| 118 | - ); |
|
| 119 | - |
|
| 120 | - # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception |
|
| 121 | - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function (string $path) use ($output, $scanMetadata): void { |
|
| 122 | - $output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
|
| 123 | - ++$this->filesCounter; |
|
| 124 | - $this->abortIfInterrupted(); |
|
| 125 | - if ($scanMetadata !== null) { |
|
| 126 | - $node = $this->rootFolder->get($path); |
|
| 127 | - $this->filesMetadataManager->refreshMetadata( |
|
| 128 | - $node, |
|
| 129 | - ($scanMetadata !== '') ? IFilesMetadataManager::PROCESS_NAMED : IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND, |
|
| 130 | - $scanMetadata |
|
| 131 | - ); |
|
| 132 | - } |
|
| 133 | - }); |
|
| 134 | - |
|
| 135 | - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output): void { |
|
| 136 | - $output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
|
| 137 | - ++$this->foldersCounter; |
|
| 138 | - $this->abortIfInterrupted(); |
|
| 139 | - }); |
|
| 140 | - |
|
| 141 | - $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output): void { |
|
| 142 | - $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); |
|
| 143 | - ++$this->errorsCounter; |
|
| 144 | - }); |
|
| 145 | - |
|
| 146 | - $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function ($fullPath) use ($output): void { |
|
| 147 | - $output->writeln("\t<error>Entry \"" . $fullPath . '" will not be accessible due to incompatible encoding</error>'); |
|
| 148 | - ++$this->errorsCounter; |
|
| 149 | - }); |
|
| 150 | - |
|
| 151 | - $this->eventDispatcher->addListener(NodeAddedToCache::class, function (): void { |
|
| 152 | - ++$this->newCounter; |
|
| 153 | - }); |
|
| 154 | - $this->eventDispatcher->addListener(FileCacheUpdated::class, function (): void { |
|
| 155 | - ++$this->updatedCounter; |
|
| 156 | - }); |
|
| 157 | - $this->eventDispatcher->addListener(NodeRemovedFromCache::class, function (): void { |
|
| 158 | - ++$this->removedCounter; |
|
| 159 | - }); |
|
| 160 | - |
|
| 161 | - try { |
|
| 162 | - if ($backgroundScan) { |
|
| 163 | - $scanner->backgroundScan($path); |
|
| 164 | - } else { |
|
| 165 | - $scanner->scan($path, $recursive, $mountFilter); |
|
| 166 | - } |
|
| 167 | - } catch (ForbiddenException $e) { |
|
| 168 | - $output->writeln("<error>Home storage for user $user not writable or 'files' subdirectory missing</error>"); |
|
| 169 | - $output->writeln(' ' . $e->getMessage()); |
|
| 170 | - $output->writeln('Make sure you\'re running the scan command only as the user the web server runs as'); |
|
| 171 | - ++$this->errorsCounter; |
|
| 172 | - } catch (InterruptedException $e) { |
|
| 173 | - # exit the function if ctrl-c has been pressed |
|
| 174 | - $output->writeln('Interrupted by user'); |
|
| 175 | - } catch (NotFoundException $e) { |
|
| 176 | - $output->writeln('<error>Path not found: ' . $e->getMessage() . '</error>'); |
|
| 177 | - ++$this->errorsCounter; |
|
| 178 | - } catch (LockedException $e) { |
|
| 179 | - if (str_starts_with($e->getPath(), 'scanner::')) { |
|
| 180 | - $output->writeln('<error>Another process is already scanning \'' . substr($e->getPath(), strlen('scanner::')) . '\'</error>'); |
|
| 181 | - } else { |
|
| 182 | - throw $e; |
|
| 183 | - } |
|
| 184 | - } catch (\Exception $e) { |
|
| 185 | - $output->writeln('<error>Exception during scan: ' . $e->getMessage() . '</error>'); |
|
| 186 | - $output->writeln('<error>' . $e->getTraceAsString() . '</error>'); |
|
| 187 | - ++$this->errorsCounter; |
|
| 188 | - } |
|
| 189 | - } |
|
| 190 | - |
|
| 191 | - public function isHomeMount(IMountPoint $mountPoint): bool { |
|
| 192 | - // any mountpoint inside '/$user/files/' |
|
| 193 | - return substr_count($mountPoint->getMountPoint(), '/') <= 3; |
|
| 194 | - } |
|
| 195 | - |
|
| 196 | - protected function execute(InputInterface $input, OutputInterface $output): int { |
|
| 197 | - $inputPath = $input->getOption('path'); |
|
| 198 | - if ($inputPath) { |
|
| 199 | - $inputPath = '/' . trim($inputPath, '/'); |
|
| 200 | - [, $user,] = explode('/', $inputPath, 3); |
|
| 201 | - $users = [$user]; |
|
| 202 | - } elseif ($input->getOption('all')) { |
|
| 203 | - $users = $this->userManager->search(''); |
|
| 204 | - } else { |
|
| 205 | - $users = $input->getArgument('user_id'); |
|
| 206 | - } |
|
| 207 | - |
|
| 208 | - # check quantity of users to be process and show it on the command line |
|
| 209 | - $users_total = count($users); |
|
| 210 | - if ($users_total === 0) { |
|
| 211 | - $output->writeln('<error>Please specify the user id to scan, --all to scan for all users or --path=...</error>'); |
|
| 212 | - return self::FAILURE; |
|
| 213 | - } |
|
| 214 | - |
|
| 215 | - $this->initTools($output); |
|
| 216 | - |
|
| 217 | - // getOption() logic on VALUE_OPTIONAL |
|
| 218 | - $metadata = null; // null if --generate-metadata is not set, empty if option have no value, value if set |
|
| 219 | - if ($input->getOption('generate-metadata') !== '') { |
|
| 220 | - $metadata = $input->getOption('generate-metadata') ?? ''; |
|
| 221 | - } |
|
| 222 | - |
|
| 223 | - $homeOnly = $input->getOption('home-only'); |
|
| 224 | - $scannedStorages = []; |
|
| 225 | - $mountFilter = function (IMountPoint $mount) use ($homeOnly, &$scannedStorages) { |
|
| 226 | - if ($homeOnly && !$this->isHomeMount($mount)) { |
|
| 227 | - return false; |
|
| 228 | - } |
|
| 229 | - |
|
| 230 | - // when scanning multiple users, the scanner might encounter the same storage multiple times (e.g. external storages, or group folders) |
|
| 231 | - // we can filter out any storage we've already scanned to avoid double work |
|
| 232 | - $storage = $mount->getStorage(); |
|
| 233 | - $storageKey = $storage->getId(); |
|
| 234 | - while ($storage->instanceOfStorage(Jail::class)) { |
|
| 235 | - $storageKey .= '/' . $storage->getUnjailedPath(''); |
|
| 236 | - $storage = $storage->getUnjailedStorage(); |
|
| 237 | - } |
|
| 238 | - if (array_key_exists($storageKey, $scannedStorages)) { |
|
| 239 | - return false; |
|
| 240 | - } |
|
| 241 | - |
|
| 242 | - $scannedStorages[$storageKey] = true; |
|
| 243 | - return true; |
|
| 244 | - }; |
|
| 245 | - |
|
| 246 | - $user_count = 0; |
|
| 247 | - foreach ($users as $user) { |
|
| 248 | - if (is_object($user)) { |
|
| 249 | - $user = $user->getUID(); |
|
| 250 | - } |
|
| 251 | - $path = $inputPath ?: '/' . $user; |
|
| 252 | - ++$user_count; |
|
| 253 | - if ($this->userManager->userExists($user)) { |
|
| 254 | - $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); |
|
| 255 | - $this->scanFiles( |
|
| 256 | - $user, |
|
| 257 | - $path, |
|
| 258 | - $metadata, |
|
| 259 | - $output, |
|
| 260 | - $mountFilter, |
|
| 261 | - $input->getOption('unscanned'), |
|
| 262 | - !$input->getOption('shallow'), |
|
| 263 | - ); |
|
| 264 | - $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); |
|
| 265 | - } else { |
|
| 266 | - $output->writeln("<error>Unknown user $user_count $user</error>"); |
|
| 267 | - $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); |
|
| 268 | - } |
|
| 269 | - |
|
| 270 | - try { |
|
| 271 | - $this->abortIfInterrupted(); |
|
| 272 | - } catch (InterruptedException $e) { |
|
| 273 | - break; |
|
| 274 | - } |
|
| 275 | - } |
|
| 276 | - |
|
| 277 | - $this->presentStats($output); |
|
| 278 | - return self::SUCCESS; |
|
| 279 | - } |
|
| 280 | - |
|
| 281 | - /** |
|
| 282 | - * Initialises some useful tools for the Command |
|
| 283 | - */ |
|
| 284 | - protected function initTools(OutputInterface $output): void { |
|
| 285 | - // Start the timer |
|
| 286 | - $this->execTime = -microtime(true); |
|
| 287 | - // Convert PHP errors to exceptions |
|
| 288 | - set_error_handler( |
|
| 289 | - fn (int $severity, string $message, string $file, int $line): bool => |
|
| 290 | - $this->exceptionErrorHandler($output, $severity, $message, $file, $line), |
|
| 291 | - E_ALL |
|
| 292 | - ); |
|
| 293 | - } |
|
| 294 | - |
|
| 295 | - /** |
|
| 296 | - * Processes PHP errors in order to be able to show them in the output |
|
| 297 | - * |
|
| 298 | - * @see https://www.php.net/manual/en/function.set-error-handler.php |
|
| 299 | - * |
|
| 300 | - * @param int $severity the level of the error raised |
|
| 301 | - * @param string $message |
|
| 302 | - * @param string $file the filename that the error was raised in |
|
| 303 | - * @param int $line the line number the error was raised |
|
| 304 | - */ |
|
| 305 | - public function exceptionErrorHandler(OutputInterface $output, int $severity, string $message, string $file, int $line): bool { |
|
| 306 | - if (($severity === E_DEPRECATED) || ($severity === E_USER_DEPRECATED)) { |
|
| 307 | - // Do not show deprecation warnings |
|
| 308 | - return false; |
|
| 309 | - } |
|
| 310 | - $e = new \ErrorException($message, 0, $severity, $file, $line); |
|
| 311 | - $output->writeln('<error>Error during scan: ' . $e->getMessage() . '</error>'); |
|
| 312 | - $output->writeln('<error>' . $e->getTraceAsString() . '</error>', OutputInterface::VERBOSITY_VERY_VERBOSE); |
|
| 313 | - ++$this->errorsCounter; |
|
| 314 | - return true; |
|
| 315 | - } |
|
| 316 | - |
|
| 317 | - protected function presentStats(OutputInterface $output): void { |
|
| 318 | - // Stop the timer |
|
| 319 | - $this->execTime += microtime(true); |
|
| 320 | - |
|
| 321 | - $this->logger->info("Completed scan of {$this->filesCounter} files in {$this->foldersCounter} folder. Found {$this->newCounter} new, {$this->updatedCounter} updated and {$this->removedCounter} removed items"); |
|
| 322 | - |
|
| 323 | - $headers = [ |
|
| 324 | - 'Folders', |
|
| 325 | - 'Files', |
|
| 326 | - 'New', |
|
| 327 | - 'Updated', |
|
| 328 | - 'Removed', |
|
| 329 | - 'Errors', |
|
| 330 | - 'Elapsed time', |
|
| 331 | - ]; |
|
| 332 | - $niceDate = $this->formatExecTime(); |
|
| 333 | - $rows = [ |
|
| 334 | - $this->foldersCounter, |
|
| 335 | - $this->filesCounter, |
|
| 336 | - $this->newCounter, |
|
| 337 | - $this->updatedCounter, |
|
| 338 | - $this->removedCounter, |
|
| 339 | - $this->errorsCounter, |
|
| 340 | - $niceDate, |
|
| 341 | - ]; |
|
| 342 | - $table = new Table($output); |
|
| 343 | - $table |
|
| 344 | - ->setHeaders($headers) |
|
| 345 | - ->setRows([$rows]); |
|
| 346 | - $table->render(); |
|
| 347 | - } |
|
| 348 | - |
|
| 349 | - |
|
| 350 | - /** |
|
| 351 | - * Formats microtime into a human-readable format |
|
| 352 | - */ |
|
| 353 | - protected function formatExecTime(): string { |
|
| 354 | - $secs = (int)round($this->execTime); |
|
| 355 | - # convert seconds into HH:MM:SS form |
|
| 356 | - return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60); |
|
| 357 | - } |
|
| 358 | - |
|
| 359 | - protected function reconnectToDatabase(OutputInterface $output): Connection { |
|
| 360 | - /** @var Connection $connection */ |
|
| 361 | - $connection = Server::get(Connection::class); |
|
| 362 | - try { |
|
| 363 | - $connection->close(); |
|
| 364 | - } catch (\Exception $ex) { |
|
| 365 | - $output->writeln("<info>Error while disconnecting from database: {$ex->getMessage()}</info>"); |
|
| 366 | - } |
|
| 367 | - while (!$connection->isConnected()) { |
|
| 368 | - try { |
|
| 369 | - $connection->connect(); |
|
| 370 | - } catch (\Exception $ex) { |
|
| 371 | - $output->writeln("<info>Error while re-connecting to database: {$ex->getMessage()}</info>"); |
|
| 372 | - sleep(60); |
|
| 373 | - } |
|
| 374 | - } |
|
| 375 | - return $connection; |
|
| 376 | - } |
|
| 38 | + protected float $execTime = 0; |
|
| 39 | + protected int $foldersCounter = 0; |
|
| 40 | + protected int $filesCounter = 0; |
|
| 41 | + protected int $errorsCounter = 0; |
|
| 42 | + protected int $newCounter = 0; |
|
| 43 | + protected int $updatedCounter = 0; |
|
| 44 | + protected int $removedCounter = 0; |
|
| 45 | + |
|
| 46 | + public function __construct( |
|
| 47 | + private IUserManager $userManager, |
|
| 48 | + private IRootFolder $rootFolder, |
|
| 49 | + private FilesMetadataManager $filesMetadataManager, |
|
| 50 | + private IEventDispatcher $eventDispatcher, |
|
| 51 | + private LoggerInterface $logger, |
|
| 52 | + ) { |
|
| 53 | + parent::__construct(); |
|
| 54 | + } |
|
| 55 | + |
|
| 56 | + protected function configure(): void { |
|
| 57 | + parent::configure(); |
|
| 58 | + |
|
| 59 | + $this |
|
| 60 | + ->setName('files:scan') |
|
| 61 | + ->setDescription('rescan filesystem') |
|
| 62 | + ->addArgument( |
|
| 63 | + 'user_id', |
|
| 64 | + InputArgument::OPTIONAL | InputArgument::IS_ARRAY, |
|
| 65 | + 'will rescan all files of the given user(s)' |
|
| 66 | + ) |
|
| 67 | + ->addOption( |
|
| 68 | + 'path', |
|
| 69 | + 'p', |
|
| 70 | + InputOption::VALUE_REQUIRED, |
|
| 71 | + 'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored' |
|
| 72 | + ) |
|
| 73 | + ->addOption( |
|
| 74 | + 'generate-metadata', |
|
| 75 | + null, |
|
| 76 | + InputOption::VALUE_OPTIONAL, |
|
| 77 | + 'Generate metadata for all scanned files; if specified only generate for named value', |
|
| 78 | + '' |
|
| 79 | + ) |
|
| 80 | + ->addOption( |
|
| 81 | + 'all', |
|
| 82 | + null, |
|
| 83 | + InputOption::VALUE_NONE, |
|
| 84 | + 'will rescan all files of all known users' |
|
| 85 | + )->addOption( |
|
| 86 | + 'unscanned', |
|
| 87 | + null, |
|
| 88 | + InputOption::VALUE_NONE, |
|
| 89 | + 'only scan files which are marked as not fully scanned' |
|
| 90 | + )->addOption( |
|
| 91 | + 'shallow', |
|
| 92 | + null, |
|
| 93 | + InputOption::VALUE_NONE, |
|
| 94 | + 'do not scan folders recursively' |
|
| 95 | + )->addOption( |
|
| 96 | + 'home-only', |
|
| 97 | + null, |
|
| 98 | + InputOption::VALUE_NONE, |
|
| 99 | + 'only scan the home storage, ignoring any mounted external storage or share' |
|
| 100 | + ); |
|
| 101 | + } |
|
| 102 | + |
|
| 103 | + protected function scanFiles( |
|
| 104 | + string $user, |
|
| 105 | + string $path, |
|
| 106 | + ?string $scanMetadata, |
|
| 107 | + OutputInterface $output, |
|
| 108 | + callable $mountFilter, |
|
| 109 | + bool $backgroundScan = false, |
|
| 110 | + bool $recursive = true, |
|
| 111 | + ): void { |
|
| 112 | + $connection = $this->reconnectToDatabase($output); |
|
| 113 | + $scanner = new Scanner( |
|
| 114 | + $user, |
|
| 115 | + new ConnectionAdapter($connection), |
|
| 116 | + Server::get(IEventDispatcher::class), |
|
| 117 | + Server::get(LoggerInterface::class) |
|
| 118 | + ); |
|
| 119 | + |
|
| 120 | + # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception |
|
| 121 | + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function (string $path) use ($output, $scanMetadata): void { |
|
| 122 | + $output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
|
| 123 | + ++$this->filesCounter; |
|
| 124 | + $this->abortIfInterrupted(); |
|
| 125 | + if ($scanMetadata !== null) { |
|
| 126 | + $node = $this->rootFolder->get($path); |
|
| 127 | + $this->filesMetadataManager->refreshMetadata( |
|
| 128 | + $node, |
|
| 129 | + ($scanMetadata !== '') ? IFilesMetadataManager::PROCESS_NAMED : IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND, |
|
| 130 | + $scanMetadata |
|
| 131 | + ); |
|
| 132 | + } |
|
| 133 | + }); |
|
| 134 | + |
|
| 135 | + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output): void { |
|
| 136 | + $output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
|
| 137 | + ++$this->foldersCounter; |
|
| 138 | + $this->abortIfInterrupted(); |
|
| 139 | + }); |
|
| 140 | + |
|
| 141 | + $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output): void { |
|
| 142 | + $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); |
|
| 143 | + ++$this->errorsCounter; |
|
| 144 | + }); |
|
| 145 | + |
|
| 146 | + $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function ($fullPath) use ($output): void { |
|
| 147 | + $output->writeln("\t<error>Entry \"" . $fullPath . '" will not be accessible due to incompatible encoding</error>'); |
|
| 148 | + ++$this->errorsCounter; |
|
| 149 | + }); |
|
| 150 | + |
|
| 151 | + $this->eventDispatcher->addListener(NodeAddedToCache::class, function (): void { |
|
| 152 | + ++$this->newCounter; |
|
| 153 | + }); |
|
| 154 | + $this->eventDispatcher->addListener(FileCacheUpdated::class, function (): void { |
|
| 155 | + ++$this->updatedCounter; |
|
| 156 | + }); |
|
| 157 | + $this->eventDispatcher->addListener(NodeRemovedFromCache::class, function (): void { |
|
| 158 | + ++$this->removedCounter; |
|
| 159 | + }); |
|
| 160 | + |
|
| 161 | + try { |
|
| 162 | + if ($backgroundScan) { |
|
| 163 | + $scanner->backgroundScan($path); |
|
| 164 | + } else { |
|
| 165 | + $scanner->scan($path, $recursive, $mountFilter); |
|
| 166 | + } |
|
| 167 | + } catch (ForbiddenException $e) { |
|
| 168 | + $output->writeln("<error>Home storage for user $user not writable or 'files' subdirectory missing</error>"); |
|
| 169 | + $output->writeln(' ' . $e->getMessage()); |
|
| 170 | + $output->writeln('Make sure you\'re running the scan command only as the user the web server runs as'); |
|
| 171 | + ++$this->errorsCounter; |
|
| 172 | + } catch (InterruptedException $e) { |
|
| 173 | + # exit the function if ctrl-c has been pressed |
|
| 174 | + $output->writeln('Interrupted by user'); |
|
| 175 | + } catch (NotFoundException $e) { |
|
| 176 | + $output->writeln('<error>Path not found: ' . $e->getMessage() . '</error>'); |
|
| 177 | + ++$this->errorsCounter; |
|
| 178 | + } catch (LockedException $e) { |
|
| 179 | + if (str_starts_with($e->getPath(), 'scanner::')) { |
|
| 180 | + $output->writeln('<error>Another process is already scanning \'' . substr($e->getPath(), strlen('scanner::')) . '\'</error>'); |
|
| 181 | + } else { |
|
| 182 | + throw $e; |
|
| 183 | + } |
|
| 184 | + } catch (\Exception $e) { |
|
| 185 | + $output->writeln('<error>Exception during scan: ' . $e->getMessage() . '</error>'); |
|
| 186 | + $output->writeln('<error>' . $e->getTraceAsString() . '</error>'); |
|
| 187 | + ++$this->errorsCounter; |
|
| 188 | + } |
|
| 189 | + } |
|
| 190 | + |
|
| 191 | + public function isHomeMount(IMountPoint $mountPoint): bool { |
|
| 192 | + // any mountpoint inside '/$user/files/' |
|
| 193 | + return substr_count($mountPoint->getMountPoint(), '/') <= 3; |
|
| 194 | + } |
|
| 195 | + |
|
| 196 | + protected function execute(InputInterface $input, OutputInterface $output): int { |
|
| 197 | + $inputPath = $input->getOption('path'); |
|
| 198 | + if ($inputPath) { |
|
| 199 | + $inputPath = '/' . trim($inputPath, '/'); |
|
| 200 | + [, $user,] = explode('/', $inputPath, 3); |
|
| 201 | + $users = [$user]; |
|
| 202 | + } elseif ($input->getOption('all')) { |
|
| 203 | + $users = $this->userManager->search(''); |
|
| 204 | + } else { |
|
| 205 | + $users = $input->getArgument('user_id'); |
|
| 206 | + } |
|
| 207 | + |
|
| 208 | + # check quantity of users to be process and show it on the command line |
|
| 209 | + $users_total = count($users); |
|
| 210 | + if ($users_total === 0) { |
|
| 211 | + $output->writeln('<error>Please specify the user id to scan, --all to scan for all users or --path=...</error>'); |
|
| 212 | + return self::FAILURE; |
|
| 213 | + } |
|
| 214 | + |
|
| 215 | + $this->initTools($output); |
|
| 216 | + |
|
| 217 | + // getOption() logic on VALUE_OPTIONAL |
|
| 218 | + $metadata = null; // null if --generate-metadata is not set, empty if option have no value, value if set |
|
| 219 | + if ($input->getOption('generate-metadata') !== '') { |
|
| 220 | + $metadata = $input->getOption('generate-metadata') ?? ''; |
|
| 221 | + } |
|
| 222 | + |
|
| 223 | + $homeOnly = $input->getOption('home-only'); |
|
| 224 | + $scannedStorages = []; |
|
| 225 | + $mountFilter = function (IMountPoint $mount) use ($homeOnly, &$scannedStorages) { |
|
| 226 | + if ($homeOnly && !$this->isHomeMount($mount)) { |
|
| 227 | + return false; |
|
| 228 | + } |
|
| 229 | + |
|
| 230 | + // when scanning multiple users, the scanner might encounter the same storage multiple times (e.g. external storages, or group folders) |
|
| 231 | + // we can filter out any storage we've already scanned to avoid double work |
|
| 232 | + $storage = $mount->getStorage(); |
|
| 233 | + $storageKey = $storage->getId(); |
|
| 234 | + while ($storage->instanceOfStorage(Jail::class)) { |
|
| 235 | + $storageKey .= '/' . $storage->getUnjailedPath(''); |
|
| 236 | + $storage = $storage->getUnjailedStorage(); |
|
| 237 | + } |
|
| 238 | + if (array_key_exists($storageKey, $scannedStorages)) { |
|
| 239 | + return false; |
|
| 240 | + } |
|
| 241 | + |
|
| 242 | + $scannedStorages[$storageKey] = true; |
|
| 243 | + return true; |
|
| 244 | + }; |
|
| 245 | + |
|
| 246 | + $user_count = 0; |
|
| 247 | + foreach ($users as $user) { |
|
| 248 | + if (is_object($user)) { |
|
| 249 | + $user = $user->getUID(); |
|
| 250 | + } |
|
| 251 | + $path = $inputPath ?: '/' . $user; |
|
| 252 | + ++$user_count; |
|
| 253 | + if ($this->userManager->userExists($user)) { |
|
| 254 | + $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); |
|
| 255 | + $this->scanFiles( |
|
| 256 | + $user, |
|
| 257 | + $path, |
|
| 258 | + $metadata, |
|
| 259 | + $output, |
|
| 260 | + $mountFilter, |
|
| 261 | + $input->getOption('unscanned'), |
|
| 262 | + !$input->getOption('shallow'), |
|
| 263 | + ); |
|
| 264 | + $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); |
|
| 265 | + } else { |
|
| 266 | + $output->writeln("<error>Unknown user $user_count $user</error>"); |
|
| 267 | + $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); |
|
| 268 | + } |
|
| 269 | + |
|
| 270 | + try { |
|
| 271 | + $this->abortIfInterrupted(); |
|
| 272 | + } catch (InterruptedException $e) { |
|
| 273 | + break; |
|
| 274 | + } |
|
| 275 | + } |
|
| 276 | + |
|
| 277 | + $this->presentStats($output); |
|
| 278 | + return self::SUCCESS; |
|
| 279 | + } |
|
| 280 | + |
|
| 281 | + /** |
|
| 282 | + * Initialises some useful tools for the Command |
|
| 283 | + */ |
|
| 284 | + protected function initTools(OutputInterface $output): void { |
|
| 285 | + // Start the timer |
|
| 286 | + $this->execTime = -microtime(true); |
|
| 287 | + // Convert PHP errors to exceptions |
|
| 288 | + set_error_handler( |
|
| 289 | + fn (int $severity, string $message, string $file, int $line): bool => |
|
| 290 | + $this->exceptionErrorHandler($output, $severity, $message, $file, $line), |
|
| 291 | + E_ALL |
|
| 292 | + ); |
|
| 293 | + } |
|
| 294 | + |
|
| 295 | + /** |
|
| 296 | + * Processes PHP errors in order to be able to show them in the output |
|
| 297 | + * |
|
| 298 | + * @see https://www.php.net/manual/en/function.set-error-handler.php |
|
| 299 | + * |
|
| 300 | + * @param int $severity the level of the error raised |
|
| 301 | + * @param string $message |
|
| 302 | + * @param string $file the filename that the error was raised in |
|
| 303 | + * @param int $line the line number the error was raised |
|
| 304 | + */ |
|
| 305 | + public function exceptionErrorHandler(OutputInterface $output, int $severity, string $message, string $file, int $line): bool { |
|
| 306 | + if (($severity === E_DEPRECATED) || ($severity === E_USER_DEPRECATED)) { |
|
| 307 | + // Do not show deprecation warnings |
|
| 308 | + return false; |
|
| 309 | + } |
|
| 310 | + $e = new \ErrorException($message, 0, $severity, $file, $line); |
|
| 311 | + $output->writeln('<error>Error during scan: ' . $e->getMessage() . '</error>'); |
|
| 312 | + $output->writeln('<error>' . $e->getTraceAsString() . '</error>', OutputInterface::VERBOSITY_VERY_VERBOSE); |
|
| 313 | + ++$this->errorsCounter; |
|
| 314 | + return true; |
|
| 315 | + } |
|
| 316 | + |
|
| 317 | + protected function presentStats(OutputInterface $output): void { |
|
| 318 | + // Stop the timer |
|
| 319 | + $this->execTime += microtime(true); |
|
| 320 | + |
|
| 321 | + $this->logger->info("Completed scan of {$this->filesCounter} files in {$this->foldersCounter} folder. Found {$this->newCounter} new, {$this->updatedCounter} updated and {$this->removedCounter} removed items"); |
|
| 322 | + |
|
| 323 | + $headers = [ |
|
| 324 | + 'Folders', |
|
| 325 | + 'Files', |
|
| 326 | + 'New', |
|
| 327 | + 'Updated', |
|
| 328 | + 'Removed', |
|
| 329 | + 'Errors', |
|
| 330 | + 'Elapsed time', |
|
| 331 | + ]; |
|
| 332 | + $niceDate = $this->formatExecTime(); |
|
| 333 | + $rows = [ |
|
| 334 | + $this->foldersCounter, |
|
| 335 | + $this->filesCounter, |
|
| 336 | + $this->newCounter, |
|
| 337 | + $this->updatedCounter, |
|
| 338 | + $this->removedCounter, |
|
| 339 | + $this->errorsCounter, |
|
| 340 | + $niceDate, |
|
| 341 | + ]; |
|
| 342 | + $table = new Table($output); |
|
| 343 | + $table |
|
| 344 | + ->setHeaders($headers) |
|
| 345 | + ->setRows([$rows]); |
|
| 346 | + $table->render(); |
|
| 347 | + } |
|
| 348 | + |
|
| 349 | + |
|
| 350 | + /** |
|
| 351 | + * Formats microtime into a human-readable format |
|
| 352 | + */ |
|
| 353 | + protected function formatExecTime(): string { |
|
| 354 | + $secs = (int)round($this->execTime); |
|
| 355 | + # convert seconds into HH:MM:SS form |
|
| 356 | + return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60); |
|
| 357 | + } |
|
| 358 | + |
|
| 359 | + protected function reconnectToDatabase(OutputInterface $output): Connection { |
|
| 360 | + /** @var Connection $connection */ |
|
| 361 | + $connection = Server::get(Connection::class); |
|
| 362 | + try { |
|
| 363 | + $connection->close(); |
|
| 364 | + } catch (\Exception $ex) { |
|
| 365 | + $output->writeln("<info>Error while disconnecting from database: {$ex->getMessage()}</info>"); |
|
| 366 | + } |
|
| 367 | + while (!$connection->isConnected()) { |
|
| 368 | + try { |
|
| 369 | + $connection->connect(); |
|
| 370 | + } catch (\Exception $ex) { |
|
| 371 | + $output->writeln("<info>Error while re-connecting to database: {$ex->getMessage()}</info>"); |
|
| 372 | + sleep(60); |
|
| 373 | + } |
|
| 374 | + } |
|
| 375 | + return $connection; |
|
| 376 | + } |
|
| 377 | 377 | } |
@@ -118,7 +118,7 @@ discard block |
||
| 118 | 118 | ); |
| 119 | 119 | |
| 120 | 120 | # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception |
| 121 | - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function (string $path) use ($output, $scanMetadata): void { |
|
| 121 | + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function(string $path) use ($output, $scanMetadata): void { |
|
| 122 | 122 | $output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
| 123 | 123 | ++$this->filesCounter; |
| 124 | 124 | $this->abortIfInterrupted(); |
@@ -132,29 +132,29 @@ discard block |
||
| 132 | 132 | } |
| 133 | 133 | }); |
| 134 | 134 | |
| 135 | - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output): void { |
|
| 135 | + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function($path) use ($output): void { |
|
| 136 | 136 | $output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE); |
| 137 | 137 | ++$this->foldersCounter; |
| 138 | 138 | $this->abortIfInterrupted(); |
| 139 | 139 | }); |
| 140 | 140 | |
| 141 | - $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output): void { |
|
| 142 | - $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); |
|
| 141 | + $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function(StorageNotAvailableException $e) use ($output): void { |
|
| 142 | + $output->writeln('Error while scanning, storage not available ('.$e->getMessage().')', OutputInterface::VERBOSITY_VERBOSE); |
|
| 143 | 143 | ++$this->errorsCounter; |
| 144 | 144 | }); |
| 145 | 145 | |
| 146 | - $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function ($fullPath) use ($output): void { |
|
| 147 | - $output->writeln("\t<error>Entry \"" . $fullPath . '" will not be accessible due to incompatible encoding</error>'); |
|
| 146 | + $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function($fullPath) use ($output): void { |
|
| 147 | + $output->writeln("\t<error>Entry \"".$fullPath.'" will not be accessible due to incompatible encoding</error>'); |
|
| 148 | 148 | ++$this->errorsCounter; |
| 149 | 149 | }); |
| 150 | 150 | |
| 151 | - $this->eventDispatcher->addListener(NodeAddedToCache::class, function (): void { |
|
| 151 | + $this->eventDispatcher->addListener(NodeAddedToCache::class, function(): void { |
|
| 152 | 152 | ++$this->newCounter; |
| 153 | 153 | }); |
| 154 | - $this->eventDispatcher->addListener(FileCacheUpdated::class, function (): void { |
|
| 154 | + $this->eventDispatcher->addListener(FileCacheUpdated::class, function(): void { |
|
| 155 | 155 | ++$this->updatedCounter; |
| 156 | 156 | }); |
| 157 | - $this->eventDispatcher->addListener(NodeRemovedFromCache::class, function (): void { |
|
| 157 | + $this->eventDispatcher->addListener(NodeRemovedFromCache::class, function(): void { |
|
| 158 | 158 | ++$this->removedCounter; |
| 159 | 159 | }); |
| 160 | 160 | |
@@ -166,24 +166,24 @@ discard block |
||
| 166 | 166 | } |
| 167 | 167 | } catch (ForbiddenException $e) { |
| 168 | 168 | $output->writeln("<error>Home storage for user $user not writable or 'files' subdirectory missing</error>"); |
| 169 | - $output->writeln(' ' . $e->getMessage()); |
|
| 169 | + $output->writeln(' '.$e->getMessage()); |
|
| 170 | 170 | $output->writeln('Make sure you\'re running the scan command only as the user the web server runs as'); |
| 171 | 171 | ++$this->errorsCounter; |
| 172 | 172 | } catch (InterruptedException $e) { |
| 173 | 173 | # exit the function if ctrl-c has been pressed |
| 174 | 174 | $output->writeln('Interrupted by user'); |
| 175 | 175 | } catch (NotFoundException $e) { |
| 176 | - $output->writeln('<error>Path not found: ' . $e->getMessage() . '</error>'); |
|
| 176 | + $output->writeln('<error>Path not found: '.$e->getMessage().'</error>'); |
|
| 177 | 177 | ++$this->errorsCounter; |
| 178 | 178 | } catch (LockedException $e) { |
| 179 | 179 | if (str_starts_with($e->getPath(), 'scanner::')) { |
| 180 | - $output->writeln('<error>Another process is already scanning \'' . substr($e->getPath(), strlen('scanner::')) . '\'</error>'); |
|
| 180 | + $output->writeln('<error>Another process is already scanning \''.substr($e->getPath(), strlen('scanner::')).'\'</error>'); |
|
| 181 | 181 | } else { |
| 182 | 182 | throw $e; |
| 183 | 183 | } |
| 184 | 184 | } catch (\Exception $e) { |
| 185 | - $output->writeln('<error>Exception during scan: ' . $e->getMessage() . '</error>'); |
|
| 186 | - $output->writeln('<error>' . $e->getTraceAsString() . '</error>'); |
|
| 185 | + $output->writeln('<error>Exception during scan: '.$e->getMessage().'</error>'); |
|
| 186 | + $output->writeln('<error>'.$e->getTraceAsString().'</error>'); |
|
| 187 | 187 | ++$this->errorsCounter; |
| 188 | 188 | } |
| 189 | 189 | } |
@@ -196,8 +196,8 @@ discard block |
||
| 196 | 196 | protected function execute(InputInterface $input, OutputInterface $output): int { |
| 197 | 197 | $inputPath = $input->getOption('path'); |
| 198 | 198 | if ($inputPath) { |
| 199 | - $inputPath = '/' . trim($inputPath, '/'); |
|
| 200 | - [, $user,] = explode('/', $inputPath, 3); |
|
| 199 | + $inputPath = '/'.trim($inputPath, '/'); |
|
| 200 | + [, $user, ] = explode('/', $inputPath, 3); |
|
| 201 | 201 | $users = [$user]; |
| 202 | 202 | } elseif ($input->getOption('all')) { |
| 203 | 203 | $users = $this->userManager->search(''); |
@@ -222,7 +222,7 @@ discard block |
||
| 222 | 222 | |
| 223 | 223 | $homeOnly = $input->getOption('home-only'); |
| 224 | 224 | $scannedStorages = []; |
| 225 | - $mountFilter = function (IMountPoint $mount) use ($homeOnly, &$scannedStorages) { |
|
| 225 | + $mountFilter = function(IMountPoint $mount) use ($homeOnly, &$scannedStorages) { |
|
| 226 | 226 | if ($homeOnly && !$this->isHomeMount($mount)) { |
| 227 | 227 | return false; |
| 228 | 228 | } |
@@ -232,7 +232,7 @@ discard block |
||
| 232 | 232 | $storage = $mount->getStorage(); |
| 233 | 233 | $storageKey = $storage->getId(); |
| 234 | 234 | while ($storage->instanceOfStorage(Jail::class)) { |
| 235 | - $storageKey .= '/' . $storage->getUnjailedPath(''); |
|
| 235 | + $storageKey .= '/'.$storage->getUnjailedPath(''); |
|
| 236 | 236 | $storage = $storage->getUnjailedStorage(); |
| 237 | 237 | } |
| 238 | 238 | if (array_key_exists($storageKey, $scannedStorages)) { |
@@ -248,7 +248,7 @@ discard block |
||
| 248 | 248 | if (is_object($user)) { |
| 249 | 249 | $user = $user->getUID(); |
| 250 | 250 | } |
| 251 | - $path = $inputPath ?: '/' . $user; |
|
| 251 | + $path = $inputPath ?: '/'.$user; |
|
| 252 | 252 | ++$user_count; |
| 253 | 253 | if ($this->userManager->userExists($user)) { |
| 254 | 254 | $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); |
@@ -308,8 +308,8 @@ discard block |
||
| 308 | 308 | return false; |
| 309 | 309 | } |
| 310 | 310 | $e = new \ErrorException($message, 0, $severity, $file, $line); |
| 311 | - $output->writeln('<error>Error during scan: ' . $e->getMessage() . '</error>'); |
|
| 312 | - $output->writeln('<error>' . $e->getTraceAsString() . '</error>', OutputInterface::VERBOSITY_VERY_VERBOSE); |
|
| 311 | + $output->writeln('<error>Error during scan: '.$e->getMessage().'</error>'); |
|
| 312 | + $output->writeln('<error>'.$e->getTraceAsString().'</error>', OutputInterface::VERBOSITY_VERY_VERBOSE); |
|
| 313 | 313 | ++$this->errorsCounter; |
| 314 | 314 | return true; |
| 315 | 315 | } |
@@ -351,9 +351,9 @@ discard block |
||
| 351 | 351 | * Formats microtime into a human-readable format |
| 352 | 352 | */ |
| 353 | 353 | protected function formatExecTime(): string { |
| 354 | - $secs = (int)round($this->execTime); |
|
| 354 | + $secs = (int) round($this->execTime); |
|
| 355 | 355 | # convert seconds into HH:MM:SS form |
| 356 | - return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60); |
|
| 356 | + return sprintf('%02d:%02d:%02d', (int) ($secs / 3600), ((int) ($secs / 60) % 60), $secs % 60); |
|
| 357 | 357 | } |
| 358 | 358 | |
| 359 | 359 | protected function reconnectToDatabase(OutputInterface $output): Connection { |