nextcloud /
server
@@ -24,120 +24,120 @@ |
||
| 24 | 24 | * @package OCA\Files\BackgroundJob |
| 25 | 25 | */ |
| 26 | 26 | class ScanFiles extends TimedJob { |
| 27 | - /** Amount of users that should get scanned per execution */ |
|
| 28 | - public const USERS_PER_SESSION = 500; |
|
| 29 | - |
|
| 30 | - public function __construct( |
|
| 31 | - private IConfig $config, |
|
| 32 | - private IEventDispatcher $dispatcher, |
|
| 33 | - private LoggerInterface $logger, |
|
| 34 | - private IDBConnection $connection, |
|
| 35 | - ITimeFactory $time, |
|
| 36 | - ) { |
|
| 37 | - parent::__construct($time); |
|
| 38 | - // Run once per 10 minutes |
|
| 39 | - $this->setInterval(60 * 10); |
|
| 40 | - } |
|
| 41 | - |
|
| 42 | - protected function runScanner(string $user): void { |
|
| 43 | - try { |
|
| 44 | - $scanner = new Scanner( |
|
| 45 | - $user, |
|
| 46 | - null, |
|
| 47 | - $this->dispatcher, |
|
| 48 | - $this->logger |
|
| 49 | - ); |
|
| 50 | - $scanner->backgroundScan(''); |
|
| 51 | - } catch (\Exception $e) { |
|
| 52 | - $this->logger->error($e->getMessage(), ['exception' => $e, 'app' => 'files']); |
|
| 53 | - } |
|
| 54 | - \OC_Util::tearDownFS(); |
|
| 55 | - } |
|
| 56 | - |
|
| 57 | - /** |
|
| 58 | - * Find a storage which have unindexed files and return a user with access to the storage |
|
| 59 | - * |
|
| 60 | - * @return string|false |
|
| 61 | - */ |
|
| 62 | - private function getUserToScan() { |
|
| 63 | - if ($this->connection->getShardDefinition('filecache')) { |
|
| 64 | - // for sharded filecache, the "LIMIT" from the normal query doesn't work |
|
| 65 | - |
|
| 66 | - // first we try it with a "LEFT JOIN" on mounts, this is fast, but might return a storage that isn't mounted. |
|
| 67 | - // we also ask for up to 10 results from different storages to increase the odds of finding a result that is mounted |
|
| 68 | - $query = $this->connection->getQueryBuilder(); |
|
| 69 | - $query->select('m.user_id') |
|
| 70 | - ->from('filecache', 'f') |
|
| 71 | - ->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage')) |
|
| 72 | - ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 73 | - ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 74 | - ->setMaxResults(10) |
|
| 75 | - ->groupBy('f.storage') |
|
| 76 | - ->runAcrossAllShards(); |
|
| 77 | - |
|
| 78 | - $result = $query->executeQuery(); |
|
| 79 | - while ($res = $result->fetch()) { |
|
| 80 | - if ($res['user_id']) { |
|
| 81 | - return $res['user_id']; |
|
| 82 | - } |
|
| 83 | - } |
|
| 84 | - |
|
| 85 | - // as a fallback, we try a slower approach where we find all mounted storages first |
|
| 86 | - // this is essentially doing the inner join manually |
|
| 87 | - $storages = $this->getAllMountedStorages(); |
|
| 88 | - |
|
| 89 | - $query = $this->connection->getQueryBuilder(); |
|
| 90 | - $query->select('m.user_id') |
|
| 91 | - ->from('filecache', 'f') |
|
| 92 | - ->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage')) |
|
| 93 | - ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 94 | - ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 95 | - ->andWhere($query->expr()->in('f.storage', $query->createNamedParameter($storages, IQueryBuilder::PARAM_INT_ARRAY))) |
|
| 96 | - ->setMaxResults(1) |
|
| 97 | - ->runAcrossAllShards(); |
|
| 98 | - return $query->executeQuery()->fetchOne(); |
|
| 99 | - } else { |
|
| 100 | - $query = $this->connection->getQueryBuilder(); |
|
| 101 | - $query->select('m.user_id') |
|
| 102 | - ->from('filecache', 'f') |
|
| 103 | - ->innerJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage')) |
|
| 104 | - ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 105 | - ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 106 | - ->setMaxResults(1) |
|
| 107 | - ->runAcrossAllShards(); |
|
| 108 | - |
|
| 109 | - return $query->executeQuery()->fetchOne(); |
|
| 110 | - } |
|
| 111 | - } |
|
| 112 | - |
|
| 113 | - private function getAllMountedStorages(): array { |
|
| 114 | - $query = $this->connection->getQueryBuilder(); |
|
| 115 | - $query->selectDistinct('storage_id') |
|
| 116 | - ->from('mounts'); |
|
| 117 | - return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); |
|
| 118 | - } |
|
| 119 | - |
|
| 120 | - /** |
|
| 121 | - * @param $argument |
|
| 122 | - * @throws \Exception |
|
| 123 | - */ |
|
| 124 | - protected function run($argument) { |
|
| 125 | - if ($this->config->getSystemValueBool('files_no_background_scan', false)) { |
|
| 126 | - return; |
|
| 127 | - } |
|
| 128 | - |
|
| 129 | - $usersScanned = 0; |
|
| 130 | - $lastUser = ''; |
|
| 131 | - $user = $this->getUserToScan(); |
|
| 132 | - while ($user && $usersScanned < self::USERS_PER_SESSION && $lastUser !== $user) { |
|
| 133 | - $this->runScanner($user); |
|
| 134 | - $lastUser = $user; |
|
| 135 | - $user = $this->getUserToScan(); |
|
| 136 | - $usersScanned += 1; |
|
| 137 | - } |
|
| 138 | - |
|
| 139 | - if ($lastUser === $user) { |
|
| 140 | - $this->logger->warning("User $user still has unscanned files after running background scan, background scan might be stopped prematurely"); |
|
| 141 | - } |
|
| 142 | - } |
|
| 27 | + /** Amount of users that should get scanned per execution */ |
|
| 28 | + public const USERS_PER_SESSION = 500; |
|
| 29 | + |
|
| 30 | + public function __construct( |
|
| 31 | + private IConfig $config, |
|
| 32 | + private IEventDispatcher $dispatcher, |
|
| 33 | + private LoggerInterface $logger, |
|
| 34 | + private IDBConnection $connection, |
|
| 35 | + ITimeFactory $time, |
|
| 36 | + ) { |
|
| 37 | + parent::__construct($time); |
|
| 38 | + // Run once per 10 minutes |
|
| 39 | + $this->setInterval(60 * 10); |
|
| 40 | + } |
|
| 41 | + |
|
| 42 | + protected function runScanner(string $user): void { |
|
| 43 | + try { |
|
| 44 | + $scanner = new Scanner( |
|
| 45 | + $user, |
|
| 46 | + null, |
|
| 47 | + $this->dispatcher, |
|
| 48 | + $this->logger |
|
| 49 | + ); |
|
| 50 | + $scanner->backgroundScan(''); |
|
| 51 | + } catch (\Exception $e) { |
|
| 52 | + $this->logger->error($e->getMessage(), ['exception' => $e, 'app' => 'files']); |
|
| 53 | + } |
|
| 54 | + \OC_Util::tearDownFS(); |
|
| 55 | + } |
|
| 56 | + |
|
| 57 | + /** |
|
| 58 | + * Find a storage which have unindexed files and return a user with access to the storage |
|
| 59 | + * |
|
| 60 | + * @return string|false |
|
| 61 | + */ |
|
| 62 | + private function getUserToScan() { |
|
| 63 | + if ($this->connection->getShardDefinition('filecache')) { |
|
| 64 | + // for sharded filecache, the "LIMIT" from the normal query doesn't work |
|
| 65 | + |
|
| 66 | + // first we try it with a "LEFT JOIN" on mounts, this is fast, but might return a storage that isn't mounted. |
|
| 67 | + // we also ask for up to 10 results from different storages to increase the odds of finding a result that is mounted |
|
| 68 | + $query = $this->connection->getQueryBuilder(); |
|
| 69 | + $query->select('m.user_id') |
|
| 70 | + ->from('filecache', 'f') |
|
| 71 | + ->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage')) |
|
| 72 | + ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 73 | + ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 74 | + ->setMaxResults(10) |
|
| 75 | + ->groupBy('f.storage') |
|
| 76 | + ->runAcrossAllShards(); |
|
| 77 | + |
|
| 78 | + $result = $query->executeQuery(); |
|
| 79 | + while ($res = $result->fetch()) { |
|
| 80 | + if ($res['user_id']) { |
|
| 81 | + return $res['user_id']; |
|
| 82 | + } |
|
| 83 | + } |
|
| 84 | + |
|
| 85 | + // as a fallback, we try a slower approach where we find all mounted storages first |
|
| 86 | + // this is essentially doing the inner join manually |
|
| 87 | + $storages = $this->getAllMountedStorages(); |
|
| 88 | + |
|
| 89 | + $query = $this->connection->getQueryBuilder(); |
|
| 90 | + $query->select('m.user_id') |
|
| 91 | + ->from('filecache', 'f') |
|
| 92 | + ->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage')) |
|
| 93 | + ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 94 | + ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 95 | + ->andWhere($query->expr()->in('f.storage', $query->createNamedParameter($storages, IQueryBuilder::PARAM_INT_ARRAY))) |
|
| 96 | + ->setMaxResults(1) |
|
| 97 | + ->runAcrossAllShards(); |
|
| 98 | + return $query->executeQuery()->fetchOne(); |
|
| 99 | + } else { |
|
| 100 | + $query = $this->connection->getQueryBuilder(); |
|
| 101 | + $query->select('m.user_id') |
|
| 102 | + ->from('filecache', 'f') |
|
| 103 | + ->innerJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage')) |
|
| 104 | + ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 105 | + ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) |
|
| 106 | + ->setMaxResults(1) |
|
| 107 | + ->runAcrossAllShards(); |
|
| 108 | + |
|
| 109 | + return $query->executeQuery()->fetchOne(); |
|
| 110 | + } |
|
| 111 | + } |
|
| 112 | + |
|
| 113 | + private function getAllMountedStorages(): array { |
|
| 114 | + $query = $this->connection->getQueryBuilder(); |
|
| 115 | + $query->selectDistinct('storage_id') |
|
| 116 | + ->from('mounts'); |
|
| 117 | + return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); |
|
| 118 | + } |
|
| 119 | + |
|
| 120 | + /** |
|
| 121 | + * @param $argument |
|
| 122 | + * @throws \Exception |
|
| 123 | + */ |
|
| 124 | + protected function run($argument) { |
|
| 125 | + if ($this->config->getSystemValueBool('files_no_background_scan', false)) { |
|
| 126 | + return; |
|
| 127 | + } |
|
| 128 | + |
|
| 129 | + $usersScanned = 0; |
|
| 130 | + $lastUser = ''; |
|
| 131 | + $user = $this->getUserToScan(); |
|
| 132 | + while ($user && $usersScanned < self::USERS_PER_SESSION && $lastUser !== $user) { |
|
| 133 | + $this->runScanner($user); |
|
| 134 | + $lastUser = $user; |
|
| 135 | + $user = $this->getUserToScan(); |
|
| 136 | + $usersScanned += 1; |
|
| 137 | + } |
|
| 138 | + |
|
| 139 | + if ($lastUser === $user) { |
|
| 140 | + $this->logger->warning("User $user still has unscanned files after running background scan, background scan might be stopped prematurely"); |
|
| 141 | + } |
|
| 142 | + } |
|
| 143 | 143 | } |
