@@ -36,195 +36,195 @@ |
||
36 | 36 | * Propagate etags and mtimes within the storage |
37 | 37 | */ |
38 | 38 | class Propagator implements IPropagator { |
39 | - public const MAX_RETRIES = 3; |
|
40 | - private $inBatch = false; |
|
41 | - |
|
42 | - private $batch = []; |
|
43 | - |
|
44 | - /** |
|
45 | - * @var \OC\Files\Storage\Storage |
|
46 | - */ |
|
47 | - protected $storage; |
|
48 | - |
|
49 | - /** |
|
50 | - * @var IDBConnection |
|
51 | - */ |
|
52 | - private $connection; |
|
53 | - |
|
54 | - /** |
|
55 | - * @var array |
|
56 | - */ |
|
57 | - private $ignore = []; |
|
58 | - |
|
59 | - public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection, array $ignore = []) { |
|
60 | - $this->storage = $storage; |
|
61 | - $this->connection = $connection; |
|
62 | - $this->ignore = $ignore; |
|
63 | - } |
|
64 | - |
|
65 | - |
|
66 | - /** |
|
67 | - * @param string $internalPath |
|
68 | - * @param int $time |
|
69 | - * @param int $sizeDifference number of bytes the file has grown |
|
70 | - */ |
|
71 | - public function propagateChange($internalPath, $time, $sizeDifference = 0) { |
|
72 | - // Do not propagate changes in ignored paths |
|
73 | - foreach ($this->ignore as $ignore) { |
|
74 | - if (strpos($internalPath, $ignore) === 0) { |
|
75 | - return; |
|
76 | - } |
|
77 | - } |
|
78 | - |
|
79 | - $storageId = (int)$this->storage->getStorageCache()->getNumericId(); |
|
80 | - |
|
81 | - $parents = $this->getParents($internalPath); |
|
82 | - |
|
83 | - if ($this->inBatch) { |
|
84 | - foreach ($parents as $parent) { |
|
85 | - $this->addToBatch($parent, $time, $sizeDifference); |
|
86 | - } |
|
87 | - return; |
|
88 | - } |
|
89 | - |
|
90 | - $parentHashes = array_map('md5', $parents); |
|
91 | - $etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag |
|
92 | - |
|
93 | - $builder = $this->connection->getQueryBuilder(); |
|
94 | - $hashParams = array_map(function ($hash) use ($builder) { |
|
95 | - return $builder->expr()->literal($hash); |
|
96 | - }, $parentHashes); |
|
97 | - |
|
98 | - $builder->update('filecache') |
|
99 | - ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int)$time, IQueryBuilder::PARAM_INT))) |
|
100 | - ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) |
|
101 | - ->andWhere($builder->expr()->in('path_hash', $hashParams)); |
|
102 | - if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { |
|
103 | - $builder->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR)); |
|
104 | - } |
|
105 | - |
|
106 | - if ($sizeDifference !== 0) { |
|
107 | - $hasCalculatedSize = $builder->expr()->gt('size', $builder->expr()->literal(-1, IQUeryBuilder::PARAM_INT)); |
|
108 | - $sizeColumn = $builder->getColumnName('size'); |
|
109 | - $newSize = $builder->func()->greatest( |
|
110 | - $builder->func()->add('size', $builder->createNamedParameter($sizeDifference)), |
|
111 | - $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) |
|
112 | - ); |
|
113 | - |
|
114 | - // Only update if row had a previously calculated size |
|
115 | - $builder->set('size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newSize ELSE $sizeColumn END")); |
|
116 | - |
|
117 | - if ($this->storage->instanceOfStorage(Encryption::class)) { |
|
118 | - // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size |
|
119 | - $hasUnencryptedSize = $builder->expr()->neq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT)); |
|
120 | - $sizeColumn = $builder->getColumnName('size'); |
|
121 | - $unencryptedSizeColumn = $builder->getColumnName('unencrypted_size'); |
|
122 | - $newUnencryptedSize = $builder->func()->greatest( |
|
123 | - $builder->func()->add( |
|
124 | - $builder->createFunction("CASE WHEN $hasUnencryptedSize THEN $unencryptedSizeColumn ELSE $sizeColumn END"), |
|
125 | - $builder->createNamedParameter($sizeDifference) |
|
126 | - ), |
|
127 | - $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) |
|
128 | - ); |
|
129 | - |
|
130 | - // Only update if row had a previously calculated size |
|
131 | - $builder->set('unencrypted_size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newUnencryptedSize ELSE $unencryptedSizeColumn END")); |
|
132 | - } |
|
133 | - } |
|
134 | - |
|
135 | - for ($i = 0; $i < self::MAX_RETRIES; $i++) { |
|
136 | - try { |
|
137 | - $builder->executeStatement(); |
|
138 | - break; |
|
139 | - } catch (RetryableException $e) { |
|
140 | - /** @var LoggerInterface $loggerInterface */ |
|
141 | - $loggerInterface = \OCP\Server::get(LoggerInterface::class); |
|
142 | - $loggerInterface->warning('Retrying propagation query after retryable exception.', [ 'exception' => $e ]); |
|
143 | - } |
|
144 | - } |
|
145 | - } |
|
146 | - |
|
147 | - protected function getParents($path) { |
|
148 | - $parts = explode('/', $path); |
|
149 | - $parent = ''; |
|
150 | - $parents = []; |
|
151 | - foreach ($parts as $part) { |
|
152 | - $parents[] = $parent; |
|
153 | - $parent = trim($parent . '/' . $part, '/'); |
|
154 | - } |
|
155 | - return $parents; |
|
156 | - } |
|
157 | - |
|
158 | - /** |
|
159 | - * Mark the beginning of a propagation batch |
|
160 | - * |
|
161 | - * Note that not all cache setups support propagation in which case this will be a noop |
|
162 | - * |
|
163 | - * Batching for cache setups that do support it has to be explicit since the cache state is not fully consistent |
|
164 | - * before the batch is committed. |
|
165 | - */ |
|
166 | - public function beginBatch() { |
|
167 | - $this->inBatch = true; |
|
168 | - } |
|
169 | - |
|
170 | - private function addToBatch($internalPath, $time, $sizeDifference) { |
|
171 | - if (!isset($this->batch[$internalPath])) { |
|
172 | - $this->batch[$internalPath] = [ |
|
173 | - 'hash' => md5($internalPath), |
|
174 | - 'time' => $time, |
|
175 | - 'size' => $sizeDifference, |
|
176 | - ]; |
|
177 | - } else { |
|
178 | - $this->batch[$internalPath]['size'] += $sizeDifference; |
|
179 | - if ($time > $this->batch[$internalPath]['time']) { |
|
180 | - $this->batch[$internalPath]['time'] = $time; |
|
181 | - } |
|
182 | - } |
|
183 | - } |
|
184 | - |
|
185 | - /** |
|
186 | - * Commit the active propagation batch |
|
187 | - */ |
|
188 | - public function commitBatch() { |
|
189 | - if (!$this->inBatch) { |
|
190 | - throw new \BadMethodCallException('Not in batch'); |
|
191 | - } |
|
192 | - $this->inBatch = false; |
|
193 | - |
|
194 | - $this->connection->beginTransaction(); |
|
195 | - |
|
196 | - $query = $this->connection->getQueryBuilder(); |
|
197 | - $storageId = (int)$this->storage->getStorageCache()->getNumericId(); |
|
198 | - |
|
199 | - $query->update('filecache') |
|
200 | - ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) |
|
201 | - ->set('etag', $query->expr()->literal(uniqid())) |
|
202 | - ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) |
|
203 | - ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))); |
|
204 | - |
|
205 | - $sizeQuery = $this->connection->getQueryBuilder(); |
|
206 | - $sizeQuery->update('filecache') |
|
207 | - ->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size'))) |
|
208 | - ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) |
|
209 | - ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))) |
|
210 | - ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->expr()->literal(-1, IQueryBuilder::PARAM_INT))); |
|
211 | - |
|
212 | - foreach ($this->batch as $item) { |
|
213 | - $query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT); |
|
214 | - $query->setParameter('hash', $item['hash']); |
|
215 | - |
|
216 | - $query->execute(); |
|
217 | - |
|
218 | - if ($item['size']) { |
|
219 | - $sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT); |
|
220 | - $sizeQuery->setParameter('hash', $item['hash']); |
|
221 | - |
|
222 | - $sizeQuery->execute(); |
|
223 | - } |
|
224 | - } |
|
225 | - |
|
226 | - $this->batch = []; |
|
227 | - |
|
228 | - $this->connection->commit(); |
|
229 | - } |
|
39 | + public const MAX_RETRIES = 3; |
|
40 | + private $inBatch = false; |
|
41 | + |
|
42 | + private $batch = []; |
|
43 | + |
|
44 | + /** |
|
45 | + * @var \OC\Files\Storage\Storage |
|
46 | + */ |
|
47 | + protected $storage; |
|
48 | + |
|
49 | + /** |
|
50 | + * @var IDBConnection |
|
51 | + */ |
|
52 | + private $connection; |
|
53 | + |
|
54 | + /** |
|
55 | + * @var array |
|
56 | + */ |
|
57 | + private $ignore = []; |
|
58 | + |
|
59 | + public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection, array $ignore = []) { |
|
60 | + $this->storage = $storage; |
|
61 | + $this->connection = $connection; |
|
62 | + $this->ignore = $ignore; |
|
63 | + } |
|
64 | + |
|
65 | + |
|
66 | + /** |
|
67 | + * @param string $internalPath |
|
68 | + * @param int $time |
|
69 | + * @param int $sizeDifference number of bytes the file has grown |
|
70 | + */ |
|
71 | + public function propagateChange($internalPath, $time, $sizeDifference = 0) { |
|
72 | + // Do not propagate changes in ignored paths |
|
73 | + foreach ($this->ignore as $ignore) { |
|
74 | + if (strpos($internalPath, $ignore) === 0) { |
|
75 | + return; |
|
76 | + } |
|
77 | + } |
|
78 | + |
|
79 | + $storageId = (int)$this->storage->getStorageCache()->getNumericId(); |
|
80 | + |
|
81 | + $parents = $this->getParents($internalPath); |
|
82 | + |
|
83 | + if ($this->inBatch) { |
|
84 | + foreach ($parents as $parent) { |
|
85 | + $this->addToBatch($parent, $time, $sizeDifference); |
|
86 | + } |
|
87 | + return; |
|
88 | + } |
|
89 | + |
|
90 | + $parentHashes = array_map('md5', $parents); |
|
91 | + $etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag |
|
92 | + |
|
93 | + $builder = $this->connection->getQueryBuilder(); |
|
94 | + $hashParams = array_map(function ($hash) use ($builder) { |
|
95 | + return $builder->expr()->literal($hash); |
|
96 | + }, $parentHashes); |
|
97 | + |
|
98 | + $builder->update('filecache') |
|
99 | + ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int)$time, IQueryBuilder::PARAM_INT))) |
|
100 | + ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) |
|
101 | + ->andWhere($builder->expr()->in('path_hash', $hashParams)); |
|
102 | + if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { |
|
103 | + $builder->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR)); |
|
104 | + } |
|
105 | + |
|
106 | + if ($sizeDifference !== 0) { |
|
107 | + $hasCalculatedSize = $builder->expr()->gt('size', $builder->expr()->literal(-1, IQUeryBuilder::PARAM_INT)); |
|
108 | + $sizeColumn = $builder->getColumnName('size'); |
|
109 | + $newSize = $builder->func()->greatest( |
|
110 | + $builder->func()->add('size', $builder->createNamedParameter($sizeDifference)), |
|
111 | + $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) |
|
112 | + ); |
|
113 | + |
|
114 | + // Only update if row had a previously calculated size |
|
115 | + $builder->set('size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newSize ELSE $sizeColumn END")); |
|
116 | + |
|
117 | + if ($this->storage->instanceOfStorage(Encryption::class)) { |
|
118 | + // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size |
|
119 | + $hasUnencryptedSize = $builder->expr()->neq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT)); |
|
120 | + $sizeColumn = $builder->getColumnName('size'); |
|
121 | + $unencryptedSizeColumn = $builder->getColumnName('unencrypted_size'); |
|
122 | + $newUnencryptedSize = $builder->func()->greatest( |
|
123 | + $builder->func()->add( |
|
124 | + $builder->createFunction("CASE WHEN $hasUnencryptedSize THEN $unencryptedSizeColumn ELSE $sizeColumn END"), |
|
125 | + $builder->createNamedParameter($sizeDifference) |
|
126 | + ), |
|
127 | + $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) |
|
128 | + ); |
|
129 | + |
|
130 | + // Only update if row had a previously calculated size |
|
131 | + $builder->set('unencrypted_size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newUnencryptedSize ELSE $unencryptedSizeColumn END")); |
|
132 | + } |
|
133 | + } |
|
134 | + |
|
135 | + for ($i = 0; $i < self::MAX_RETRIES; $i++) { |
|
136 | + try { |
|
137 | + $builder->executeStatement(); |
|
138 | + break; |
|
139 | + } catch (RetryableException $e) { |
|
140 | + /** @var LoggerInterface $loggerInterface */ |
|
141 | + $loggerInterface = \OCP\Server::get(LoggerInterface::class); |
|
142 | + $loggerInterface->warning('Retrying propagation query after retryable exception.', [ 'exception' => $e ]); |
|
143 | + } |
|
144 | + } |
|
145 | + } |
|
146 | + |
|
147 | + protected function getParents($path) { |
|
148 | + $parts = explode('/', $path); |
|
149 | + $parent = ''; |
|
150 | + $parents = []; |
|
151 | + foreach ($parts as $part) { |
|
152 | + $parents[] = $parent; |
|
153 | + $parent = trim($parent . '/' . $part, '/'); |
|
154 | + } |
|
155 | + return $parents; |
|
156 | + } |
|
157 | + |
|
158 | + /** |
|
159 | + * Mark the beginning of a propagation batch |
|
160 | + * |
|
161 | + * Note that not all cache setups support propagation in which case this will be a noop |
|
162 | + * |
|
163 | + * Batching for cache setups that do support it has to be explicit since the cache state is not fully consistent |
|
164 | + * before the batch is committed. |
|
165 | + */ |
|
166 | + public function beginBatch() { |
|
167 | + $this->inBatch = true; |
|
168 | + } |
|
169 | + |
|
170 | + private function addToBatch($internalPath, $time, $sizeDifference) { |
|
171 | + if (!isset($this->batch[$internalPath])) { |
|
172 | + $this->batch[$internalPath] = [ |
|
173 | + 'hash' => md5($internalPath), |
|
174 | + 'time' => $time, |
|
175 | + 'size' => $sizeDifference, |
|
176 | + ]; |
|
177 | + } else { |
|
178 | + $this->batch[$internalPath]['size'] += $sizeDifference; |
|
179 | + if ($time > $this->batch[$internalPath]['time']) { |
|
180 | + $this->batch[$internalPath]['time'] = $time; |
|
181 | + } |
|
182 | + } |
|
183 | + } |
|
184 | + |
|
185 | + /** |
|
186 | + * Commit the active propagation batch |
|
187 | + */ |
|
188 | + public function commitBatch() { |
|
189 | + if (!$this->inBatch) { |
|
190 | + throw new \BadMethodCallException('Not in batch'); |
|
191 | + } |
|
192 | + $this->inBatch = false; |
|
193 | + |
|
194 | + $this->connection->beginTransaction(); |
|
195 | + |
|
196 | + $query = $this->connection->getQueryBuilder(); |
|
197 | + $storageId = (int)$this->storage->getStorageCache()->getNumericId(); |
|
198 | + |
|
199 | + $query->update('filecache') |
|
200 | + ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) |
|
201 | + ->set('etag', $query->expr()->literal(uniqid())) |
|
202 | + ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) |
|
203 | + ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))); |
|
204 | + |
|
205 | + $sizeQuery = $this->connection->getQueryBuilder(); |
|
206 | + $sizeQuery->update('filecache') |
|
207 | + ->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size'))) |
|
208 | + ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) |
|
209 | + ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))) |
|
210 | + ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->expr()->literal(-1, IQueryBuilder::PARAM_INT))); |
|
211 | + |
|
212 | + foreach ($this->batch as $item) { |
|
213 | + $query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT); |
|
214 | + $query->setParameter('hash', $item['hash']); |
|
215 | + |
|
216 | + $query->execute(); |
|
217 | + |
|
218 | + if ($item['size']) { |
|
219 | + $sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT); |
|
220 | + $sizeQuery->setParameter('hash', $item['hash']); |
|
221 | + |
|
222 | + $sizeQuery->execute(); |
|
223 | + } |
|
224 | + } |
|
225 | + |
|
226 | + $this->batch = []; |
|
227 | + |
|
228 | + $this->connection->commit(); |
|
229 | + } |
|
230 | 230 | } |
@@ -76,7 +76,7 @@ discard block |
||
76 | 76 | } |
77 | 77 | } |
78 | 78 | |
79 | - $storageId = (int)$this->storage->getStorageCache()->getNumericId(); |
|
79 | + $storageId = (int) $this->storage->getStorageCache()->getNumericId(); |
|
80 | 80 | |
81 | 81 | $parents = $this->getParents($internalPath); |
82 | 82 | |
@@ -91,12 +91,12 @@ discard block |
||
91 | 91 | $etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag |
92 | 92 | |
93 | 93 | $builder = $this->connection->getQueryBuilder(); |
94 | - $hashParams = array_map(function ($hash) use ($builder) { |
|
94 | + $hashParams = array_map(function($hash) use ($builder) { |
|
95 | 95 | return $builder->expr()->literal($hash); |
96 | 96 | }, $parentHashes); |
97 | 97 | |
98 | 98 | $builder->update('filecache') |
99 | - ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int)$time, IQueryBuilder::PARAM_INT))) |
|
99 | + ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int) $time, IQueryBuilder::PARAM_INT))) |
|
100 | 100 | ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) |
101 | 101 | ->andWhere($builder->expr()->in('path_hash', $hashParams)); |
102 | 102 | if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { |
@@ -139,7 +139,7 @@ discard block |
||
139 | 139 | } catch (RetryableException $e) { |
140 | 140 | /** @var LoggerInterface $loggerInterface */ |
141 | 141 | $loggerInterface = \OCP\Server::get(LoggerInterface::class); |
142 | - $loggerInterface->warning('Retrying propagation query after retryable exception.', [ 'exception' => $e ]); |
|
142 | + $loggerInterface->warning('Retrying propagation query after retryable exception.', ['exception' => $e]); |
|
143 | 143 | } |
144 | 144 | } |
145 | 145 | } |
@@ -150,7 +150,7 @@ discard block |
||
150 | 150 | $parents = []; |
151 | 151 | foreach ($parts as $part) { |
152 | 152 | $parents[] = $parent; |
153 | - $parent = trim($parent . '/' . $part, '/'); |
|
153 | + $parent = trim($parent.'/'.$part, '/'); |
|
154 | 154 | } |
155 | 155 | return $parents; |
156 | 156 | } |
@@ -194,7 +194,7 @@ discard block |
||
194 | 194 | $this->connection->beginTransaction(); |
195 | 195 | |
196 | 196 | $query = $this->connection->getQueryBuilder(); |
197 | - $storageId = (int)$this->storage->getStorageCache()->getNumericId(); |
|
197 | + $storageId = (int) $this->storage->getStorageCache()->getNumericId(); |
|
198 | 198 | |
199 | 199 | $query->update('filecache') |
200 | 200 | ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) |