These results are based on our legacy PHP analysis, consider migrating to our new PHP analysis engine instead. Learn more
1 | <?php |
||
2 | /** |
||
3 | * External storage in SQL database. |
||
4 | * |
||
5 | * This program is free software; you can redistribute it and/or modify |
||
6 | * it under the terms of the GNU General Public License as published by |
||
7 | * the Free Software Foundation; either version 2 of the License, or |
||
8 | * (at your option) any later version. |
||
9 | * |
||
10 | * This program is distributed in the hope that it will be useful, |
||
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
||
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||
13 | * GNU General Public License for more details. |
||
14 | * |
||
15 | * You should have received a copy of the GNU General Public License along |
||
16 | * with this program; if not, write to the Free Software Foundation, Inc., |
||
17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
||
18 | * http://www.gnu.org/copyleft/gpl.html |
||
19 | * |
||
20 | * @file |
||
21 | */ |
||
22 | |||
23 | /** |
||
24 | * DB accessable external objects. |
||
25 | * |
||
26 | * In this system, each store "location" maps to a database "cluster". |
||
27 | * The clusters must be defined in the normal LBFactory configuration. |
||
28 | * |
||
29 | * @ingroup ExternalStorage |
||
30 | */ |
||
31 | class ExternalStoreDB extends ExternalStoreMedium { |
||
32 | /** |
||
33 | * The provided URL is in the form of DB://cluster/id |
||
34 | * or DB://cluster/id/itemid for concatened storage. |
||
35 | * |
||
36 | * @param string $url |
||
37 | * @return string|bool False if missing |
||
38 | * @see ExternalStoreMedium::fetchFromURL() |
||
39 | */ |
||
40 | public function fetchFromURL( $url ) { |
||
41 | list( $cluster, $id, $itemID ) = $this->parseURL( $url ); |
||
42 | $ret = $this->fetchBlob( $cluster, $id, $itemID ); |
||
43 | |||
44 | if ( $itemID !== false && $ret !== false ) { |
||
45 | return $ret->getItem( $itemID ); |
||
46 | } |
||
47 | |||
48 | return $ret; |
||
49 | } |
||
50 | |||
51 | /** |
||
52 | * Fetch data from given external store URLs. |
||
53 | * The provided URLs are in the form of DB://cluster/id |
||
54 | * or DB://cluster/id/itemid for concatened storage. |
||
55 | * |
||
56 | * @param array $urls An array of external store URLs |
||
57 | * @return array A map from url to stored content. Failed results |
||
58 | * are not represented. |
||
59 | */ |
||
60 | public function batchFetchFromURLs( array $urls ) { |
||
61 | $batched = $inverseUrlMap = []; |
||
62 | View Code Duplication | foreach ( $urls as $url ) { |
|
63 | list( $cluster, $id, $itemID ) = $this->parseURL( $url ); |
||
64 | $batched[$cluster][$id][] = $itemID; |
||
65 | // false $itemID gets cast to int, but should be ok |
||
66 | // since we do === from the $itemID in $batched |
||
67 | $inverseUrlMap[$cluster][$id][$itemID] = $url; |
||
68 | } |
||
69 | $ret = []; |
||
70 | foreach ( $batched as $cluster => $batchByCluster ) { |
||
71 | $res = $this->batchFetchBlobs( $cluster, $batchByCluster ); |
||
72 | /** @var HistoryBlob $blob */ |
||
73 | foreach ( $res as $id => $blob ) { |
||
74 | View Code Duplication | foreach ( $batchByCluster[$id] as $itemID ) { |
|
75 | $url = $inverseUrlMap[$cluster][$id][$itemID]; |
||
76 | if ( $itemID === false ) { |
||
77 | $ret[$url] = $blob; |
||
78 | } else { |
||
79 | $ret[$url] = $blob->getItem( $itemID ); |
||
80 | } |
||
81 | } |
||
82 | } |
||
83 | } |
||
84 | |||
85 | return $ret; |
||
86 | } |
||
87 | |||
88 | public function store( $location, $data ) { |
||
89 | $dbw = $this->getMaster( $location ); |
||
90 | $id = $dbw->nextSequenceValue( 'blob_blob_id_seq' ); |
||
91 | $dbw->insert( $this->getTable( $dbw ), |
||
92 | [ 'blob_id' => $id, 'blob_text' => $data ], |
||
93 | __METHOD__ ); |
||
94 | $id = $dbw->insertId(); |
||
95 | if ( !$id ) { |
||
96 | throw new MWException( __METHOD__ . ': no insert ID' ); |
||
97 | } |
||
98 | |||
99 | return "DB://$location/$id"; |
||
100 | } |
||
101 | |||
102 | /** |
||
103 | * Get a LoadBalancer for the specified cluster |
||
104 | * |
||
105 | * @param string $cluster Cluster name |
||
106 | * @return LoadBalancer |
||
107 | */ |
||
108 | function getLoadBalancer( $cluster ) { |
||
109 | $wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false; |
||
110 | |||
111 | return wfGetLBFactory()->getExternalLB( $cluster, $wiki ); |
||
0 ignored issues
–
show
|
|||
112 | } |
||
113 | |||
114 | /** |
||
115 | * Get a replica DB connection for the specified cluster |
||
116 | * |
||
117 | * @param string $cluster Cluster name |
||
118 | * @return IDatabase |
||
119 | */ |
||
120 | function getSlave( $cluster ) { |
||
121 | global $wgDefaultExternalStore; |
||
122 | |||
123 | $wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false; |
||
124 | $lb = $this->getLoadBalancer( $cluster ); |
||
125 | |||
126 | if ( !in_array( "DB://" . $cluster, (array)$wgDefaultExternalStore ) ) { |
||
127 | wfDebug( "read only external store\n" ); |
||
128 | $lb->allowLagged( true ); |
||
129 | } else { |
||
130 | wfDebug( "writable external store\n" ); |
||
131 | } |
||
132 | |||
133 | $db = $lb->getConnectionRef( DB_REPLICA, [], $wiki ); |
||
134 | $db->clearFlag( DBO_TRX ); // sanity |
||
135 | |||
136 | return $db; |
||
137 | } |
||
138 | |||
139 | /** |
||
140 | * Get a master database connection for the specified cluster |
||
141 | * |
||
142 | * @param string $cluster Cluster name |
||
143 | * @return IDatabase |
||
144 | */ |
||
145 | function getMaster( $cluster ) { |
||
146 | $wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false; |
||
147 | $lb = $this->getLoadBalancer( $cluster ); |
||
148 | |||
149 | $db = $lb->getConnectionRef( DB_MASTER, [], $wiki ); |
||
150 | $db->clearFlag( DBO_TRX ); // sanity |
||
151 | |||
152 | return $db; |
||
153 | } |
||
154 | |||
155 | /** |
||
156 | * Get the 'blobs' table name for this database |
||
157 | * |
||
158 | * @param IDatabase $db |
||
159 | * @return string Table name ('blobs' by default) |
||
160 | */ |
||
161 | function getTable( $db ) { |
||
162 | $table = $db->getLBInfo( 'blobs table' ); |
||
163 | if ( is_null( $table ) ) { |
||
164 | $table = 'blobs'; |
||
165 | } |
||
166 | |||
167 | return $table; |
||
168 | } |
||
169 | |||
170 | /** |
||
171 | * Fetch a blob item out of the database; a cache of the last-loaded |
||
172 | * blob will be kept so that multiple loads out of a multi-item blob |
||
173 | * can avoid redundant database access and decompression. |
||
174 | * @param string $cluster |
||
175 | * @param string $id |
||
176 | * @param string $itemID |
||
177 | * @return HistoryBlob|bool Returns false if missing |
||
178 | * @private |
||
179 | */ |
||
180 | function fetchBlob( $cluster, $id, $itemID ) { |
||
181 | /** |
||
182 | * One-step cache variable to hold base blobs; operations that |
||
183 | * pull multiple revisions may often pull multiple times from |
||
184 | * the same blob. By keeping the last-used one open, we avoid |
||
185 | * redundant unserialization and decompression overhead. |
||
186 | */ |
||
187 | static $externalBlobCache = []; |
||
188 | |||
189 | $cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/"; |
||
190 | if ( isset( $externalBlobCache[$cacheID] ) ) { |
||
191 | wfDebugLog( 'ExternalStoreDB-cache', |
||
192 | "ExternalStoreDB::fetchBlob cache hit on $cacheID" ); |
||
193 | |||
194 | return $externalBlobCache[$cacheID]; |
||
195 | } |
||
196 | |||
197 | wfDebugLog( 'ExternalStoreDB-cache', |
||
198 | "ExternalStoreDB::fetchBlob cache miss on $cacheID" ); |
||
199 | |||
200 | $dbr = $this->getSlave( $cluster ); |
||
201 | $ret = $dbr->selectField( $this->getTable( $dbr ), |
||
202 | 'blob_text', [ 'blob_id' => $id ], __METHOD__ ); |
||
203 | if ( $ret === false ) { |
||
204 | wfDebugLog( 'ExternalStoreDB', |
||
205 | "ExternalStoreDB::fetchBlob master fallback on $cacheID" ); |
||
206 | // Try the master |
||
207 | $dbw = $this->getMaster( $cluster ); |
||
208 | $ret = $dbw->selectField( $this->getTable( $dbw ), |
||
209 | 'blob_text', [ 'blob_id' => $id ], __METHOD__ ); |
||
210 | if ( $ret === false ) { |
||
211 | wfDebugLog( 'ExternalStoreDB', |
||
212 | "ExternalStoreDB::fetchBlob master failed to find $cacheID" ); |
||
213 | } |
||
214 | } |
||
215 | if ( $itemID !== false && $ret !== false ) { |
||
216 | // Unserialise object; caller extracts item |
||
217 | $ret = unserialize( $ret ); |
||
218 | } |
||
219 | |||
220 | $externalBlobCache = [ $cacheID => $ret ]; |
||
221 | |||
222 | return $ret; |
||
223 | } |
||
224 | |||
225 | /** |
||
226 | * Fetch multiple blob items out of the database |
||
227 | * |
||
228 | * @param string $cluster A cluster name valid for use with LBFactory |
||
229 | * @param array $ids A map from the blob_id's to look for to the requested itemIDs in the blobs |
||
230 | * @return array A map from the blob_id's requested to their content. |
||
231 | * Unlocated ids are not represented |
||
232 | */ |
||
233 | function batchFetchBlobs( $cluster, array $ids ) { |
||
234 | $dbr = $this->getSlave( $cluster ); |
||
235 | $res = $dbr->select( $this->getTable( $dbr ), |
||
236 | [ 'blob_id', 'blob_text' ], [ 'blob_id' => array_keys( $ids ) ], __METHOD__ ); |
||
237 | $ret = []; |
||
238 | if ( $res !== false ) { |
||
239 | $this->mergeBatchResult( $ret, $ids, $res ); |
||
240 | } |
||
241 | if ( $ids ) { |
||
242 | wfDebugLog( __CLASS__, __METHOD__ . |
||
243 | " master fallback on '$cluster' for: " . |
||
244 | implode( ',', array_keys( $ids ) ) ); |
||
245 | // Try the master |
||
246 | $dbw = $this->getMaster( $cluster ); |
||
247 | $res = $dbw->select( $this->getTable( $dbr ), |
||
248 | [ 'blob_id', 'blob_text' ], |
||
249 | [ 'blob_id' => array_keys( $ids ) ], |
||
250 | __METHOD__ ); |
||
251 | if ( $res === false ) { |
||
252 | wfDebugLog( __CLASS__, __METHOD__ . " master failed on '$cluster'" ); |
||
253 | } else { |
||
254 | $this->mergeBatchResult( $ret, $ids, $res ); |
||
255 | } |
||
256 | } |
||
257 | if ( $ids ) { |
||
258 | wfDebugLog( __CLASS__, __METHOD__ . |
||
259 | " master on '$cluster' failed locating items: " . |
||
260 | implode( ',', array_keys( $ids ) ) ); |
||
261 | } |
||
262 | |||
263 | return $ret; |
||
264 | } |
||
265 | |||
266 | /** |
||
267 | * Helper function for self::batchFetchBlobs for merging master/replica DB results |
||
268 | * @param array &$ret Current self::batchFetchBlobs return value |
||
269 | * @param array &$ids Map from blob_id to requested itemIDs |
||
270 | * @param mixed $res DB result from Database::select |
||
271 | */ |
||
272 | private function mergeBatchResult( array &$ret, array &$ids, $res ) { |
||
273 | foreach ( $res as $row ) { |
||
274 | $id = $row->blob_id; |
||
275 | $itemIDs = $ids[$id]; |
||
276 | unset( $ids[$id] ); // to track if everything is found |
||
277 | if ( count( $itemIDs ) === 1 && reset( $itemIDs ) === false ) { |
||
278 | // single result stored per blob |
||
279 | $ret[$id] = $row->blob_text; |
||
280 | } else { |
||
281 | // multi result stored per blob |
||
282 | $ret[$id] = unserialize( $row->blob_text ); |
||
283 | } |
||
284 | } |
||
285 | } |
||
286 | |||
287 | /** |
||
288 | * @param string $url |
||
289 | * @return array |
||
290 | */ |
||
291 | protected function parseURL( $url ) { |
||
292 | $path = explode( '/', $url ); |
||
293 | |||
294 | return [ |
||
295 | $path[2], // cluster |
||
296 | $path[3], // id |
||
297 | isset( $path[4] ) ? $path[4] : false // itemID |
||
298 | ]; |
||
299 | } |
||
300 | } |
||
301 |
This function has been deprecated. The supplier of the file has supplied an explanatory message.
The explanatory message should give you some clue as to whether and when the function will be removed from the class and what other function to use instead.