Issues in ExternalStoreDB.php - All Issues - Inspection of "Daily Inspection: Merge "Add GENDER support to pro..." - wikimedia/mediawiki - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Branch — master (939199)

unknown

created 2016-11-04 17:19 UTC

includes/externalstore/ExternalStoreDB.php (1 issue)

Labels

Deprecated Code 1

Severity

Minor 1

Andrew Garrett 1

Upgrade to new PHP Analysis Engine

These results are based on our legacy PHP analysis, consider migrating to our new PHP analysis engine instead. Learn more

<?php
/**
 * External storage in SQL database.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 * http://www.gnu.org/copyleft/gpl.html
 *
 * @file
 */

/**
 * DB accessable external objects.
 *
 * In this system, each store "location" maps to a database "cluster".
 * The clusters must be defined in the normal LBFactory configuration.
 *
 * @ingroup ExternalStorage
 */
class ExternalStoreDB extends ExternalStoreMedium {
	/**
	 * The provided URL is in the form of DB://cluster/id
	 * or DB://cluster/id/itemid for concatened storage.
	 *
	 * @param string $url
	 * @return string|bool False if missing
	 * @see ExternalStoreMedium::fetchFromURL()
	 */
	public function fetchFromURL( $url ) {
		list( $cluster, $id, $itemID ) = $this->parseURL( $url );
		$ret = $this->fetchBlob( $cluster, $id, $itemID );

		if ( $itemID !== false && $ret !== false ) {
			return $ret->getItem( $itemID );
		}

		return $ret;
	}

	/**
	 * Fetch data from given external store URLs.
	 * The provided URLs are in the form of DB://cluster/id
	 * or DB://cluster/id/itemid for concatened storage.
	 *
	 * @param array $urls An array of external store URLs
	 * @return array A map from url to stored content. Failed results
	 *     are not represented.
	 */
	public function batchFetchFromURLs( array $urls ) {
		$batched = $inverseUrlMap = [];
		foreach ( $urls as $url ) {
			list( $cluster, $id, $itemID ) = $this->parseURL( $url );
			$batched[$cluster][$id][] = $itemID;
			// false $itemID gets cast to int, but should be ok
			// since we do === from the $itemID in $batched
			$inverseUrlMap[$cluster][$id][$itemID] = $url;
		}
		$ret = [];
		foreach ( $batched as $cluster => $batchByCluster ) {
			$res = $this->batchFetchBlobs( $cluster, $batchByCluster );
			/** @var HistoryBlob $blob */
			foreach ( $res as $id => $blob ) {
				foreach ( $batchByCluster[$id] as $itemID ) {
					$url = $inverseUrlMap[$cluster][$id][$itemID];
					if ( $itemID === false ) {
						$ret[$url] = $blob;
					} else {
						$ret[$url] = $blob->getItem( $itemID );
					}
				}
			}
		}

		return $ret;
	}

	public function store( $location, $data ) {
		$dbw = $this->getMaster( $location );
		$id = $dbw->nextSequenceValue( 'blob_blob_id_seq' );
		$dbw->insert( $this->getTable( $dbw ),
			[ 'blob_id' => $id, 'blob_text' => $data ],
			__METHOD__ );
		$id = $dbw->insertId();
		if ( !$id ) {
			throw new MWException( __METHOD__ . ': no insert ID' );
		}

		return "DB://$location/$id";
	}

	/**
	 * Get a LoadBalancer for the specified cluster
	 *
	 * @param string $cluster Cluster name
	 * @return LoadBalancer
	 */
	function getLoadBalancer( $cluster ) {
		$wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false;

		return wfGetLBFactory()->getExternalLB( $cluster, $wiki );

	}

	/**
	 * Get a replica DB connection for the specified cluster
	 *
	 * @param string $cluster Cluster name
	 * @return IDatabase
	 */
	function getSlave( $cluster ) {
		global $wgDefaultExternalStore;

		$wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false;
		$lb = $this->getLoadBalancer( $cluster );

		if ( !in_array( "DB://" . $cluster, (array)$wgDefaultExternalStore ) ) {
			wfDebug( "read only external store\n" );
			$lb->allowLagged( true );
		} else {
			wfDebug( "writable external store\n" );
		}

		$db = $lb->getConnectionRef( DB_REPLICA, [], $wiki );
		$db->clearFlag( DBO_TRX ); // sanity

		return $db;
	}

	/**
	 * Get a master database connection for the specified cluster
	 *
	 * @param string $cluster Cluster name
	 * @return IDatabase
	 */
	function getMaster( $cluster ) {
		$wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false;
		$lb = $this->getLoadBalancer( $cluster );

		$db = $lb->getConnectionRef( DB_MASTER, [], $wiki );
		$db->clearFlag( DBO_TRX ); // sanity

		return $db;
	}

	/**
	 * Get the 'blobs' table name for this database
	 *
	 * @param IDatabase $db
	 * @return string Table name ('blobs' by default)
	 */
	function getTable( $db ) {
		$table = $db->getLBInfo( 'blobs table' );
		if ( is_null( $table ) ) {
			$table = 'blobs';
		}

		return $table;
	}

	/**
	 * Fetch a blob item out of the database; a cache of the last-loaded
	 * blob will be kept so that multiple loads out of a multi-item blob
	 * can avoid redundant database access and decompression.
	 * @param string $cluster
	 * @param string $id
	 * @param string $itemID
	 * @return HistoryBlob|bool Returns false if missing
	 * @private
	 */
	function fetchBlob( $cluster, $id, $itemID ) {
		/**
		 * One-step cache variable to hold base blobs; operations that
		 * pull multiple revisions may often pull multiple times from
		 * the same blob. By keeping the last-used one open, we avoid
		 * redundant unserialization and decompression overhead.
		 */
		static $externalBlobCache = [];

		$cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/";
		if ( isset( $externalBlobCache[$cacheID] ) ) {
			wfDebugLog( 'ExternalStoreDB-cache',
				"ExternalStoreDB::fetchBlob cache hit on $cacheID" );

			return $externalBlobCache[$cacheID];
		}

		wfDebugLog( 'ExternalStoreDB-cache',
			"ExternalStoreDB::fetchBlob cache miss on $cacheID" );

		$dbr = $this->getSlave( $cluster );
		$ret = $dbr->selectField( $this->getTable( $dbr ),
			'blob_text', [ 'blob_id' => $id ], __METHOD__ );
		if ( $ret === false ) {
			wfDebugLog( 'ExternalStoreDB',
				"ExternalStoreDB::fetchBlob master fallback on $cacheID" );
			// Try the master
			$dbw = $this->getMaster( $cluster );
			$ret = $dbw->selectField( $this->getTable( $dbw ),
				'blob_text', [ 'blob_id' => $id ], __METHOD__ );
			if ( $ret === false ) {
				wfDebugLog( 'ExternalStoreDB',
					"ExternalStoreDB::fetchBlob master failed to find $cacheID" );
			}
		}
		if ( $itemID !== false && $ret !== false ) {
			// Unserialise object; caller extracts item
			$ret = unserialize( $ret );
		}

		$externalBlobCache = [ $cacheID => $ret ];

		return $ret;
	}

	/**
	 * Fetch multiple blob items out of the database
	 *
	 * @param string $cluster A cluster name valid for use with LBFactory
	 * @param array $ids A map from the blob_id's to look for to the requested itemIDs in the blobs
	 * @return array A map from the blob_id's requested to their content.
	 *   Unlocated ids are not represented
	 */
	function batchFetchBlobs( $cluster, array $ids ) {
		$dbr = $this->getSlave( $cluster );
		$res = $dbr->select( $this->getTable( $dbr ),
			[ 'blob_id', 'blob_text' ], [ 'blob_id' => array_keys( $ids ) ], __METHOD__ );
		$ret = [];
		if ( $res !== false ) {
			$this->mergeBatchResult( $ret, $ids, $res );
		}
		if ( $ids ) {
			wfDebugLog( __CLASS__, __METHOD__ .
				" master fallback on '$cluster' for: " .
				implode( ',', array_keys( $ids ) ) );
			// Try the master
			$dbw = $this->getMaster( $cluster );
			$res = $dbw->select( $this->getTable( $dbr ),
				[ 'blob_id', 'blob_text' ],
				[ 'blob_id' => array_keys( $ids ) ],
				__METHOD__ );
			if ( $res === false ) {
				wfDebugLog( __CLASS__, __METHOD__ . " master failed on '$cluster'" );
			} else {
				$this->mergeBatchResult( $ret, $ids, $res );
			}
		}
		if ( $ids ) {
			wfDebugLog( __CLASS__, __METHOD__ .
				" master on '$cluster' failed locating items: " .
				implode( ',', array_keys( $ids ) ) );
		}

		return $ret;
	}

	/**
	 * Helper function for self::batchFetchBlobs for merging master/replica DB results
	 * @param array &$ret Current self::batchFetchBlobs return value
	 * @param array &$ids Map from blob_id to requested itemIDs
	 * @param mixed $res DB result from Database::select
	 */
	private function mergeBatchResult( array &$ret, array &$ids, $res ) {
		foreach ( $res as $row ) {
			$id = $row->blob_id;
			$itemIDs = $ids[$id];
			unset( $ids[$id] ); // to track if everything is found
			if ( count( $itemIDs ) === 1 && reset( $itemIDs ) === false ) {
				// single result stored per blob
				$ret[$id] = $row->blob_text;
			} else {
				// multi result stored per blob
				$ret[$id] = unserialize( $row->blob_text );
			}
		}
	}

	/**
	 * @param string $url
	 * @return array
	 */
	protected function parseURL( $url ) {
		$path = explode( '/', $url );

		return [
			$path[2], // cluster
			$path[3], // id
			isset( $path[4] ) ? $path[4] : false // itemID
		];
	}
}


1		<?php
2		/**
3		* External storage in SQL database.
4		*
5		* This program is free software; you can redistribute it and/or modify
6		* it under the terms of the GNU General Public License as published by
7		* the Free Software Foundation; either version 2 of the License, or
8		* (at your option) any later version.
9		*
10		* This program is distributed in the hope that it will be useful,
11		* but WITHOUT ANY WARRANTY; without even the implied warranty of
12		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13		* GNU General Public License for more details.
14		*
15		* You should have received a copy of the GNU General Public License along
16		* with this program; if not, write to the Free Software Foundation, Inc.,
17		* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18		* http://www.gnu.org/copyleft/gpl.html
19		*
20		* @file
21		*/
22
23		/**
24		* DB accessable external objects.
25		*
26		* In this system, each store "location" maps to a database "cluster".
27		* The clusters must be defined in the normal LBFactory configuration.
28		*
29		* @ingroup ExternalStorage
30		*/
31		class ExternalStoreDB extends ExternalStoreMedium {
32		/**
33		* The provided URL is in the form of DB://cluster/id
34		* or DB://cluster/id/itemid for concatened storage.
35		*
36		* @param string $url
37		* @return string\|bool False if missing
38		* @see ExternalStoreMedium::fetchFromURL()
39		*/
40		public function fetchFromURL( $url ) {
41		list( $cluster, $id, $itemID ) = $this->parseURL( $url );
42		$ret = $this->fetchBlob( $cluster, $id, $itemID );
43
44		if ( $itemID !== false && $ret !== false ) {
45		return $ret->getItem( $itemID );
46		}
47
48		return $ret;
49		}
50
51		/**
52		* Fetch data from given external store URLs.
53		* The provided URLs are in the form of DB://cluster/id
54		* or DB://cluster/id/itemid for concatened storage.
55		*
56		* @param array $urls An array of external store URLs
57		* @return array A map from url to stored content. Failed results
58		* are not represented.
59		*/
60		public function batchFetchFromURLs( array $urls ) {
61		$batched = $inverseUrlMap = [];
62	View Code Duplication	foreach ( $urls as $url ) {
63		list( $cluster, $id, $itemID ) = $this->parseURL( $url );
64		$batched[$cluster][$id][] = $itemID;
65		// false $itemID gets cast to int, but should be ok
66		// since we do === from the $itemID in $batched
67		$inverseUrlMap[$cluster][$id][$itemID] = $url;
68		}
69		$ret = [];
70		foreach ( $batched as $cluster => $batchByCluster ) {
71		$res = $this->batchFetchBlobs( $cluster, $batchByCluster );
72		/** @var HistoryBlob $blob */
73		foreach ( $res as $id => $blob ) {
74	View Code Duplication	foreach ( $batchByCluster[$id] as $itemID ) {
75		$url = $inverseUrlMap[$cluster][$id][$itemID];
76		if ( $itemID === false ) {
77		$ret[$url] = $blob;
78		} else {
79		$ret[$url] = $blob->getItem( $itemID );
80		}
81		}
82		}
83		}
84
85		return $ret;
86		}
87
88		public function store( $location, $data ) {
89		$dbw = $this->getMaster( $location );
90		$id = $dbw->nextSequenceValue( 'blob_blob_id_seq' );
91		$dbw->insert( $this->getTable( $dbw ),
92		[ 'blob_id' => $id, 'blob_text' => $data ],
93		__METHOD__ );
94		$id = $dbw->insertId();
95		if ( !$id ) {
96		throw new MWException( __METHOD__ . ': no insert ID' );
97		}
98
99		return "DB://$location/$id";
100		}
101
102		/**
103		* Get a LoadBalancer for the specified cluster
104		*
105		* @param string $cluster Cluster name
106		* @return LoadBalancer
107		*/
108		function getLoadBalancer( $cluster ) {
109		$wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false;
110
111		return wfGetLBFactory()->getExternalLB( $cluster, $wiki );
		0 ignored issues – show Deprecated Code introduced 2016-04-11 17:35 UTC by Report Bug Copy Issue Report Show Similar Issues like this The function `wfGetLBFactory()` has been deprecated with message: since 1.27, use MediaWikiServices::getDBLoadBalancerFactory() instead. This function has been deprecated. The supplier of the file has supplied an explanatory message. The explanatory message should give you some clue as to whether and when the function will be removed from the class and what other function to use instead. Loading history...
112		}
113
114		/**
115		* Get a replica DB connection for the specified cluster
116		*
117		* @param string $cluster Cluster name
118		* @return IDatabase
119		*/
120		function getSlave( $cluster ) {
121		global $wgDefaultExternalStore;
122
123		$wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false;
124		$lb = $this->getLoadBalancer( $cluster );
125
126		if ( !in_array( "DB://" . $cluster, (array)$wgDefaultExternalStore ) ) {
127		wfDebug( "read only external store\n" );
128		$lb->allowLagged( true );
129		} else {
130		wfDebug( "writable external store\n" );
131		}
132
133		$db = $lb->getConnectionRef( DB_REPLICA, [], $wiki );
134		$db->clearFlag( DBO_TRX ); // sanity
135
136		return $db;
137		}
138
139		/**
140		* Get a master database connection for the specified cluster
141		*
142		* @param string $cluster Cluster name
143		* @return IDatabase
144		*/
145		function getMaster( $cluster ) {
146		$wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false;
147		$lb = $this->getLoadBalancer( $cluster );
148
149		$db = $lb->getConnectionRef( DB_MASTER, [], $wiki );
150		$db->clearFlag( DBO_TRX ); // sanity
151
152		return $db;
153		}
154
155		/**
156		* Get the 'blobs' table name for this database
157		*
158		* @param IDatabase $db
159		* @return string Table name ('blobs' by default)
160		*/
161		function getTable( $db ) {
162		$table = $db->getLBInfo( 'blobs table' );
163		if ( is_null( $table ) ) {
164		$table = 'blobs';
165		}
166
167		return $table;
168		}
169
170		/**
171		* Fetch a blob item out of the database; a cache of the last-loaded
172		* blob will be kept so that multiple loads out of a multi-item blob
173		* can avoid redundant database access and decompression.
174		* @param string $cluster
175		* @param string $id
176		* @param string $itemID
177		* @return HistoryBlob\|bool Returns false if missing
178		* @private
179		*/
180		function fetchBlob( $cluster, $id, $itemID ) {
181		/**
182		* One-step cache variable to hold base blobs; operations that
183		* pull multiple revisions may often pull multiple times from
184		* the same blob. By keeping the last-used one open, we avoid
185		* redundant unserialization and decompression overhead.
186		*/
187		static $externalBlobCache = [];
188
189		$cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/";
190		if ( isset( $externalBlobCache[$cacheID] ) ) {
191		wfDebugLog( 'ExternalStoreDB-cache',
192		"ExternalStoreDB::fetchBlob cache hit on $cacheID" );
193
194		return $externalBlobCache[$cacheID];
195		}
196
197		wfDebugLog( 'ExternalStoreDB-cache',
198		"ExternalStoreDB::fetchBlob cache miss on $cacheID" );
199
200		$dbr = $this->getSlave( $cluster );
201		$ret = $dbr->selectField( $this->getTable( $dbr ),
202		'blob_text', [ 'blob_id' => $id ], __METHOD__ );
203		if ( $ret === false ) {
204		wfDebugLog( 'ExternalStoreDB',
205		"ExternalStoreDB::fetchBlob master fallback on $cacheID" );
206		// Try the master
207		$dbw = $this->getMaster( $cluster );
208		$ret = $dbw->selectField( $this->getTable( $dbw ),
209		'blob_text', [ 'blob_id' => $id ], __METHOD__ );
210		if ( $ret === false ) {
211		wfDebugLog( 'ExternalStoreDB',
212		"ExternalStoreDB::fetchBlob master failed to find $cacheID" );
213		}
214		}
215		if ( $itemID !== false && $ret !== false ) {
216		// Unserialise object; caller extracts item
217		$ret = unserialize( $ret );
218		}
219
220		$externalBlobCache = [ $cacheID => $ret ];
221
222		return $ret;
223		}
224
225		/**
226		* Fetch multiple blob items out of the database
227		*
228		* @param string $cluster A cluster name valid for use with LBFactory
229		* @param array $ids A map from the blob_id's to look for to the requested itemIDs in the blobs
230		* @return array A map from the blob_id's requested to their content.
231		* Unlocated ids are not represented
232		*/
233		function batchFetchBlobs( $cluster, array $ids ) {
234		$dbr = $this->getSlave( $cluster );
235		$res = $dbr->select( $this->getTable( $dbr ),
236		[ 'blob_id', 'blob_text' ], [ 'blob_id' => array_keys( $ids ) ], __METHOD__ );
237		$ret = [];
238		if ( $res !== false ) {
239		$this->mergeBatchResult( $ret, $ids, $res );
240		}
241		if ( $ids ) {
242		wfDebugLog( __CLASS__, __METHOD__ .
243		" master fallback on '$cluster' for: " .
244		implode( ',', array_keys( $ids ) ) );
245		// Try the master
246		$dbw = $this->getMaster( $cluster );
247		$res = $dbw->select( $this->getTable( $dbr ),
248		[ 'blob_id', 'blob_text' ],
249		[ 'blob_id' => array_keys( $ids ) ],
250		__METHOD__ );
251		if ( $res === false ) {
252		wfDebugLog( __CLASS__, __METHOD__ . " master failed on '$cluster'" );
253		} else {
254		$this->mergeBatchResult( $ret, $ids, $res );
255		}
256		}
257		if ( $ids ) {
258		wfDebugLog( __CLASS__, __METHOD__ .
259		" master on '$cluster' failed locating items: " .
260		implode( ',', array_keys( $ids ) ) );
261		}
262
263		return $ret;
264		}
265
266		/**
267		* Helper function for self::batchFetchBlobs for merging master/replica DB results
268		* @param array &$ret Current self::batchFetchBlobs return value
269		* @param array &$ids Map from blob_id to requested itemIDs
270		* @param mixed $res DB result from Database::select
271		*/
272		private function mergeBatchResult( array &$ret, array &$ids, $res ) {
273		foreach ( $res as $row ) {
274		$id = $row->blob_id;
275		$itemIDs = $ids[$id];
276		unset( $ids[$id] ); // to track if everything is found
277		if ( count( $itemIDs ) === 1 && reset( $itemIDs ) === false ) {
278		// single result stored per blob
279		$ret[$id] = $row->blob_text;
280		} else {
281		// multi result stored per blob
282		$ret[$id] = unserialize( $row->blob_text );
283		}
284		}
285		}
286
287		/**
288		* @param string $url
289		* @return array
290		*/
291		protected function parseURL( $url ) {
292		$path = explode( '/', $url );
293
294		return [
295		$path[2], // cluster
296		$path[3], // id
297		isset( $path[4] ) ? $path[4] : false // itemID
298		];
299		}
300		}
301

wikimedia / mediawiki

Branch — master (939199)

includes/externalstore/ExternalStoreDB.php (1 issue)

Labels

Severity

Introduced By

Upgrade to new PHP Analysis Engine

Duplication Side-by-Side

Filter issues like