Completed
Branch master (8d5465)
by
unknown
31:25
created

LoadBalancer::parentInfo()   A

Complexity

Conditions 1
Paths 1

Size

Total Lines 3
Code Lines 2

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
cc 1
eloc 2
nc 1
nop 1
dl 0
loc 3
rs 10
c 0
b 0
f 0
1
<?php
2
/**
3
 * Database load balancing.
4
 *
5
 * This program is free software; you can redistribute it and/or modify
6
 * it under the terms of the GNU General Public License as published by
7
 * the Free Software Foundation; either version 2 of the License, or
8
 * (at your option) any later version.
9
 *
10
 * This program is distributed in the hope that it will be useful,
11
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
 * GNU General Public License for more details.
14
 *
15
 * You should have received a copy of the GNU General Public License along
16
 * with this program; if not, write to the Free Software Foundation, Inc.,
17
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18
 * http://www.gnu.org/copyleft/gpl.html
19
 *
20
 * @file
21
 * @ingroup Database
22
 */
23
24
/**
25
 * Database load balancing object
26
 *
27
 * @todo document
28
 * @ingroup Database
29
 */
30
class LoadBalancer {
31
	/** @var array[] Map of (server index => server config array) */
32
	private $mServers;
33
	/** @var array[] Map of (local/foreignUsed/foreignFree => server index => DatabaseBase array) */
34
	private $mConns;
35
	/** @var array Map of (server index => weight) */
36
	private $mLoads;
37
	/** @var array[] Map of (group => server index => weight) */
38
	private $mGroupLoads;
39
	/** @var bool Whether to disregard replica DB lag as a factor in replica DB selection */
40
	private $mAllowLagged;
41
	/** @var integer Seconds to spend waiting on replica DB lag to resolve */
42
	private $mWaitTimeout;
43
	/** @var array LBFactory information */
44
	private $mParentInfo;
0 ignored issues
show
Unused Code introduced by
The property $mParentInfo is not used and could be removed.

This check marks private properties in classes that are never used. Those properties can be removed.

Loading history...
45
46
	/** @var string The LoadMonitor subclass name */
47
	private $mLoadMonitorClass;
48
	/** @var LoadMonitor */
49
	private $mLoadMonitor;
50
	/** @var BagOStuff */
51
	private $srvCache;
52
	/** @var WANObjectCache */
53
	private $wanCache;
54
	/** @var TransactionProfiler */
55
	protected $trxProfiler;
56
57
	/** @var bool|DatabaseBase Database connection that caused a problem */
58
	private $mErrorConnection;
59
	/** @var integer The generic (not query grouped) replica DB index (of $mServers) */
60
	private $mReadIndex;
61
	/** @var bool|DBMasterPos False if not set */
62
	private $mWaitForPos;
63
	/** @var bool Whether the generic reader fell back to a lagged replica DB */
64
	private $laggedReplicaMode = false;
65
	/** @var bool Whether the generic reader fell back to a lagged replica DB */
66
	private $allReplicasDownMode = false;
67
	/** @var string The last DB selection or connection error */
68
	private $mLastError = 'Unknown error';
69
	/** @var string|bool Reason the LB is read-only or false if not */
70
	private $readOnlyReason = false;
71
	/** @var integer Total connections opened */
72
	private $connsOpened = 0;
73
	/** @var string|bool String if a requested DBO_TRX transaction round is active */
74
	private $trxRoundId = false;
75
	/** @var array[] Map of (name => callable) */
76
	private $trxRecurringCallbacks = [];
77
78
	/** @var integer Warn when this many connection are held */
79
	const CONN_HELD_WARN_THRESHOLD = 10;
80
	/** @var integer Default 'max lag' when unspecified */
81
	const MAX_LAG_DEFAULT = 10;
82
	/** @var integer Max time to wait for a replica DB to catch up (e.g. ChronologyProtector) */
83
	const POS_WAIT_TIMEOUT = 10;
84
	/** @var integer Seconds to cache master server read-only status */
85
	const TTL_CACHE_READONLY = 5;
86
87
	/**
88
	 * @var boolean
89
	 */
90
	private $disabled = false;
91
92
	/**
93
	 * @param array $params Array with keys:
94
	 *  - servers : Required. Array of server info structures.
95
	 *  - loadMonitor : Name of a class used to fetch server lag and load.
96
	 *  - readOnlyReason : Reason the master DB is read-only if so [optional]
97
	 *  - waitTimeout : Maximum time to wait for replicas for consistency [optional]
98
	 *  - srvCache : BagOStuff object [optional]
99
	 *  - wanCache : WANObjectCache object [optional]
100
	 * @throws MWException
101
	 */
102
	public function __construct( array $params ) {
103
		if ( !isset( $params['servers'] ) ) {
104
			throw new MWException( __CLASS__ . ': missing servers parameter' );
105
		}
106
		$this->mServers = $params['servers'];
107
		$this->mWaitTimeout = isset( $params['waitTimeout'] )
108
			? $params['waitTimeout']
109
			: self::POS_WAIT_TIMEOUT;
110
111
		$this->mReadIndex = -1;
112
		$this->mWriteIndex = -1;
0 ignored issues
show
Bug introduced by
The property mWriteIndex does not exist. Did you maybe forget to declare it?

In PHP it is possible to write to properties without declaring them. For example, the following is perfectly valid PHP code:

class MyClass { }

$x = new MyClass();
$x->foo = true;

Generally, it is a good practice to explictly declare properties to avoid accidental typos and provide IDE auto-completion:

class MyClass {
    public $foo;
}

$x = new MyClass();
$x->foo = true;
Loading history...
113
		$this->mConns = [
0 ignored issues
show
Documentation Bug introduced by
It seems like array('local' => array()...oreignFree' => array()) of type array<string,array,{"loc..."foreignFree":"array"}> is incompatible with the declared type array<integer,array> of property $mConns.

Our type inference engine has found an assignment to a property that is incompatible with the declared type of that property.

Either this assignment is in error or the assigned type should be added to the documentation/type hint for that property..

Loading history...
114
			'local' => [],
115
			'foreignUsed' => [],
116
			'foreignFree' => [] ];
117
		$this->mLoads = [];
118
		$this->mWaitForPos = false;
119
		$this->mErrorConnection = false;
120
		$this->mAllowLagged = false;
121
122 View Code Duplication
		if ( isset( $params['readOnlyReason'] ) && is_string( $params['readOnlyReason'] ) ) {
123
			$this->readOnlyReason = $params['readOnlyReason'];
124
		}
125
126
		if ( isset( $params['loadMonitor'] ) ) {
127
			$this->mLoadMonitorClass = $params['loadMonitor'];
128
		} else {
129
			$master = reset( $params['servers'] );
130
			if ( isset( $master['type'] ) && $master['type'] === 'mysql' ) {
131
				$this->mLoadMonitorClass = 'LoadMonitorMySQL';
132
			} else {
133
				$this->mLoadMonitorClass = 'LoadMonitorNull';
134
			}
135
		}
136
137
		foreach ( $params['servers'] as $i => $server ) {
138
			$this->mLoads[$i] = $server['load'];
139
			if ( isset( $server['groupLoads'] ) ) {
140
				foreach ( $server['groupLoads'] as $group => $ratio ) {
141
					if ( !isset( $this->mGroupLoads[$group] ) ) {
142
						$this->mGroupLoads[$group] = [];
143
					}
144
					$this->mGroupLoads[$group][$i] = $ratio;
145
				}
146
			}
147
		}
148
149
		if ( isset( $params['srvCache'] ) ) {
150
			$this->srvCache = $params['srvCache'];
151
		} else {
152
			$this->srvCache = new EmptyBagOStuff();
153
		}
154
		if ( isset( $params['wanCache'] ) ) {
155
			$this->wanCache = $params['wanCache'];
156
		} else {
157
			$this->wanCache = WANObjectCache::newEmpty();
158
		}
159
		if ( isset( $params['trxProfiler'] ) ) {
160
			$this->trxProfiler = $params['trxProfiler'];
161
		} else {
162
			$this->trxProfiler = new TransactionProfiler();
163
		}
164
	}
165
166
	/**
167
	 * Get a LoadMonitor instance
168
	 *
169
	 * @return LoadMonitor
170
	 */
171
	private function getLoadMonitor() {
172
		if ( !isset( $this->mLoadMonitor ) ) {
173
			$class = $this->mLoadMonitorClass;
174
			$this->mLoadMonitor = new $class( $this );
175
		}
176
177
		return $this->mLoadMonitor;
178
	}
179
180
	/**
181
	 * @param array $loads
182
	 * @param bool|string $wiki Wiki to get non-lagged for
183
	 * @param int $maxLag Restrict the maximum allowed lag to this many seconds
184
	 * @return bool|int|string
185
	 */
186
	private function getRandomNonLagged( array $loads, $wiki = false, $maxLag = INF ) {
187
		$lags = $this->getLagTimes( $wiki );
188
189
		# Unset excessively lagged servers
190
		foreach ( $lags as $i => $lag ) {
191
			if ( $i != 0 ) {
192
				# How much lag this server nominally is allowed to have
193
				$maxServerLag = isset( $this->mServers[$i]['max lag'] )
194
					? $this->mServers[$i]['max lag']
195
					: self::MAX_LAG_DEFAULT; // default
196
				# Constrain that futher by $maxLag argument
197
				$maxServerLag = min( $maxServerLag, $maxLag );
198
199
				$host = $this->getServerName( $i );
200
				if ( $lag === false && !is_infinite( $maxServerLag ) ) {
201
					wfDebugLog( 'replication', "Server $host (#$i) is not replicating?" );
202
					unset( $loads[$i] );
203
				} elseif ( $lag > $maxServerLag ) {
204
					wfDebugLog( 'replication', "Server $host (#$i) has >= $lag seconds of lag" );
205
					unset( $loads[$i] );
206
				}
207
			}
208
		}
209
210
		# Find out if all the replica DBs with non-zero load are lagged
211
		$sum = 0;
212
		foreach ( $loads as $load ) {
213
			$sum += $load;
214
		}
215
		if ( $sum == 0 ) {
216
			# No appropriate DB servers except maybe the master and some replica DBs with zero load
217
			# Do NOT use the master
218
			# Instead, this function will return false, triggering read-only mode,
219
			# and a lagged replica DB will be used instead.
220
			return false;
221
		}
222
223
		if ( count( $loads ) == 0 ) {
224
			return false;
225
		}
226
227
		# Return a random representative of the remainder
228
		return ArrayUtils::pickRandom( $loads );
229
	}
230
231
	/**
232
	 * Get the index of the reader connection, which may be a replica DB
233
	 * This takes into account load ratios and lag times. It should
234
	 * always return a consistent index during a given invocation
235
	 *
236
	 * Side effect: opens connections to databases
237
	 * @param string|bool $group Query group, or false for the generic reader
238
	 * @param string|bool $wiki Wiki ID, or false for the current wiki
239
	 * @throws MWException
240
	 * @return bool|int|string
241
	 */
242
	public function getReaderIndex( $group = false, $wiki = false ) {
243
		global $wgDBtype;
244
245
		# @todo FIXME: For now, only go through all this for mysql databases
246
		if ( $wgDBtype != 'mysql' ) {
247
			return $this->getWriterIndex();
248
		}
249
250
		if ( count( $this->mServers ) == 1 ) {
251
			# Skip the load balancing if there's only one server
252
			return 0;
253
		} elseif ( $group === false && $this->mReadIndex >= 0 ) {
254
			# Shortcut if generic reader exists already
255
			return $this->mReadIndex;
256
		}
257
258
		# Find the relevant load array
259
		if ( $group !== false ) {
260
			if ( isset( $this->mGroupLoads[$group] ) ) {
261
				$nonErrorLoads = $this->mGroupLoads[$group];
262
			} else {
263
				# No loads for this group, return false and the caller can use some other group
264
				wfDebugLog( 'connect', __METHOD__ . ": no loads for group $group\n" );
265
266
				return false;
267
			}
268
		} else {
269
			$nonErrorLoads = $this->mLoads;
270
		}
271
272
		if ( !count( $nonErrorLoads ) ) {
273
			throw new MWException( "Empty server array given to LoadBalancer" );
274
		}
275
276
		# Scale the configured load ratios according to the dynamic load (if the load monitor supports it)
277
		$this->getLoadMonitor()->scaleLoads( $nonErrorLoads, $group, $wiki );
278
279
		$laggedReplicaMode = false;
280
281
		# No server found yet
282
		$i = false;
283
		# First try quickly looking through the available servers for a server that
284
		# meets our criteria
285
		$currentLoads = $nonErrorLoads;
286
		while ( count( $currentLoads ) ) {
287
			if ( $this->mAllowLagged || $laggedReplicaMode ) {
288
				$i = ArrayUtils::pickRandom( $currentLoads );
289
			} else {
290
				$i = false;
291
				if ( $this->mWaitForPos && $this->mWaitForPos->asOfTime() ) {
292
					# ChronologyProtecter causes mWaitForPos to be set via sessions.
293
					# This triggers doWait() after connect, so it's especially good to
294
					# avoid lagged servers so as to avoid just blocking in that method.
295
					$ago = microtime( true ) - $this->mWaitForPos->asOfTime();
296
					# Aim for <= 1 second of waiting (being too picky can backfire)
297
					$i = $this->getRandomNonLagged( $currentLoads, $wiki, $ago + 1 );
298
				}
299
				if ( $i === false ) {
300
					# Any server with less lag than it's 'max lag' param is preferable
301
					$i = $this->getRandomNonLagged( $currentLoads, $wiki );
302
				}
303
				if ( $i === false && count( $currentLoads ) != 0 ) {
304
					# All replica DBs lagged. Switch to read-only mode
305
					wfDebugLog( 'replication', "All replica DBs lagged. Switch to read-only mode" );
306
					$i = ArrayUtils::pickRandom( $currentLoads );
307
					$laggedReplicaMode = true;
308
				}
309
			}
310
311
			if ( $i === false ) {
312
				# pickRandom() returned false
313
				# This is permanent and means the configuration or the load monitor
314
				# wants us to return false.
315
				wfDebugLog( 'connect', __METHOD__ . ": pickRandom() returned false" );
316
317
				return false;
318
			}
319
320
			$serverName = $this->getServerName( $i );
321
			wfDebugLog( 'connect', __METHOD__ . ": Using reader #$i: $serverName..." );
322
323
			$conn = $this->openConnection( $i, $wiki );
324
			if ( !$conn ) {
325
				wfDebugLog( 'connect', __METHOD__ . ": Failed connecting to $i/$wiki" );
326
				unset( $nonErrorLoads[$i] );
327
				unset( $currentLoads[$i] );
328
				$i = false;
329
				continue;
330
			}
331
332
			// Decrement reference counter, we are finished with this connection.
333
			// It will be incremented for the caller later.
334
			if ( $wiki !== false ) {
335
				$this->reuseConnection( $conn );
336
			}
337
338
			# Return this server
339
			break;
340
		}
341
342
		# If all servers were down, quit now
343
		if ( !count( $nonErrorLoads ) ) {
344
			wfDebugLog( 'connect', "All servers down" );
345
		}
346
347
		if ( $i !== false ) {
348
			# Replica DB connection successful.
349
			# Wait for the session master pos for a short time.
350
			if ( $this->mWaitForPos && $i > 0 ) {
351
				$this->doWait( $i );
352
			}
353
			if ( $this->mReadIndex <= 0 && $this->mLoads[$i] > 0 && $group === false ) {
354
				$this->mReadIndex = $i;
0 ignored issues
show
Documentation Bug introduced by
It seems like $i can also be of type string. However, the property $mReadIndex is declared as type integer. Maybe add an additional type check?

Our type inference engine has found a suspicous assignment of a value to a property. This check raises an issue when a value that can be of a mixed type is assigned to a property that is type hinted more strictly.

For example, imagine you have a variable $accountId that can either hold an Id object or false (if there is no account id yet). Your code now assigns that value to the id property of an instance of the Account class. This class holds a proper account, so the id value must no longer be false.

Either this assignment is in error or a type check should be added for that assignment.

class Id
{
    public $id;

    public function __construct($id)
    {
        $this->id = $id;
    }

}

class Account
{
    /** @var  Id $id */
    public $id;
}

$account_id = false;

if (starsAreRight()) {
    $account_id = new Id(42);
}

$account = new Account();
if ($account instanceof Id)
{
    $account->id = $account_id;
}
Loading history...
355
				# Record if the generic reader index is in "lagged replica DB" mode
356
				if ( $laggedReplicaMode ) {
357
					$this->laggedReplicaMode = true;
358
				}
359
			}
360
			$serverName = $this->getServerName( $i );
361
			wfDebugLog( 'connect', __METHOD__ .
362
				": using server $serverName for group '$group'\n" );
363
		}
364
365
		return $i;
366
	}
367
368
	/**
369
	 * Set the master wait position
370
	 * If a DB_REPLICA connection has been opened already, waits
371
	 * Otherwise sets a variable telling it to wait if such a connection is opened
372
	 * @param DBMasterPos $pos
373
	 */
374
	public function waitFor( $pos ) {
375
		$this->mWaitForPos = $pos;
376
		$i = $this->mReadIndex;
377
378
		if ( $i > 0 ) {
379
			if ( !$this->doWait( $i ) ) {
380
				$this->laggedReplicaMode = true;
381
			}
382
		}
383
	}
384
385
	/**
386
	 * Set the master wait position and wait for a "generic" replica DB to catch up to it
387
	 *
388
	 * This can be used a faster proxy for waitForAll()
389
	 *
390
	 * @param DBMasterPos $pos
391
	 * @param int $timeout Max seconds to wait; default is mWaitTimeout
392
	 * @return bool Success (able to connect and no timeouts reached)
393
	 * @since 1.26
394
	 */
395
	public function waitForOne( $pos, $timeout = null ) {
396
		$this->mWaitForPos = $pos;
397
398
		$i = $this->mReadIndex;
399
		if ( $i <= 0 ) {
400
			// Pick a generic replica DB if there isn't one yet
401
			$readLoads = $this->mLoads;
402
			unset( $readLoads[$this->getWriterIndex()] ); // replica DBs only
403
			$readLoads = array_filter( $readLoads ); // with non-zero load
404
			$i = ArrayUtils::pickRandom( $readLoads );
405
		}
406
407 View Code Duplication
		if ( $i > 0 ) {
408
			$ok = $this->doWait( $i, true, $timeout );
409
		} else {
410
			$ok = true; // no applicable loads
411
		}
412
413
		return $ok;
414
	}
415
416
	/**
417
	 * Set the master wait position and wait for ALL replica DBs to catch up to it
418
	 * @param DBMasterPos $pos
419
	 * @param int $timeout Max seconds to wait; default is mWaitTimeout
420
	 * @return bool Success (able to connect and no timeouts reached)
421
	 */
422
	public function waitForAll( $pos, $timeout = null ) {
423
		$this->mWaitForPos = $pos;
424
		$serverCount = count( $this->mServers );
425
426
		$ok = true;
427
		for ( $i = 1; $i < $serverCount; $i++ ) {
428 View Code Duplication
			if ( $this->mLoads[$i] > 0 ) {
429
				$ok = $this->doWait( $i, true, $timeout ) && $ok;
430
			}
431
		}
432
433
		return $ok;
434
	}
435
436
	/**
437
	 * Get any open connection to a given server index, local or foreign
438
	 * Returns false if there is no connection open
439
	 *
440
	 * @param int $i Server index
441
	 * @return DatabaseBase|bool False on failure
442
	 */
443
	public function getAnyOpenConnection( $i ) {
444
		foreach ( $this->mConns as $connsByServer ) {
445
			if ( !empty( $connsByServer[$i] ) ) {
446
				return reset( $connsByServer[$i] );
447
			}
448
		}
449
450
		return false;
451
	}
452
453
	/**
454
	 * Wait for a given replica DB to catch up to the master pos stored in $this
455
	 * @param int $index Server index
456
	 * @param bool $open Check the server even if a new connection has to be made
457
	 * @param int $timeout Max seconds to wait; default is mWaitTimeout
458
	 * @return bool
459
	 */
460
	protected function doWait( $index, $open = false, $timeout = null ) {
461
		$close = false; // close the connection afterwards
462
463
		// Check if we already know that the DB has reached this point
464
		$server = $this->getServerName( $index );
465
		$key = $this->srvCache->makeGlobalKey( __CLASS__, 'last-known-pos', $server );
466
		/** @var DBMasterPos $knownReachedPos */
467
		$knownReachedPos = $this->srvCache->get( $key );
468
		if ( $knownReachedPos && $knownReachedPos->hasReached( $this->mWaitForPos ) ) {
0 ignored issues
show
Bug introduced by
It seems like $this->mWaitForPos can also be of type boolean; however, DBMasterPos::hasReached() does only seem to accept object<DBMasterPos>, maybe add an additional type check?

If a method or function can return multiple different values and unless you are sure that you only can receive a single value in this context, we recommend to add an additional type check:

/**
 * @return array|string
 */
function returnsDifferentValues($x) {
    if ($x) {
        return 'foo';
    }

    return array();
}

$x = returnsDifferentValues($y);
if (is_array($x)) {
    // $x is an array.
}

If this a common case that PHP Analyzer should handle natively, please let us know by opening an issue.

Loading history...
469
			wfDebugLog( 'replication', __METHOD__ .
470
				": replica DB $server known to be caught up (pos >= $knownReachedPos).\n" );
471
			return true;
472
		}
473
474
		// Find a connection to wait on, creating one if needed and allowed
475
		$conn = $this->getAnyOpenConnection( $index );
476
		if ( !$conn ) {
477
			if ( !$open ) {
478
				wfDebugLog( 'replication', __METHOD__ . ": no connection open for $server\n" );
479
480
				return false;
481
			} else {
482
				$conn = $this->openConnection( $index, '' );
483
				if ( !$conn ) {
484
					wfDebugLog( 'replication', __METHOD__ . ": failed to connect to $server\n" );
485
486
					return false;
487
				}
488
				// Avoid connection spam in waitForAll() when connections
489
				// are made just for the sake of doing this lag check.
490
				$close = true;
491
			}
492
		}
493
494
		wfDebugLog( 'replication', __METHOD__ . ": Waiting for replica DB $server to catch up...\n" );
495
		$timeout = $timeout ?: $this->mWaitTimeout;
496
		$result = $conn->masterPosWait( $this->mWaitForPos, $timeout );
0 ignored issues
show
Bug introduced by
It seems like $this->mWaitForPos can also be of type boolean; however, DatabaseBase::masterPosWait() does only seem to accept object<DBMasterPos>, maybe add an additional type check?

If a method or function can return multiple different values and unless you are sure that you only can receive a single value in this context, we recommend to add an additional type check:

/**
 * @return array|string
 */
function returnsDifferentValues($x) {
    if ($x) {
        return 'foo';
    }

    return array();
}

$x = returnsDifferentValues($y);
if (is_array($x)) {
    // $x is an array.
}

If this a common case that PHP Analyzer should handle natively, please let us know by opening an issue.

Loading history...
497
498
		if ( $result == -1 || is_null( $result ) ) {
499
			// Timed out waiting for replica DB, use master instead
500
			$msg = __METHOD__ . ": Timed out waiting on $server pos {$this->mWaitForPos}";
501
			wfDebugLog( 'replication', "$msg\n" );
502
			wfDebugLog( 'DBPerformance', "$msg:\n" . wfBacktrace( true ) );
503
			$ok = false;
504
		} else {
505
			wfDebugLog( 'replication', __METHOD__ . ": Done\n" );
506
			$ok = true;
507
			// Remember that the DB reached this point
508
			$this->srvCache->set( $key, $this->mWaitForPos, BagOStuff::TTL_DAY );
509
		}
510
511
		if ( $close ) {
512
			$this->closeConnection( $conn );
0 ignored issues
show
Bug introduced by
It seems like $conn can also be of type boolean; however, LoadBalancer::closeConnection() does only seem to accept object<DatabaseBase>, maybe add an additional type check?

If a method or function can return multiple different values and unless you are sure that you only can receive a single value in this context, we recommend to add an additional type check:

/**
 * @return array|string
 */
function returnsDifferentValues($x) {
    if ($x) {
        return 'foo';
    }

    return array();
}

$x = returnsDifferentValues($y);
if (is_array($x)) {
    // $x is an array.
}

If this a common case that PHP Analyzer should handle natively, please let us know by opening an issue.

Loading history...
513
		}
514
515
		return $ok;
516
	}
517
518
	/**
519
	 * Get a connection by index
520
	 * This is the main entry point for this class.
521
	 *
522
	 * @param int $i Server index
523
	 * @param array|string|bool $groups Query group(s), or false for the generic reader
524
	 * @param string|bool $wiki Wiki ID, or false for the current wiki
525
	 *
526
	 * @throws MWException
527
	 * @return DatabaseBase
528
	 */
529
	public function getConnection( $i, $groups = [], $wiki = false ) {
530
		if ( $i === null || $i === false ) {
531
			throw new MWException( 'Attempt to call ' . __METHOD__ .
532
				' with invalid server index' );
533
		}
534
535
		if ( $wiki === wfWikiID() ) {
536
			$wiki = false;
537
		}
538
539
		$groups = ( $groups === false || $groups === [] )
540
			? [ false ] // check one "group": the generic pool
541
			: (array)$groups;
542
543
		$masterOnly = ( $i == DB_MASTER || $i == $this->getWriterIndex() );
544
		$oldConnsOpened = $this->connsOpened; // connections open now
545
546
		if ( $i == DB_MASTER ) {
547
			$i = $this->getWriterIndex();
548
		} else {
549
			# Try to find an available server in any the query groups (in order)
550
			foreach ( $groups as $group ) {
551
				$groupIndex = $this->getReaderIndex( $group, $wiki );
552
				if ( $groupIndex !== false ) {
553
					$i = $groupIndex;
554
					break;
555
				}
556
			}
557
		}
558
559
		# Operation-based index
560
		if ( $i == DB_REPLICA ) {
561
			$this->mLastError = 'Unknown error'; // reset error string
562
			# Try the general server pool if $groups are unavailable.
563
			$i = in_array( false, $groups, true )
564
				? false // don't bother with this if that is what was tried above
565
				: $this->getReaderIndex( false, $wiki );
566
			# Couldn't find a working server in getReaderIndex()?
567
			if ( $i === false ) {
568
				$this->mLastError = 'No working replica DB server: ' . $this->mLastError;
569
570
				return $this->reportConnectionError();
571
			}
572
		}
573
574
		# Now we have an explicit index into the servers array
575
		$conn = $this->openConnection( $i, $wiki );
576
		if ( !$conn ) {
577
			return $this->reportConnectionError();
578
		}
579
580
		# Profile any new connections that happen
581
		if ( $this->connsOpened > $oldConnsOpened ) {
582
			$host = $conn->getServer();
583
			$dbname = $conn->getDBname();
584
			$trxProf = Profiler::instance()->getTransactionProfiler();
585
			$trxProf->recordConnection( $host, $dbname, $masterOnly );
586
		}
587
588
		if ( $masterOnly ) {
589
			# Make master-requested DB handles inherit any read-only mode setting
590
			$conn->setLBInfo( 'readOnlyReason', $this->getReadOnlyReason( $wiki, $conn ) );
0 ignored issues
show
Bug introduced by
It seems like $conn defined by $this->openConnection($i, $wiki) on line 575 can also be of type boolean; however, LoadBalancer::getReadOnlyReason() does only seem to accept null|object<DatabaseBase>, maybe add an additional type check?

If a method or function can return multiple different values and unless you are sure that you only can receive a single value in this context, we recommend to add an additional type check:

/**
 * @return array|string
 */
function returnsDifferentValues($x) {
    if ($x) {
        return 'foo';
    }

    return array();
}

$x = returnsDifferentValues($y);
if (is_array($x)) {
    // $x is an array.
}

If this a common case that PHP Analyzer should handle natively, please let us know by opening an issue.

Loading history...
591
		}
592
593
		return $conn;
594
	}
595
596
	/**
597
	 * Mark a foreign connection as being available for reuse under a different
598
	 * DB name or prefix. This mechanism is reference-counted, and must be called
599
	 * the same number of times as getConnection() to work.
600
	 *
601
	 * @param DatabaseBase $conn
602
	 * @throws MWException
603
	 */
604
	public function reuseConnection( $conn ) {
605
		$serverIndex = $conn->getLBInfo( 'serverIndex' );
606
		$refCount = $conn->getLBInfo( 'foreignPoolRefCount' );
607
		if ( $serverIndex === null || $refCount === null ) {
608
			/**
609
			 * This can happen in code like:
610
			 *   foreach ( $dbs as $db ) {
611
			 *     $conn = $lb->getConnection( DB_REPLICA, [], $db );
612
			 *     ...
613
			 *     $lb->reuseConnection( $conn );
614
			 *   }
615
			 * When a connection to the local DB is opened in this way, reuseConnection()
616
			 * should be ignored
617
			 */
618
			return;
619
		}
620
621
		$dbName = $conn->getDBname();
622
		$prefix = $conn->tablePrefix();
623
		if ( strval( $prefix ) !== '' ) {
624
			$wiki = "$dbName-$prefix";
625
		} else {
626
			$wiki = $dbName;
627
		}
628
		if ( $this->mConns['foreignUsed'][$serverIndex][$wiki] !== $conn ) {
629
			throw new MWException( __METHOD__ . ": connection not found, has " .
630
				"the connection been freed already?" );
631
		}
632
		$conn->setLBInfo( 'foreignPoolRefCount', --$refCount );
633
		if ( $refCount <= 0 ) {
634
			$this->mConns['foreignFree'][$serverIndex][$wiki] = $conn;
635
			unset( $this->mConns['foreignUsed'][$serverIndex][$wiki] );
636
			wfDebug( __METHOD__ . ": freed connection $serverIndex/$wiki\n" );
637
		} else {
638
			wfDebug( __METHOD__ . ": reference count for $serverIndex/$wiki reduced to $refCount\n" );
639
		}
640
	}
641
642
	/**
643
	 * Get a database connection handle reference
644
	 *
645
	 * The handle's methods wrap simply wrap those of a DatabaseBase handle
646
	 *
647
	 * @see LoadBalancer::getConnection() for parameter information
648
	 *
649
	 * @param int $db
650
	 * @param array|string|bool $groups Query group(s), or false for the generic reader
651
	 * @param string|bool $wiki Wiki ID, or false for the current wiki
652
	 * @return DBConnRef
653
	 */
654
	public function getConnectionRef( $db, $groups = [], $wiki = false ) {
655
		return new DBConnRef( $this, $this->getConnection( $db, $groups, $wiki ) );
656
	}
657
658
	/**
659
	 * Get a database connection handle reference without connecting yet
660
	 *
661
	 * The handle's methods wrap simply wrap those of a DatabaseBase handle
662
	 *
663
	 * @see LoadBalancer::getConnection() for parameter information
664
	 *
665
	 * @param int $db
666
	 * @param array|string|bool $groups Query group(s), or false for the generic reader
667
	 * @param string|bool $wiki Wiki ID, or false for the current wiki
668
	 * @return DBConnRef
669
	 */
670
	public function getLazyConnectionRef( $db, $groups = [], $wiki = false ) {
671
		return new DBConnRef( $this, [ $db, $groups, $wiki ] );
672
	}
673
674
	/**
675
	 * Open a connection to the server given by the specified index
676
	 * Index must be an actual index into the array.
677
	 * If the server is already open, returns it.
678
	 *
679
	 * On error, returns false, and the connection which caused the
680
	 * error will be available via $this->mErrorConnection.
681
	 *
682
	 * @note If disable() was called on this LoadBalancer, this method will throw a DBAccessError.
683
	 *
684
	 * @param int $i Server index
685
	 * @param string|bool $wiki Wiki ID, or false for the current wiki
686
	 * @return DatabaseBase|bool Returns false on errors
687
	 */
688
	public function openConnection( $i, $wiki = false ) {
689
		if ( $wiki !== false ) {
690
			$conn = $this->openForeignConnection( $i, $wiki );
0 ignored issues
show
Bug introduced by
It seems like $wiki defined by parameter $wiki on line 688 can also be of type boolean; however, LoadBalancer::openForeignConnection() does only seem to accept string, maybe add an additional type check?

This check looks at variables that have been passed in as parameters and are passed out again to other methods.

If the outgoing method call has stricter type requirements than the method itself, an issue is raised.

An additional type check may prevent trouble.

Loading history...
691
		} elseif ( isset( $this->mConns['local'][$i][0] ) ) {
692
			$conn = $this->mConns['local'][$i][0];
693
		} else {
694
			$server = $this->mServers[$i];
695
			$server['serverIndex'] = $i;
696
			$conn = $this->reallyOpenConnection( $server, false );
697
			$serverName = $this->getServerName( $i );
698
			if ( $conn->isOpen() ) {
699
				wfDebugLog( 'connect', "Connected to database $i at $serverName\n" );
700
				$this->mConns['local'][$i][0] = $conn;
701
			} else {
702
				wfDebugLog( 'connect', "Failed to connect to database $i at $serverName\n" );
703
				$this->mErrorConnection = $conn;
704
				$conn = false;
705
			}
706
		}
707
708
		if ( $conn && !$conn->isOpen() ) {
709
			// Connection was made but later unrecoverably lost for some reason.
710
			// Do not return a handle that will just throw exceptions on use,
711
			// but let the calling code (e.g. getReaderIndex) try another server.
712
			// See DatabaseMyslBase::ping() for how this can happen.
713
			$this->mErrorConnection = $conn;
714
			$conn = false;
715
		}
716
717
		return $conn;
718
	}
719
720
	/**
721
	 * Open a connection to a foreign DB, or return one if it is already open.
722
	 *
723
	 * Increments a reference count on the returned connection which locks the
724
	 * connection to the requested wiki. This reference count can be
725
	 * decremented by calling reuseConnection().
726
	 *
727
	 * If a connection is open to the appropriate server already, but with the wrong
728
	 * database, it will be switched to the right database and returned, as long as
729
	 * it has been freed first with reuseConnection().
730
	 *
731
	 * On error, returns false, and the connection which caused the
732
	 * error will be available via $this->mErrorConnection.
733
	 *
734
	 * @note If disable() was called on this LoadBalancer, this method will throw a DBAccessError.
735
	 *
736
	 * @param int $i Server index
737
	 * @param string $wiki Wiki ID to open
738
	 * @return DatabaseBase
739
	 */
740
	private function openForeignConnection( $i, $wiki ) {
741
		list( $dbName, $prefix ) = wfSplitWikiID( $wiki );
742
		if ( isset( $this->mConns['foreignUsed'][$i][$wiki] ) ) {
743
			// Reuse an already-used connection
744
			$conn = $this->mConns['foreignUsed'][$i][$wiki];
745
			wfDebug( __METHOD__ . ": reusing connection $i/$wiki\n" );
746
		} elseif ( isset( $this->mConns['foreignFree'][$i][$wiki] ) ) {
747
			// Reuse a free connection for the same wiki
748
			$conn = $this->mConns['foreignFree'][$i][$wiki];
749
			unset( $this->mConns['foreignFree'][$i][$wiki] );
750
			$this->mConns['foreignUsed'][$i][$wiki] = $conn;
751
			wfDebug( __METHOD__ . ": reusing free connection $i/$wiki\n" );
752
		} elseif ( !empty( $this->mConns['foreignFree'][$i] ) ) {
753
			// Reuse a connection from another wiki
754
			$conn = reset( $this->mConns['foreignFree'][$i] );
755
			$oldWiki = key( $this->mConns['foreignFree'][$i] );
756
757
			// The empty string as a DB name means "don't care".
758
			// DatabaseMysqlBase::open() already handle this on connection.
759
			if ( $dbName !== '' && !$conn->selectDB( $dbName ) ) {
760
				$this->mLastError = "Error selecting database $dbName on server " .
761
					$conn->getServer() . " from client host " . wfHostname() . "\n";
762
				$this->mErrorConnection = $conn;
763
				$conn = false;
764
			} else {
765
				$conn->tablePrefix( $prefix );
766
				unset( $this->mConns['foreignFree'][$i][$oldWiki] );
767
				$this->mConns['foreignUsed'][$i][$wiki] = $conn;
768
				wfDebug( __METHOD__ . ": reusing free connection from $oldWiki for $wiki\n" );
769
			}
770
		} else {
771
			// Open a new connection
772
			$server = $this->mServers[$i];
773
			$server['serverIndex'] = $i;
774
			$server['foreignPoolRefCount'] = 0;
775
			$server['foreign'] = true;
776
			$conn = $this->reallyOpenConnection( $server, $dbName );
777
			if ( !$conn->isOpen() ) {
778
				wfDebug( __METHOD__ . ": error opening connection for $i/$wiki\n" );
779
				$this->mErrorConnection = $conn;
780
				$conn = false;
781
			} else {
782
				$conn->tablePrefix( $prefix );
783
				$this->mConns['foreignUsed'][$i][$wiki] = $conn;
784
				wfDebug( __METHOD__ . ": opened new connection for $i/$wiki\n" );
785
			}
786
		}
787
788
		// Increment reference count
789
		if ( $conn ) {
790
			$refCount = $conn->getLBInfo( 'foreignPoolRefCount' );
791
			$conn->setLBInfo( 'foreignPoolRefCount', $refCount + 1 );
792
		}
793
794
		return $conn;
795
	}
796
797
	/**
798
	 * Test if the specified index represents an open connection
799
	 *
800
	 * @param int $index Server index
801
	 * @access private
802
	 * @return bool
803
	 */
804
	private function isOpen( $index ) {
805
		if ( !is_integer( $index ) ) {
806
			return false;
807
		}
808
809
		return (bool)$this->getAnyOpenConnection( $index );
810
	}
811
812
	/**
813
	 * Really opens a connection. Uncached.
814
	 * Returns a Database object whether or not the connection was successful.
815
	 * @access private
816
	 *
817
	 * @param array $server
818
	 * @param bool $dbNameOverride
819
	 * @throws MWException
820
	 * @return DatabaseBase
821
	 */
822
	protected function reallyOpenConnection( $server, $dbNameOverride = false ) {
823
		if ( $this->disabled ) {
824
			throw new DBAccessError();
825
		}
826
827
		if ( !is_array( $server ) ) {
828
			throw new MWException( 'You must update your load-balancing configuration. ' .
829
				'See DefaultSettings.php entry for $wgDBservers.' );
830
		}
831
832
		if ( $dbNameOverride !== false ) {
833
			$server['dbname'] = $dbNameOverride;
834
		}
835
836
		// Let the handle know what the cluster master is (e.g. "db1052")
837
		$masterName = $this->getServerName( $this->getWriterIndex() );
838
		$server['clusterMasterHost'] = $masterName;
839
840
		// Log when many connection are made on requests
841
		if ( ++$this->connsOpened >= self::CONN_HELD_WARN_THRESHOLD ) {
842
			wfDebugLog( 'DBPerformance', __METHOD__ . ": " .
843
				"{$this->connsOpened}+ connections made (master=$masterName)\n" .
844
				wfBacktrace( true ) );
845
		}
846
847
		# Create object
848
		try {
849
			$db = DatabaseBase::factory( $server['type'], $server );
850
		} catch ( DBConnectionError $e ) {
851
			// FIXME: This is probably the ugliest thing I have ever done to
852
			// PHP. I'm half-expecting it to segfault, just out of disgust. -- TS
853
			$db = $e->db;
854
		}
855
856
		$db->setLBInfo( $server );
857
		$db->setLazyMasterHandle(
858
			$this->getLazyConnectionRef( DB_MASTER, [], $db->getWikiID() )
859
		);
860
		$db->setTransactionProfiler( $this->trxProfiler );
861
862
		if ( $server['serverIndex'] === $this->getWriterIndex() ) {
863
			if ( $this->trxRoundId !== false ) {
864
				$this->applyTransactionRoundFlags( $db );
865
			}
866
			foreach ( $this->trxRecurringCallbacks as $name => $callback ) {
867
				$db->setTransactionListener( $name, $callback );
868
			}
869
		}
870
871
		return $db;
872
	}
873
874
	/**
875
	 * @throws DBConnectionError
876
	 * @return bool
877
	 */
878
	private function reportConnectionError() {
879
		$conn = $this->mErrorConnection; // The connection which caused the error
880
		$context = [
881
			'method' => __METHOD__,
882
			'last_error' => $this->mLastError,
883
		];
884
885
		if ( !is_object( $conn ) ) {
886
			// No last connection, probably due to all servers being too busy
887
			wfLogDBError(
888
				"LB failure with no last connection. Connection error: {last_error}",
889
				$context
890
			);
891
892
			// If all servers were busy, mLastError will contain something sensible
893
			throw new DBConnectionError( null, $this->mLastError );
894
		} else {
895
			$context['db_server'] = $conn->getProperty( 'mServer' );
896
			wfLogDBError(
897
				"Connection error: {last_error} ({db_server})",
898
				$context
899
			);
900
901
			// throws DBConnectionError
902
			$conn->reportConnectionError( "{$this->mLastError} ({$context['db_server']})" );
903
		}
904
905
		return false; /* not reached */
906
	}
907
908
	/**
909
	 * @return int
910
	 * @since 1.26
911
	 */
912
	public function getWriterIndex() {
913
		return 0;
914
	}
915
916
	/**
917
	 * Returns true if the specified index is a valid server index
918
	 *
919
	 * @param string $i
920
	 * @return bool
921
	 */
922
	public function haveIndex( $i ) {
923
		return array_key_exists( $i, $this->mServers );
924
	}
925
926
	/**
927
	 * Returns true if the specified index is valid and has non-zero load
928
	 *
929
	 * @param string $i
930
	 * @return bool
931
	 */
932
	public function isNonZeroLoad( $i ) {
933
		return array_key_exists( $i, $this->mServers ) && $this->mLoads[$i] != 0;
934
	}
935
936
	/**
937
	 * Get the number of defined servers (not the number of open connections)
938
	 *
939
	 * @return int
940
	 */
941
	public function getServerCount() {
942
		return count( $this->mServers );
943
	}
944
945
	/**
946
	 * Get the host name or IP address of the server with the specified index
947
	 * Prefer a readable name if available.
948
	 * @param string $i
949
	 * @return string
950
	 */
951
	public function getServerName( $i ) {
952
		if ( isset( $this->mServers[$i]['hostName'] ) ) {
953
			$name = $this->mServers[$i]['hostName'];
954
		} elseif ( isset( $this->mServers[$i]['host'] ) ) {
955
			$name = $this->mServers[$i]['host'];
956
		} else {
957
			$name = '';
958
		}
959
960
		return ( $name != '' ) ? $name : 'localhost';
961
	}
962
963
	/**
964
	 * Return the server info structure for a given index, or false if the index is invalid.
965
	 * @param int $i
966
	 * @return array|bool
967
	 */
968
	public function getServerInfo( $i ) {
969
		if ( isset( $this->mServers[$i] ) ) {
970
			return $this->mServers[$i];
971
		} else {
972
			return false;
973
		}
974
	}
975
976
	/**
977
	 * Sets the server info structure for the given index. Entry at index $i
978
	 * is created if it doesn't exist
979
	 * @param int $i
980
	 * @param array $serverInfo
981
	 */
982
	public function setServerInfo( $i, array $serverInfo ) {
983
		$this->mServers[$i] = $serverInfo;
984
	}
985
986
	/**
987
	 * Get the current master position for chronology control purposes
988
	 * @return DBMasterPos|bool Returns false if not applicable
989
	 */
990
	public function getMasterPos() {
991
		# If this entire request was served from a replica DB without opening a connection to the
992
		# master (however unlikely that may be), then we can fetch the position from the replica DB.
993
		$masterConn = $this->getAnyOpenConnection( $this->getWriterIndex() );
994
		if ( !$masterConn ) {
995
			$serverCount = count( $this->mServers );
996
			for ( $i = 1; $i < $serverCount; $i++ ) {
997
				$conn = $this->getAnyOpenConnection( $i );
998
				if ( $conn ) {
999
					return $conn->getSlavePos();
0 ignored issues
show
Bug introduced by
The method getSlavePos cannot be called on $conn (of type boolean).

Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects.

Loading history...
1000
				}
1001
			}
1002
		} else {
1003
			return $masterConn->getMasterPos();
1004
		}
1005
1006
		return false;
1007
	}
1008
1009
	/**
1010
	 * Disable this load balancer. All connections are closed, and any attempt to
1011
	 * open a new connection will result in a DBAccessError.
1012
	 *
1013
	 * @since 1.27
1014
	 */
1015
	public function disable() {
1016
		$this->closeAll();
1017
		$this->disabled = true;
1018
	}
1019
1020
	/**
1021
	 * Close all open connections
1022
	 */
1023
	public function closeAll() {
1024
		$this->forEachOpenConnection( function ( DatabaseBase $conn ) {
1025
			$conn->close();
1026
		} );
1027
1028
		$this->mConns = [
0 ignored issues
show
Documentation Bug introduced by
It seems like array('local' => array()...oreignUsed' => array()) of type array<string,array,{"loc..."foreignUsed":"array"}> is incompatible with the declared type array<integer,array> of property $mConns.

Our type inference engine has found an assignment to a property that is incompatible with the declared type of that property.

Either this assignment is in error or the assigned type should be added to the documentation/type hint for that property..

Loading history...
1029
			'local' => [],
1030
			'foreignFree' => [],
1031
			'foreignUsed' => [],
1032
		];
1033
		$this->connsOpened = 0;
1034
	}
1035
1036
	/**
1037
	 * Close a connection
1038
	 *
1039
	 * Using this function makes sure the LoadBalancer knows the connection is closed.
1040
	 * If you use $conn->close() directly, the load balancer won't update its state.
1041
	 *
1042
	 * @param DatabaseBase $conn
1043
	 */
1044
	public function closeConnection( DatabaseBase $conn ) {
1045
		$serverIndex = $conn->getLBInfo( 'serverIndex' ); // second index level of mConns
1046
		foreach ( $this->mConns as $type => $connsByServer ) {
1047
			if ( !isset( $connsByServer[$serverIndex] ) ) {
1048
				continue;
1049
			}
1050
1051
			foreach ( $connsByServer[$serverIndex] as $i => $trackedConn ) {
1052
				if ( $conn === $trackedConn ) {
1053
					unset( $this->mConns[$type][$serverIndex][$i] );
1054
					--$this->connsOpened;
1055
					break 2;
1056
				}
1057
			}
1058
		}
1059
1060
		$conn->close();
1061
	}
1062
1063
	/**
1064
	 * Commit transactions on all open connections
1065
	 * @param string $fname Caller name
1066
	 * @throws DBExpectedError
1067
	 */
1068
	public function commitAll( $fname = __METHOD__ ) {
1069
		$failures = [];
1070
1071
		$restore = ( $this->trxRoundId !== false );
1072
		$this->trxRoundId = false;
1073
		$this->forEachOpenConnection(
1074
			function ( DatabaseBase $conn ) use ( $fname, $restore, &$failures ) {
1075
				try {
1076
					$conn->commit( $fname, $conn::FLUSHING_ALL_PEERS );
1077
				} catch ( DBError $e ) {
1078
					MWExceptionHandler::logException( $e );
1079
					$failures[] = "{$conn->getServer()}: {$e->getMessage()}";
1080
				}
1081
				if ( $restore && $conn->getLBInfo( 'master' ) ) {
1082
					$this->undoTransactionRoundFlags( $conn );
1083
				}
1084
			}
1085
		);
1086
1087
		if ( $failures ) {
1088
			throw new DBExpectedError(
1089
				null,
1090
				"Commit failed on server(s) " . implode( "\n", array_unique( $failures ) )
1091
			);
1092
		}
1093
	}
1094
1095
	/**
1096
	 * Perform all pre-commit callbacks that remain part of the atomic transactions
1097
	 * and disable any post-commit callbacks until runMasterPostTrxCallbacks()
1098
	 * @since 1.28
1099
	 */
1100
	public function finalizeMasterChanges() {
1101
		$this->forEachOpenMasterConnection( function ( DatabaseBase $conn ) {
1102
			// Any error should cause all DB transactions to be rolled back together
1103
			$conn->setTrxEndCallbackSuppression( false );
1104
			$conn->runOnTransactionPreCommitCallbacks();
1105
			// Defer post-commit callbacks until COMMIT finishes for all DBs
1106
			$conn->setTrxEndCallbackSuppression( true );
1107
		} );
1108
	}
1109
1110
	/**
1111
	 * Perform all pre-commit checks for things like replication safety
1112
	 * @param array $options Includes:
1113
	 *   - maxWriteDuration : max write query duration time in seconds
1114
	 * @throws DBTransactionError
1115
	 * @since 1.28
1116
	 */
1117
	public function approveMasterChanges( array $options ) {
1118
		$limit = isset( $options['maxWriteDuration'] ) ? $options['maxWriteDuration'] : 0;
1119
		$this->forEachOpenMasterConnection( function ( DatabaseBase $conn ) use ( $limit ) {
1120
			// If atomic sections or explicit transactions are still open, some caller must have
1121
			// caught an exception but failed to properly rollback any changes. Detect that and
1122
			// throw and error (causing rollback).
1123
			if ( $conn->explicitTrxActive() ) {
1124
				throw new DBTransactionError(
1125
					$conn,
1126
					"Explicit transaction still active. A caller may have caught an error."
1127
				);
1128
			}
1129
			// Assert that the time to replicate the transaction will be sane.
1130
			// If this fails, then all DB transactions will be rollback back together.
1131
			$time = $conn->pendingWriteQueryDuration( $conn::ESTIMATE_DB_APPLY );
1132
			if ( $limit > 0 && $time > $limit ) {
1133
				throw new DBTransactionError(
1134
					$conn,
1135
					wfMessage( 'transaction-duration-limit-exceeded', $time, $limit )->text()
1136
				);
1137
			}
1138
			// If a connection sits idle while slow queries execute on another, that connection
1139
			// may end up dropped before the commit round is reached. Ping servers to detect this.
1140
			if ( $conn->writesOrCallbacksPending() && !$conn->ping() ) {
1141
				throw new DBTransactionError(
1142
					$conn,
1143
					"A connection to the {$conn->getDBname()} database was lost before commit."
1144
				);
1145
			}
1146
		} );
1147
	}
1148
1149
	/**
1150
	 * Flush any master transaction snapshots and set DBO_TRX (if DBO_DEFAULT is set)
1151
	 *
1152
	 * The DBO_TRX setting will be reverted to the default in each of these methods:
1153
	 *   - commitMasterChanges()
1154
	 *   - rollbackMasterChanges()
1155
	 *   - commitAll()
1156
	 * This allows for custom transaction rounds from any outer transaction scope.
1157
	 *
1158
	 * @param string $fname
1159
	 * @throws DBExpectedError
1160
	 * @since 1.28
1161
	 */
1162
	public function beginMasterChanges( $fname = __METHOD__ ) {
1163
		if ( $this->trxRoundId !== false ) {
1164
			throw new DBTransactionError(
1165
				null,
1166
				"$fname: Transaction round '{$this->trxRoundId}' already started."
1167
			);
1168
		}
1169
		$this->trxRoundId = $fname;
1170
1171
		$failures = [];
1172
		$this->forEachOpenMasterConnection(
1173
			function ( DatabaseBase $conn ) use ( $fname, &$failures ) {
1174
				$conn->setTrxEndCallbackSuppression( true );
1175
				try {
1176
					$conn->flushSnapshot( $fname );
1177
				} catch ( DBError $e ) {
1178
					MWExceptionHandler::logException( $e );
1179
					$failures[] = "{$conn->getServer()}: {$e->getMessage()}";
1180
				}
1181
				$conn->setTrxEndCallbackSuppression( false );
1182
				$this->applyTransactionRoundFlags( $conn );
1183
			}
1184
		);
1185
1186 View Code Duplication
		if ( $failures ) {
1187
			throw new DBExpectedError(
1188
				null,
1189
				"$fname: Flush failed on server(s) " . implode( "\n", array_unique( $failures ) )
1190
			);
1191
		}
1192
	}
1193
1194
	/**
1195
	 * Issue COMMIT on all master connections where writes where done
1196
	 * @param string $fname Caller name
1197
	 * @throws DBExpectedError
1198
	 */
1199
	public function commitMasterChanges( $fname = __METHOD__ ) {
1200
		$failures = [];
1201
1202
		$restore = ( $this->trxRoundId !== false );
1203
		$this->trxRoundId = false;
1204
		$this->forEachOpenMasterConnection(
1205
			function ( DatabaseBase $conn ) use ( $fname, $restore, &$failures ) {
1206
				try {
1207
					if ( $conn->writesOrCallbacksPending() ) {
1208
						$conn->commit( $fname, $conn::FLUSHING_ALL_PEERS );
1209
					} elseif ( $restore ) {
1210
						$conn->flushSnapshot( $fname );
1211
					}
1212
				} catch ( DBError $e ) {
1213
					MWExceptionHandler::logException( $e );
1214
					$failures[] = "{$conn->getServer()}: {$e->getMessage()}";
1215
				}
1216
				if ( $restore ) {
1217
					$this->undoTransactionRoundFlags( $conn );
1218
				}
1219
			}
1220
		);
1221
1222 View Code Duplication
		if ( $failures ) {
1223
			throw new DBExpectedError(
1224
				null,
1225
				"$fname: Commit failed on server(s) " . implode( "\n", array_unique( $failures ) )
1226
			);
1227
		}
1228
	}
1229
1230
	/**
1231
	 * Issue all pending post-COMMIT/ROLLBACK callbacks
1232
	 * @param integer $type IDatabase::TRIGGER_* constant
1233
	 * @return Exception|null The first exception or null if there were none
1234
	 * @since 1.28
1235
	 */
1236
	public function runMasterPostTrxCallbacks( $type ) {
1237
		$e = null; // first exception
1238
		$this->forEachOpenMasterConnection( function ( DatabaseBase $conn ) use ( $type, &$e ) {
1239
			$conn->setTrxEndCallbackSuppression( false );
1240
			if ( $conn->writesOrCallbacksPending() ) {
1241
				// This happens if onTransactionIdle() callbacks leave callbacks on *another* DB
1242
				// (which finished its callbacks already). Warn and recover in this case. Let the
1243
				// callbacks run in the final commitMasterChanges() in LBFactory::shutdown().
1244
				wfWarn( __METHOD__ . ": did not expect writes/callbacks pending." );
1245
				return;
1246
			} elseif ( $conn->trxLevel() ) {
1247
				// This happens for single-DB setups where DB_REPLICA uses the master DB,
1248
				// thus leaving an implicit read-only transaction open at this point. It
1249
				// also happens if onTransactionIdle() callbacks leave implicit transactions
1250
				// open on *other* DBs (which is slightly improper). Let these COMMIT on the
1251
				// next call to commitMasterChanges(), possibly in LBFactory::shutdown().
1252
				return;
1253
			}
1254
			try {
1255
				$conn->runOnTransactionIdleCallbacks( $type );
1256
			} catch ( Exception $ex ) {
1257
				$e = $e ?: $ex;
1258
			}
1259
			try {
1260
				$conn->runTransactionListenerCallbacks( $type );
1261
			} catch ( Exception $ex ) {
1262
				$e = $e ?: $ex;
1263
			}
1264
		} );
1265
1266
		return $e;
1267
	}
1268
1269
	/**
1270
	 * Issue ROLLBACK only on master, only if queries were done on connection
1271
	 * @param string $fname Caller name
1272
	 * @throws DBExpectedError
1273
	 * @since 1.23
1274
	 */
1275
	public function rollbackMasterChanges( $fname = __METHOD__ ) {
1276
		$restore = ( $this->trxRoundId !== false );
1277
		$this->trxRoundId = false;
1278
		$this->forEachOpenMasterConnection(
1279
			function ( DatabaseBase $conn ) use ( $fname, $restore ) {
1280
				if ( $conn->writesOrCallbacksPending() ) {
1281
					$conn->rollback( $fname, $conn::FLUSHING_ALL_PEERS );
1282
				}
1283
				if ( $restore ) {
1284
					$this->undoTransactionRoundFlags( $conn );
1285
				}
1286
			}
1287
		);
1288
	}
1289
1290
	/**
1291
	 * Suppress all pending post-COMMIT/ROLLBACK callbacks
1292
	 * @return Exception|null The first exception or null if there were none
1293
	 * @since 1.28
1294
	 */
1295
	public function suppressTransactionEndCallbacks() {
1296
		$this->forEachOpenMasterConnection( function ( DatabaseBase $conn ) {
1297
			$conn->setTrxEndCallbackSuppression( true );
1298
		} );
1299
	}
1300
1301
	/**
1302
	 * @param DatabaseBase $conn
1303
	 */
1304
	private function applyTransactionRoundFlags( DatabaseBase $conn ) {
1305
		if ( $conn->getFlag( DBO_DEFAULT ) ) {
1306
			// DBO_TRX is controlled entirely by CLI mode presence with DBO_DEFAULT.
1307
			// Force DBO_TRX even in CLI mode since a commit round is expected soon.
1308
			$conn->setFlag( DBO_TRX, $conn::REMEMBER_PRIOR );
1309
			// If config has explicitly requested DBO_TRX be either on or off by not
1310
			// setting DBO_DEFAULT, then respect that. Forcing no transactions is useful
1311
			// for things like blob stores (ExternalStore) which want auto-commit mode.
1312
		}
1313
	}
1314
1315
	/**
1316
	 * @param DatabaseBase $conn
1317
	 */
1318
	private function undoTransactionRoundFlags( DatabaseBase $conn ) {
1319
		if ( $conn->getFlag( DBO_DEFAULT ) ) {
1320
			$conn->restoreFlags( $conn::RESTORE_PRIOR );
1321
		}
1322
	}
1323
1324
	/**
1325
	 * Commit all replica DB transactions so as to flush any REPEATABLE-READ or SSI snapshot
1326
	 *
1327
	 * @param string $fname Caller name
1328
	 * @since 1.28
1329
	 */
1330
	public function flushReplicaSnapshots( $fname = __METHOD__ ) {
1331
		$this->forEachOpenReplicaConnection( function ( DatabaseBase $conn ) {
1332
			$conn->flushSnapshot( __METHOD__ );
1333
		} );
1334
	}
1335
1336
	/**
1337
	 * @return bool Whether a master connection is already open
1338
	 * @since 1.24
1339
	 */
1340
	public function hasMasterConnection() {
1341
		return $this->isOpen( $this->getWriterIndex() );
1342
	}
1343
1344
	/**
1345
	 * Determine if there are pending changes in a transaction by this thread
1346
	 * @since 1.23
1347
	 * @return bool
1348
	 */
1349
	public function hasMasterChanges() {
1350
		$pending = 0;
1351
		$this->forEachOpenMasterConnection( function ( DatabaseBase $conn ) use ( &$pending ) {
1352
			$pending |= $conn->writesOrCallbacksPending();
1353
		} );
1354
1355
		return (bool)$pending;
1356
	}
1357
1358
	/**
1359
	 * Get the timestamp of the latest write query done by this thread
1360
	 * @since 1.25
1361
	 * @return float|bool UNIX timestamp or false
1362
	 */
1363
	public function lastMasterChangeTimestamp() {
1364
		$lastTime = false;
1365
		$this->forEachOpenMasterConnection( function ( DatabaseBase $conn ) use ( &$lastTime ) {
1366
			$lastTime = max( $lastTime, $conn->lastDoneWrites() );
1367
		} );
1368
1369
		return $lastTime;
1370
	}
1371
1372
	/**
1373
	 * Check if this load balancer object had any recent or still
1374
	 * pending writes issued against it by this PHP thread
1375
	 *
1376
	 * @param float $age How many seconds ago is "recent" [defaults to mWaitTimeout]
1377
	 * @return bool
1378
	 * @since 1.25
1379
	 */
1380
	public function hasOrMadeRecentMasterChanges( $age = null ) {
1381
		$age = ( $age === null ) ? $this->mWaitTimeout : $age;
1382
1383
		return ( $this->hasMasterChanges()
1384
			|| $this->lastMasterChangeTimestamp() > microtime( true ) - $age );
1385
	}
1386
1387
	/**
1388
	 * Get the list of callers that have pending master changes
1389
	 *
1390
	 * @return string[] List of method names
1391
	 * @since 1.27
1392
	 */
1393
	public function pendingMasterChangeCallers() {
1394
		$fnames = [];
1395
		$this->forEachOpenMasterConnection( function ( DatabaseBase $conn ) use ( &$fnames ) {
1396
			$fnames = array_merge( $fnames, $conn->pendingWriteCallers() );
1397
		} );
1398
1399
		return $fnames;
1400
	}
1401
1402
	/**
1403
	 * @note This method will trigger a DB connection if not yet done
1404
	 * @param string|bool $wiki Wiki ID, or false for the current wiki
1405
	 * @return bool Whether the generic connection for reads is highly "lagged"
1406
	 */
1407
	public function getLaggedReplicaMode( $wiki = false ) {
1408
		// No-op if there is only one DB (also avoids recursion)
1409
		if ( !$this->laggedReplicaMode && $this->getServerCount() > 1 ) {
1410
			try {
1411
				// See if laggedReplicaMode gets set
1412
				$conn = $this->getConnection( DB_REPLICA, false, $wiki );
1413
				$this->reuseConnection( $conn );
1414
			} catch ( DBConnectionError $e ) {
1415
				// Avoid expensive re-connect attempts and failures
1416
				$this->allReplicasDownMode = true;
1417
				$this->laggedReplicaMode = true;
1418
			}
1419
		}
1420
1421
		return $this->laggedReplicaMode;
1422
	}
1423
1424
	/**
1425
	 * @param bool $wiki
1426
	 * @return bool
1427
	 * @deprecated 1.28; use getLaggedReplicaMode()
1428
	 */
1429
	public function getLaggedSlaveMode( $wiki = false ) {
1430
		return $this->getLaggedReplicaMode( $wiki );
1431
	}
1432
1433
	/**
1434
	 * @note This method will never cause a new DB connection
1435
	 * @return bool Whether any generic connection used for reads was highly "lagged"
1436
	 * @since 1.28
1437
	 */
1438
	public function laggedReplicaUsed() {
1439
		return $this->laggedReplicaMode;
1440
	}
1441
1442
	/**
1443
	 * @return bool
1444
	 * @since 1.27
1445
	 * @deprecated Since 1.28; use laggedReplicaUsed()
1446
	 */
1447
	public function laggedSlaveUsed() {
1448
		return $this->laggedReplicaUsed();
1449
	}
1450
1451
	/**
1452
	 * @note This method may trigger a DB connection if not yet done
1453
	 * @param string|bool $wiki Wiki ID, or false for the current wiki
1454
	 * @param DatabaseBase|null DB master connection; used to avoid loops [optional]
1455
	 * @return string|bool Reason the master is read-only or false if it is not
1456
	 * @since 1.27
1457
	 */
1458
	public function getReadOnlyReason( $wiki = false, DatabaseBase $conn = null ) {
1459
		if ( $this->readOnlyReason !== false ) {
1460
			return $this->readOnlyReason;
1461
		} elseif ( $this->getLaggedReplicaMode( $wiki ) ) {
1462
			if ( $this->allReplicasDownMode ) {
1463
				return 'The database has been automatically locked ' .
1464
					'until the replica database servers become available';
1465
			} else {
1466
				return 'The database has been automatically locked ' .
1467
					'while the replica database servers catch up to the master.';
1468
			}
1469
		} elseif ( $this->masterRunningReadOnly( $wiki, $conn ) ) {
0 ignored issues
show
Bug introduced by
It seems like $wiki defined by parameter $wiki on line 1458 can also be of type boolean; however, LoadBalancer::masterRunningReadOnly() does only seem to accept string, maybe add an additional type check?

This check looks at variables that have been passed in as parameters and are passed out again to other methods.

If the outgoing method call has stricter type requirements than the method itself, an issue is raised.

An additional type check may prevent trouble.

Loading history...
1470
			return 'The database master is running in read-only mode.';
1471
		}
1472
1473
		return false;
1474
	}
1475
1476
	/**
1477
	 * @param string $wiki Wiki ID, or false for the current wiki
1478
	 * @param DatabaseBase|null DB master connectionl used to avoid loops [optional]
1479
	 * @return bool
1480
	 */
1481
	private function masterRunningReadOnly( $wiki, DatabaseBase $conn = null ) {
1482
		$cache = $this->wanCache;
1483
		$masterServer = $this->getServerName( $this->getWriterIndex() );
1484
1485
		return (bool)$cache->getWithSetCallback(
1486
			$cache->makeGlobalKey( __CLASS__, 'server-read-only', $masterServer ),
1487
			self::TTL_CACHE_READONLY,
1488
			function () use ( $wiki, $conn ) {
1489
				$this->trxProfiler->setSilenced( true );
1490
				try {
1491
					$dbw = $conn ?: $this->getConnection( DB_MASTER, [], $wiki );
1492
					$readOnly = (int)$dbw->serverIsReadOnly();
1493
				} catch ( DBError $e ) {
1494
					$readOnly = 0;
1495
				}
1496
				$this->trxProfiler->setSilenced( false );
1497
				return $readOnly;
1498
			},
1499
			[ 'pcTTL' => $cache::TTL_PROC_LONG, 'busyValue' => 0 ]
1500
		);
1501
	}
1502
1503
	/**
1504
	 * Disables/enables lag checks
1505
	 * @param null|bool $mode
1506
	 * @return bool
1507
	 */
1508
	public function allowLagged( $mode = null ) {
1509
		if ( $mode === null ) {
1510
			return $this->mAllowLagged;
1511
		}
1512
		$this->mAllowLagged = $mode;
1513
1514
		return $this->mAllowLagged;
1515
	}
1516
1517
	/**
1518
	 * @return bool
1519
	 */
1520
	public function pingAll() {
1521
		$success = true;
1522
		$this->forEachOpenConnection( function ( DatabaseBase $conn ) use ( &$success ) {
1523
			if ( !$conn->ping() ) {
1524
				$success = false;
1525
			}
1526
		} );
1527
1528
		return $success;
1529
	}
1530
1531
	/**
1532
	 * Call a function with each open connection object
1533
	 * @param callable $callback
1534
	 * @param array $params
1535
	 */
1536 View Code Duplication
	public function forEachOpenConnection( $callback, array $params = [] ) {
1537
		foreach ( $this->mConns as $connsByServer ) {
1538
			foreach ( $connsByServer as $serverConns ) {
1539
				foreach ( $serverConns as $conn ) {
1540
					$mergedParams = array_merge( [ $conn ], $params );
1541
					call_user_func_array( $callback, $mergedParams );
1542
				}
1543
			}
1544
		}
1545
	}
1546
1547
	/**
1548
	 * Call a function with each open connection object to a master
1549
	 * @param callable $callback
1550
	 * @param array $params
1551
	 * @since 1.28
1552
	 */
1553
	public function forEachOpenMasterConnection( $callback, array $params = [] ) {
1554
		$masterIndex = $this->getWriterIndex();
1555
		foreach ( $this->mConns as $connsByServer ) {
1556
			if ( isset( $connsByServer[$masterIndex] ) ) {
1557
				/** @var DatabaseBase $conn */
1558
				foreach ( $connsByServer[$masterIndex] as $conn ) {
1559
					$mergedParams = array_merge( [ $conn ], $params );
1560
					call_user_func_array( $callback, $mergedParams );
1561
				}
1562
			}
1563
		}
1564
	}
1565
1566
	/**
1567
	 * Call a function with each open replica DB connection object
1568
	 * @param callable $callback
1569
	 * @param array $params
1570
	 * @since 1.28
1571
	 */
1572 View Code Duplication
	public function forEachOpenReplicaConnection( $callback, array $params = [] ) {
1573
		foreach ( $this->mConns as $connsByServer ) {
1574
			foreach ( $connsByServer as $i => $serverConns ) {
1575
				if ( $i === $this->getWriterIndex() ) {
1576
					continue; // skip master
1577
				}
1578
				foreach ( $serverConns as $conn ) {
1579
					$mergedParams = array_merge( [ $conn ], $params );
1580
					call_user_func_array( $callback, $mergedParams );
1581
				}
1582
			}
1583
		}
1584
	}
1585
1586
	/**
1587
	 * Get the hostname and lag time of the most-lagged replica DB
1588
	 *
1589
	 * This is useful for maintenance scripts that need to throttle their updates.
1590
	 * May attempt to open connections to replica DBs on the default DB. If there is
1591
	 * no lag, the maximum lag will be reported as -1.
1592
	 *
1593
	 * @param bool|string $wiki Wiki ID, or false for the default database
1594
	 * @return array ( host, max lag, index of max lagged host )
1595
	 */
1596
	public function getMaxLag( $wiki = false ) {
1597
		$maxLag = -1;
1598
		$host = '';
1599
		$maxIndex = 0;
1600
1601
		if ( $this->getServerCount() <= 1 ) {
1602
			return [ $host, $maxLag, $maxIndex ]; // no replication = no lag
1603
		}
1604
1605
		$lagTimes = $this->getLagTimes( $wiki );
1606
		foreach ( $lagTimes as $i => $lag ) {
1607
			if ( $this->mLoads[$i] > 0 && $lag > $maxLag ) {
1608
				$maxLag = $lag;
1609
				$host = $this->mServers[$i]['host'];
1610
				$maxIndex = $i;
1611
			}
1612
		}
1613
1614
		return [ $host, $maxLag, $maxIndex ];
1615
	}
1616
1617
	/**
1618
	 * Get an estimate of replication lag (in seconds) for each server
1619
	 *
1620
	 * Results are cached for a short time in memcached/process cache
1621
	 *
1622
	 * Values may be "false" if replication is too broken to estimate
1623
	 *
1624
	 * @param string|bool $wiki
1625
	 * @return int[] Map of (server index => float|int|bool)
1626
	 */
1627
	public function getLagTimes( $wiki = false ) {
1628
		if ( $this->getServerCount() <= 1 ) {
1629
			return [ 0 => 0 ]; // no replication = no lag
1630
		}
1631
1632
		# Send the request to the load monitor
1633
		return $this->getLoadMonitor()->getLagTimes( array_keys( $this->mServers ), $wiki );
0 ignored issues
show
Bug introduced by
It seems like $wiki defined by parameter $wiki on line 1627 can also be of type boolean; however, LoadMonitor::getLagTimes() does only seem to accept string, maybe add an additional type check?

This check looks at variables that have been passed in as parameters and are passed out again to other methods.

If the outgoing method call has stricter type requirements than the method itself, an issue is raised.

An additional type check may prevent trouble.

Loading history...
1634
	}
1635
1636
	/**
1637
	 * Get the lag in seconds for a given connection, or zero if this load
1638
	 * balancer does not have replication enabled.
1639
	 *
1640
	 * This should be used in preference to Database::getLag() in cases where
1641
	 * replication may not be in use, since there is no way to determine if
1642
	 * replication is in use at the connection level without running
1643
	 * potentially restricted queries such as SHOW SLAVE STATUS. Using this
1644
	 * function instead of Database::getLag() avoids a fatal error in this
1645
	 * case on many installations.
1646
	 *
1647
	 * @param IDatabase $conn
1648
	 * @return int|bool Returns false on error
1649
	 */
1650
	public function safeGetLag( IDatabase $conn ) {
1651
		if ( $this->getServerCount() == 1 ) {
1652
			return 0;
1653
		} else {
1654
			return $conn->getLag();
1655
		}
1656
	}
1657
1658
	/**
1659
	 * Wait for a replica DB to reach a specified master position
1660
	 *
1661
	 * This will connect to the master to get an accurate position if $pos is not given
1662
	 *
1663
	 * @param IDatabase $conn Replica DB
1664
	 * @param DBMasterPos|bool $pos Master position; default: current position
1665
	 * @param integer|null $timeout Timeout in seconds [optional]
1666
	 * @return bool Success
1667
	 * @since 1.27
1668
	 */
1669
	public function safeWaitForMasterPos( IDatabase $conn, $pos = false, $timeout = null ) {
1670
		if ( $this->getServerCount() == 1 || !$conn->getLBInfo( 'replica' ) ) {
1671
			return true; // server is not a replica DB
1672
		}
1673
1674
		$pos = $pos ?: $this->getConnection( DB_MASTER )->getMasterPos();
1675
		if ( !( $pos instanceof DBMasterPos ) ) {
1676
			return false; // something is misconfigured
1677
		}
1678
1679
		$timeout = $timeout ?: $this->mWaitTimeout;
1680
		$result = $conn->masterPosWait( $pos, $timeout );
1681
		if ( $result == -1 || is_null( $result ) ) {
1682
			$msg = __METHOD__ . ": Timed out waiting on {$conn->getServer()} pos {$pos}";
1683
			wfDebugLog( 'replication', "$msg\n" );
1684
			wfDebugLog( 'DBPerformance', "$msg:\n" . wfBacktrace( true ) );
1685
			$ok = false;
1686
		} else {
1687
			wfDebugLog( 'replication', __METHOD__ . ": Done\n" );
1688
			$ok = true;
1689
		}
1690
1691
		return $ok;
1692
	}
1693
1694
	/**
1695
	 * Clear the cache for slag lag delay times
1696
	 *
1697
	 * This is only used for testing
1698
	 */
1699
	public function clearLagTimeCache() {
1700
		$this->getLoadMonitor()->clearCaches();
1701
	}
1702
1703
	/**
1704
	 * Set a callback via DatabaseBase::setTransactionListener() on
1705
	 * all current and future master connections of this load balancer
1706
	 *
1707
	 * @param string $name Callback name
1708
	 * @param callable|null $callback
1709
	 * @since 1.28
1710
	 */
1711
	public function setTransactionListener( $name, callable $callback = null ) {
1712
		if ( $callback ) {
1713
			$this->trxRecurringCallbacks[$name] = $callback;
1714
		} else {
1715
			unset( $this->trxRecurringCallbacks[$name] );
1716
		}
1717
		$this->forEachOpenMasterConnection(
1718
			function ( DatabaseBase $conn ) use ( $name, $callback ) {
1719
				$conn->setTransactionListener( $name, $callback );
1720
			}
1721
		);
1722
	}
1723
}
1724