wikimedia /
mediawiki
Complex classes like WANObjectCache often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes. You can also have a look at the cohesion graph to spot any un-connected, or weakly-connected components.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use WANObjectCache, and based on these observations, apply Extract Interface, too.
| 1 | <?php |
||
| 76 | class WANObjectCache implements IExpiringStore, LoggerAwareInterface { |
||
| 77 | /** @var BagOStuff The local datacenter cache */ |
||
| 78 | protected $cache; |
||
| 79 | /** @var HashBagOStuff[] Map of group PHP instance caches */ |
||
| 80 | protected $processCaches = []; |
||
| 81 | /** @var string Purge channel name */ |
||
| 82 | protected $purgeChannel; |
||
| 83 | /** @var EventRelayer Bus that handles purge broadcasts */ |
||
| 84 | protected $purgeRelayer; |
||
| 85 | /** @var LoggerInterface */ |
||
| 86 | protected $logger; |
||
| 87 | |||
| 88 | /** @var int ERR_* constant for the "last error" registry */ |
||
| 89 | protected $lastRelayError = self::ERR_NONE; |
||
| 90 | |||
| 91 | /** @var mixed[] Temporary warm-up cache */ |
||
| 92 | private $warmupCache = []; |
||
| 93 | |||
| 94 | /** Max time expected to pass between delete() and DB commit finishing */ |
||
| 95 | const MAX_COMMIT_DELAY = 3; |
||
| 96 | /** Max replication+snapshot lag before applying TTL_LAGGED or disallowing set() */ |
||
| 97 | const MAX_READ_LAG = 7; |
||
| 98 | /** Seconds to tombstone keys on delete() */ |
||
| 99 | const HOLDOFF_TTL = 11; // MAX_COMMIT_DELAY + MAX_READ_LAG + 1 |
||
| 100 | |||
| 101 | /** Seconds to keep dependency purge keys around */ |
||
| 102 | const CHECK_KEY_TTL = self::TTL_YEAR; |
||
| 103 | /** Seconds to keep lock keys around */ |
||
| 104 | const LOCK_TTL = 10; |
||
| 105 | /** Default remaining TTL at which to consider pre-emptive regeneration */ |
||
| 106 | const LOW_TTL = 30; |
||
| 107 | /** Default time-since-expiry on a miss that makes a key "hot" */ |
||
| 108 | const LOCK_TSE = 1; |
||
| 109 | |||
| 110 | /** Never consider performing "popularity" refreshes until a key reaches this age */ |
||
| 111 | const AGE_NEW = 60; |
||
| 112 | /** The time length of the "popularity" refresh window for hot keys */ |
||
| 113 | const HOT_TTR = 900; |
||
| 114 | /** Hits/second for a refresh to be expected within the "popularity" window */ |
||
| 115 | const HIT_RATE_HIGH = 1; |
||
| 116 | /** Seconds to ramp up to the "popularity" refresh chance after a key is no longer new */ |
||
| 117 | const RAMPUP_TTL = 30; |
||
| 118 | |||
| 119 | /** Idiom for getWithSetCallback() callbacks to avoid calling set() */ |
||
| 120 | const TTL_UNCACHEABLE = -1; |
||
| 121 | /** Idiom for getWithSetCallback() callbacks to 'lockTSE' logic */ |
||
| 122 | const TSE_NONE = -1; |
||
| 123 | /** Max TTL to store keys when a data sourced is lagged */ |
||
| 124 | const TTL_LAGGED = 30; |
||
| 125 | /** Idiom for delete() for "no hold-off" */ |
||
| 126 | const HOLDOFF_NONE = 0; |
||
| 127 | /** Idiom for getWithSetCallback() for "no minimum required as-of timestamp" */ |
||
| 128 | const MIN_TIMESTAMP_NONE = 0.0; |
||
| 129 | |||
| 130 | /** Tiny negative float to use when CTL comes up >= 0 due to clock skew */ |
||
| 131 | const TINY_NEGATIVE = -0.000001; |
||
| 132 | |||
| 133 | /** Cache format version number */ |
||
| 134 | const VERSION = 1; |
||
| 135 | |||
| 136 | const FLD_VERSION = 0; // key to cache version number |
||
| 137 | const FLD_VALUE = 1; // key to the cached value |
||
| 138 | const FLD_TTL = 2; // key to the original TTL |
||
| 139 | const FLD_TIME = 3; // key to the cache time |
||
| 140 | const FLD_FLAGS = 4; // key to the flags bitfield |
||
| 141 | const FLD_HOLDOFF = 5; // key to any hold-off TTL |
||
| 142 | |||
| 143 | /** @var integer Treat this value as expired-on-arrival */ |
||
| 144 | const FLG_STALE = 1; |
||
| 145 | |||
| 146 | const ERR_NONE = 0; // no error |
||
| 147 | const ERR_NO_RESPONSE = 1; // no response |
||
| 148 | const ERR_UNREACHABLE = 2; // can't connect |
||
| 149 | const ERR_UNEXPECTED = 3; // response gave some error |
||
| 150 | const ERR_RELAY = 4; // relay broadcast failed |
||
| 151 | |||
| 152 | const VALUE_KEY_PREFIX = 'WANCache:v:'; |
||
| 153 | const INTERIM_KEY_PREFIX = 'WANCache:i:'; |
||
| 154 | const TIME_KEY_PREFIX = 'WANCache:t:'; |
||
| 155 | const MUTEX_KEY_PREFIX = 'WANCache:m:'; |
||
| 156 | |||
| 157 | const PURGE_VAL_PREFIX = 'PURGED:'; |
||
| 158 | |||
| 159 | const VFLD_DATA = 'WOC:d'; // key to the value of versioned data |
||
| 160 | const VFLD_VERSION = 'WOC:v'; // key to the version of the value present |
||
| 161 | |||
| 162 | const PC_PRIMARY = 'primary:1000'; // process cache name and max key count |
||
| 163 | |||
| 164 | const DEFAULT_PURGE_CHANNEL = 'wancache-purge'; |
||
| 165 | |||
| 166 | /** |
||
| 167 | * @param array $params |
||
| 168 | * - cache : BagOStuff object for a persistent cache |
||
| 169 | * - channels : Map of (action => channel string). Actions include "purge". |
||
| 170 | * - relayers : Map of (action => EventRelayer object). Actions include "purge". |
||
| 171 | * - logger : LoggerInterface object |
||
| 172 | */ |
||
| 173 | public function __construct( array $params ) { |
||
| 183 | |||
| 184 | public function setLogger( LoggerInterface $logger ) { |
||
| 187 | |||
| 188 | /** |
||
| 189 | * Get an instance that wraps EmptyBagOStuff |
||
| 190 | * |
||
| 191 | * @return WANObjectCache |
||
| 192 | */ |
||
| 193 | public static function newEmpty() { |
||
| 200 | |||
| 201 | /** |
||
| 202 | * Fetch the value of a key from cache |
||
| 203 | * |
||
| 204 | * If supplied, $curTTL is set to the remaining TTL (current time left): |
||
| 205 | * - a) INF; if $key exists, has no TTL, and is not expired by $checkKeys |
||
| 206 | * - b) float (>=0); if $key exists, has a TTL, and is not expired by $checkKeys |
||
| 207 | * - c) float (<0); if $key is tombstoned, stale, or existing but expired by $checkKeys |
||
| 208 | * - d) null; if $key does not exist and is not tombstoned |
||
| 209 | * |
||
| 210 | * If a key is tombstoned, $curTTL will reflect the time since delete(). |
||
| 211 | * |
||
| 212 | * The timestamp of $key will be checked against the last-purge timestamp |
||
| 213 | * of each of $checkKeys. Those $checkKeys not in cache will have the last-purge |
||
| 214 | * initialized to the current timestamp. If any of $checkKeys have a timestamp |
||
| 215 | * greater than that of $key, then $curTTL will reflect how long ago $key |
||
| 216 | * became invalid. Callers can use $curTTL to know when the value is stale. |
||
| 217 | * The $checkKeys parameter allow mass invalidations by updating a single key: |
||
| 218 | * - a) Each "check" key represents "last purged" of some source data |
||
| 219 | * - b) Callers pass in relevant "check" keys as $checkKeys in get() |
||
| 220 | * - c) When the source data that "check" keys represent changes, |
||
| 221 | * the touchCheckKey() method is called on them |
||
| 222 | * |
||
| 223 | * Source data entities might exists in a DB that uses snapshot isolation |
||
| 224 | * (e.g. the default REPEATABLE-READ in innoDB). Even for mutable data, that |
||
| 225 | * isolation can largely be maintained by doing the following: |
||
| 226 | * - a) Calling delete() on entity change *and* creation, before DB commit |
||
| 227 | * - b) Keeping transaction duration shorter than delete() hold-off TTL |
||
| 228 | * |
||
| 229 | * However, pre-snapshot values might still be seen if an update was made |
||
| 230 | * in a remote datacenter but the purge from delete() didn't relay yet. |
||
| 231 | * |
||
| 232 | * Consider using getWithSetCallback() instead of get() and set() cycles. |
||
| 233 | * That method has cache slam avoiding features for hot/expensive keys. |
||
| 234 | * |
||
| 235 | * @param string $key Cache key |
||
| 236 | * @param mixed $curTTL Approximate TTL left on the key if present/tombstoned [returned] |
||
| 237 | * @param array $checkKeys List of "check" keys |
||
| 238 | * @param float &$asOf UNIX timestamp of cached value; null on failure [returned] |
||
| 239 | * @return mixed Value of cache key or false on failure |
||
| 240 | */ |
||
| 241 | final public function get( $key, &$curTTL = null, array $checkKeys = [], &$asOf = null ) { |
||
| 250 | |||
| 251 | /** |
||
| 252 | * Fetch the value of several keys from cache |
||
| 253 | * |
||
| 254 | * @see WANObjectCache::get() |
||
| 255 | * |
||
| 256 | * @param array $keys List of cache keys |
||
| 257 | * @param array $curTTLs Map of (key => approximate TTL left) for existing keys [returned] |
||
| 258 | * @param array $checkKeys List of check keys to apply to all $keys. May also apply "check" |
||
| 259 | * keys to specific cache keys only by using cache keys as keys in the $checkKeys array. |
||
| 260 | * @param float[] &$asOfs Map of (key => UNIX timestamp of cached value; null on failure) |
||
| 261 | * @return array Map of (key => value) for keys that exist |
||
| 262 | */ |
||
| 263 | final public function getMulti( |
||
| 343 | |||
| 344 | /** |
||
| 345 | * @since 1.27 |
||
| 346 | * @param array $timeKeys List of prefixed time check keys |
||
| 347 | * @param array $wrappedValues |
||
| 348 | * @param float $now |
||
| 349 | * @return array List of purge value arrays |
||
| 350 | */ |
||
| 351 | private function processCheckKeys( array $timeKeys, array $wrappedValues, $now ) { |
||
| 367 | |||
| 368 | /** |
||
| 369 | * Set the value of a key in cache |
||
| 370 | * |
||
| 371 | * Simply calling this method when source data changes is not valid because |
||
| 372 | * the changes do not replicate to the other WAN sites. In that case, delete() |
||
| 373 | * should be used instead. This method is intended for use on cache misses. |
||
| 374 | * |
||
| 375 | * If the data was read from a snapshot-isolated transactions (e.g. the default |
||
| 376 | * REPEATABLE-READ in innoDB), use 'since' to avoid the following race condition: |
||
| 377 | * - a) T1 starts |
||
| 378 | * - b) T2 updates a row, calls delete(), and commits |
||
| 379 | * - c) The HOLDOFF_TTL passes, expiring the delete() tombstone |
||
| 380 | * - d) T1 reads the row and calls set() due to a cache miss |
||
| 381 | * - e) Stale value is stuck in cache |
||
| 382 | * |
||
| 383 | * Setting 'lag' and 'since' help avoids keys getting stuck in stale states. |
||
| 384 | * |
||
| 385 | * Example usage: |
||
| 386 | * @code |
||
| 387 | * $dbr = wfGetDB( DB_REPLICA ); |
||
| 388 | * $setOpts = Database::getCacheSetOptions( $dbr ); |
||
| 389 | * // Fetch the row from the DB |
||
| 390 | * $row = $dbr->selectRow( ... ); |
||
| 391 | * $key = $cache->makeKey( 'building', $buildingId ); |
||
| 392 | * $cache->set( $key, $row, $cache::TTL_DAY, $setOpts ); |
||
| 393 | * @endcode |
||
| 394 | * |
||
| 395 | * @param string $key Cache key |
||
| 396 | * @param mixed $value |
||
| 397 | * @param integer $ttl Seconds to live. Special values are: |
||
| 398 | * - WANObjectCache::TTL_INDEFINITE: Cache forever |
||
| 399 | * @param array $opts Options map: |
||
| 400 | * - lag : Seconds of replica DB lag. Typically, this is either the replica DB lag |
||
| 401 | * before the data was read or, if applicable, the replica DB lag before |
||
| 402 | * the snapshot-isolated transaction the data was read from started. |
||
| 403 | * Use false to indicate that replication is not running. |
||
| 404 | * Default: 0 seconds |
||
| 405 | * - since : UNIX timestamp of the data in $value. Typically, this is either |
||
| 406 | * the current time the data was read or (if applicable) the time when |
||
| 407 | * the snapshot-isolated transaction the data was read from started. |
||
| 408 | * Default: 0 seconds |
||
| 409 | * - pending : Whether this data is possibly from an uncommitted write transaction. |
||
| 410 | * Generally, other threads should not see values from the future and |
||
| 411 | * they certainly should not see ones that ended up getting rolled back. |
||
| 412 | * Default: false |
||
| 413 | * - lockTSE : if excessive replication/snapshot lag is detected, then store the value |
||
| 414 | * with this TTL and flag it as stale. This is only useful if the reads for |
||
| 415 | * this key use getWithSetCallback() with "lockTSE" set. |
||
| 416 | * Default: WANObjectCache::TSE_NONE |
||
| 417 | * - staleTTL : Seconds to keep the key around if it is stale. The get()/getMulti() |
||
| 418 | * methods return such stale values with a $curTTL of 0, and getWithSetCallback() |
||
| 419 | * will call the regeneration callback in such cases, passing in the old value |
||
| 420 | * and its as-of time to the callback. This is useful if adaptiveTTL() is used |
||
| 421 | * on the old value's as-of time when it is verified as still being correct. |
||
| 422 | * Default: 0. |
||
| 423 | * @note Options added in 1.28: staleTTL |
||
| 424 | * @return bool Success |
||
| 425 | */ |
||
| 426 | final public function set( $key, $value, $ttl = 0, array $opts = [] ) { |
||
| 475 | |||
| 476 | /** |
||
| 477 | * Purge a key from all datacenters |
||
| 478 | * |
||
| 479 | * This should only be called when the underlying data (being cached) |
||
| 480 | * changes in a significant way. This deletes the key and starts a hold-off |
||
| 481 | * period where the key cannot be written to for a few seconds (HOLDOFF_TTL). |
||
| 482 | * This is done to avoid the following race condition: |
||
| 483 | * - a) Some DB data changes and delete() is called on a corresponding key |
||
| 484 | * - b) A request refills the key with a stale value from a lagged DB |
||
| 485 | * - c) The stale value is stuck there until the key is expired/evicted |
||
| 486 | * |
||
| 487 | * This is implemented by storing a special "tombstone" value at the cache |
||
| 488 | * key that this class recognizes; get() calls will return false for the key |
||
| 489 | * and any set() calls will refuse to replace tombstone values at the key. |
||
| 490 | * For this to always avoid stale value writes, the following must hold: |
||
| 491 | * - a) Replication lag is bounded to being less than HOLDOFF_TTL; or |
||
| 492 | * - b) If lag is higher, the DB will have gone into read-only mode already |
||
| 493 | * |
||
| 494 | * Note that set() can also be lag-aware and lower the TTL if it's high. |
||
| 495 | * |
||
| 496 | * When using potentially long-running ACID transactions, a good pattern is |
||
| 497 | * to use a pre-commit hook to issue the delete. This means that immediately |
||
| 498 | * after commit, callers will see the tombstone in cache upon purge relay. |
||
| 499 | * It also avoids the following race condition: |
||
| 500 | * - a) T1 begins, changes a row, and calls delete() |
||
| 501 | * - b) The HOLDOFF_TTL passes, expiring the delete() tombstone |
||
| 502 | * - c) T2 starts, reads the row and calls set() due to a cache miss |
||
| 503 | * - d) T1 finally commits |
||
| 504 | * - e) Stale value is stuck in cache |
||
| 505 | * |
||
| 506 | * Example usage: |
||
| 507 | * @code |
||
| 508 | * $dbw->startAtomic( __METHOD__ ); // start of request |
||
| 509 | * ... <execute some stuff> ... |
||
| 510 | * // Update the row in the DB |
||
| 511 | * $dbw->update( ... ); |
||
| 512 | * $key = $cache->makeKey( 'homes', $homeId ); |
||
| 513 | * // Purge the corresponding cache entry just before committing |
||
| 514 | * $dbw->onTransactionPreCommitOrIdle( function() use ( $cache, $key ) { |
||
| 515 | * $cache->delete( $key ); |
||
| 516 | * } ); |
||
| 517 | * ... <execute some stuff> ... |
||
| 518 | * $dbw->endAtomic( __METHOD__ ); // end of request |
||
| 519 | * @endcode |
||
| 520 | * |
||
| 521 | * The $ttl parameter can be used when purging values that have not actually changed |
||
| 522 | * recently. For example, a cleanup script to purge cache entries does not really need |
||
| 523 | * a hold-off period, so it can use HOLDOFF_NONE. Likewise for user-requested purge. |
||
| 524 | * Note that $ttl limits the effective range of 'lockTSE' for getWithSetCallback(). |
||
| 525 | * |
||
| 526 | * If called twice on the same key, then the last hold-off TTL takes precedence. For |
||
| 527 | * idempotence, the $ttl should not vary for different delete() calls on the same key. |
||
| 528 | * |
||
| 529 | * @param string $key Cache key |
||
| 530 | * @param integer $ttl Tombstone TTL; Default: WANObjectCache::HOLDOFF_TTL |
||
| 531 | * @return bool True if the item was purged or not found, false on failure |
||
| 532 | */ |
||
| 533 | final public function delete( $key, $ttl = self::HOLDOFF_TTL ) { |
||
| 546 | |||
| 547 | /** |
||
| 548 | * Fetch the value of a timestamp "check" key |
||
| 549 | * |
||
| 550 | * The key will be *initialized* to the current time if not set, |
||
| 551 | * so only call this method if this behavior is actually desired |
||
| 552 | * |
||
| 553 | * The timestamp can be used to check whether a cached value is valid. |
||
| 554 | * Callers should not assume that this returns the same timestamp in |
||
| 555 | * all datacenters due to relay delays. |
||
| 556 | * |
||
| 557 | * The level of staleness can roughly be estimated from this key, but |
||
| 558 | * if the key was evicted from cache, such calculations may show the |
||
| 559 | * time since expiry as ~0 seconds. |
||
| 560 | * |
||
| 561 | * Note that "check" keys won't collide with other regular keys. |
||
| 562 | * |
||
| 563 | * @param string $key |
||
| 564 | * @return float UNIX timestamp of the check key |
||
| 565 | */ |
||
| 566 | final public function getCheckKeyTime( $key ) { |
||
| 584 | |||
| 585 | /** |
||
| 586 | * Purge a "check" key from all datacenters, invalidating keys that use it |
||
| 587 | * |
||
| 588 | * This should only be called when the underlying data (being cached) |
||
| 589 | * changes in a significant way, and it is impractical to call delete() |
||
| 590 | * on all keys that should be changed. When get() is called on those |
||
| 591 | * keys, the relevant "check" keys must be supplied for this to work. |
||
| 592 | * |
||
| 593 | * The "check" key essentially represents a last-modified field. |
||
| 594 | * When touched, the field will be updated on all cache servers. |
||
| 595 | * Keys using it via get(), getMulti(), or getWithSetCallback() will |
||
| 596 | * be invalidated. It is treated as being HOLDOFF_TTL seconds in the future |
||
| 597 | * by those methods to avoid race conditions where dependent keys get updated |
||
| 598 | * with stale values (e.g. from a DB replica DB). |
||
| 599 | * |
||
| 600 | * This is typically useful for keys with hardcoded names or in some cases |
||
| 601 | * dynamically generated names where a low number of combinations exist. |
||
| 602 | * When a few important keys get a large number of hits, a high cache |
||
| 603 | * time is usually desired as well as "lockTSE" logic. The resetCheckKey() |
||
| 604 | * method is less appropriate in such cases since the "time since expiry" |
||
| 605 | * cannot be inferred, causing any get() after the reset to treat the key |
||
| 606 | * as being "hot", resulting in more stale value usage. |
||
| 607 | * |
||
| 608 | * Note that "check" keys won't collide with other regular keys. |
||
| 609 | * |
||
| 610 | * @see WANObjectCache::get() |
||
| 611 | * @see WANObjectCache::getWithSetCallback() |
||
| 612 | * @see WANObjectCache::resetCheckKey() |
||
| 613 | * |
||
| 614 | * @param string $key Cache key |
||
| 615 | * @param int $holdoff HOLDOFF_TTL or HOLDOFF_NONE constant |
||
| 616 | * @return bool True if the item was purged or not found, false on failure |
||
| 617 | */ |
||
| 618 | final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) { |
||
| 622 | |||
| 623 | /** |
||
| 624 | * Delete a "check" key from all datacenters, invalidating keys that use it |
||
| 625 | * |
||
| 626 | * This is similar to touchCheckKey() in that keys using it via get(), getMulti(), |
||
| 627 | * or getWithSetCallback() will be invalidated. The differences are: |
||
| 628 | * - a) The "check" key will be deleted from all caches and lazily |
||
| 629 | * re-initialized when accessed (rather than set everywhere) |
||
| 630 | * - b) Thus, dependent keys will be known to be invalid, but not |
||
| 631 | * for how long (they are treated as "just" purged), which |
||
| 632 | * effects any lockTSE logic in getWithSetCallback() |
||
| 633 | * - c) Since "check" keys are initialized only on the server the key hashes |
||
| 634 | * to, any temporary ejection of that server will cause the value to be |
||
| 635 | * seen as purged as a new server will initialize the "check" key. |
||
| 636 | * |
||
| 637 | * The advantage is that this does not place high TTL keys on every cache |
||
| 638 | * server, making it better for code that will cache many different keys |
||
| 639 | * and either does not use lockTSE or uses a low enough TTL anyway. |
||
| 640 | * |
||
| 641 | * This is typically useful for keys with dynamically generated names |
||
| 642 | * where a high number of combinations exist. |
||
| 643 | * |
||
| 644 | * Note that "check" keys won't collide with other regular keys. |
||
| 645 | * |
||
| 646 | * @see WANObjectCache::get() |
||
| 647 | * @see WANObjectCache::getWithSetCallback() |
||
| 648 | * @see WANObjectCache::touchCheckKey() |
||
| 649 | * |
||
| 650 | * @param string $key Cache key |
||
| 651 | * @return bool True if the item was purged or not found, false on failure |
||
| 652 | */ |
||
| 653 | final public function resetCheckKey( $key ) { |
||
| 657 | |||
| 658 | /** |
||
| 659 | * Method to fetch/regenerate cache keys |
||
| 660 | * |
||
| 661 | * On cache miss, the key will be set to the callback result via set() |
||
| 662 | * (unless the callback returns false) and that result will be returned. |
||
| 663 | * The arguments supplied to the callback are: |
||
| 664 | * - $oldValue : current cache value or false if not present |
||
| 665 | * - &$ttl : a reference to the TTL which can be altered |
||
| 666 | * - &$setOpts : a reference to options for set() which can be altered |
||
| 667 | * - $oldAsOf : generation UNIX timestamp of $oldValue or null if not present (since 1.28) |
||
| 668 | * |
||
| 669 | * It is strongly recommended to set the 'lag' and 'since' fields to avoid race conditions |
||
| 670 | * that can cause stale values to get stuck at keys. Usually, callbacks ignore the current |
||
| 671 | * value, but it can be used to maintain "most recent X" values that come from time or |
||
| 672 | * sequence based source data, provided that the "as of" id/time is tracked. Note that |
||
| 673 | * preemptive regeneration and $checkKeys can result in a non-false current value. |
||
| 674 | * |
||
| 675 | * Usage of $checkKeys is similar to get() and getMulti(). However, rather than the caller |
||
| 676 | * having to inspect a "current time left" variable (e.g. $curTTL, $curTTLs), a cache |
||
| 677 | * regeneration will automatically be triggered using the callback. |
||
| 678 | * |
||
| 679 | * The simplest way to avoid stampedes for hot keys is to use |
||
| 680 | * the 'lockTSE' option in $opts. If cache purges are needed, also: |
||
| 681 | * - a) Pass $key into $checkKeys |
||
| 682 | * - b) Use touchCheckKey( $key ) instead of delete( $key ) |
||
| 683 | * |
||
| 684 | * Example usage (typical key): |
||
| 685 | * @code |
||
| 686 | * $catInfo = $cache->getWithSetCallback( |
||
| 687 | * // Key to store the cached value under |
||
| 688 | * $cache->makeKey( 'cat-attributes', $catId ), |
||
| 689 | * // Time-to-live (in seconds) |
||
| 690 | * $cache::TTL_MINUTE, |
||
| 691 | * // Function that derives the new key value |
||
| 692 | * function ( $oldValue, &$ttl, array &$setOpts ) { |
||
| 693 | * $dbr = wfGetDB( DB_REPLICA ); |
||
| 694 | * // Account for any snapshot/replica DB lag |
||
| 695 | * $setOpts += Database::getCacheSetOptions( $dbr ); |
||
| 696 | * |
||
| 697 | * return $dbr->selectRow( ... ); |
||
| 698 | * } |
||
| 699 | * ); |
||
| 700 | * @endcode |
||
| 701 | * |
||
| 702 | * Example usage (key that is expensive and hot): |
||
| 703 | * @code |
||
| 704 | * $catConfig = $cache->getWithSetCallback( |
||
| 705 | * // Key to store the cached value under |
||
| 706 | * $cache->makeKey( 'site-cat-config' ), |
||
| 707 | * // Time-to-live (in seconds) |
||
| 708 | * $cache::TTL_DAY, |
||
| 709 | * // Function that derives the new key value |
||
| 710 | * function ( $oldValue, &$ttl, array &$setOpts ) { |
||
| 711 | * $dbr = wfGetDB( DB_REPLICA ); |
||
| 712 | * // Account for any snapshot/replica DB lag |
||
| 713 | * $setOpts += Database::getCacheSetOptions( $dbr ); |
||
| 714 | * |
||
| 715 | * return CatConfig::newFromRow( $dbr->selectRow( ... ) ); |
||
| 716 | * }, |
||
| 717 | * [ |
||
| 718 | * // Calling touchCheckKey() on this key invalidates the cache |
||
| 719 | * 'checkKeys' => [ $cache->makeKey( 'site-cat-config' ) ], |
||
| 720 | * // Try to only let one datacenter thread manage cache updates at a time |
||
| 721 | * 'lockTSE' => 30, |
||
| 722 | * // Avoid querying cache servers multiple times in a web request |
||
| 723 | * 'pcTTL' => $cache::TTL_PROC_LONG |
||
| 724 | * ] |
||
| 725 | * ); |
||
| 726 | * @endcode |
||
| 727 | * |
||
| 728 | * Example usage (key with dynamic dependencies): |
||
| 729 | * @code |
||
| 730 | * $catState = $cache->getWithSetCallback( |
||
| 731 | * // Key to store the cached value under |
||
| 732 | * $cache->makeKey( 'cat-state', $cat->getId() ), |
||
| 733 | * // Time-to-live (seconds) |
||
| 734 | * $cache::TTL_HOUR, |
||
| 735 | * // Function that derives the new key value |
||
| 736 | * function ( $oldValue, &$ttl, array &$setOpts ) { |
||
| 737 | * // Determine new value from the DB |
||
| 738 | * $dbr = wfGetDB( DB_REPLICA ); |
||
| 739 | * // Account for any snapshot/replica DB lag |
||
| 740 | * $setOpts += Database::getCacheSetOptions( $dbr ); |
||
| 741 | * |
||
| 742 | * return CatState::newFromResults( $dbr->select( ... ) ); |
||
| 743 | * }, |
||
| 744 | * [ |
||
| 745 | * // The "check" keys that represent things the value depends on; |
||
| 746 | * // Calling touchCheckKey() on any of them invalidates the cache |
||
| 747 | * 'checkKeys' => [ |
||
| 748 | * $cache->makeKey( 'sustenance-bowls', $cat->getRoomId() ), |
||
| 749 | * $cache->makeKey( 'people-present', $cat->getHouseId() ), |
||
| 750 | * $cache->makeKey( 'cat-laws', $cat->getCityId() ), |
||
| 751 | * ] |
||
| 752 | * ] |
||
| 753 | * ); |
||
| 754 | * @endcode |
||
| 755 | * |
||
| 756 | * Example usage (hot key holding most recent 100 events): |
||
| 757 | * @code |
||
| 758 | * $lastCatActions = $cache->getWithSetCallback( |
||
| 759 | * // Key to store the cached value under |
||
| 760 | * $cache->makeKey( 'cat-last-actions', 100 ), |
||
| 761 | * // Time-to-live (in seconds) |
||
| 762 | * 10, |
||
| 763 | * // Function that derives the new key value |
||
| 764 | * function ( $oldValue, &$ttl, array &$setOpts ) { |
||
| 765 | * $dbr = wfGetDB( DB_REPLICA ); |
||
| 766 | * // Account for any snapshot/replica DB lag |
||
| 767 | * $setOpts += Database::getCacheSetOptions( $dbr ); |
||
| 768 | * |
||
| 769 | * // Start off with the last cached list |
||
| 770 | * $list = $oldValue ?: []; |
||
| 771 | * // Fetch the last 100 relevant rows in descending order; |
||
| 772 | * // only fetch rows newer than $list[0] to reduce scanning |
||
| 773 | * $rows = iterator_to_array( $dbr->select( ... ) ); |
||
| 774 | * // Merge them and get the new "last 100" rows |
||
| 775 | * return array_slice( array_merge( $new, $list ), 0, 100 ); |
||
| 776 | * }, |
||
| 777 | * [ |
||
| 778 | * // Try to only let one datacenter thread manage cache updates at a time |
||
| 779 | * 'lockTSE' => 30, |
||
| 780 | * // Use a magic value when no cache value is ready rather than stampeding |
||
| 781 | * 'busyValue' => 'computing' |
||
| 782 | * ] |
||
| 783 | * ); |
||
| 784 | * @endcode |
||
| 785 | * |
||
| 786 | * @see WANObjectCache::get() |
||
| 787 | * @see WANObjectCache::set() |
||
| 788 | * |
||
| 789 | * @param string $key Cache key |
||
| 790 | * @param integer $ttl Seconds to live for key updates. Special values are: |
||
| 791 | * - WANObjectCache::TTL_INDEFINITE: Cache forever |
||
| 792 | * - WANObjectCache::TTL_UNCACHEABLE: Do not cache at all |
||
| 793 | * @param callable $callback Value generation function |
||
| 794 | * @param array $opts Options map: |
||
| 795 | * - checkKeys: List of "check" keys. The key at $key will be seen as invalid when either |
||
| 796 | * touchCheckKey() or resetCheckKey() is called on any of these keys. |
||
| 797 | * Default: []. |
||
| 798 | * - lockTSE: If the key is tombstoned or expired (by checkKeys) less than this many seconds |
||
| 799 | * ago, then try to have a single thread handle cache regeneration at any given time. |
||
| 800 | * Other threads will try to use stale values if possible. If, on miss, the time since |
||
| 801 | * expiration is low, the assumption is that the key is hot and that a stampede is worth |
||
| 802 | * avoiding. Setting this above WANObjectCache::HOLDOFF_TTL makes no difference. The |
||
| 803 | * higher this is set, the higher the worst-case staleness can be. |
||
| 804 | * Use WANObjectCache::TSE_NONE to disable this logic. |
||
| 805 | * Default: WANObjectCache::TSE_NONE. |
||
| 806 | * - busyValue: If no value exists and another thread is currently regenerating it, use this |
||
| 807 | * as a fallback value (or a callback to generate such a value). This assures that cache |
||
| 808 | * stampedes cannot happen if the value falls out of cache. This can be used as insurance |
||
| 809 | * against cache regeneration becoming very slow for some reason (greater than the TTL). |
||
| 810 | * Default: null. |
||
| 811 | * - pcTTL: Process cache the value in this PHP instance for this many seconds. This avoids |
||
| 812 | * network I/O when a key is read several times. This will not cache when the callback |
||
| 813 | * returns false, however. Note that any purges will not be seen while process cached; |
||
| 814 | * since the callback should use replica DBs and they may be lagged or have snapshot |
||
| 815 | * isolation anyway, this should not typically matter. |
||
| 816 | * Default: WANObjectCache::TTL_UNCACHEABLE. |
||
| 817 | * - pcGroup: Process cache group to use instead of the primary one. If set, this must be |
||
| 818 | * of the format ALPHANUMERIC_NAME:MAX_KEY_SIZE, e.g. "mydata:10". Use this for storing |
||
| 819 | * large values, small yet numerous values, or some values with a high cost of eviction. |
||
| 820 | * It is generally preferable to use a class constant when setting this value. |
||
| 821 | * This has no effect unless pcTTL is used. |
||
| 822 | * Default: WANObjectCache::PC_PRIMARY. |
||
| 823 | * - version: Integer version number. This allows for callers to make breaking changes to |
||
| 824 | * how values are stored while maintaining compatability and correct cache purges. New |
||
| 825 | * versions are stored alongside older versions concurrently. Avoid storing class objects |
||
| 826 | * however, as this reduces compatibility (due to serialization). |
||
| 827 | * Default: null. |
||
| 828 | * - minAsOf: Reject values if they were generated before this UNIX timestamp. |
||
| 829 | * This is useful if the source of a key is suspected of having possibly changed |
||
| 830 | * recently, and the caller wants any such changes to be reflected. |
||
| 831 | * Default: WANObjectCache::MIN_TIMESTAMP_NONE. |
||
| 832 | * - hotTTR: Expected time-till-refresh for keys that average ~1 hit/second. |
||
| 833 | * This should be greater than "ageNew". Keys with higher hit rates will regenerate |
||
| 834 | * more often. This is useful when a popular key is changed but the cache purge was |
||
| 835 | * delayed or lost. Seldom used keys are rarely affected by this setting, unless an |
||
| 836 | * extremely low "hotTTR" value is passed in. |
||
| 837 | * Default: WANObjectCache::HOT_TTR. |
||
| 838 | * - lowTTL: Consider pre-emptive updates when the current TTL (seconds) of the key is less |
||
| 839 | * than this. It becomes more likely over time, becoming certain once the key is expired. |
||
| 840 | * Default: WANObjectCache::LOW_TTL. |
||
| 841 | * - ageNew: Consider popularity refreshes only once a key reaches this age in seconds. |
||
| 842 | * Default: WANObjectCache::AGE_NEW. |
||
| 843 | * @return mixed Value found or written to the key |
||
| 844 | * @note Options added in 1.28: version, busyValue, hotTTR, ageNew, pcGroup, minAsOf |
||
| 845 | * @note Callable type hints are not used to avoid class-autoloading |
||
| 846 | */ |
||
| 847 | final public function getWithSetCallback( $key, $ttl, $callback, array $opts = [] ) { |
||
| 913 | |||
| 914 | /** |
||
| 915 | * Do the actual I/O for getWithSetCallback() when needed |
||
| 916 | * |
||
| 917 | * @see WANObjectCache::getWithSetCallback() |
||
| 918 | * |
||
| 919 | * @param string $key |
||
| 920 | * @param integer $ttl |
||
| 921 | * @param callback $callback |
||
| 922 | * @param array $opts Options map for getWithSetCallback() |
||
| 923 | * @param float &$asOf Cache generation timestamp of returned value [returned] |
||
| 924 | * @return mixed |
||
| 925 | * @note Callable type hints are not used to avoid class-autoloading |
||
| 926 | */ |
||
| 927 | protected function doGetWithSetCallback( $key, $ttl, $callback, array $opts, &$asOf = null ) { |
||
| 1028 | |||
| 1029 | /** |
||
| 1030 | * Method to fetch/regenerate multiple cache keys at once |
||
| 1031 | * |
||
| 1032 | * This works the same as getWithSetCallback() except: |
||
| 1033 | * - a) The $keys argument expects the result of WANObjectCache::makeMultiKeys() |
||
| 1034 | * - b) The $callback argument expects a callback taking the following arguments: |
||
| 1035 | * - $id: ID of an entity to query |
||
| 1036 | * - $oldValue : the prior cache value or false if none was present |
||
| 1037 | * - &$ttl : a reference to the new value TTL in seconds |
||
| 1038 | * - &$setOpts : a reference to options for set() which can be altered |
||
| 1039 | * - $oldAsOf : generation UNIX timestamp of $oldValue or null if not present |
||
| 1040 | * Aside from the additional $id argument, the other arguments function the same |
||
| 1041 | * way they do in getWithSetCallback(). |
||
| 1042 | * - c) The return value is a map of (cache key => value) in the order of $keyedIds |
||
| 1043 | * |
||
| 1044 | * @see WANObjectCache::getWithSetCallback() |
||
| 1045 | * |
||
| 1046 | * Example usage: |
||
| 1047 | * @code |
||
| 1048 | * $rows = $cache->getMultiWithSetCallback( |
||
| 1049 | * // Map of cache keys to entitiy IDs |
||
| 1050 | * $cache->makeMultiKeys( |
||
| 1051 | * $this->fileVersionIds(), |
||
| 1052 | * function ( $id, WANObjectCache $cache ) { |
||
| 1053 | * return $cache->makeKey( 'file-version', $id ); |
||
| 1054 | * } |
||
| 1055 | * ), |
||
| 1056 | * // Time-to-live (in seconds) |
||
| 1057 | * $cache::TTL_DAY, |
||
| 1058 | * // Function that derives the new key value |
||
| 1059 | * return function ( $id, $oldValue, &$ttl, array &$setOpts ) { |
||
| 1060 | * $dbr = wfGetDB( DB_REPLICA ); |
||
| 1061 | * // Account for any snapshot/replica DB lag |
||
| 1062 | * $setOpts += Database::getCacheSetOptions( $dbr ); |
||
| 1063 | * |
||
| 1064 | * // Load the row for this file |
||
| 1065 | * $row = $dbr->selectRow( 'file', '*', [ 'id' => $id ], __METHOD__ ); |
||
| 1066 | * |
||
| 1067 | * return $row ? (array)$row : false; |
||
| 1068 | * }, |
||
| 1069 | * [ |
||
| 1070 | * // Process cache for 30 seconds |
||
| 1071 | * 'pcTTL' => 30, |
||
| 1072 | * // Use a dedicated 500 item cache (initialized on-the-fly) |
||
| 1073 | * 'pcGroup' => 'file-versions:500' |
||
| 1074 | * ] |
||
| 1075 | * ); |
||
| 1076 | * $files = array_map( [ __CLASS__, 'newFromRow' ], $rows ); |
||
| 1077 | * @endcode |
||
| 1078 | * |
||
| 1079 | * @param ArrayIterator $keyedIds Result of WANObjectCache::makeMultiKeys() |
||
| 1080 | * @param integer $ttl Seconds to live for key updates |
||
| 1081 | * @param callable $callback Callback the yields entity regeneration callbacks |
||
| 1082 | * @param array $opts Options map |
||
| 1083 | * @return array Map of (cache key => value) in the same order as $keyedIds |
||
| 1084 | * @since 1.28 |
||
| 1085 | */ |
||
| 1086 | final public function getMultiWithSetCallback( |
||
| 1117 | |||
| 1118 | /** |
||
| 1119 | * @see BagOStuff::makeKey() |
||
| 1120 | * @param string ... Key component |
||
| 1121 | * @return string |
||
| 1122 | * @since 1.27 |
||
| 1123 | */ |
||
| 1124 | public function makeKey() { |
||
| 1127 | |||
| 1128 | /** |
||
| 1129 | * @see BagOStuff::makeGlobalKey() |
||
| 1130 | * @param string ... Key component |
||
| 1131 | * @return string |
||
| 1132 | * @since 1.27 |
||
| 1133 | */ |
||
| 1134 | public function makeGlobalKey() { |
||
| 1137 | |||
| 1138 | /** |
||
| 1139 | * @param array $entities List of entity IDs |
||
| 1140 | * @param callable $keyFunc Callback yielding a key from (entity ID, this WANObjectCache) |
||
| 1141 | * @return ArrayIterator Iterator yielding (cache key => entity ID) in $entities order |
||
| 1142 | * @since 1.28 |
||
| 1143 | */ |
||
| 1144 | public function makeMultiKeys( array $entities, callable $keyFunc ) { |
||
| 1152 | |||
| 1153 | /** |
||
| 1154 | * Get the "last error" registered; clearLastError() should be called manually |
||
| 1155 | * @return int ERR_* class constant for the "last error" registry |
||
| 1156 | */ |
||
| 1157 | final public function getLastError() { |
||
| 1179 | |||
| 1180 | /** |
||
| 1181 | * Clear the "last error" registry |
||
| 1182 | */ |
||
| 1183 | final public function clearLastError() { |
||
| 1187 | |||
| 1188 | /** |
||
| 1189 | * Clear the in-process caches; useful for testing |
||
| 1190 | * |
||
| 1191 | * @since 1.27 |
||
| 1192 | */ |
||
| 1193 | public function clearProcessCache() { |
||
| 1196 | |||
| 1197 | /** |
||
| 1198 | * @param integer $flag ATTR_* class constant |
||
| 1199 | * @return integer QOS_* class constant |
||
| 1200 | * @since 1.28 |
||
| 1201 | */ |
||
| 1202 | public function getQoS( $flag ) { |
||
| 1205 | |||
| 1206 | /** |
||
| 1207 | * Get a TTL that is higher for objects that have not changed recently |
||
| 1208 | * |
||
| 1209 | * This is useful for keys that get explicit purges and DB or purge relay |
||
| 1210 | * lag is a potential concern (especially how it interacts with CDN cache) |
||
| 1211 | * |
||
| 1212 | * Example usage: |
||
| 1213 | * @code |
||
| 1214 | * // Last-modified time of page |
||
| 1215 | * $mtime = wfTimestamp( TS_UNIX, $page->getTimestamp() ); |
||
| 1216 | * // Get adjusted TTL. If $mtime is 3600 seconds ago and $minTTL/$factor left at |
||
| 1217 | * // defaults, then $ttl is 3600 * .2 = 720. If $minTTL was greater than 720, then |
||
| 1218 | * // $ttl would be $minTTL. If $maxTTL was smaller than 720, $ttl would be $maxTTL. |
||
| 1219 | * $ttl = $cache->adaptiveTTL( $mtime, $cache::TTL_DAY ); |
||
| 1220 | * @endcode |
||
| 1221 | * |
||
| 1222 | * @param integer|float $mtime UNIX timestamp |
||
| 1223 | * @param integer $maxTTL Maximum TTL (seconds) |
||
| 1224 | * @param integer $minTTL Minimum TTL (seconds); Default: 30 |
||
| 1225 | * @param float $factor Value in the range (0,1); Default: .2 |
||
| 1226 | * @return integer Adaptive TTL |
||
| 1227 | * @since 1.28 |
||
| 1228 | */ |
||
| 1229 | public function adaptiveTTL( $mtime, $maxTTL, $minTTL = 30, $factor = .2 ) { |
||
| 1242 | |||
| 1243 | /** |
||
| 1244 | * Do the actual async bus purge of a key |
||
| 1245 | * |
||
| 1246 | * This must set the key to "PURGED:<UNIX timestamp>:<holdoff>" |
||
| 1247 | * |
||
| 1248 | * @param string $key Cache key |
||
| 1249 | * @param integer $ttl How long to keep the tombstone [seconds] |
||
| 1250 | * @param integer $holdoff HOLDOFF_* constant controlling how long to ignore sets for this key |
||
| 1251 | * @return bool Success |
||
| 1252 | */ |
||
| 1253 | protected function relayPurge( $key, $ttl, $holdoff ) { |
||
| 1277 | |||
| 1278 | /** |
||
| 1279 | * Do the actual async bus delete of a key |
||
| 1280 | * |
||
| 1281 | * @param string $key Cache key |
||
| 1282 | * @return bool Success |
||
| 1283 | */ |
||
| 1284 | protected function relayDelete( $key ) { |
||
| 1302 | |||
| 1303 | /** |
||
| 1304 | * Check if a key should be regenerated (using random probability) |
||
| 1305 | * |
||
| 1306 | * This returns false if $curTTL >= $lowTTL. Otherwise, the chance |
||
| 1307 | * of returning true increases steadily from 0% to 100% as the $curTTL |
||
| 1308 | * moves from $lowTTL to 0 seconds. This handles widely varying |
||
| 1309 | * levels of cache access traffic. |
||
| 1310 | * |
||
| 1311 | * @param float $curTTL Approximate TTL left on the key if present |
||
| 1312 | * @param float $lowTTL Consider a refresh when $curTTL is less than this |
||
| 1313 | * @return bool |
||
| 1314 | */ |
||
| 1315 | protected function worthRefreshExpiring( $curTTL, $lowTTL ) { |
||
| 1326 | |||
| 1327 | /** |
||
| 1328 | * Check if a key is due for randomized regeneration due to its popularity |
||
| 1329 | * |
||
| 1330 | * This is used so that popular keys can preemptively refresh themselves for higher |
||
| 1331 | * consistency (especially in the case of purge loss/delay). Unpopular keys can remain |
||
| 1332 | * in cache with their high nominal TTL. This means popular keys keep good consistency, |
||
| 1333 | * whether the data changes frequently or not, and long-tail keys get to stay in cache |
||
| 1334 | * and get hits too. Similar to worthRefreshExpiring(), randomization is used. |
||
| 1335 | * |
||
| 1336 | * @param float $asOf UNIX timestamp of the value |
||
| 1337 | * @param integer $ageNew Age of key when this might recommend refreshing (seconds) |
||
| 1338 | * @param integer $timeTillRefresh Age of key when it should be refreshed if popular (seconds) |
||
| 1339 | * @return bool |
||
| 1340 | */ |
||
| 1341 | protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh ) { |
||
| 1360 | |||
| 1361 | /** |
||
| 1362 | * Check whether $value is appropriately versioned and not older than $minTime (if set) |
||
| 1363 | * |
||
| 1364 | * @param array $value |
||
| 1365 | * @param bool $versioned |
||
| 1366 | * @param float $asOf The time $value was generated |
||
| 1367 | * @param float $minTime The last time the main value was generated (0.0 if unknown) |
||
| 1368 | * @return bool |
||
| 1369 | */ |
||
| 1370 | protected function isValid( $value, $versioned, $asOf, $minTime ) { |
||
| 1379 | |||
| 1380 | /** |
||
| 1381 | * Do not use this method outside WANObjectCache |
||
| 1382 | * |
||
| 1383 | * @param mixed $value |
||
| 1384 | * @param integer $ttl [0=forever] |
||
| 1385 | * @param float $now Unix Current timestamp just before calling set() |
||
| 1386 | * @return array |
||
| 1387 | */ |
||
| 1388 | protected function wrap( $value, $ttl, $now ) { |
||
| 1396 | |||
| 1397 | /** |
||
| 1398 | * Do not use this method outside WANObjectCache |
||
| 1399 | * |
||
| 1400 | * @param array|string|bool $wrapped |
||
| 1401 | * @param float $now Unix Current timestamp (preferrably pre-query) |
||
| 1402 | * @return array (mixed; false if absent/invalid, current time left) |
||
| 1403 | */ |
||
| 1404 | protected function unwrap( $wrapped, $now ) { |
||
| 1436 | |||
| 1437 | /** |
||
| 1438 | * @param array $keys |
||
| 1439 | * @param string $prefix |
||
| 1440 | * @return string[] |
||
| 1441 | */ |
||
| 1442 | protected static function prefixCacheKeys( array $keys, $prefix ) { |
||
| 1450 | |||
| 1451 | /** |
||
| 1452 | * @param string $value Wrapped value like "PURGED:<timestamp>:<holdoff>" |
||
| 1453 | * @return array|bool Array containing a UNIX timestamp (float) and holdoff period (integer), |
||
| 1454 | * or false if value isn't a valid purge value |
||
| 1455 | */ |
||
| 1456 | protected static function parsePurgeValue( $value ) { |
||
| 1475 | |||
| 1476 | /** |
||
| 1477 | * @param float $timestamp |
||
| 1478 | * @param int $holdoff In seconds |
||
| 1479 | * @return string Wrapped purge value |
||
| 1480 | */ |
||
| 1481 | protected function makePurgeValue( $timestamp, $holdoff ) { |
||
| 1484 | |||
| 1485 | /** |
||
| 1486 | * @param string $group |
||
| 1487 | * @return HashBagOStuff |
||
| 1488 | */ |
||
| 1489 | protected function getProcessCache( $group ) { |
||
| 1497 | } |
||
| 1498 |
