ReplicaMasterAwareRecordIdsAcquirer::findExistingRecords() - Code Metrics - Inspection of "[WIP] Add database term ids acquirer" - wmde/mediawiki-term-store - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#11)

by Alaa

created 2019-05-15 12:56 UTC

findExistingRecords() A

↳ Parent: ReplicaMasterAwareRecordIdsAcquirer

Complexity

Conditions	3
Paths	3

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
dl	0
loc	32
rs	9.408
c	0
b	0
f	0
cc	3
nc	3
nop	2

<?php

namespace Wikibase\TermStore\MediaWiki\PackagePrivate\Util;

use AppendIterator;
use ArrayIterator;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use Wikimedia\Rdbms\DBQueryError;
use Wikimedia\Rdbms\IDatabase;

/**
 * Allows acquiring ids of records in database table,
 * by insecting a given read-only replica database for initially
 * finding those records, falling back to a insert non-existing
 * records into a read-write master databas and getting those
 * ids as well from the master database.
 */
class ReplicaMasterAwareRecordIdsAcquirer {

	/**
	 * @var Database $dbMaster
	 */
	private $dbMaster;

	/**
	 * @var Database $dbReplica
	 */
	private $dbReplica;

	/** @var string $table */
	private $table;

	/** @var string $idColumn */
	private $idColumn;

	/** @var LoggerInterface $logger */
	private $logger;

	/**
	 * @param IDatabase $dbMaster master database to insert non-existing records into
	 * @param IDatabase $dbReplica replica database to initially query existing records in
	 * @param string $table the name of the table this acquirer is for
	 * @param string $idColumn the name of the column that contains the desired ids
	 * @param LoggerInterface $logger
	 */
	public function __construct(
		IDatabase $dbMaster,
		IDatabase $dbReplica,
		string $table,
		string $idColumn,
		LoggerInterface $logger = null
	) {
		$this->dbMaster = $dbMaster;

		$this->dbReplica = $dbReplica;

		$this->table = $table;
		$this->idColumn = $idColumn;
		$this->logger = $logger ?? new NullLogger();
class Id
{
    public $id;

    public function __construct($id)
    {
        $this->id = $id;
    }

}

class Account
{
    /** @var  Id $id */
    public $id;
}

$account_id = false;

if (starsAreRight()) {
    $account_id = new Id(42);
}

$account = new Account();
if ($account instanceof Id)
{
    $account->id = $account_id;
}
	}

	/**
	 * Acquire ids of needed records in the table, inserting non-existing
	 * ones into master database.
	 *
	 * Note 1: this function assumes that all records given in $neededRecords specify
	 * the same columns. If some records specify less, more or different columns than
	 * the first one does, the behavior is not defined.
     *
     * Note 2: this function assumes that all records given in $neededRecords have
     * their values as strings. If some values are of different type (e.g. integer ids)
     * this can cause infinite loops due to mismatch in identifying records selected in
     * database with their corresponding needed records. The first element keys will be 
     * used as the set of columns to select in database and to provide back in the returned array.
	 *
	 * @param array $neededRecords array of records to be looked-up or inserted.
	 *	Each entry in this array should an associative array of column => value pairs.
	 *	Example:
	 *	[
	 *		[ 'columnA' => 'valueA1', 'columnB' => 'valueB1' ],
	 *		[ 'columnA' => 'valueA2', 'columnB' => 'valueB2' ],
	 *		...
	 *	]
	 *
	 * @return array the array of input recrods along with their ids
	 *	Example:
	 *	[
	 *		[ 'columnA' => 'valueA1', 'columnB' => 'valueB1', 'idColumn' => '1' ],
	 *		[ 'columnA' => 'valueA2', 'columnB' => 'valueB2', 'idColumn' => '2' ],
	 *		...
	 *	]
	 */
	public function acquireIds( array $neededRecords ) {
		$existingRecords = $this->findExistingRecords( $this->dbReplica, $neededRecords );
		$neededRecords = $this->filterNonExistingRecords( $neededRecords, $existingRecords );

		while ( !empty( $neededRecords ) ) {
			$this->insertNonExistingRecords( $this->dbMaster, $neededRecords );
			$existingRecords = array_merge(
				$existingRecords,
				$this->findExistingRecords( $this->dbMaster, $neededRecords )
			);
			$neededRecords = $this->filterNonExistingRecords( $neededRecords, $existingRecords );
		}

		return $existingRecords;
	}

	private function findExistingRecords( IDatabase $db, array $neededRecords ): array {
		$recordsSelectConditions = array_map( function ( $record ) use ( $db ) {
			return $db->makeList( $record, IDatabase::LIST_AND );
		}, $neededRecords );

        /*
         * Todo, related to Note 1 on self::acquireIds(): 
         * this class can allow for specifying a different set of columns to select
         * and return back from self::acquireIds(). This set of columns can be added as
         * an optional argument to self::acquireIds() for instance, the current solution
         * in here can be a fallback when that isn't given.
         */
		$selectColumns = array_keys( $neededRecords[0] );
		$selectColumns[] = $this->idColumn;

		$existingRows = $db->select(
			$this->table,
			$selectColumns,
			$db->makeList( $recordsSelectConditions, IDatabase::LIST_OR )
		);

		$existingRecords = [];
		foreach ( $existingRows as $row ) {
			$existingRecord = [];
			foreach ( $selectColumns as $column ) {
				$existingRecord[$column] = $row->$column;
			}
			$existingRecords[] = $existingRecord;
		}

		return $existingRecords;
	}

	private function insertNonExistingRecords( IDatabase $db, array $neededRecords ) {
		$uniqueRecords = [];
		foreach ( $neededRecords as $record ) {
			$recordHash = $this->calcRecordHash( $record );
			if ( !isset( $uniqueRecords[$recordHash] ) ) {
				$uniqueRecords[$recordHash] = $record;
			}
		}

		try {
			$db->insert( $this->table, array_values( $uniqueRecords ) );
		} catch ( DBQueryError $dbError ) {

			$this->logger->info(
				'{method}: Inserting records {records} into {table} failed. {exception}',
				[
					'exception' => $dbError,
					'table' => $this->table,
					'records' => $uniqueRecords
				]
			);
		}
	}

	private function filterNonExistingRecords( $neededRecords, $existingRecords ): array {
		$existingRecordsHashes = [];
		foreach ( $existingRecords as $record ) {
			unset( $record[$this->idColumn] );
			$recordHash = $this->calcRecordHash( $record );
			$existingRecordsHashes[$recordHash] = true;
		}

		$nonExistingRecords = [];
		foreach ( $neededRecords as $record ) {
			$recordHash = $this->calcRecordHash( $record );

			if ( !isset( $existingRecordsHashes[$recordHash] ) ) {
				$nonExistingRecords[] = $record;
			}
		}

		return $nonExistingRecords;
	}
    
    /**
     * Implementation detail, related to Note 2 on self::acquireIds():
     * this function relies on the fact that the given set of needed records will have
     * all values as strings in order to produce hashes that match up correctly with
     * selected records in database, because database selection will always return
     * values as strings.
     */
	private function calcRecordHash( $record ) {
		ksort( $record );
		return md5( serialize( $record ) );
	}

}


1			<?php
2
3			namespace Wikibase\TermStore\MediaWiki\PackagePrivate\Util;
4
5			use AppendIterator;
6			use ArrayIterator;
7			use Psr\Log\LoggerInterface;
8			use Psr\Log\NullLogger;
9			use Wikimedia\Rdbms\DBQueryError;
10			use Wikimedia\Rdbms\IDatabase;
11
12			/**
13			* Allows acquiring ids of records in database table,
14			* by insecting a given read-only replica database for initially
15			* finding those records, falling back to a insert non-existing
16			* records into a read-write master databas and getting those
17			* ids as well from the master database.
18			*/
19			class ReplicaMasterAwareRecordIdsAcquirer {
20
21			/**
22			* @var Database $dbMaster
23			*/
24			private $dbMaster;
25
26			/**
27			* @var Database $dbReplica
28			*/
29			private $dbReplica;
30
31			/** @var string $table */
32			private $table;
33
34			/** @var string $idColumn */
35			private $idColumn;
36
37			/** @var LoggerInterface $logger */
38			private $logger;
39
40			/**
41			* @param IDatabase $dbMaster master database to insert non-existing records into
42			* @param IDatabase $dbReplica replica database to initially query existing records in
43			* @param string $table the name of the table this acquirer is for
44			* @param string $idColumn the name of the column that contains the desired ids
45			* @param LoggerInterface $logger
46			*/
47			public function __construct(
48			IDatabase $dbMaster,
49			IDatabase $dbReplica,
50			string $table,
51			string $idColumn,
52			LoggerInterface $logger = null
53			) {
54			$this->dbMaster = $dbMaster;
			0 ignored issues – show Documentation Bug introduced 2019-05-15 12:47 UTC by Report Bug Copy Issue Report It seems like `$dbMaster` of type `object<Wikimedia\Rdbms\IDatabase>` is incompatible with the declared type `object<Wikibase\TermStor...ePrivate\Util\Database>` of property `$dbMaster`. Our type inference engine has found an assignment to a property that is incompatible with the declared type of that property. Either this assignment is in error or the assigned type should be added to the documentation/type hint for that property.. Loading history...
55			$this->dbReplica = $dbReplica;
			0 ignored issues – show Documentation Bug introduced 2019-05-15 12:47 UTC by Report Bug Copy Issue Report It seems like `$dbReplica` of type `object<Wikimedia\Rdbms\IDatabase>` is incompatible with the declared type `object<Wikibase\TermStor...ePrivate\Util\Database>` of property `$dbReplica`. Our type inference engine has found an assignment to a property that is incompatible with the declared type of that property. Either this assignment is in error or the assigned type should be added to the documentation/type hint for that property.. Loading history...
56			$this->table = $table;
57			$this->idColumn = $idColumn;
58			$this->logger = $logger ?? new NullLogger();
			0 ignored issues – show Documentation Bug introduced 2019-05-15 07:19 UTC by Report Bug Copy Issue Report It seems like `$logger ?? new \Psr\Log\NullLogger()` can also be of type `object<Psr\Log\NullLogger>`. However, the property `$logger` is declared as type `object<Psr\Log\LoggerInterface>`. Maybe add an additional type check? Our type inference engine has found a suspicous assignment of a value to a property. This check raises an issue when a value that can be of a mixed type is assigned to a property that is type hinted more strictly. For example, imagine you have a variable `$accountId` that can either hold an Id object or false (if there is no account id yet). Your code now assigns that value to the `id` property of an instance of the `Account` class. This class holds a proper account, so the id value must no longer be false. Either this assignment is in error or a type check should be added for that assignment. class Id { public $id; public function __construct($id) { $this->id = $id; } } class Account { /** @var Id $id */ public $id; } $account_id = false; if (starsAreRight()) { $account_id = new Id(42); } $account = new Account(); if ($account instanceof Id) { $account->id = $account_id; } Loading history...
59			}
60
61			/**
62			* Acquire ids of needed records in the table, inserting non-existing
63			* ones into master database.
64			*
65			* Note 1: this function assumes that all records given in $neededRecords specify
66			* the same columns. If some records specify less, more or different columns than
67			* the first one does, the behavior is not defined.
68			*
69			* Note 2: this function assumes that all records given in $neededRecords have
70			* their values as strings. If some values are of different type (e.g. integer ids)
71			* this can cause infinite loops due to mismatch in identifying records selected in
72			* database with their corresponding needed records. The first element keys will be
73			* used as the set of columns to select in database and to provide back in the returned array.
74			*
75			* @param array $neededRecords array of records to be looked-up or inserted.
76			* Each entry in this array should an associative array of column => value pairs.
77			* Example:
78			* [
79			* [ 'columnA' => 'valueA1', 'columnB' => 'valueB1' ],
80			* [ 'columnA' => 'valueA2', 'columnB' => 'valueB2' ],
81			* ...
82			* ]
83			*
84			* @return array the array of input recrods along with their ids
85			* Example:
86			* [
87			* [ 'columnA' => 'valueA1', 'columnB' => 'valueB1', 'idColumn' => '1' ],
88			* [ 'columnA' => 'valueA2', 'columnB' => 'valueB2', 'idColumn' => '2' ],
89			* ...
90			* ]
91			*/
92			public function acquireIds( array $neededRecords ) {
93			$existingRecords = $this->findExistingRecords( $this->dbReplica, $neededRecords );
94			$neededRecords = $this->filterNonExistingRecords( $neededRecords, $existingRecords );
95
96			while ( !empty( $neededRecords ) ) {
97			$this->insertNonExistingRecords( $this->dbMaster, $neededRecords );
98			$existingRecords = array_merge(
99			$existingRecords,
100			$this->findExistingRecords( $this->dbMaster, $neededRecords )
101			);
102			$neededRecords = $this->filterNonExistingRecords( $neededRecords, $existingRecords );
103			}
104
105			return $existingRecords;
106			}
107
108			private function findExistingRecords( IDatabase $db, array $neededRecords ): array {
109			$recordsSelectConditions = array_map( function ( $record ) use ( $db ) {
110			return $db->makeList( $record, IDatabase::LIST_AND );
111			}, $neededRecords );
112
113			/*
114			* Todo, related to Note 1 on self::acquireIds():
115			* this class can allow for specifying a different set of columns to select
116			* and return back from self::acquireIds(). This set of columns can be added as
117			* an optional argument to self::acquireIds() for instance, the current solution
118			* in here can be a fallback when that isn't given.
119			*/
120			$selectColumns = array_keys( $neededRecords[0] );
121			$selectColumns[] = $this->idColumn;
122
123			$existingRows = $db->select(
124			$this->table,
125			$selectColumns,
126			$db->makeList( $recordsSelectConditions, IDatabase::LIST_OR )
127			);
128
129			$existingRecords = [];
130			foreach ( $existingRows as $row ) {
131			$existingRecord = [];
132			foreach ( $selectColumns as $column ) {
133			$existingRecord[$column] = $row->$column;
134			}
135			$existingRecords[] = $existingRecord;
136			}
137
138			return $existingRecords;
139			}
140
141			private function insertNonExistingRecords( IDatabase $db, array $neededRecords ) {
142			$uniqueRecords = [];
143			foreach ( $neededRecords as $record ) {
144			$recordHash = $this->calcRecordHash( $record );
145			if ( !isset( $uniqueRecords[$recordHash] ) ) {
146			$uniqueRecords[$recordHash] = $record;
147			}
148			}
149
150			try {
151			$db->insert( $this->table, array_values( $uniqueRecords ) );
152			} catch ( DBQueryError $dbError ) {
			0 ignored issues – show Bug introduced 2019-05-15 07:42 UTC by Report Bug Copy Issue Report The class `Wikimedia\Rdbms\DBQueryError` does not exist. Did you forget a USE statement, or did you not list all dependencies? Scrutinizer analyzes your `composer.json`/`composer.lock` file if available to determine the classes, and functions that are defined by your dependencies. It seems like the listed class was neither found in your dependencies, nor was it found in the analyzed files in your repository. If you are using some other form of dependency management, you might want to disable this analysis. Loading history...
153			$this->logger->info(
154			'{method}: Inserting records {records} into {table} failed. {exception}',
155			[
156			'exception' => $dbError,
157			'table' => $this->table,
158			'records' => $uniqueRecords
159			]
160			);
161			}
162			}
163
164			private function filterNonExistingRecords( $neededRecords, $existingRecords ): array {
165			$existingRecordsHashes = [];
166			foreach ( $existingRecords as $record ) {
167			unset( $record[$this->idColumn] );
168			$recordHash = $this->calcRecordHash( $record );
169			$existingRecordsHashes[$recordHash] = true;
170			}
171
172			$nonExistingRecords = [];
173			foreach ( $neededRecords as $record ) {
174			$recordHash = $this->calcRecordHash( $record );
175
176			if ( !isset( $existingRecordsHashes[$recordHash] ) ) {
177			$nonExistingRecords[] = $record;
178			}
179			}
180
181			return $nonExistingRecords;
182			}
183
184			/**
185			* Implementation detail, related to Note 2 on self::acquireIds():
186			* this function relies on the fact that the given set of needed records will have
187			* all values as strings in order to produce hashes that match up correctly with
188			* selected records in database, because database selection will always return
189			* values as strings.
190			*/
191			private function calcRecordHash( $record ) {
192			ksort( $record );
193			return md5( serialize( $record ) );
194			}
195
196			}
197

wmde / mediawiki-term-store

Pull Request — master (#11)

findExistingRecords() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like