SemanticMediaWiki / SemanticMediaWiki

src/SQLStore/QueryEngine/QueryEngine.php

<?php

namespace SMW\SQLStore\QueryEngine;

use RuntimeException;
use SMW\DIWikiPage;
use SMW\InvalidPredefinedPropertyException;
use SMW\Query\DebugOutputFormatter as QueryDebugOutputFormatter;
use SMW\Query\Language\Conjunction;
use SMW\Query\Language\SomeProperty;
use SMW\Query\Language\ThingDescription;
use SMWDataItem as DataItem;
use SMWPropertyValue as PropertyValue;
use SMWQuery as Query;
use SMWQueryResult as QueryResult;
use SMWSql3SmwIds;
use SMWSQLStore3 as SQLStore;
use SMW\QueryEngine as QueryEngineInterface;

/**
 * Class that implements query answering for SQLStore.
 *
 * @license GNU GPL v2+
 * @since 2.2
 *
 * @author Markus Krötzsch
 * @author Jeroen De Dauw
 * @author mwjames
 */
class QueryEngine implements QueryEngineInterface {

	/**
	 * @var SQLStore
	 */
	private $store;

	/**
	 * Query mode copied from given query. Some submethods act differently when
	 * in Query::MODE_DEBUG.
	 *
	 * @var int
	 */
	private $queryMode;

	/**
	 * Array of generated QuerySegment query descriptions (index => object)
	 *
	 * @var QuerySegment[]
	 */
	private $querySegmentList = array();

	/**
	 * Array of sorting requests ("Property_name" => "ASC"/"DESC"). Used during
	 * query processing (where these property names are searched while compiling
	 * the query conditions).
	 *
	 * @var string[]
	 */
	private $sortKeys;

	/**
	 * Local collection of error strings, passed on to callers if possible.
	 *
	 * @var string[]
	 */
	private $errors = array();

	/**
	 * @var QuerySegmentListBuilder
	 */
	private $querySegmentListBuilder = null;

	/**
	 * @var QuerySegmentListProcessor
	 */
	private $querySegmentListProcessor = null;

	/**
	 * @var EngineOptions
	 */
	private $engineOptions = null;

	/**
	 * @since 2.2
	 *
	 * @param SQLStore $parentStore
	 * @param QuerySegmentListBuilder $querySegmentListBuilder
	 * @param QuerySegmentListProcessor $querySegmentListProcessor
	 * @param EngineOptions $engineOptions
	 */
	public function __construct( SQLStore $parentStore, QuerySegmentListBuilder $querySegmentListBuilder, QuerySegmentListProcessor $querySegmentListProcessor, EngineOptions $engineOptions ) {
		$this->store = $parentStore;
		$this->querySegmentListBuilder = $querySegmentListBuilder;
		$this->querySegmentListProcessor = $querySegmentListProcessor;
		$this->engineOptions = $engineOptions;
	}

	/**
	 * @since 2.2
	 *
	 * @return QuerySegmentListBuilder
	 */
	public function getQuerySegmentListBuilder() {
		return $this->querySegmentListBuilder;
	}

	/**
	 * @since 2.2
	 *
	 * @return QuerySegmentListProcessor
	 */
	public function getQuerySegmentListProcessor() {
		return $this->querySegmentListProcessor;
	}

	/**
	 * The new SQL store's implementation of query answering. This function
	 * works in two stages: First, the nested conditions of the given query
	 * object are preprocessed to compute an abstract representation of the
	 * SQL query that is to be executed. Since query conditions correspond to
	 * joins with property tables in most cases, this abstract representation
	 * is essentially graph-like description of how property tables are joined.
	 * Moreover, this graph is tree-shaped, since all query conditions are
	 * tree-shaped. Each part of this abstract query structure is represented
	 * by an QuerySegment object in the array m_queries.
	 *
	 * As a second stage of processing, the thus prepared SQL query is actually
	 * executed. Typically, this means that the joins are collapsed into one
	 * SQL query to retrieve results. In some cases, such as in dbug mode, the
	 * execution might be restricted and not actually perform the whole query.
	 *
	 * The two-stage process helps to separate tasks, and it also allows for
	 * better optimisations: it is left to the execution engine how exactly the
	 * query result is to be obtained. For example, one could pre-compute
	 * partial suib-results in temporary tables (or even cache them somewhere),
	 * instead of passing one large join query to the DB (of course, it might
	 * be large only if the configuration of SMW allows it). For some DBMS, a
	 * step-wise execution of the query might lead to better performance, since
	 * it exploits the tree-structure of the joins, which is important for fast
	 * processing -- not all DBMS might be able in seeing this by themselves.
	 *
	 * @param Query $query
	 *
	 * @return mixed depends on $query->querymode
	 */
	public function getQueryResult( Query $query ) {

		if ( ( !$this->engineOptions->get( 'smwgIgnoreQueryErrors' ) || $query->getDescription() instanceof ThingDescription ) &&
		     $query->querymode != Query::MODE_DEBUG &&
		     count( $query->getErrors() ) > 0 ) {
			return new QueryResult( $query->getDescription()->getPrintrequests(), $query, array(), $this->store, false );
			// NOTE: we check this here to prevent unnecessary work, but we check
			// it after query processing below again in case more errors occurred.
		} elseif ( $query->querymode == Query::MODE_NONE || $query->getLimit() < 1 ) {
			// don't query, but return something to printer
			return new QueryResult( $query->getDescription()->getPrintrequests(), $query, array(), $this->store, true );
		}

		$db = $this->store->getConnection( 'mw.db.queryengine' );

		$this->queryMode = $query->querymode;
		$this->querySegmentList = array();

		$this->errors = array();
		QuerySegment::$qnum = 0;
		$this->sortKeys = $query->sortkeys;

		// Anchor IT_TABLE as root element
		$rootSegmentNumber = QuerySegment::$qnum;
		$rootSegment = new QuerySegment();
		$rootSegment->joinTable = SMWSql3SmwIds::TABLE_NAME;
		$rootSegment->joinfield = "$rootSegment->alias.smw_id";

		$this->querySegmentListBuilder->addQuerySegment( $rootSegment );

		// *** First compute abstract representation of the query (compilation) ***//
		$this->querySegmentListBuilder->setSortKeys( $this->sortKeys );
		$this->querySegmentListBuilder->getQuerySegmentFrom( $query->getDescription() ); // compile query, build query "plan"

It seems like $query->getDescription() targeting SMWQuery::getDescription() can also be of type null; however, SMW\SQLStore\QueryEngine\QuerySegmentListBuilder::getQuerySegmentFrom() does only seem to accept object<SMW\Query\Language\Description>, maybe add an additional type check?

		$qid = $this->querySegmentListBuilder->getLastQuerySegmentId();
		$this->querySegmentList = $this->querySegmentListBuilder->getQuerySegmentList();
		$this->errors = $this->querySegmentListBuilder->getErrors();

		if ( $qid < 0 ) { // no valid/supported condition; ensure that at least only proper pages are delivered
			$qid = $rootSegmentNumber;
			$q = $this->querySegmentList[$rootSegmentNumber];
			$q->where = "$q->alias.smw_iw!=" . $db->addQuotes( SMW_SQL3_SMWIW_OUTDATED ) . " AND $q->alias.smw_iw!=" . $db->addQuotes( SMW_SQL3_SMWREDIIW ) . " AND $q->alias.smw_iw!=" . $db->addQuotes( SMW_SQL3_SMWBORDERIW ) . " AND $q->alias.smw_iw!=" . $db->addQuotes( SMW_SQL3_SMWINTDEFIW );
			$this->querySegmentList[$rootSegmentNumber] = $q;
		}

		if ( isset( $this->querySegmentList[$qid]->joinTable ) && $this->querySegmentList[$qid]->joinTable != SMWSql3SmwIds::TABLE_NAME ) {
			// manually make final root query (to retrieve namespace,title):
			$rootid = $rootSegmentNumber;
			$qobj = $this->querySegmentList[$rootSegmentNumber];
			$qobj->components = array( $qid => "$qobj->alias.smw_id" );
			$qobj->sortfields = $this->querySegmentList[$qid]->sortfields;
			$this->querySegmentList[$rootSegmentNumber] = $qobj;
		} else { // not such a common case, but worth avoiding the additional inner join:
			$rootid = $qid;
		}

		// var_dump( json_encode( $this->querySegmentList, JSON_PRETTY_PRINT ) );

45% of this comment could be valid code. Did you maybe forget this after debugging?

		// Include order conditions (may extend query if needed for sorting):
		if ( $this->engineOptions->get( 'smwgQSortingSupport' ) ) {
			$this->applyOrderConditions( $rootid );
		}

		// Possibly stop if new errors happened:
		if ( !$this->engineOptions->get( 'smwgIgnoreQueryErrors' ) &&
				$query->querymode != Query::MODE_DEBUG &&
				count( $this->errors ) > 0 ) {
			$query->addErrors( $this->errors );
			return new QueryResult( $query->getDescription()->getPrintrequests(), $query, array(), $this->store, false );
		}

		// *** Now execute the computed query ***//
		$this->querySegmentListProcessor->setQueryMode( $this->queryMode );
		$this->querySegmentListProcessor->setQuerySegmentList( $this->querySegmentList );

		// execute query tree, resolve all dependencies
		$this->querySegmentListProcessor->doResolveQueryDependenciesById( $rootid );

		$this->applyExtraWhereCondition( $rootid );

		switch ( $query->querymode ) {
			case Query::MODE_DEBUG:
				$result = $this->getDebugQueryResult( $query, $rootid );
			break;
			case Query::MODE_COUNT:
				$result = $this->getCountQueryResult( $query, $rootid );

The expression $this->getCountQueryResult($query, $rootid); of type integer|SMWQueryResult adds the type integer to the return on line 240 which is incompatible with the return type declared by the interface SMW\QueryEngine::getQueryResult of type SMWQueryResult|string.

			break;
			default:
				$result = $this->getInstanceQueryResult( $query, $rootid );
			break;
		}

		$this->querySegmentListProcessor->cleanUp();
		$query->addErrors( $this->errors );

		return $result;
	}

	/**
	 * Using a preprocessed internal query description referenced by $rootid, compute
	 * the proper debug output for the given query.
	 *
	 * @param Query $query
	 * @param integer $rootid
	 *
	 * @return string
	 */
	private function getDebugQueryResult( Query $query, $rootid ) {

		$qobj = $this->querySegmentList[$rootid];
		$entries = array();

		$sqlOptions = $this->getSQLOptions( $query, $rootid );

		$entries['SQL Query'] = '';
		$entries['SQL Explain'] = '';

		$this->doExecuteDebugQueryResult( $qobj, $sqlOptions, $entries );
		$auxtables = '';

		foreach ( $this->querySegmentListProcessor->getListOfResolvedQueries() as $table => $log ) {
			$auxtables .= "<li>Temporary table $table";
			foreach ( $log as $q ) {
				$auxtables .= "<br />&#160;&#160;<tt>$q</tt>";
			}
			$auxtables .= '</li>';
		}

		if ( $auxtables ) {
			$entries['Auxilliary Tables'] = "<ul>$auxtables</ul>";
		} else {
			$entries['Auxilliary Tables'] = 'No auxilliary tables used.';
		}

		return QueryDebugOutputFormatter::getStringFrom( 'SQLStore', $entries, $query );
	}

	private function doExecuteDebugQueryResult( $qobj, $sqlOptions, &$entries ) {

		if ( !isset( $qobj->joinfield ) || $qobj->joinfield === '' ) {
			return $entries['SQL Query'] = 'Empty result, no SQL query created.';
		}

		$db = $this->store->getConnection( 'mw.db.queryengine' );
		list( $startOpts, $useIndex, $tailOpts ) = $db->makeSelectOptions( $sqlOptions );

		$sql = "SELECT DISTINCT ".
			"$qobj->alias.smw_id AS id," .
			"$qobj->alias.smw_title AS t," .
			"$qobj->alias.smw_namespace AS ns," .
			"$qobj->alias.smw_iw AS iw," .
			"$qobj->alias.smw_subobject AS so," .
			"$qobj->alias.smw_sortkey AS sortkey " .
			"FROM " .
			$db->tableName( $qobj->joinTable ) . " AS $qobj->alias" . $qobj->from .
			( $qobj->where === '' ? '':' WHERE ' ) . $qobj->where . "$tailOpts $startOpts $useIndex ".
			"LIMIT " . $sqlOptions['LIMIT'] . ' ' .
			"OFFSET " . $sqlOptions['OFFSET'];

		$res = $db->query(
			'EXPLAIN '. $sql,
			__METHOD__
		);

		$entries['SQL Explain'] = QueryDebugOutputFormatter::doFormatSQLExplainOutput( $db->getType(), $res );
		$entries['SQL Query'] = QueryDebugOutputFormatter::doFormatSQLStatement( $sql, $qobj->alias );

		$db->freeResult( $res );
	}

	/**
	 * Using a preprocessed internal query description referenced by $rootid, compute
	 * the proper counting output for the given query.
	 *
	 * @param Query $query
	 * @param integer $rootid
	 *
	 * @return integer
	 */
	private function getCountQueryResult( Query $query, $rootid ) {

		$queryResult = new QueryResult(
			$query->getDescription()->getPrintrequests(),
			$query,
			array(),
			$this->store,
			false
		);

		$queryResult->setCountValue( 0 );

		$qobj = $this->querySegmentList[$rootid];

		if ( $qobj->joinfield === '' ) { // empty result, no query needed
			return 0;
		}

		$db = $this->store->getConnection( 'mw.db.queryengine' );

		$sql_options = array( 'LIMIT' => $query->getLimit() + 1, 'OFFSET' => $query->getOffset() );

		$res = $db->select(
			$db->tableName( $qobj->joinTable ) . " AS $qobj->alias" . $qobj->from,
			"COUNT(DISTINCT $qobj->alias.smw_id) AS count",
			$qobj->where,
			__METHOD__,
			$sql_options
		);

		$row = $db->fetchObject( $res );

		$count = $row->count;
		$db->freeResult( $res );

		$queryResult->setCountValue( $count );

		return $queryResult;
	}

	/**
	 * Using a preprocessed internal query description referenced by $rootid,
	 * compute the proper result instance output for the given query.
	 * @todo The SQL standard requires us to select all fields by which we sort, leading
	 * to wrong results regarding the given limit: the user expects limit to be applied to
	 * the number of distinct pages, but we can use DISTINCT only to whole rows. Thus, if
	 * rows contain sortfields, then pages with multiple values for that field are distinct
	 * and appear multiple times in the result. Filtering duplicates in post processing
	 * would still allow such duplicates to push aside wanted values, leading to less than
	 * "limit" results although there would have been "limit" really distinct results. For
	 * this reason, we select sortfields only for POSTGRES. MySQL is able to perform what
	 * we want here. It would be nice if we could eliminate the bug in POSTGRES as well.
	 *
	 * @param Query $query
	 * @param integer $rootid
	 *
	 * @return QueryResult
	 */
	private function getInstanceQueryResult( Query $query, $rootid ) {

		$db = $this->store->getConnection( 'mw.db.queryengine' );
		$dbType = $db->getType();

		$qobj = $this->querySegmentList[$rootid];

		if ( $qobj->joinfield === '' ) { // empty result, no query needed
			$result = new QueryResult( $query->getDescription()->getPrintrequests(), $query, array(), $this->store, false );
			return $result;
		}

		$sql_options = $this->getSQLOptions( $query, $rootid );

		// Selecting those is required in standard SQL (but MySQL does not require it).
		$sortfields = implode( $qobj->sortfields, ',' );

		$res = $db->select(
			$db->tableName( $qobj->joinTable ) . " AS $qobj->alias" . $qobj->from,
			"DISTINCT $qobj->alias.smw_id AS id,$qobj->alias.smw_title AS t,$qobj->alias.smw_namespace AS ns,$qobj->alias.smw_iw AS iw,$qobj->alias.smw_subobject AS so,$qobj->alias.smw_sortkey AS sortkey" .
			  ( $dbType == 'postgres' ? ( ( $sortfields ? ',' : '' ) . $sortfields ) : '' ),
			$qobj->where,
			__METHOD__,
			$sql_options
		);

		$qr = array();

		$count = 0; // the number of fetched results ( != number of valid results in array $qr)
		$missedCount = 0;
		$dataItemCache = array();
		$logToTable = array();
		$hasFurtherResults = false;

		$prs = $query->getDescription()->getPrintrequests();

		$diHandler = $this->store->getDataItemHandlerForDIType( DataItem::TYPE_WIKIPAGE );

		while ( ( $count < $query->getLimit() ) && ( $row = $db->fetchObject( $res ) ) ) {
			if ( $row->iw === '' || $row->iw{0} != ':' )  {

				// Catch exception for non-existing predefined properties that
				// still registered within non-updated pages (@see bug 48711)
				try {
					$dataItem = $diHandler->dataItemFromDBKeys( array(
						$row->t,
						intval( $row->ns ),
						$row->iw,
						'',
						$row->so
					) );
				} catch ( InvalidPredefinedPropertyException $e ) {
					$logToTable[$row->t] = "issue creating a {$row->t} dataitem from a database row";
					wfDebugLog( 'smw', __METHOD__ . ' ' . $e->getMessage() . "\n" );
					$dataItem = '';
				}

				if ( $dataItem instanceof DIWikiPage && !isset( $dataItemCache[$dataItem->getHash()] ) ) {
					$count++;
					$dataItemCache[$dataItem->getHash()] = true;
					$qr[] = $dataItem;
					// These IDs are usually needed for displaying the page (esp. if more property values are displayed):
					$this->store->smwIds->setCache( $row->t, $row->ns, $row->iw, $row->so, $row->id, $row->sortkey );
				} else {
					$missedCount++;
					$logToTable[$row->t] = "skip result for {$row->t} existing cache entry / query " . $query->getHash();
				}
			} else {
				$missedCount++;
				$logToTable[$row->t] = "skip result for {$row->t} due to an internal `{$row->iw}` pointer / query " . $query->getHash();
			}
		}

		if ( $db->fetchObject( $res ) ) {
			$count++;
		}

		if ( $logToTable !== array() ) {
			wfDebugLog( 'smw', __METHOD__ . ' ' . implode( ',', $logToTable ) . "\n" );
		}

		if ( $count > $query->getLimit() || ( $count + $missedCount ) > $query->getLimit() ) {
			$hasFurtherResults = true;
		};

		$db->freeResult( $res );
		$result = new QueryResult( $prs, $query, $qr, $this->store, $hasFurtherResults );

		return $result;
	}

	/**
	 * This function modifies the given query object at $qid to account for all ordering conditions
	 * in the Query $query. It is always required that $qid is the id of a query that joins with
	 * SMW IDs table so that the field alias.smw_title is $available for default sorting.
	 *
	 * @param integer $qid
	 */
	private function applyOrderConditions( $qid ) {
		$qobj = $this->querySegmentList[$qid];

		$extraProperties = $this->collectedRequiredExtraPropertyDescriptions( $qobj );

		if ( count( $extraProperties ) > 0 ) {
			$this->compileAccordingConditionsAndHackThemIntoQobj( $extraProperties, $qobj, $qid );
		}
	}

	private function applyExtraWhereCondition( $qid ) {

		$db = $this->store->getConnection( 'mw.db.queryengine' );

		if ( !isset( $this->querySegmentList[$qid] ) ) {
			return null;
		}

		$qobj = $this->querySegmentList[$qid];

		// Filter elements that should never appear in a result set
		$extraWhereCondition = array(
			'del'  => "$qobj->alias.smw_iw!=" . $db->addQuotes( SMW_SQL3_SMWIW_OUTDATED ) . " AND $qobj->alias.smw_iw!=" . $db->addQuotes( SMW_SQL3_SMWDELETEIW ),
			'redi' => "$qobj->alias.smw_iw!=" . $db->addQuotes( SMW_SQL3_SMWREDIIW )
		);

		if ( strpos( $qobj->where, SMW_SQL3_SMWIW_OUTDATED ) === false ) {
			$qobj->where .= $qobj->where === '' ? $extraWhereCondition['del'] : " AND " . $extraWhereCondition['del'];
		}

		if ( strpos( $qobj->where, SMW_SQL3_SMWREDIIW ) === false ) {
			$qobj->where .= $qobj->where === '' ? $extraWhereCondition['redi'] : " AND " . $extraWhereCondition['redi'];
		}

		$this->querySegmentList[$qid] = $qobj;
	}

	private function collectedRequiredExtraPropertyDescriptions( $qobj ) {
		$extraProperties = array();

		foreach ( $this->sortKeys as $propkey => $order ) {

			if ( !is_string( $propkey ) ) {
				throw new RuntimeException( "Expected a string value as sortkey" );
			}

			if ( !array_key_exists( $propkey, $qobj->sortfields ) ) { // Find missing property to sort by.
				if ( $propkey === '' ) { // Sort by first result column (page titles).
					$qobj->sortfields[$propkey] = "$qobj->alias.smw_sortkey";
				} elseif ( $propkey === '#' ) { // Sort by first result column (page titles).
					// PHP7 showed a rather erratic behaviour where in cases
					// the sortkey contains the same string for comparison, the
					// result returned from the DB was mixed in order therefore
					// using # as indicator to search for additional fields if
					// no specific property is given (see test cases in #1534)
					$qobj->sortfields[$propkey] = "$qobj->alias.smw_sortkey,$qobj->alias.smw_title,$qobj->alias.smw_subobject";
				} else { // Try to extend query.
					$sortprop = PropertyValue::makeUserProperty( $propkey );

					if ( $sortprop->isValid() ) {
						$extraProperties[] = new SomeProperty( $sortprop->getDataItem(), new ThingDescription() );

$sortprop->getDataItem() of type object<SMWDataItem> is not a sub-type of object<SMW\DIProperty>. It seems like you assume a child class of the class SMWDataItem to be always present.

					}
				}
			}
		}

		return $extraProperties;
	}

	private function compileAccordingConditionsAndHackThemIntoQobj( array $extraProperties, $qobj, $qid ) {
		$this->querySegmentListBuilder->setSortKeys( $this->sortKeys );
		$this->querySegmentListBuilder->getQuerySegmentFrom( new Conjunction( $extraProperties ) );

		$newQuerySegmentId = $this->querySegmentListBuilder->getLastQuerySegmentId();
		$this->querySegmentList = $this->querySegmentListBuilder->getQuerySegmentList();
		$this->errors = $this->querySegmentListBuilder->getErrors();

		$newQuerySegment = $this->querySegmentList[$newQuerySegmentId]; // This is always an QuerySegment::Q_CONJUNCTION ...

		foreach ( $newQuerySegment->components as $cid => $field ) { // ... so just re-wire its dependencies
			$qobj->components[$cid] = $qobj->joinfield;
			$qobj->sortfields = array_merge( $qobj->sortfields, $this->querySegmentList[$cid]->sortfields );
		}

		$this->querySegmentList[$qid] = $qobj;
	}

	/**
	 * Get a SQL option array for the given query and preprocessed query object at given id.
	 *
	 * @param Query $query
	 * @param integer $rootId
	 *
	 * @return array
	 */
	private function getSQLOptions( Query $query, $rootId ) {

		$result = array( 'LIMIT' => $query->getLimit() + 5, 'OFFSET' => $query->getOffset() );

		// Build ORDER BY options using discovered sorting fields.
		if ( $this->engineOptions->get( 'smwgQSortingSupport' ) ) {
			$qobj = $this->querySegmentList[$rootId];
			$type = $this->store->getConnection( 'mw.db.queryengine' )->getType();

			foreach ( $this->sortKeys as $propkey => $order ) {

				if ( !is_string( $propkey ) ) {
					throw new RuntimeException( "Expected a string value as sortkey" );
				}

				// #835
				// SELECT DISTINCT and ORDER BY RANDOM causes an issue for postgres
				// Disable RANDOM support for postgres
				if ( $type === 'postgres' ) {
					$this->engineOptions->set( 'smwgQRandSortingSupport', false );
				}

				if ( ( $order != 'RANDOM' ) && array_key_exists( $propkey, $qobj->sortfields ) ) { // Field was successfully added.
					$result['ORDER BY'] = ( array_key_exists( 'ORDER BY', $result ) ? $result['ORDER BY'] . ', ' : '' ) . $qobj->sortfields[$propkey] . " $order ";
				} elseif ( ( $order == 'RANDOM' ) && $this->engineOptions->get( 'smwgQRandSortingSupport' ) ) {
					$result['ORDER BY'] = ( array_key_exists( 'ORDER BY', $result ) ? $result['ORDER BY'] . ', ' : '' ) . ' RAND() ';
				}
			}
		}
		return $result;
	}

}