wikimedia / mediawiki

maintenance/purgeChangedFiles.php

<?php

For compatibility and reusability of your code, PSR1 recommends that a file should introduce either new symbols (like classes, functions, etc.) or have side-effects (like outputting something, or including other files), but not both at the same time. The first symbol is defined on line 32 and the first side effect is on line 24.

/**
 * Scan the logging table and purge affected files within a timeframe.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 * http://www.gnu.org/copyleft/gpl.html
 *
 * @file
 * @ingroup Maintenance
 */

require_once __DIR__ . '/Maintenance.php';

/**
 * Maintenance script that scans the deletion log and purges affected files
 * within a timeframe.
 *
 * @ingroup Maintenance
 */
class PurgeChangedFiles extends Maintenance {
	/**
	 * Mapping from type option to log type and actions.
	 * @var array
	 */
	private static $typeMappings = [
		'created' => [
			'upload' => [ 'upload' ],
			'import' => [ 'upload', 'interwiki' ],
		],
		'deleted' => [
			'delete' => [ 'delete', 'revision' ],
			'suppress' => [ 'delete', 'revision' ],
		],
		'modified' => [
			'upload' => [ 'overwrite', 'revert' ],
			'move' => [ 'move', 'move_redir' ],
		],
	];

	/**
	 * @var string
	 */
	private $startTimestamp;

	/**
	 * @var string
	 */
	private $endTimestamp;

	public function __construct() {
		parent::__construct();
		$this->addDescription( 'Scan the logging table and purge files and thumbnails.' );
		$this->addOption( 'starttime', 'Starting timestamp', true, true );
		$this->addOption( 'endtime', 'Ending timestamp', true, true );
		$this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' .
			implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true );
		$this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
		$this->addOption( 'dry-run', 'Do not send purge requests' );
		$this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
		$this->addOption( 'verbose', 'Show more output', false, false, 'v' );
		$this->setBatchSize( 100 );
	}

	public function execute() {
		global $wgHTCPRouting;

		if ( $this->hasOption( 'htcp-dest' ) ) {
			$parts = explode( ':', $this->getOption( 'htcp-dest' ) );
			if ( count( $parts ) < 2 ) {
				// Add default htcp port
				$parts[] = '4827';
			}

			// Route all HTCP messages to provided host:port
			$wgHTCPRouting = [
				'' => [ 'host' => $parts[0], 'port' => $parts[1] ],
			];
			$this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
		}

		// Find out which actions we should be concerned with
		$typeOpt = $this->getOption( 'type', 'all' );
		$validTypes = array_keys( self::$typeMappings );
		if ( $typeOpt === 'all' ) {
			// Convert 'all' to all registered types
			$typeOpt = implode( ',', $validTypes );
		}
		$typeList = explode( ',', $typeOpt );
		foreach ( $typeList as $type ) {
			if ( !in_array( $type, $validTypes ) ) {
				$this->error( "\nERROR: Unknown type: {$type}\n" );
				$this->maybeHelp( true );
			}
		}

		// Validate the timestamps
		$dbr = $this->getDB( DB_REPLICA );
		$this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) );
		$this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) );

		if ( $this->startTimestamp > $this->endTimestamp ) {
			$this->error( "\nERROR: starttime after endtime\n" );
			$this->maybeHelp( true );
		}

		// Turn on verbose when dry-run is enabled
		if ( $this->hasOption( 'dry-run' ) ) {
			$this->mOptions['verbose'] = 1;
		}

		$this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" );
		foreach ( $typeList as $type ) {
			$this->verbose( "Checking for {$type} files...\n" );
			$this->purgeFromLogType( $type );
			if ( !$this->hasOption( 'dry-run' ) ) {
				$this->verbose( "...{$type} files purged.\n\n" );
			}
		}
	}

	/**
	 * Purge cache and thumbnails for changes of the given type.
	 *
	 * @param string $type Type of change to find
	 */
	protected function purgeFromLogType( $type ) {
		$repo = RepoGroup::singleton()->getLocalRepo();
		$dbr = $this->getDB( DB_REPLICA );

		foreach ( self::$typeMappings[$type] as $logType => $logActions ) {
			$this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" );

			$res = $dbr->select(
				'logging',
				[ 'log_title', 'log_timestamp', 'log_params' ],
				[
					'log_namespace' => NS_FILE,
					'log_type' => $logType,
					'log_action' => $logActions,
					'log_timestamp >= ' . $dbr->addQuotes( $this->startTimestamp ),
					'log_timestamp <= ' . $dbr->addQuotes( $this->endTimestamp ),
				],
				__METHOD__
			);

			$bSize = 0;
			foreach ( $res as $row ) {
				$file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) );

				if ( $this->hasOption( 'dry-run' ) ) {
					$this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" );
					continue;
				}

				// Purge current version and its thumbnails
				$file->purgeCache();
				// Purge the old versions and their thumbnails
				foreach ( $file->getHistory() as $oldFile ) {
					$oldFile->purgeCache();
				}

				if ( $logType === 'delete' ) {
					// If there is an orphaned storage file... delete it
					if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) {
						$dpath = $this->getDeletedPath( $repo, $file );
						if ( $repo->fileExists( $dpath ) ) {
							// Sanity check to avoid data loss
							$repo->getBackend()->delete( [ 'src' => $file->getPath() ] );
							$this->verbose( "Deleted orphan file: {$file->getPath()}.\n" );
						} else {
							$this->error( "File was not deleted: {$file->getPath()}.\n" );
						}
					}

					// Purge items from fileachive table (rows are likely here)
					$this->purgeFromArchiveTable( $repo, $file );
				} elseif ( $logType === 'move' ) {
					// Purge the target file as well

					$params = unserialize( $row->log_params );
					if ( isset( $params['4::target'] ) ) {
						$target = $params['4::target'];
						$targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) );
						$targetFile->purgeCache();
						$this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" );
					}
				}

				$this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" );

				if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->mBatchSize ) {
					$bSize = 0;
					// sleep-per-batch is milliseconds, usleep wants micro seconds.
					usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
				}
			}
		}
	}

	protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) {
		$dbr = $repo->getSlaveDB();
		$res = $dbr->select(
			'filearchive',
			[ 'fa_archive_name' ],
			[ 'fa_name' => $file->getName() ],
			__METHOD__
		);

		foreach ( $res as $row ) {
			if ( $row->fa_archive_name === null ) {
				// Was not an old version (current version names checked already)
				continue;
			}
			$ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name );
			// If there is an orphaned storage file still there...delete it
			if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) {

It seems like $ofile->getPath() targeting File::getPath() can also be of type boolean; however, FileRepo::fileExists() does only seem to accept string, maybe add an additional type check?

				$dpath = $this->getDeletedPath( $repo, $ofile );
				if ( $repo->fileExists( $dpath ) ) {
					// Sanity check to avoid data loss
					$repo->getBackend()->delete( [ 'src' => $ofile->getPath() ] );
					$this->output( "Deleted orphan file: {$ofile->getPath()}.\n" );
				} else {
					$this->error( "File was not deleted: {$ofile->getPath()}.\n" );
				}
			}
			$file->purgeOldThumbnails( $row->fa_archive_name );
		}
	}

	protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) {
		$hash = $repo->getFileSha1( $file->getPath() );

It seems like $file->getPath() targeting File::getPath() can also be of type boolean; however, FileRepo::getFileSha1() does only seem to accept string, maybe add an additional type check?

		$key = "{$hash}.{$file->getExtension()}";

		return $repo->getDeletedHashPath( $key ) . $key;
	}

	/**
	 * Send an output message iff the 'verbose' option has been provided.
	 *
	 * @param string $msg Message to output
	 */
	protected function verbose( $msg ) {
		if ( $this->hasOption( 'verbose' ) ) {
			$this->output( $msg );
		}
	}
}

$maintClass = "PurgeChangedFiles";
require_once RUN_MAINTENANCE_IF_MAIN;