PageRepository::getHTMLContent() - Code Metrics - Inspection of "Improve handling of upstream errors" - x-tools/xtools - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — bad-gateway ( 773331 )

by MusikAnimal

created 2023-05-01 22:31 UTC

PageRepository::getHTMLContent() B

↳ Parent: PageRepository

Complexity

Conditions	8
Paths	12

Size

Total Lines	25
Code Lines	17

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	8
eloc	17
nc	12
nop	2
dl	0
loc	25
rs	8.4444
c	0
b	0
f	0

<?php

declare(strict_types = 1);

namespace App\Repository;

use App\Exception\BadGatewayException;
use App\Model\Page;
use App\Model\Project;
use App\Model\User;
use DateTime;
use Doctrine\DBAL\Driver\ResultStatement;
use GuzzleHttp\Exception\ClientException;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Exception\ServerException;
use GuzzleHttp\RequestOptions;
use Symfony\Component\HttpFoundation\Response;

/**
 * A PageRepository fetches data about Pages, either singularly or for multiple.
 * Despite the name, this does not have a direct correlation with the Pages tool.
 * @codeCoverageIgnore
 */
class PageRepository extends Repository
{
    /**
     * Get metadata about a single page from the API.
     * @param Project $project The project to which the page belongs.
     * @param string $pageTitle Page title.
     * @return string[]|null Array with some of the following keys: pageid, title, missing, displaytitle, url.
     *   Returns null if page does not exist.
     */
    public function getPageInfo(Project $project, string $pageTitle): ?array
    {
        $info = $this->getPagesInfo($project, [$pageTitle]);
        return null !== $info ? array_shift($info) : null;
    }

    /**
     * Get metadata about a set of pages from the API.
     * @param Project $project The project to which the pages belong.
     * @param string[] $pageTitles Array of page titles.
     * @return array|null Array keyed by the page names, each element with some of the following keys: pageid,
     *   title, missing, displaytitle, url. Returns null if page does not exist.
     */
    public function getPagesInfo(Project $project, array $pageTitles): ?array
    {
        $params = [
            'prop' => 'info|pageprops',
            'inprop' => 'protection|talkid|watched|watchers|notificationtimestamp|subjectid|url|displaytitle',
            'converttitles' => '',
            'titles' => join('|', $pageTitles),
            'formatversion' => 2,
        ];

        $res = $this->executeApiRequest($project, $params);
        $result = [];
        if (isset($res['query']['pages'])) {
            foreach ($res['query']['pages'] as $pageInfo) {
                $result[$pageInfo['title']] = $pageInfo;
            }
        } else {
            return null;
        }
        return $result;
    }

    /**
     * Get the full page text of a set of pages.
     * @param Project $project The project to which the pages belong.
     * @param string[] $pageTitles Array of page titles.
     * @return string[] Array keyed by the page names, with the page text as the values.
     */
    public function getPagesWikitext(Project $project, array $pageTitles): array
    {
        $params = [
            'prop' => 'revisions',
            'rvprop' => 'content',
            'titles' => join('|', $pageTitles),
            'formatversion' => 2,
        ];
        $res = $this->executeApiRequest($project, $params);
        $result = [];

        if (!isset($res['query']['pages'])) {
            return [];
        }

        foreach ($res['query']['pages'] as $page) {
            if (isset($page['revisions'][0]['content'])) {
                $result[$page['title']] = $page['revisions'][0]['content'];
            } else {
                $result[$page['title']] = '';
            }
        }

        return $result;
    }

    /**
     * Get revisions of a single page.
     * @param Page $page The page.
     * @param User|null $user Specify to get only revisions by the given user.
     * @param false|int $start
     * @param false|int $end
     * @return string[] Each member with keys: id, timestamp, length.
     */
    public function getRevisions(Page $page, ?User $user = null, $start = false, $end = false): array
    {
        $cacheKey = $this->getCacheKey(func_get_args(), 'page_revisions');
        if ($this->cache->hasItem($cacheKey)) {
            return $this->cache->getItem($cacheKey)->get();
        }

        $stmt = $this->getRevisionsStmt($page, $user, null, null, $start, $end);
        $result = $stmt->fetchAllAssociative();

        // Cache and return.
        return $this->setCache($cacheKey, $result);
    }

    /**
     * Get the statement for a single revision, so that you can iterate row by row.
     * @param Page $page The page.
     * @param User|null $user Specify to get only revisions by the given user.
     * @param ?int $limit Max number of revisions to process.
     * @param ?int $numRevisions Number of revisions, if known. This is used solely to determine the
     *   OFFSET if we are given a $limit (see below). If $limit is set and $numRevisions is not set,
     *   a separate query is ran to get the number of revisions.
     * @param false|int $start
     * @param false|int $end
     * @return ResultStatement
     */
    public function getRevisionsStmt(
        Page $page,
        ?User $user = null,
        ?int $limit = null,
        ?int $numRevisions = null,
        $start = false,
        $end = false
    ): ResultStatement {
        $revTable = $this->getTableName(
            $page->getProject()->getDatabaseName(),
            'revision',
            $user ? null : '' // Use 'revision' if there's no user, otherwise default to revision_userindex
        );
        $commentTable = $page->getProject()->getTableName('comment');
        $actorTable = $page->getProject()->getTableName('actor');
        $userClause = $user ? "revs.rev_actor = :actorId AND " : "";

        $limitClause = '';
        if (intval($limit) > 0 && isset($numRevisions)) {
            $limitClause = "LIMIT $limit";
        }

        $dateConditions = $this->getDateConditions($start, $end, false, 'revs.');

        $sql = "SELECT * FROM (
                    SELECT
                        revs.rev_id AS id,
                        revs.rev_timestamp AS timestamp,
                        revs.rev_minor_edit AS minor,
                        revs.rev_len AS length,
                        (CAST(revs.rev_len AS SIGNED) - IFNULL(parentrevs.rev_len, 0)) AS length_change,
                        actor_user AS user_id,
                        actor_name AS username,
                        comment_text AS `comment`,
                        revs.rev_sha1 AS sha,
                        revs.rev_deleted AS rev_deleted
                    FROM $revTable AS revs
                    LEFT JOIN $actorTable ON revs.rev_actor = actor_id
                    LEFT JOIN $revTable AS parentrevs ON (revs.rev_parent_id = parentrevs.rev_id)
                    LEFT OUTER JOIN $commentTable ON comment_id = revs.rev_comment_id
                    WHERE $userClause revs.rev_page = :pageid $dateConditions
                    ORDER BY revs.rev_timestamp DESC
                    $limitClause
                ) a
                ORDER BY timestamp ASC";

        $params = ['pageid' => $page->getId()];
        if ($user) {
            $params['actorId'] = $user->getActorId($page->getProject());
        }

        return $this->executeProjectsQuery($page->getProject(), $sql, $params);
    }

    /**
     * Get a count of the number of revisions of a single page
     * @param Page $page The page.
     * @param User|null $user Specify to only count revisions by the given user.
     * @param false|int $start
     * @param false|int $end
     * @return int
     */
    public function getNumRevisions(Page $page, ?User $user = null, $start = false, $end = false): int
    {
        $cacheKey = $this->getCacheKey(func_get_args(), 'page_numrevisions');
        if ($this->cache->hasItem($cacheKey)) {
            return $this->cache->getItem($cacheKey)->get();
        }

        // In this case revision is faster than revision_userindex if we're not querying by user.
        $revTable = $page->getProject()->getTableName(
            'revision',
            $user && $this->isWMF ? '_userindex' : ''
        );
        $userClause = $user ? "rev_actor = :actorId AND " : "";

        $dateConditions = $this->getDateConditions($start, $end);

        $sql = "SELECT COUNT(*)
                FROM $revTable
                WHERE $userClause rev_page = :pageid $dateConditions";
        $params = ['pageid' => $page->getId()];
        if ($user) {
            $params['rev_actor'] = $user->getActorId($page->getProject());
        }

        $result = (int)$this->executeProjectsQuery($page->getProject(), $sql, $params)->fetchOne();

        // Cache and return.
        return $this->setCache($cacheKey, $result);
    }

    /**
     * Get any CheckWiki errors of a single page
     * @param Page $page
     * @return array Results from query
     */
    public function getCheckWikiErrors(Page $page): array
    {
        // Only support mainspace on Labs installations
        if (0 !== $page->getNamespace() || !$this->isWMF) {
            return [];
        }

        $sql = "SELECT error, notice, found, name_trans AS name, prio, text_trans AS explanation
                FROM s51080__checkwiki_p.cw_error a
                JOIN s51080__checkwiki_p.cw_overview_errors b
                WHERE a.project = b.project
                AND a.project = :dbName
                AND a.title = :title
                AND a.error = b.id
                AND a.ok = 0";

        // remove _p if present
        $dbName = preg_replace('/_p$/', '', $page->getProject()->getDatabaseName());

        // Page title without underscores (str_replace just to be sure)
        $pageTitle = str_replace('_', ' ', $page->getTitle());

        $conn = $this->getToolsConnection();
        return $conn->executeQuery($sql, [
            'dbName' => $dbName,
            'title' => $pageTitle,
        ])->fetchAllAssociative();
    }

    /**
     * Get basic wikidata on the page: label and description.
     * @param Page $page
     * @return string[][] In the format:
     *    [[
     *         'term' => string such as 'label',
     *         'term_text' => string (value for 'label'),
     *     ], ... ]
     */
    public function getWikidataInfo(Page $page): array
    {
        if (empty($page->getWikidataId())) {
            return [];
        }

        $wikidataId = ltrim($page->getWikidataId(), 'Q');
        $lang = $page->getProject()->getLang();
        $wdp = 'wikidatawiki_p';

        $sql = "SELECT wby_name AS term, wbx_text AS term_text
                FROM $wdp.wbt_item_terms
                JOIN $wdp.wbt_term_in_lang ON wbit_term_in_lang_id = wbtl_id
                JOIN $wdp.wbt_type ON wbtl_type_id = wby_id
                JOIN $wdp.wbt_text_in_lang ON wbtl_text_in_lang_id = wbxl_id
                JOIN $wdp.wbt_text ON wbxl_text_id = wbx_id
                WHERE wbit_item_id = :wikidataId
                AND wby_name IN ('label', 'description')
                AND wbxl_language = :lang";

        return $this->executeProjectsQuery('wikidatawiki', $sql, [
            'lang' => $lang,
            'wikidataId' => $wikidataId,
        ])->fetchAllAssociative();
    }

    /**
     * Get or count all wikidata items for the given page,
     *     not just languages of sister projects
     * @param Page $page
     * @param bool $count Set to true to get only a COUNT
     * @return string[]|int Records as returend by the DB,
     *                      or raw COUNT of the records.
     */
    public function getWikidataItems(Page $page, bool $count = false)
    {
        if (!$page->getWikidataId()) {
            return $count ? 0 : [];
        }

        $wikidataId = ltrim($page->getWikidataId(), 'Q');

        $sql = "SELECT " . ($count ? 'COUNT(*) AS count' : '*') . "
                FROM wikidatawiki_p.wb_items_per_site
                WHERE ips_item_id = :wikidataId";

        $result = $this->executeProjectsQuery('wikidatawiki', $sql, [
            'wikidataId' => $wikidataId,
        ])->fetchAllAssociative();

        return $count ? (int) $result[0]['count'] : $result;
    }

    /**
     * Get number of in and outgoing links and redirects to the given page.
     * @param Page $page
     * @return string[] Counts with the keys 'links_ext_count', 'links_out_count',
     *                  'links_in_count' and 'redirects_count'
     */
    public function countLinksAndRedirects(Page $page): array
    {
        $externalLinksTable = $this->getTableName($page->getProject()->getDatabaseName(), 'externallinks');
        $pageLinksTable = $this->getTableName($page->getProject()->getDatabaseName(), 'pagelinks');
        $redirectTable = $this->getTableName($page->getProject()->getDatabaseName(), 'redirect');

        $sql = "SELECT COUNT(*) AS value, 'links_ext' AS type
                FROM $externalLinksTable WHERE el_from = :id
                UNION
                SELECT COUNT(*) AS value, 'links_out' AS type
                FROM $pageLinksTable WHERE pl_from = :id
                UNION
                SELECT COUNT(*) AS value, 'links_in' AS type
                FROM $pageLinksTable WHERE pl_namespace = :namespace AND pl_title = :title
                UNION
                SELECT COUNT(*) AS value, 'redirects' AS type
                FROM $redirectTable WHERE rd_namespace = :namespace AND rd_title = :title";

        $params = [
            'id' => $page->getId(),
            'title' => str_replace(' ', '_', $page->getTitleWithoutNamespace()),
            'namespace' => $page->getNamespace(),
        ];

        $res = $this->executeProjectsQuery($page->getProject(), $sql, $params);
        $data = [];

        // Transform to associative array by 'type'
        foreach ($res as $row) {
            $data[$row['type'] . '_count'] = (int)$row['value'];
        }

        return $data;
    }

    /**
     * Count wikidata items for the given page, not just languages of sister projects
     * @param Page $page
     * @return int Number of records.
     */
    public function countWikidataItems(Page $page): int
    {
        return $this->getWikidataItems($page, true);
    }

    /**
     * Get page views for the given page and timeframe.
     * @fixme use Symfony Guzzle package.
     * @param Page $page
     * @param string|DateTime $start In the format YYYYMMDD
     * @param string|DateTime $end In the format YYYYMMDD
     * @return string[][][]
     * @throws BadGatewayException
     */
    public function getPageviews(Page $page, $start, $end): array
    {
        // Pull from cache for each call during the same request.
        // FIXME: This is fine for now as we only fetch pageviews for one page at a time,
        //   but if that ever changes we'll need to use APCu cache or otherwise respect $page, $start and $end.
        //   Better of course would be to move to a Symfony CachingHttpClient instead of Guzzle across the board.
        static $pageviews;
        if (isset($pageviews)) {
            return $pageviews;
        }

        $title = rawurlencode(str_replace(' ', '_', $page->getTitle()));

        if ($start instanceof DateTime) {
            $start = $start->format('Ymd');
        } else {
            $start = (new DateTime($start))->format('Ymd');
        }
        if ($end instanceof DateTime) {
            $end = $end->format('Ymd');
        } else {
            $end = (new DateTime($end))->format('Ymd');
        }

        $project = $page->getProject()->getDomain();

        $url = 'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/' .
            "$project/all-access/user/$title/daily/$start/$end";

        try {
            $res = $this->guzzle->request('GET', $url, [
                // Five seconds should be plenty...
                RequestOptions::CONNECT_TIMEOUT => 5,
            ]);
            $pageviews = json_decode($res->getBody()->getContents(), true);
            return $pageviews;
        } catch (ServerException|ConnectException $e) {
            throw new BadGatewayException('api-error-wikimedia', ['Pageviews'], $e);
        }
    }

    /**
     * Get the full HTML content of the the page.
     * @param Page $page
     * @param int|null $revId What revision to query for.
     * @return string
     */
    public function getHTMLContent(Page $page, ?int $revId = null): string
    {
        if ($this->isWMF) {
            $domain = $page->getProject()->getDomain();
            $url = "https://$domain/api/rest_v1/page/html/" . urlencode(str_replace(' ', '_', $page->getTitle()));
            if (null !== $revId) {
                $url .= "/$revId";
            }
        } else {
            $url = $page->getUrl();
            if (null !== $revId) {
                $url .= "?oldid=$revId";
            }
        }

        try {
            return $this->guzzle->request('GET', $url)
                ->getBody()
                ->getContents();
        } catch (ServerException|ClientException $e) {
            if ($e instanceof ClientException && $page->exists() && Response::HTTP_NOT_FOUND === $e->getCode()) {
                // Sometimes the REST API throws 404s when the page does in fact exist.
                throw new BadGatewayException('api-error-wikimedia', ['Wikimedia REST'], $e);
            }
            throw new BadGatewayException('api-error-wikimedia', ['Wikimedia REST'], $e);
        }
    }

    /**
     * Get the ID of the revision of a page at the time of the given DateTime.
     * @param Page $page
     * @param DateTime $date
     * @return int
     */
    public function getRevisionIdAtDate(Page $page, DateTime $date): int
    {
        $revisionTable = $page->getProject()->getTableName('revision');
        $pageId = $page->getId();
        $datestamp = $date->format('YmdHis');
        $sql = "SELECT MAX(rev_id)
                FROM $revisionTable
                WHERE rev_timestamp <= $datestamp
                AND rev_page = $pageId LIMIT 1;";
        $resultQuery = $this->getProjectsConnection($page->getProject())
            ->executeQuery($sql);
        return (int)$resultQuery->fetchOne();
    }

    /**
     * Get HTML display titles of a set of pages (or the normal title if there's no display title).
     * This will send t/50 API requests where t is the number of titles supplied.
     * @param Project $project The project.
     * @param string[] $pageTitles The titles to fetch.
     * @return string[] Keys are the original supplied title, and values are the display titles.
     * @static
     */
    public function displayTitles(Project $project, array $pageTitles): array
    {
        $displayTitles = [];
        $numPages = count($pageTitles);

        for ($n = 0; $n < $numPages; $n += 50) {
            $titleSlice = array_slice($pageTitles, $n, 50);
            $res = $this->guzzle->request('GET', $project->getApiUrl(), ['query' => [
                'action' => 'query',
                'prop' => 'info|pageprops',
                'inprop' => 'displaytitle',
                'titles' => join('|', $titleSlice),
                'format' => 'json',
            ]]);
            $result = json_decode($res->getBody()->getContents(), true);

            // Extract normalization info.
            $normalized = [];
            if (isset($result['query']['normalized'])) {
                array_map(
                    function ($e) use (&$normalized): void {
                        $normalized[$e['to']] = $e['from'];
                    },
                    $result['query']['normalized']
                );
            }

            // Match up the normalized titles with the display titles and the original titles.
            foreach ($result['query']['pages'] as $pageInfo) {
                $displayTitle = $pageInfo['pageprops']['displaytitle'] ?? $pageInfo['title'];
                $origTitle = $normalized[$pageInfo['title']] ?? $pageInfo['title'];
                $displayTitles[$origTitle] = $displayTitle;
            }
        }

        return $displayTitles;
    }
}


1			<?php
2
3			declare(strict_types = 1);
4
5			namespace App\Repository;
6
7			use App\Exception\BadGatewayException;
8			use App\Model\Page;
9			use App\Model\Project;
10			use App\Model\User;
11			use DateTime;
12			use Doctrine\DBAL\Driver\ResultStatement;
13			use GuzzleHttp\Exception\ClientException;
14			use GuzzleHttp\Exception\ConnectException;
15			use GuzzleHttp\Exception\ServerException;
16			use GuzzleHttp\RequestOptions;
17			use Symfony\Component\HttpFoundation\Response;
18
19			/**
20			* A PageRepository fetches data about Pages, either singularly or for multiple.
21			* Despite the name, this does not have a direct correlation with the Pages tool.
22			* @codeCoverageIgnore
23			*/
24			class PageRepository extends Repository
25			{
26			/**
27			* Get metadata about a single page from the API.
28			* @param Project $project The project to which the page belongs.
29			* @param string $pageTitle Page title.
30			* @return string[]\|null Array with some of the following keys: pageid, title, missing, displaytitle, url.
31			* Returns null if page does not exist.
32			*/
33			public function getPageInfo(Project $project, string $pageTitle): ?array
34			{
35			$info = $this->getPagesInfo($project, [$pageTitle]);
36			return null !== $info ? array_shift($info) : null;
37			}
38
39			/**
40			* Get metadata about a set of pages from the API.
41			* @param Project $project The project to which the pages belong.
42			* @param string[] $pageTitles Array of page titles.
43			* @return array\|null Array keyed by the page names, each element with some of the following keys: pageid,
44			* title, missing, displaytitle, url. Returns null if page does not exist.
45			*/
46			public function getPagesInfo(Project $project, array $pageTitles): ?array
47			{
48			$params = [
49			'prop' => 'info\|pageprops',
50			'inprop' => 'protection\|talkid\|watched\|watchers\|notificationtimestamp\|subjectid\|url\|displaytitle',
51			'converttitles' => '',
52			'titles' => join('\|', $pageTitles),
53			'formatversion' => 2,
54			];
55
56			$res = $this->executeApiRequest($project, $params);
57			$result = [];
58			if (isset($res['query']['pages'])) {
59			foreach ($res['query']['pages'] as $pageInfo) {
60			$result[$pageInfo['title']] = $pageInfo;
61			}
62			} else {
63			return null;
64			}
65			return $result;
66			}
67
68			/**
69			* Get the full page text of a set of pages.
70			* @param Project $project The project to which the pages belong.
71			* @param string[] $pageTitles Array of page titles.
72			* @return string[] Array keyed by the page names, with the page text as the values.
73			*/
74			public function getPagesWikitext(Project $project, array $pageTitles): array
75			{
76			$params = [
77			'prop' => 'revisions',
78			'rvprop' => 'content',
79			'titles' => join('\|', $pageTitles),
80			'formatversion' => 2,
81			];
82			$res = $this->executeApiRequest($project, $params);
83			$result = [];
84
85			if (!isset($res['query']['pages'])) {
86			return [];
87			}
88
89			foreach ($res['query']['pages'] as $page) {
90			if (isset($page['revisions'][0]['content'])) {
91			$result[$page['title']] = $page['revisions'][0]['content'];
92			} else {
93			$result[$page['title']] = '';
94			}
95			}
96
97			return $result;
98			}
99
100			/**
101			* Get revisions of a single page.
102			* @param Page $page The page.
103			* @param User\|null $user Specify to get only revisions by the given user.
104			* @param false\|int $start
105			* @param false\|int $end
106			* @return string[] Each member with keys: id, timestamp, length.
107			*/
108			public function getRevisions(Page $page, ?User $user = null, $start = false, $end = false): array
109			{
110			$cacheKey = $this->getCacheKey(func_get_args(), 'page_revisions');
111			if ($this->cache->hasItem($cacheKey)) {
112			return $this->cache->getItem($cacheKey)->get();
113			}
114
115			$stmt = $this->getRevisionsStmt($page, $user, null, null, $start, $end);
116			$result = $stmt->fetchAllAssociative();
117
118			// Cache and return.
119			return $this->setCache($cacheKey, $result);
120			}
121
122			/**
123			* Get the statement for a single revision, so that you can iterate row by row.
124			* @param Page $page The page.
125			* @param User\|null $user Specify to get only revisions by the given user.
126			* @param ?int $limit Max number of revisions to process.
127			* @param ?int $numRevisions Number of revisions, if known. This is used solely to determine the
128			* OFFSET if we are given a $limit (see below). If $limit is set and $numRevisions is not set,
129			* a separate query is ran to get the number of revisions.
130			* @param false\|int $start
131			* @param false\|int $end
132			* @return ResultStatement
133			*/
134			public function getRevisionsStmt(
135			Page $page,
136			?User $user = null,
137			?int $limit = null,
138			?int $numRevisions = null,
139			$start = false,
140			$end = false
141			): ResultStatement {
142			$revTable = $this->getTableName(
143			$page->getProject()->getDatabaseName(),
144			'revision',
145			$user ? null : '' // Use 'revision' if there's no user, otherwise default to revision_userindex
146			);
147			$commentTable = $page->getProject()->getTableName('comment');
148			$actorTable = $page->getProject()->getTableName('actor');
149			$userClause = $user ? "revs.rev_actor = :actorId AND " : "";
150
151			$limitClause = '';
152			if (intval($limit) > 0 && isset($numRevisions)) {
153			$limitClause = "LIMIT $limit";
154			}
155
156			$dateConditions = $this->getDateConditions($start, $end, false, 'revs.');
157
158			$sql = "SELECT * FROM (
159			SELECT
160			revs.rev_id AS id,
161			revs.rev_timestamp AS timestamp,
162			revs.rev_minor_edit AS minor,
163			revs.rev_len AS length,
164			(CAST(revs.rev_len AS SIGNED) - IFNULL(parentrevs.rev_len, 0)) AS length_change,
165			actor_user AS user_id,
166			actor_name AS username,
167			comment_text AS `comment`,
168			revs.rev_sha1 AS sha,
169			revs.rev_deleted AS rev_deleted
170			FROM $revTable AS revs
171			LEFT JOIN $actorTable ON revs.rev_actor = actor_id
172			LEFT JOIN $revTable AS parentrevs ON (revs.rev_parent_id = parentrevs.rev_id)
173			LEFT OUTER JOIN $commentTable ON comment_id = revs.rev_comment_id
174			WHERE $userClause revs.rev_page = :pageid $dateConditions
175			ORDER BY revs.rev_timestamp DESC
176			$limitClause
177			) a
178			ORDER BY timestamp ASC";
179
180			$params = ['pageid' => $page->getId()];
181			if ($user) {
182			$params['actorId'] = $user->getActorId($page->getProject());
183			}
184
185			return $this->executeProjectsQuery($page->getProject(), $sql, $params);
186			}
187
188			/**
189			* Get a count of the number of revisions of a single page
190			* @param Page $page The page.
191			* @param User\|null $user Specify to only count revisions by the given user.
192			* @param false\|int $start
193			* @param false\|int $end
194			* @return int
195			*/
196			public function getNumRevisions(Page $page, ?User $user = null, $start = false, $end = false): int
197			{
198			$cacheKey = $this->getCacheKey(func_get_args(), 'page_numrevisions');
199			if ($this->cache->hasItem($cacheKey)) {
200			return $this->cache->getItem($cacheKey)->get();
201			}
202
203			// In this case revision is faster than revision_userindex if we're not querying by user.
204			$revTable = $page->getProject()->getTableName(
205			'revision',
206			$user && $this->isWMF ? '_userindex' : ''
207			);
208			$userClause = $user ? "rev_actor = :actorId AND " : "";
209
210			$dateConditions = $this->getDateConditions($start, $end);
211
212			$sql = "SELECT COUNT(*)
213			FROM $revTable
214			WHERE $userClause rev_page = :pageid $dateConditions";
215			$params = ['pageid' => $page->getId()];
216			if ($user) {
217			$params['rev_actor'] = $user->getActorId($page->getProject());
218			}
219
220			$result = (int)$this->executeProjectsQuery($page->getProject(), $sql, $params)->fetchOne();
221
222			// Cache and return.
223			return $this->setCache($cacheKey, $result);
224			}
225
226			/**
227			* Get any CheckWiki errors of a single page
228			* @param Page $page
229			* @return array Results from query
230			*/
231			public function getCheckWikiErrors(Page $page): array
232			{
233			// Only support mainspace on Labs installations
234			if (0 !== $page->getNamespace() \|\| !$this->isWMF) {
235			return [];
236			}
237
238			$sql = "SELECT error, notice, found, name_trans AS name, prio, text_trans AS explanation
239			FROM s51080__checkwiki_p.cw_error a
240			JOIN s51080__checkwiki_p.cw_overview_errors b
241			WHERE a.project = b.project
242			AND a.project = :dbName
243			AND a.title = :title
244			AND a.error = b.id
245			AND a.ok = 0";
246
247			// remove _p if present
248			$dbName = preg_replace('/_p$/', '', $page->getProject()->getDatabaseName());
249
250			// Page title without underscores (str_replace just to be sure)
251			$pageTitle = str_replace('_', ' ', $page->getTitle());
252
253			$conn = $this->getToolsConnection();
254			return $conn->executeQuery($sql, [
255			'dbName' => $dbName,
256			'title' => $pageTitle,
257			])->fetchAllAssociative();
258			}
259
260			/**
261			* Get basic wikidata on the page: label and description.
262			* @param Page $page
263			* @return string[][] In the format:
264			* [[
265			* 'term' => string such as 'label',
266			* 'term_text' => string (value for 'label'),
267			* ], ... ]
268			*/
269			public function getWikidataInfo(Page $page): array
270			{
271			if (empty($page->getWikidataId())) {
272			return [];
273			}
274
275			$wikidataId = ltrim($page->getWikidataId(), 'Q');
276			$lang = $page->getProject()->getLang();
277			$wdp = 'wikidatawiki_p';
278
279			$sql = "SELECT wby_name AS term, wbx_text AS term_text
280			FROM $wdp.wbt_item_terms
281			JOIN $wdp.wbt_term_in_lang ON wbit_term_in_lang_id = wbtl_id
282			JOIN $wdp.wbt_type ON wbtl_type_id = wby_id
283			JOIN $wdp.wbt_text_in_lang ON wbtl_text_in_lang_id = wbxl_id
284			JOIN $wdp.wbt_text ON wbxl_text_id = wbx_id
285			WHERE wbit_item_id = :wikidataId
286			AND wby_name IN ('label', 'description')
287			AND wbxl_language = :lang";
288
289			return $this->executeProjectsQuery('wikidatawiki', $sql, [
290			'lang' => $lang,
291			'wikidataId' => $wikidataId,
292			])->fetchAllAssociative();
293			}
294
295			/**
296			* Get or count all wikidata items for the given page,
297			* not just languages of sister projects
298			* @param Page $page
299			* @param bool $count Set to true to get only a COUNT
300			* @return string[]\|int Records as returend by the DB,
301			* or raw COUNT of the records.
302			*/
303			public function getWikidataItems(Page $page, bool $count = false)
304			{
305			if (!$page->getWikidataId()) {
306			return $count ? 0 : [];
307			}
308
309			$wikidataId = ltrim($page->getWikidataId(), 'Q');
310
311			$sql = "SELECT " . ($count ? 'COUNT() AS count' : '') . "
312			FROM wikidatawiki_p.wb_items_per_site
313			WHERE ips_item_id = :wikidataId";
314
315			$result = $this->executeProjectsQuery('wikidatawiki', $sql, [
316			'wikidataId' => $wikidataId,
317			])->fetchAllAssociative();
318
319			return $count ? (int) $result[0]['count'] : $result;
320			}
321
322			/**
323			* Get number of in and outgoing links and redirects to the given page.
324			* @param Page $page
325			* @return string[] Counts with the keys 'links_ext_count', 'links_out_count',
326			* 'links_in_count' and 'redirects_count'
327			*/
328			public function countLinksAndRedirects(Page $page): array
329			{
330			$externalLinksTable = $this->getTableName($page->getProject()->getDatabaseName(), 'externallinks');
331			$pageLinksTable = $this->getTableName($page->getProject()->getDatabaseName(), 'pagelinks');
332			$redirectTable = $this->getTableName($page->getProject()->getDatabaseName(), 'redirect');
333
334			$sql = "SELECT COUNT(*) AS value, 'links_ext' AS type
335			FROM $externalLinksTable WHERE el_from = :id
336			UNION
337			SELECT COUNT(*) AS value, 'links_out' AS type
338			FROM $pageLinksTable WHERE pl_from = :id
339			UNION
340			SELECT COUNT(*) AS value, 'links_in' AS type
341			FROM $pageLinksTable WHERE pl_namespace = :namespace AND pl_title = :title
342			UNION
343			SELECT COUNT(*) AS value, 'redirects' AS type
344			FROM $redirectTable WHERE rd_namespace = :namespace AND rd_title = :title";
345
346			$params = [
347			'id' => $page->getId(),
348			'title' => str_replace(' ', '_', $page->getTitleWithoutNamespace()),
349			'namespace' => $page->getNamespace(),
350			];
351
352			$res = $this->executeProjectsQuery($page->getProject(), $sql, $params);
353			$data = [];
354
355			// Transform to associative array by 'type'
356			foreach ($res as $row) {
357			$data[$row['type'] . '_count'] = (int)$row['value'];
358			}
359
360			return $data;
361			}
362
363			/**
364			* Count wikidata items for the given page, not just languages of sister projects
365			* @param Page $page
366			* @return int Number of records.
367			*/
368			public function countWikidataItems(Page $page): int
369			{
370			return $this->getWikidataItems($page, true);
371			}
372
373			/**
374			* Get page views for the given page and timeframe.
375			* @fixme use Symfony Guzzle package.
376			* @param Page $page
377			* @param string\|DateTime $start In the format YYYYMMDD
378			* @param string\|DateTime $end In the format YYYYMMDD
379			* @return string[][][]
380			* @throws BadGatewayException
381			*/
382			public function getPageviews(Page $page, $start, $end): array
383			{
384			// Pull from cache for each call during the same request.
385			// FIXME: This is fine for now as we only fetch pageviews for one page at a time,
386			// but if that ever changes we'll need to use APCu cache or otherwise respect $page, $start and $end.
387			// Better of course would be to move to a Symfony CachingHttpClient instead of Guzzle across the board.
388			static $pageviews;
389			if (isset($pageviews)) {
390			return $pageviews;
391			}
392
393			$title = rawurlencode(str_replace(' ', '_', $page->getTitle()));
394
395			if ($start instanceof DateTime) {
396			$start = $start->format('Ymd');
397			} else {
398			$start = (new DateTime($start))->format('Ymd');
399			}
400			if ($end instanceof DateTime) {
401			$end = $end->format('Ymd');
402			} else {
403			$end = (new DateTime($end))->format('Ymd');
404			}
405
406			$project = $page->getProject()->getDomain();
407
408			$url = 'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/' .
409			"$project/all-access/user/$title/daily/$start/$end";
410
411			try {
412			$res = $this->guzzle->request('GET', $url, [
413			// Five seconds should be plenty...
414			RequestOptions::CONNECT_TIMEOUT => 5,
415			]);
416			$pageviews = json_decode($res->getBody()->getContents(), true);
417			return $pageviews;
418			} catch (ServerException\|ConnectException $e) {
419			throw new BadGatewayException('api-error-wikimedia', ['Pageviews'], $e);
420			}
421			}
422
423			/**
424			* Get the full HTML content of the the page.
425			* @param Page $page
426			* @param int\|null $revId What revision to query for.
427			* @return string
428			*/
429			public function getHTMLContent(Page $page, ?int $revId = null): string
430			{
431			if ($this->isWMF) {
432			$domain = $page->getProject()->getDomain();
433			$url = "https://$domain/api/rest_v1/page/html/" . urlencode(str_replace(' ', '_', $page->getTitle()));
434			if (null !== $revId) {
435			$url .= "/$revId";
436			}
437			} else {
438			$url = $page->getUrl();
439			if (null !== $revId) {
440			$url .= "?oldid=$revId";
441			}
442			}
443
444			try {
445			return $this->guzzle->request('GET', $url)
446			->getBody()
447			->getContents();
448			} catch (ServerException\|ClientException $e) {
449			if ($e instanceof ClientException && $page->exists() && Response::HTTP_NOT_FOUND === $e->getCode()) {
450			// Sometimes the REST API throws 404s when the page does in fact exist.
451			throw new BadGatewayException('api-error-wikimedia', ['Wikimedia REST'], $e);
452			}
453			throw new BadGatewayException('api-error-wikimedia', ['Wikimedia REST'], $e);
454			}
455			}
456
457			/**
458			* Get the ID of the revision of a page at the time of the given DateTime.
459			* @param Page $page
460			* @param DateTime $date
461			* @return int
462			*/
463			public function getRevisionIdAtDate(Page $page, DateTime $date): int
464			{
465			$revisionTable = $page->getProject()->getTableName('revision');
466			$pageId = $page->getId();
467			$datestamp = $date->format('YmdHis');
468			$sql = "SELECT MAX(rev_id)
469			FROM $revisionTable
470			WHERE rev_timestamp <= $datestamp
471			AND rev_page = $pageId LIMIT 1;";
472			$resultQuery = $this->getProjectsConnection($page->getProject())
473			->executeQuery($sql);
474			return (int)$resultQuery->fetchOne();
475			}
476
477			/**
478			* Get HTML display titles of a set of pages (or the normal title if there's no display title).
479			* This will send t/50 API requests where t is the number of titles supplied.
480			* @param Project $project The project.
481			* @param string[] $pageTitles The titles to fetch.
482			* @return string[] Keys are the original supplied title, and values are the display titles.
483			* @static
484			*/
485			public function displayTitles(Project $project, array $pageTitles): array
486			{
487			$displayTitles = [];
488			$numPages = count($pageTitles);
489
490			for ($n = 0; $n < $numPages; $n += 50) {
491			$titleSlice = array_slice($pageTitles, $n, 50);
492			$res = $this->guzzle->request('GET', $project->getApiUrl(), ['query' => [
493			'action' => 'query',
494			'prop' => 'info\|pageprops',
495			'inprop' => 'displaytitle',
496			'titles' => join('\|', $titleSlice),
497			'format' => 'json',
498			]]);
499			$result = json_decode($res->getBody()->getContents(), true);
500
501			// Extract normalization info.
502			$normalized = [];
503			if (isset($result['query']['normalized'])) {
504			array_map(
505			function ($e) use (&$normalized): void {
506			$normalized[$e['to']] = $e['from'];
507			},
508			$result['query']['normalized']
509			);
510			}
511
512			// Match up the normalized titles with the display titles and the original titles.
513			foreach ($result['query']['pages'] as $pageInfo) {
514			$displayTitle = $pageInfo['pageprops']['displaytitle'] ?? $pageInfo['title'];
515			$origTitle = $normalized[$pageInfo['title']] ?? $pageInfo['title'];
516			$displayTitles[$origTitle] = $displayTitle;
517			}
518			}
519
520			return $displayTitles;
521			}
522			}
523

x-tools / xtools

Push — bad-gateway ( 773331 )

PageRepository::getHTMLContent() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like