StaticSiteRewriteLinksTask::linkIsJunk() - Code Metrics - phptek/silverstripe-exodus - Measure and Improve Code Quality continuously with Scrutinizer

StaticSiteRewriteLinksTask::linkIsJunk() A
last analyzed 2023-04-18 19:31 UTC

↳ Parent: StaticSiteRewriteLinksTask
Complexity

Conditions	1
Paths	1
Size

Total Lines	3
Code Lines	1
Duplication

Lines	0
Ratio	0 %
Importance

Changes
Metric	Value
eloc	1
c	0
b	0
f	0
dl	0
loc	3
rs	10
cc	1
nc	1
nop	1
<?php

namespace PhpTek\Exodus\Task;

use PhpTek\Exodus\Model\FailedURLRewriteObject;
use PhpTek\Exodus\Model\FailedURLRewriteSummary;
use PhpTek\Exodus\Model\StaticSiteContentSource;
use PhpTek\Exodus\Model\StaticSiteImportDataObject;
use PhpTek\Exodus\Tool\StaticSiteLinkRewriter;
use PhpTek\Exodus\Tool\StaticSiteUtils;
use PhpTek\Exodus\Report\FailedURLRewriteReport;
use SilverStripe\Assets\File;
use SilverStripe\Dev\BuildTask;
use SilverStripe\Control\Director;
use SilverStripe\Control\Controller;
use SilverStripe\ORM\DataObject;
use SilverStripe\Versioned\Versioned;
use SilverStripe\Forms\FormField;

/**
 * Rewrites content-links found in `img.src` and `a.href` HTML element attributes,
 * which were originally imported via {@link StaticSiteImporter}.
 *
 * The task takes two arguments:
 *
 * - ImportID:
 *   Allows the rewriter to know which content to rewrite, when duplicate imports exist.
 *
 * - SourceID
 *   Allows the rewriter to fetch the correct content relative to the given source of scraped URLs.
 *
 * - SilentRun
 *   Allows the rewriter to run silently, without displaying anything to stdout. This is the
 *   default for when the task is run from the front-end.
 *
 * All rewrite failures are written to an individual {@link FailedURLRewriteObject} DataObject to power the
 * CMS {@link FailedURLRewriteReport}. See also {@link FailedURLRewriteSummary}.
 *
 * @author Sam Minnee <[email protected]>
 * @author Russell Michell <[email protected]>
 * @author Michael Parkhill <[email protected]>
 * @package phptek/silverstripe-exodus
 */
class StaticSiteRewriteLinksTask extends BuildTask
{
    /**
     * An inexhaustive list of non http(s) URI schemes which we don't want to try to normalise.
     *
     * @see http://en.wikipedia.org/wiki/URI_scheme
     * @var array
     */
    private static $non_http_uri_schemes = [
        'mailto',
        'tel',
        'htp',
        'ftp',
        'ftps',
        'res',
        'skype',
        'ssh',
        'telnet',
        'gopher',
        'rtsp',
    ];

    /**
     *
     * @var string The prefix to use for the log-file summary.
     */
    private static $summary_prefix = 'Import No.';

    /**
     * @var string
     */
    protected $description = 'Rewrites imported links into SilverStripe compatible format.';

    /**
     *
     * @var string
     */
    public $currentPageID = null;

    /**
     * Stores the dodgy URLs for later analysis
     *
     * @var array
     */
    public $listFailedRewrites = [];

    /**
     * The ID number of the StaticSiteContentSource which has the links to be rewritten
     *
     * @var string
     */
    protected $contentSourceID;

    /**
     * The import identifier
     *
     * @var string
     */
    protected $contentImportID;

    /**
     *
     * @var
     */
    protected $silentRun;

    /**
     * The StaticSiteContentSource which has the links to be rewritten
     *
     * @var StaticSiteContentSource
     */
    protected $contentSource = null;

    /**
     *
     * @var string
     */
    protected $newLine = '';

    /**
     * Starts the task
     *
     * @param HTTPRequest $request The request parameter passed from the task initiator, browser or CLI

     * @return null | void
     */
    public function run($request)
    {
        $this->newLine = Director::is_cli() ? PHP_EOL : '<br/>';

        // Get the StaticSiteContentSource and Import ID from request parameters
        $this->contentSourceID = trim($request->getVar('SourceID') ?? '');
        $this->contentImportID = trim($request->getVar('ImportID') ?? '');
        $this->silentRun = trim($request->getVar('SilentRun') ?? '');

        if (!$this->checkInputs()) {
            return;
        }

        // Load imported objects and filter on ImportID, to know which links should be re-written.
        $pages = $this->contentSource->Pages()->filter('StaticSiteImportID', $this->contentImportID);
        $files = $this->contentSource->Files()->filter('StaticSiteImportID', $this->contentImportID);

        $this->printMessage("Processing Import: {$pages->count()} pages, {$files->count()} files", 'NOTICE');

        if ($pages->count() == 0) {
            $this->printMessage("Nothing to rewrite! Did you forget to run an import?", 'NOTICE');
        }

        $pageLookup = $pages;
        $fileLookup = $files->map('StaticSiteURL', 'ID');
        $task = $this;

        // Callback for URL rewriter, called from StaticSiteLinkRewriter and passed through $callback($url)
        $rewriter = StaticSiteLinkRewriter::create(function ($url) use ($pageLookup, $fileLookup, $task) {
            $origUrl = $url;
            $anchor = '';

            if (strpos($url, '#') !== false) {
                list($url, $anchor) = explode('#', $url, 2);
            }

            // Just return now if the url is un-rewritable
            if ($task->urlUnrewritable($url)) {
                // If it's being ignored, log it for a summary used in the CMS report.
                $task->pushFailedRewrite($task, $url);
                return;
            }

            // Strip the trailing slash, if any
            $url = substr($url, -1) == '/' ? rtrim($url, '/') : $url;

            // The keys to use when URL-matching in $pageLookup and $fileLookup
            $pageMapKey = Controller::join_links($task->contentSource->BaseUrl, parse_url($url, PHP_URL_PATH));

            $fileMapKey = Controller::join_links($task->contentSource->BaseUrl, $origUrl);

            if (strlen($anchor)) {
                $task->printMessage("\tFound anchor: '#$anchor' (Removed for matching)");
            }

            $task->printMessage("\tPage-key: '$pageMapKey'");
            $task->printMessage("\tFile-key: '$fileMapKey'");

            $fileLookup = $fileLookup->toArray();

            /*
             * Rewrite SiteTree links by replacing the phpQuery processed Page-URL
             * with a CMS shortcode or anchor if one is found in the 'Content' field.
             */
            if ($siteTreeObject = $pageLookup->find('StaticSiteURL', $pageMapKey)) {
                $output = '[sitetree_link,id=' . $siteTreeObject->ID . ']';
                $task->printMessage("\tFound: SiteTree ID#" . $siteTreeObject->ID, null, $output);
                $anchorPattern = "<[\w]+\s+(name|id)=('|\")?" . $anchor . "('|\")?";

                if (strlen($anchor) && preg_match("#$anchorPattern#mi", $siteTreeObject->Content)) {
                    $output = "#$anchor";
                }

                return $output;
                // Rewrite Asset links by replacing phpQuery processed URLs with the appropriate filename.
            } elseif (isset($fileLookup[$fileMapKey]) && $fileID = $fileLookup[$fileMapKey]) {
                if ($file = DataObject::get_by_id(File::class, $fileID)) {
                    $output = $file->RelativeLink();
                    $task->printMessage("\tFound: File ID#" . $fileID, null, $output);

                    return $output;
                }
            } else {
                // Log failures for later analysis
                $this->pushFailedRewrite($task, $url);
                $task->printMessage("\tRewriter failed. See detail below:");
            }

            return $origUrl;
        });

        // Perform rewriting
        $changedFields = 0;

        foreach ($pages as $i => $page) {
            // For the rewriter, so it has some context for urls that couldn't be re-writen.
            $this->currentPageID = $page->ID;
            $url = $page->StaticSiteURL;

            // Get the schema that matches the page's legacy url
            // TODO should arg 2 for getSchemaForURL() be based on user-input and not baked-in?
            if ($schema = $this->contentSource->getSchemaForURL($url, 'text/html')) {
                $fields = [];

                foreach ($schema->ImportRules() as $rule) {
                    if (!$rule->PlainText) {
                        $fields[] = $rule->FieldName;
                    } else {
                        $this->printMessage("\tPlain-text enabled in import rule. Skipping.");
                    }
                }

                $fields = array_unique($fields);
            } else {
                $this->printMessage("\tNo schema found for {$page->URLSegment}", 'WARNING');
                continue;
            }

            $modified = false;

            foreach ($fields as $field) {
                $task->printMessage("START Rewriter for links in: '$url'");
                $newContent = $rewriter->rewriteInContent($page->$field);
                // Square-brackets are converted upstream, so change them back.
                // Leave this well-alone. Removing it will blank all imported content and titles!
                $fieldContent = str_replace(array('%5B', '%5D'), ['[', ']'], $newContent);

                // If rewrite succeeded, then the content returned should differ from the original
                if ($fieldContent != $page->$field) {
                    $changedFields++;
                    $this->printMessage("\tChanged field: '$field' on page: \"{$page->Title}\" ID: {$page->ID}");
                    $page->$field = $fieldContent;
                    $modified = true;
                } else {
                    $task->printMessage("\tNothing to rewrite");
                }

                $task->printMessage("END Rewriter for links in: '$url'");
            }

            if ($modified) {
                Versioned::set_reading_mode('Stage.Stage');
                $page->write();
                $page->publishRecursive();
            }
        }

        $newLine = $this->newLine;
        $this->printMessage("{$newLine}Complete.");
        $this->printMessage("Amended $changedFields content fields for {$pages->count()} pages and {$files->count()} files processed.");

        $msgNextSteps = " - Not all links will get fixed. It's recommended to also run a 3rd party link-checker over your imported content.";
        $msgSeeReport = " - Check the CMS \"" . singleton(FailedURLRewriteReport::class)->title() . "\" report for a summary of failed link-rewrites.";

        $this->printMessage("Tips:");
        $this->printMessage("{$newLine}$msgNextSteps");
        $this->printMessage($msgSeeReport);
        $this->writeFailedRewrites();
    }

    /**
     * Prints notices and warnings and aggregates them into two lists for later analysis,
     * depending on $level and whether you're using the CLI or a browser to run the task.
     *
     * @param string $message The message to log
     * @param string $level The log level, e.g. NOTICE or WARNING
     * @param string $url The url which was being re-written
     * @return mixed null|void
     */
    public function printMessage($message, $level = null, $url = null)
    {
        singleton(StaticSiteUtils::class)->log("$level$message$url", '', '');

        if ($this->silentRun) {
            return;
        }

        $url = ($url ? '(' . $url . ') ' : '');
        $level = ($level ? '[' . $level . '] ' : '');

        echo "$level$message$url" . PHP_EOL;
    }

    /**
     * Write failed rewrites to the {@link BadImportLog} for later analysis by users
     * via the CMS' Report admin.
     *
     * @return void
     * @todo What to do with report summaries when a task for the same import is re-run?
     */
    public function writeFailedRewrites()
    {
        $importID = 0;
        $postProcessed = [];

        $uniq = ($this->listFailedRewrites);
        foreach ($uniq as $failure) {
            $importID = $failure['ImportID']; // Will be the same value each time

            /*
             * Prevent the same bad-link (for the same Import & container page)
             * being written.
             */
            $failureExists = DataObject::get(FailedURLRewriteObject::class)->filter(array(
                'ImportID' => $importID,
                'OrigUrl' => $failure['OrigUrl'],
                'ContainedInID' => $failure['ContainedInID']
            ));

            if ($failureExists->count() > 0) {
                continue;
            }

            $failedURLObj = FailedURLRewriteObject::create();
            $failedURLObj->BadLinkType = $failure['BadLinkType'];
            $failedURLObj->ImportID = $failure['ImportID'];
            $failedURLObj->ContainedInID = $failure['ContainedInID'];
            $failedURLObj->OrigUrl = $failure['OrigUrl'];
            $failedURLObj->write();
        }

        $summaryText = self::$summary_prefix . $this->contentImportID;
        foreach ($this->countFailureTypes() as $label => $payload) {
            $label = FormField::name_to_label($label) . ': ';
            $summaryText .= ' ' . $label . $payload['count'] . ' ' . $payload['desc'] . PHP_EOL;
        }

        // Write the summary, to be shown at the top of the report
        $summary = FailedURLRewriteSummary::create();
        $summary->Text = $summaryText;
        $summary->ImportID = $importID ? $importID : 0;
        $summary->write();
    }

    /**
     * Returns an array of totals of all the failed URLs, in different categories according to:
     * - No. Non $baseURL http(s) URLs
     * - No. Non http(s) URI schemes (e.g. mailto, tel etc)
     * - No. URLs not imported
     * - No. Junk URLs (i.e. those not matching any of the above)
     *
     * @return array
     * @todo too many URLs being collected in $this->listFailedRewrites
     */
    public function countFailureTypes()
    {
        $rawData = $this->listFailedRewrites;
        $countThirdParty = $countBadScheme = $countNotImported = $countJunk = $countUnknown = 0;
        foreach ($rawData as $data) {
            $url = $data['OrigUrl'];
            if ($this->linkIsJunk($url)) {
                ++$countJunk;
                continue;
            } elseif ($this->linkIsThirdParty($url)) {
                ++$countThirdParty;
                continue;
            } elseif ($this->linkIsBadScheme($url)) {
                ++$countBadScheme;
                continue;
            } elseif ($this->linkIsNotImported($url)) {
                ++$countNotImported;
                continue;
            } else {
                ++$countUnknown;
            }
        }
        return [
            'Total failed link rewrites'    => ['count' => count($rawData), 'desc' => ''],
            'ThirdParty'        => ['count' => $countThirdParty, 'desc' => '(Links to external websites)'],
            'BadScheme'         => ['count' => $countBadScheme, 'desc' => '(Links with bad/unimportable scheme)'],
            'NotImported'       => ['count' => $countNotImported, 'desc' => '(Links to pages or assets that were not imported)'],
            'Junk'                      => ['count' => $countJunk, 'desc' => '(Junk links)'],
            'Unknown'           => ['count' => $countUnknown, 'desc' => '(Not categorisable)'],
        ];
    }

    /**
     * Detects if a link is to a third-party website.
     *
     * @param string $link
     * @return boolean
     */
    public function linkIsThirdParty($link)
    {
        $link = trim($link);
        return (bool)preg_match("#^http(s)?://#", $link);
    }

    /**
     * Detects if a link uses an unsupported protocol (e.g. mailto, tel etc)
     *
     * @param string $link
     * @return boolean
     */
    public function linkIsBadScheme($link)
    {
        $nonHTTPSchemes = implode('|', self::$non_http_uri_schemes);
        $badScheme = preg_match("#^($nonHTTPSchemes):#", $link);
        $alreadyImported = $this->linkIsAlreadyRewritten($link);
        return (bool)($badScheme || $alreadyImported);
    }

    /**
     * After rewrite task is run, link doesn't match a valid CMS link shortcode.
     *
     * @param string $link
     * @return boolean
     */
    public function linkIsNotImported($link)
    {
        return (bool)(stristr($link, 'sitetree') === false && stristr($link, 'assets') === false);
    }

    /**
     * Detects if a link has already been re-written.
     *
     * @param string $link
     * @return boolean
     */
    public function linkIsAlreadyRewritten($link)
    {
        return (bool)(stristr($link, 'sitetree') !== false || stristr($link, 'assets') !== false);
    }

    /**
     * Link begins with non-legitimate character
     *
     * @param string $link
     * @return boolean
     */
    public function linkIsJunk($link)
    {
        return (bool)preg_match("#^[^\[\/a-zA-Z\d].+#", $link);
    }

    /**
     * What kind of bad link is $link? The returned string should match the ENUM
     * values on FailedURLRewriteObject
     *
     * @param string $link
     * @return string
     * @todo can we add a check for links with anchors to other pages?
     */
    public function badLinkType($link)
    {
        if ($this->linkIsJunk($link)) {
            return 'Junk';
        }
        if ($this->linkIsThirdParty($link)) {
            return 'ThirdParty';
        }
        if ($this->linkIsBadScheme($link)) {
            return 'BadScheme';
        }
        if ($this->linkIsNotImported($link)) {
            return 'NotImported';
        }
        return 'Unknown';
    }

    /**
     * Whether or not to ingore a URL. Returns true if a URL is either:
     *
     *  - An empty string
     *  - A non-HTTP scheme like an email link see: self::$non_http_uri_schemes
     *  - A CMS sitetree shortcode or file/image asset path, e.g. [sitetree_link,id=1234] or assets/Images/logo.gif
     *  - An absolute url, i.e. anything that begins with 'http'
     *
     * @param string $url A URL
     * @return boolean true if the url can be ignored
     */
    public function urlUnrewritable($url)
    {
        $url = trim($url);

        // Link is "Junk"
        if ($this->linkIsJunk($url)) {
            $this->printMessage("\tIgnoring: $url (Junk URL)");
            return true;
        }

        // Not an HTTP protocol
        if ($this->linkIsBadScheme($url)) {
            $this->printMessage("\tIgnoring: $url (Non-HTTP URL)");
            return true;
        }

        // Is external or an absolute url
        if ($this->linkIsThirdParty($url)) {
            $this->printMessage("\tIgnoring: $url (3rd party URL)");
            return true;
        }

        // Has already been processed
        if ($this->linkIsAlreadyRewritten($url)) {
            $this->printMessage("\tIgnoring: $url (CMS link, already converted)");
            return true;
        }
        return false;
    }

    /**
     * Set the ID number of the StaticSiteContentSource
     *
     * @param int $contentSourceID
     * @return void
     */
    public function setContentSourceID($contentSourceID)
    {
        $this->contentSourceID = $contentSourceID;
    }

    /**
     * Checks the user-passed data is cotia.
     *
     * @return boolean
     */
    public function checkInputs(): bool
    {
        $hasSid = ($this->contentSourceID && is_numeric($this->contentSourceID));
        $hasIid = ($this->contentImportID && is_numeric($this->contentImportID));

        if (!$hasSid || !$hasIid) {
            $this->printTaskInfo();
            return false;
        }

        // Load the content source using the passed content-source ID and Import ID
        $this->contentSource = (StaticSiteContentSource::get()->byID($this->contentSourceID));
        $this->contentSource = (StaticSiteContentSource::get()->byID(/** @scrutinizer ignore-type */ $this->contentSourceID));
        $contentImport = (StaticSiteImportDataObject::get()->byID($this->contentImportID));

        if (!$this->contentSource) {
            $this->printMessage("No content-source found via SourceID: " . $this->contentSourceID, 'WARNING');
            return false;
        }

        if (!$contentImport) {
            $this->printMessage("No content-import found via ImportID: " . $this->contentImportID, 'WARNING');
            return false;
        }

        return true;
    }

    /**
     * Prints information on the options available for running the task and
     * debugging and usage examples.
     *
     * @return void
     */
    public function printTaskInfo()
    {
        $msgFragment = (Director::is_cli() ? '' : '?') . 'SourceID=(number) ImportID=(number)';
        $this->printMessage("Choose a SourceID and an ImportID e.g. $msgFragment", 'WARNING');
        $newLine = $this->newLine;

        // List the content sources to prompt user for selection
        if ($contentSources = StaticSiteContentSource::get()) {
            $this->printMessage($newLine . 'Available content-sources:' . $newLine);
            foreach ($contentSources as $i => $contentSource) {
                $this->printMessage("\tdev/tasks/" . __CLASS__ . ' SourceID=' . $contentSource->ID . ' ImportID=<number>');
            }
            echo $newLine;
            if (Director::is_cli()) {
                $this->printMessage('Available command line options: ' . $newLine);
                $this->printMessage("\tSourceID=<number> \t\tThe ID of the original crawl.");
                $this->printMessage("\tImportID=<number> \t\tThe ID of the import to use.");
                $this->printMessage("\tSHOW=pages \tPrint the contents of the pages map.");
                $this->printMessage("\tSHOW=files \tPrint the contents of the files map.");
            }
            echo $newLine;
        }
    }

    /**
     * Build an array of failed URL rewrites for later reporting.
     *
     * @param StaticSiteRewriteLinksTask $obj
     * @param string $link
     * @return void
     */
    protected function pushFailedRewrite($obj, $link): void
    {
        array_push($obj->listFailedRewrites, [
            'OrigUrl' => $link,
            'ImportID' => $obj->contentImportID,
            'ContainedInID' => $obj->currentPageID,
            'BadLinkType' => $obj->badLinkType($link),
        ]);
    }
}

phptek / silverstripe-exodus

StaticSiteRewriteLinksTask::linkIsJunk() A last analyzed 2023-04-18 19:31 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

StaticSiteRewriteLinksTask::linkIsJunk() A
last analyzed 2023-04-18 19:31 UTC
