phptek / silverstripe-exodus

src/Task/StaticSiteRewriteLinksTask.php

<?php

namespace PhpTek\Exodus\Task;

use PhpTek\Exodus\Model\FailedURLRewriteObject;
use PhpTek\Exodus\Model\FailedURLRewriteSummary;
use PhpTek\Exodus\Model\StaticSiteContentSource;
use PhpTek\Exodus\Model\StaticSiteImportDataObject;
use PhpTek\Exodus\Tool\StaticSiteLinkRewriter;
use PhpTek\Exodus\Tool\StaticSiteUtils;
use PhpTek\Exodus\Report\FailedURLRewriteReport;
use SilverStripe\Assets\File;
use SilverStripe\Dev\BuildTask;
use SilverStripe\Control\Director;
use SilverStripe\Control\Controller;
use SilverStripe\ORM\DataObject;
use SilverStripe\Versioned\Versioned;
use SilverStripe\Forms\FormField;

/**
 * Rewrites content-links found in `img.src` and `a.href` HTML element attributes,
 * which were originally imported via {@link StaticSiteImporter}.
 *
 * The task takes two arguments:
 *
 * - ImportID:
 *   Allows the rewriter to know which content to rewrite, when duplicate imports exist.
 *
 * - SourceID
 *   Allows the rewriter to fetch the correct content relative to the given source of scraped URLs.
 *
 * - SilentRun
 *   Allows the rewriter to run silently, without displaying anything to stdout. This is the
 *   default for when the task is run from the front-end.
 *
 * All rewrite failures are written to an individual {@link FailedURLRewriteObject} DataObject to power the
 * CMS {@link FailedURLRewriteReport}. See also {@link FailedURLRewriteSummary}.
 *
 * @author Sam Minnee <sam@silverstripe.com>
 * @author Russell Michell <russ@theruss.com>
 * @author Michael Parkhill <mike@silverstripe.com>
 * @package phptek/silverstripe-exodus
 */
class StaticSiteRewriteLinksTask extends BuildTask
{
    /**
     * An inexhaustive list of non http(s) URI schemes which we don't want to try to normalise.
     *
     * @see http://en.wikipedia.org/wiki/URI_scheme
     * @var array
     */
    private static $non_http_uri_schemes = [
        'mailto',
        'tel',
        'htp',
        'ftp',
        'ftps',
        'res',
        'skype',
        'ssh',
        'telnet',
        'gopher',
        'rtsp',
    ];

    /**
     *
     * @var string The prefix to use for the log-file summary.
     */
    private static $summary_prefix = 'Import No.';

    /**
     * @var string
     */
    protected $description = 'Rewrites imported links into SilverStripe compatible format.';

    /**
     *
     * @var string
     */
    public $currentPageID = null;

    /**
     * Stores the dodgy URLs for later analysis
     *
     * @var array
     */
    public $listFailedRewrites = [];

    /**
     * The ID number of the StaticSiteContentSource which has the links to be rewritten
     *
     * @var string
     */
    protected $contentSourceID;

    /**
     * The import identifier
     *
     * @var string
     */
    protected $contentImportID;

    /**
     *
     * @var
     */
    protected $silentRun;

    /**
     * The StaticSiteContentSource which has the links to be rewritten
     *
     * @var StaticSiteContentSource
     */
    protected $contentSource = null;

    /**
     *
     * @var string
     */
    protected $newLine = '';

    /**
     * Starts the task
     *
     * @param HTTPRequest $request The request parameter passed from the task initiator, browser or CLI

The type PhpTek\Exodus\Task\HTTPRequest was not found. Did you mean HTTPRequest? If so, make sure to prefix the type with \.

     * @return null | void
     */
    public function run($request)
    {
        $this->newLine = Director::is_cli() ? PHP_EOL : '<br/>';

        // Get the StaticSiteContentSource and Import ID from request parameters
        $this->contentSourceID = trim($request->getVar('SourceID') ?? '');
        $this->contentImportID = trim($request->getVar('ImportID') ?? '');
        $this->silentRun = trim($request->getVar('SilentRun') ?? '');

        if (!$this->checkInputs()) {
            return;
        }

        // Load imported objects and filter on ImportID, to know which links should be re-written.
        $pages = $this->contentSource->Pages()->filter('StaticSiteImportID', $this->contentImportID);
        $files = $this->contentSource->Files()->filter('StaticSiteImportID', $this->contentImportID);

        $this->printMessage("Processing Import: {$pages->count()} pages, {$files->count()} files", 'NOTICE');

        if ($pages->count() == 0) {
            $this->printMessage("Nothing to rewrite! Did you forget to run an import?", 'NOTICE');
        }

        $pageLookup = $pages;
        $fileLookup = $files->map('StaticSiteURL', 'ID');
        $task = $this;

        // Callback for URL rewriter, called from StaticSiteLinkRewriter and passed through $callback($url)
        $rewriter = StaticSiteLinkRewriter::create(function ($url) use ($pageLookup, $fileLookup, $task) {
            $origUrl = $url;
            $anchor = '';

            if (strpos($url, '#') !== false) {
                list($url, $anchor) = explode('#', $url, 2);
            }

            // Just return now if the url is un-rewritable
            if ($task->urlUnrewritable($url)) {
                // If it's being ignored, log it for a summary used in the CMS report.
                $task->pushFailedRewrite($task, $url);
                return;
            }

            // Strip the trailing slash, if any
            $url = substr($url, -1) == '/' ? rtrim($url, '/') : $url;

            // The keys to use when URL-matching in $pageLookup and $fileLookup
            $pageMapKey = Controller::join_links($task->contentSource->BaseUrl, parse_url($url, PHP_URL_PATH));

The property BaseUrl does not exist on PhpTek\Exodus\Model\StaticSiteContentSource. Since you implemented __get, consider adding a @property annotation.

            $fileMapKey = Controller::join_links($task->contentSource->BaseUrl, $origUrl);

            if (strlen($anchor)) {
                $task->printMessage("\tFound anchor: '#$anchor' (Removed for matching)");
            }

            $task->printMessage("\tPage-key: '$pageMapKey'");
            $task->printMessage("\tFile-key: '$fileMapKey'");

            $fileLookup = $fileLookup->toArray();

            /*
             * Rewrite SiteTree links by replacing the phpQuery processed Page-URL
             * with a CMS shortcode or anchor if one is found in the 'Content' field.
             */
            if ($siteTreeObject = $pageLookup->find('StaticSiteURL', $pageMapKey)) {
                $output = '[sitetree_link,id=' . $siteTreeObject->ID . ']';
                $task->printMessage("\tFound: SiteTree ID#" . $siteTreeObject->ID, null, $output);
                $anchorPattern = "<[\w]+\s+(name|id)=('|\")?" . $anchor . "('|\")?";

                if (strlen($anchor) && preg_match("#$anchorPattern#mi", $siteTreeObject->Content)) {
                    $output = "#$anchor";
                }

                return $output;
                // Rewrite Asset links by replacing phpQuery processed URLs with the appropriate filename.
            } elseif (isset($fileLookup[$fileMapKey]) && $fileID = $fileLookup[$fileMapKey]) {
                if ($file = DataObject::get_by_id(File::class, $fileID)) {
                    $output = $file->RelativeLink();
                    $task->printMessage("\tFound: File ID#" . $fileID, null, $output);

                    return $output;
                }
            } else {
                // Log failures for later analysis
                $this->pushFailedRewrite($task, $url);
                $task->printMessage("\tRewriter failed. See detail below:");
            }

            return $origUrl;
        });

        // Perform rewriting
        $changedFields = 0;

        foreach ($pages as $i => $page) {
            // For the rewriter, so it has some context for urls that couldn't be re-writen.
            $this->currentPageID = $page->ID;
            $url = $page->StaticSiteURL;

            // Get the schema that matches the page's legacy url
            // TODO should arg 2 for getSchemaForURL() be based on user-input and not baked-in?
            if ($schema = $this->contentSource->getSchemaForURL($url, 'text/html')) {
                $fields = [];

                foreach ($schema->ImportRules() as $rule) {
                    if (!$rule->PlainText) {
                        $fields[] = $rule->FieldName;
                    } else {
                        $this->printMessage("\tPlain-text enabled in import rule. Skipping.");
                    }
                }

                $fields = array_unique($fields);
            } else {
                $this->printMessage("\tNo schema found for {$page->URLSegment}", 'WARNING');
                continue;
            }

            $modified = false;

            foreach ($fields as $field) {
                $task->printMessage("START Rewriter for links in: '$url'");
                $newContent = $rewriter->rewriteInContent($page->$field);
                // Square-brackets are converted upstream, so change them back.
                // Leave this well-alone. Removing it will blank all imported content and titles!
                $fieldContent = str_replace(array('%5B', '%5D'), ['[', ']'], $newContent);

                // If rewrite succeeded, then the content returned should differ from the original
                if ($fieldContent != $page->$field) {
                    $changedFields++;
                    $this->printMessage("\tChanged field: '$field' on page: \"{$page->Title}\" ID: {$page->ID}");
                    $page->$field = $fieldContent;
                    $modified = true;
                } else {
                    $task->printMessage("\tNothing to rewrite");
                }

                $task->printMessage("END Rewriter for links in: '$url'");
            }

            if ($modified) {
                Versioned::set_reading_mode('Stage.Stage');
                $page->write();
                $page->publishRecursive();
            }
        }

        $newLine = $this->newLine;
        $this->printMessage("{$newLine}Complete.");
        $this->printMessage("Amended $changedFields content fields for {$pages->count()} pages and {$files->count()} files processed.");

        $msgNextSteps = " - Not all links will get fixed. It's recommended to also run a 3rd party link-checker over your imported content.";
        $msgSeeReport = " - Check the CMS \"" . singleton(FailedURLRewriteReport::class)->title() . "\" report for a summary of failed link-rewrites.";

        $this->printMessage("Tips:");
        $this->printMessage("{$newLine}$msgNextSteps");
        $this->printMessage($msgSeeReport);
        $this->writeFailedRewrites();
    }

    /**
     * Prints notices and warnings and aggregates them into two lists for later analysis,
     * depending on $level and whether you're using the CLI or a browser to run the task.
     *
     * @param string $message The message to log
     * @param string $level The log level, e.g. NOTICE or WARNING
     * @param string $url The url which was being re-written
     * @return mixed null|void
     */
    public function printMessage($message, $level = null, $url = null)
    {
        singleton(StaticSiteUtils::class)->log("$level$message$url", '', '');

        if ($this->silentRun) {
            return;
        }

        $url = ($url ? '(' . $url . ') ' : '');
        $level = ($level ? '[' . $level . '] ' : '');

        echo "$level$message$url" . PHP_EOL;
    }

    /**
     * Write failed rewrites to the {@link BadImportLog} for later analysis by users
     * via the CMS' Report admin.
     *
     * @return void
     * @todo What to do with report summaries when a task for the same import is re-run?
     */
    public function writeFailedRewrites()
    {
        $importID = 0;
        $postProcessed = [];

The assignment to $postProcessed is dead and can be removed.

        $uniq = ($this->listFailedRewrites);
        foreach ($uniq as $failure) {
            $importID = $failure['ImportID']; // Will be the same value each time

            /*
             * Prevent the same bad-link (for the same Import & container page)
             * being written.
             */
            $failureExists = DataObject::get(FailedURLRewriteObject::class)->filter(array(
                'ImportID' => $importID,
                'OrigUrl' => $failure['OrigUrl'],
                'ContainedInID' => $failure['ContainedInID']
            ));

            if ($failureExists->count() > 0) {
                continue;
            }

            $failedURLObj = FailedURLRewriteObject::create();
            $failedURLObj->BadLinkType = $failure['BadLinkType'];
            $failedURLObj->ImportID = $failure['ImportID'];
            $failedURLObj->ContainedInID = $failure['ContainedInID'];
            $failedURLObj->OrigUrl = $failure['OrigUrl'];
            $failedURLObj->write();
        }

        $summaryText = self::$summary_prefix . $this->contentImportID;
        foreach ($this->countFailureTypes() as $label => $payload) {
            $label = FormField::name_to_label($label) . ': ';
            $summaryText .= ' ' . $label . $payload['count'] . ' ' . $payload['desc'] . PHP_EOL;
        }

        // Write the summary, to be shown at the top of the report
        $summary = FailedURLRewriteSummary::create();
        $summary->Text = $summaryText;
        $summary->ImportID = $importID ? $importID : 0;
        $summary->write();
    }

    /**
     * Returns an array of totals of all the failed URLs, in different categories according to:
     * - No. Non $baseURL http(s) URLs
     * - No. Non http(s) URI schemes (e.g. mailto, tel etc)
     * - No. URLs not imported
     * - No. Junk URLs (i.e. those not matching any of the above)
     *
     * @return array
     * @todo too many URLs being collected in $this->listFailedRewrites
     */
    public function countFailureTypes()
    {
        $rawData = $this->listFailedRewrites;
        $countThirdParty = $countBadScheme = $countNotImported = $countJunk = $countUnknown = 0;
        foreach ($rawData as $data) {
            $url = $data['OrigUrl'];
            if ($this->linkIsJunk($url)) {
                ++$countJunk;
                continue;
            } elseif ($this->linkIsThirdParty($url)) {
                ++$countThirdParty;
                continue;
            } elseif ($this->linkIsBadScheme($url)) {
                ++$countBadScheme;
                continue;
            } elseif ($this->linkIsNotImported($url)) {
                ++$countNotImported;
                continue;
            } else {
                ++$countUnknown;
            }
        }
        return [
            'Total failed link rewrites'    => ['count' => count($rawData), 'desc' => ''],
            'ThirdParty'        => ['count' => $countThirdParty, 'desc' => '(Links to external websites)'],
            'BadScheme'         => ['count' => $countBadScheme, 'desc' => '(Links with bad/unimportable scheme)'],
            'NotImported'       => ['count' => $countNotImported, 'desc' => '(Links to pages or assets that were not imported)'],
            'Junk'                      => ['count' => $countJunk, 'desc' => '(Junk links)'],
            'Unknown'           => ['count' => $countUnknown, 'desc' => '(Not categorisable)'],
        ];
    }

    /**
     * Detects if a link is to a third-party website.
     *
     * @param string $link
     * @return boolean
     */
    public function linkIsThirdParty($link)
    {
        $link = trim($link);
        return (bool)preg_match("#^http(s)?://#", $link);
    }

    /**
     * Detects if a link uses an unsupported protocol (e.g. mailto, tel etc)
     *
     * @param string $link
     * @return boolean
     */
    public function linkIsBadScheme($link)
    {
        $nonHTTPSchemes = implode('|', self::$non_http_uri_schemes);
        $badScheme = preg_match("#^($nonHTTPSchemes):#", $link);
        $alreadyImported = $this->linkIsAlreadyRewritten($link);
        return (bool)($badScheme || $alreadyImported);
    }

    /**
     * After rewrite task is run, link doesn't match a valid CMS link shortcode.
     *
     * @param string $link
     * @return boolean
     */
    public function linkIsNotImported($link)
    {
        return (bool)(stristr($link, 'sitetree') === false && stristr($link, 'assets') === false);
    }

    /**
     * Detects if a link has already been re-written.
     *
     * @param string $link
     * @return boolean
     */
    public function linkIsAlreadyRewritten($link)
    {
        return (bool)(stristr($link, 'sitetree') !== false || stristr($link, 'assets') !== false);
    }

    /**
     * Link begins with non-legitimate character
     *
     * @param string $link
     * @return boolean
     */
    public function linkIsJunk($link)
    {
        return (bool)preg_match("#^[^\[\/a-zA-Z\d].+#", $link);
    }

    /**
     * What kind of bad link is $link? The returned string should match the ENUM
     * values on FailedURLRewriteObject
     *
     * @param string $link
     * @return string
     * @todo can we add a check for links with anchors to other pages?
     */
    public function badLinkType($link)
    {
        if ($this->linkIsJunk($link)) {
            return 'Junk';
        }
        if ($this->linkIsThirdParty($link)) {
            return 'ThirdParty';
        }
        if ($this->linkIsBadScheme($link)) {
            return 'BadScheme';
        }
        if ($this->linkIsNotImported($link)) {
            return 'NotImported';
        }
        return 'Unknown';
    }

    /**
     * Whether or not to ingore a URL. Returns true if a URL is either:
     *
     *  - An empty string
     *  - A non-HTTP scheme like an email link see: self::$non_http_uri_schemes
     *  - A CMS sitetree shortcode or file/image asset path, e.g. [sitetree_link,id=1234] or assets/Images/logo.gif
     *  - An absolute url, i.e. anything that begins with 'http'
     *
     * @param string $url A URL
     * @return boolean true if the url can be ignored
     */
    public function urlUnrewritable($url)
    {
        $url = trim($url);

        // Link is "Junk"
        if ($this->linkIsJunk($url)) {
            $this->printMessage("\tIgnoring: $url (Junk URL)");
            return true;
        }

        // Not an HTTP protocol
        if ($this->linkIsBadScheme($url)) {
            $this->printMessage("\tIgnoring: $url (Non-HTTP URL)");
            return true;
        }

        // Is external or an absolute url
        if ($this->linkIsThirdParty($url)) {
            $this->printMessage("\tIgnoring: $url (3rd party URL)");
            return true;
        }

        // Has already been processed
        if ($this->linkIsAlreadyRewritten($url)) {
            $this->printMessage("\tIgnoring: $url (CMS link, already converted)");
            return true;
        }
        return false;
    }

    /**
     * Set the ID number of the StaticSiteContentSource
     *
     * @param int $contentSourceID
     * @return void
     */
    public function setContentSourceID($contentSourceID)
    {
        $this->contentSourceID = $contentSourceID;
    }

    /**
     * Checks the user-passed data is cotia.
     *
     * @return boolean
     */
    public function checkInputs(): bool
    {
        $hasSid = ($this->contentSourceID && is_numeric($this->contentSourceID));
        $hasIid = ($this->contentImportID && is_numeric($this->contentImportID));

        if (!$hasSid || !$hasIid) {
            $this->printTaskInfo();
            return false;
        }

        // Load the content source using the passed content-source ID and Import ID
        $this->contentSource = (StaticSiteContentSource::get()->byID($this->contentSourceID));

$this->contentSourceID of type string is incompatible with the type integer expected by parameter $id of SilverStripe\ORM\DataList::byID().

        $contentImport = (StaticSiteImportDataObject::get()->byID($this->contentImportID));

        if (!$this->contentSource) {
            $this->printMessage("No content-source found via SourceID: " . $this->contentSourceID, 'WARNING');
            return false;
        }

        if (!$contentImport) {
            $this->printMessage("No content-import found via ImportID: " . $this->contentImportID, 'WARNING');
            return false;
        }

        return true;
    }

    /**
     * Prints information on the options available for running the task and
     * debugging and usage examples.
     *
     * @return void
     */
    public function printTaskInfo()
    {
        $msgFragment = (Director::is_cli() ? '' : '?') . 'SourceID=(number) ImportID=(number)';
        $this->printMessage("Choose a SourceID and an ImportID e.g. $msgFragment", 'WARNING');
        $newLine = $this->newLine;

        // List the content sources to prompt user for selection
        if ($contentSources = StaticSiteContentSource::get()) {
            $this->printMessage($newLine . 'Available content-sources:' . $newLine);
            foreach ($contentSources as $i => $contentSource) {
                $this->printMessage("\tdev/tasks/" . __CLASS__ . ' SourceID=' . $contentSource->ID . ' ImportID=<number>');
            }
            echo $newLine;
            if (Director::is_cli()) {
                $this->printMessage('Available command line options: ' . $newLine);
                $this->printMessage("\tSourceID=<number> \t\tThe ID of the original crawl.");
                $this->printMessage("\tImportID=<number> \t\tThe ID of the import to use.");
                $this->printMessage("\tSHOW=pages \tPrint the contents of the pages map.");
                $this->printMessage("\tSHOW=files \tPrint the contents of the files map.");
            }
            echo $newLine;
        }
    }

    /**
     * Build an array of failed URL rewrites for later reporting.
     *
     * @param StaticSiteRewriteLinksTask $obj
     * @param string $link
     * @return void
     */
    protected function pushFailedRewrite($obj, $link): void
    {
        array_push($obj->listFailedRewrites, [
            'OrigUrl' => $link,
            'ImportID' => $obj->contentImportID,
            'ContainedInID' => $obj->currentPageID,
            'BadLinkType' => $obj->badLinkType($link),
        ]);
    }
}