andreskrey\Readability\Readability::rateNodes()

Complexity

Conditions	50
Paths	> 20000

Size

Total Lines	285
Code Lines	130

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	0	Features	1

<?php
    private function rateNodes($nodes)
    {
        $this->logger->info('[Rating] Rating nodes...');

        $candidates = [];

        /** @var DOMElement $node */
        foreach ($nodes as $node) {
            if (is_null($node->parentNode)) {
                continue;
            }

            // Discard nodes with less than 25 characters, without blank space
            if (mb_strlen($node->getTextContent(true)) < 25) {
                continue;
            }

            $ancestors = $node->getNodeAncestors();

            // Exclude nodes with no ancestor
            if (count($ancestors) === 0) {
                continue;
            }

            // Start with a point for the paragraph itself as a base.
            $contentScore = 1;

            // Add points for any commas within this paragraph.
            $contentScore += count(explode(',', $node->getTextContent(true)));

            // For every 100 characters in this paragraph, add another point. Up to 3 points.
            $contentScore += min(floor(mb_strlen($node->getTextContent(true)) / 100), 3);

            $this->logger->debug(sprintf('[Rating] Node score %s, content: \'%s\'', $contentScore, substr($node->nodeValue, 0, 128)));

            /** @var $ancestor DOMElement */
            foreach ($ancestors as $level => $ancestor) {
                $this->logger->debug('[Rating] Found ancestor, initializing and adding it as a candidate...');
                if (!$ancestor->isInitialized()) {
                    $ancestor->initializeNode($this->configuration->getWeightClasses());
                    $candidates[] = $ancestor;
                }

                /*
                 * Node score divider:
                 *  - parent:             1 (no division)
                 *  - grandparent:        2
                 *  - great grandparent+: ancestor level * 3
                 */

                if ($level === 0) {
                    $scoreDivider = 1;
                } elseif ($level === 1) {
                    $scoreDivider = 2;
                } else {
                    $scoreDivider = $level * 3;
                }

                $currentScore = $ancestor->contentScore;
                $ancestor->contentScore = $currentScore + ($contentScore / $scoreDivider);

                $this->logger->debug(sprintf('[Rating] Ancestor score %s, value: \'%s\'', $ancestor->contentScore, substr($ancestor->nodeValue, 0, 128)));
            }
        }

        /*
         * After we've calculated scores, loop through all of the possible
         * candidate nodes we found and find the one with the highest score.
         */

        $topCandidates = [];
        foreach ($candidates as $candidate) {

            /*
             * Scale the final candidates score based on link density. Good content
             * should have a relatively small link density (5% or less) and be mostly
             * unaffected by this operation.
             */

            $candidate->contentScore = $candidate->contentScore * (1 - $candidate->getLinkDensity());

            for ($i = 0; $i < $this->configuration->getMaxTopCandidates(); $i++) {
                $aTopCandidate = isset($topCandidates[$i]) ? $topCandidates[$i] : null;

                if (!$aTopCandidate || $candidate->contentScore > $aTopCandidate->contentScore) {
                    array_splice($topCandidates, $i, 0, [$candidate]);
                    if (count($topCandidates) > $this->configuration->getMaxTopCandidates()) {
                        array_pop($topCandidates);
                    }
                    break;
                }
            }
        }

        $topCandidate = isset($topCandidates[0]) ? $topCandidates[0] : null;
        $parentOfTopCandidate = null;

        /*
         * If we still have no top candidate, just use the body as a last resort.
         * We also have to copy the body node so it is something we can modify.
         */

        if ($topCandidate === null || $topCandidate->nodeName === 'body') {
            $this->logger->info('[Rating] No top candidate found or top candidate is the body tag. Moving all child nodes to a new DIV node.');

            // Move all of the page's children into topCandidate
            $topCandidate = new DOMDocument('1.0', 'utf-8');
            $topCandidate->encoding = 'UTF-8';
            $topCandidate->appendChild($topCandidate->createElement('div', ''));
            $kids = $this->dom->getElementsByTagName('body')->item(0)->childNodes;

            // Cannot be foreached, don't ask me why.
            for ($i = 0; $i < $kids->length; $i++) {
                $import = $topCandidate->importNode($kids->item($i), true);
                $topCandidate->firstChild->appendChild($import);
            }

            // Candidate must be created using firstChild to grab the DOMElement instead of the DOMDocument.
            $topCandidate = $topCandidate->firstChild;
        } elseif ($topCandidate) {
            $this->logger->info(sprintf('[Rating] Found top candidate, score: %s', $topCandidate->contentScore));
            // Find a better top candidate node if it contains (at least three) nodes which belong to `topCandidates` array
            // and whose scores are quite closed with current `topCandidate` node.
            $alternativeCandidateAncestors = [];
            for ($i = 1; $i < count($topCandidates); $i++) {
                // In some cases we may end up with a top candidate with zero content score. To avoid dividing by zero
                // we have to use max() and replace zero with a low value like 0.1
                if ($topCandidates[$i]->contentScore / max($topCandidate->contentScore, 0.1) >= 0.75) {
                    array_push($alternativeCandidateAncestors, $topCandidates[$i]->getNodeAncestors(false));
                }
            }

            $MINIMUM_TOPCANDIDATES = 3;
            if (count($alternativeCandidateAncestors) >= $MINIMUM_TOPCANDIDATES) {
                $parentOfTopCandidate = $topCandidate->parentNode;

                // Check if we are actually dealing with a DOMNode and not a DOMDocument node or higher
                while ($parentOfTopCandidate->nodeName !== 'body' && $parentOfTopCandidate->nodeType === XML_ELEMENT_NODE) {
                    $listsContainingThisAncestor = 0;
                    for ($ancestorIndex = 0; $ancestorIndex < count($alternativeCandidateAncestors) && $listsContainingThisAncestor < $MINIMUM_TOPCANDIDATES; $ancestorIndex++) {
                        $listsContainingThisAncestor += (int)in_array($parentOfTopCandidate, $alternativeCandidateAncestors[$ancestorIndex]);
                    }
                    if ($listsContainingThisAncestor >= $MINIMUM_TOPCANDIDATES) {
                        $topCandidate = $parentOfTopCandidate;
                        break;
                    }
                    $parentOfTopCandidate = $parentOfTopCandidate->parentNode;
                }
            }

            /*
             * Because of our bonus system, parents of candidates might have scores
             * themselves. They get half of the node. There won't be nodes with higher
             * scores than our topCandidate, but if we see the score going *up* in the first
             * few steps up the tree, that's a decent sign that there might be more content
             * lurking in other places that we want to unify in. The sibling stuff
             * below does some of that - but only if we've looked high enough up the DOM
             * tree.
             */

            $parentOfTopCandidate = $topCandidate->parentNode;
            $lastScore = $topCandidate->contentScore;

            // The scores shouldn't get too low.
            $scoreThreshold = $lastScore / 3;

            /* @var DOMElement $parentOfTopCandidate */
            while ($parentOfTopCandidate->nodeName !== 'body') {
                $parentScore = $parentOfTopCandidate->contentScore;
                if ($parentScore < $scoreThreshold) {
                    break;
                }

                if ($parentScore > $lastScore) {
                    // Alright! We found a better parent to use.
                    $topCandidate = $parentOfTopCandidate;
                    $this->logger->info('[Rating] Found a better top candidate.');
                    break;
                }
                $lastScore = $parentOfTopCandidate->contentScore;
                $parentOfTopCandidate = $parentOfTopCandidate->parentNode;
            }

            // If the top candidate is the only child, use parent instead. This will help sibling
            // joining logic when adjacent content is actually located in parent's sibling node.
            $parentOfTopCandidate = $topCandidate->parentNode;
            while ($parentOfTopCandidate->nodeName !== 'body' && count(NodeUtility::filterTextNodes($parentOfTopCandidate->childNodes)) === 1) {
                $topCandidate = $parentOfTopCandidate;
                $parentOfTopCandidate = $topCandidate->parentNode;
            }
        }

        /*
         * Now that we have the top candidate, look through its siblings for content
         * that might also be related. Things like preambles, content split by ads
         * that we removed, etc.
         */

        $this->logger->info('[Rating] Creating final article content document...');

        $articleContent = new DOMDocument('1.0', 'utf-8');
        $articleContent->createElement('div');

        $siblingScoreThreshold = max(10, $topCandidate->contentScore * 0.2);
        // Keep potential top candidate's parent node to try to get text direction of it later.
        $parentOfTopCandidate = $topCandidate->parentNode;
        $siblings = $parentOfTopCandidate->childNodes;

        $hasContent = false;

        $this->logger->info('[Rating] Adding top candidate siblings...');

        /* @var DOMElement $sibling */
        // Can't foreach here because down there we might change the tag name and that causes the foreach to skip items
        for ($i = 0; $i < $siblings->length; $i++) {
            $sibling = $siblings[$i];
            $append = false;

            if ($sibling === $topCandidate) {
                $this->logger->debug('[Rating] Sibling is equal to the top candidate, adding to the final article...');

                $append = true;
            } else {
                $contentBonus = 0;

                // Give a bonus if sibling nodes and top candidates have the example same classname
                if ($sibling->getAttribute('class') === $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') !== '') {
                    $contentBonus += $topCandidate->contentScore * 0.2;
                }
                if ($sibling->contentScore + $contentBonus >= $siblingScoreThreshold) {
                    $append = true;
                } elseif ($sibling->nodeName === 'p') {
                    $linkDensity = $sibling->getLinkDensity();
                    $nodeContent = $sibling->getTextContent(true);

                    if (mb_strlen($nodeContent) > 80 && $linkDensity < 0.25) {
                        $append = true;
                    } elseif ($nodeContent && mb_strlen($nodeContent) < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent)) {
                        $append = true;
                    }
                }
            }

            if ($append) {
                $this->logger->debug(sprintf('[Rating] Appending sibling to final article, content is: \'%s\'', substr($sibling->nodeValue, 0, 128)));

                $hasContent = true;

                if (!in_array(strtolower($sibling->nodeName), $this->alterToDIVExceptions)) {
                    /*
                     * We have a node that isn't a common block level element, like a form or td tag.
                     * Turn it into a div so it doesn't get filtered out later by accident.
                     */
                    $sibling = NodeUtility::setNodeTag($sibling, 'div');
                }

                $import = $articleContent->importNode($sibling, true);
                $articleContent->appendChild($import);

                /*
                 * No node shifting needs to be check because when calling getChildren, an array is made with the
                 * children of the parent node, instead of using the DOMElement childNodes function, which, when used
                 * along with appendChild, would shift the nodes position and the current foreach will behave in
                 * unpredictable ways.
                 */
            }
        }

        $articleContent = $this->prepArticle($articleContent);

        if ($hasContent) {
            // Find out text direction from ancestors of final top candidate.
            $ancestors = array_merge([$parentOfTopCandidate, $topCandidate], $parentOfTopCandidate->getNodeAncestors());
            foreach ($ancestors as $ancestor) {
                $articleDir = $ancestor->getAttribute('dir');
                if ($articleDir) {
                    $this->setDirection($articleDir);
                    $this->logger->debug(sprintf('[Rating] Found article direction: %s', $articleDir));
                    break;
                }
            }

            return $articleContent;
        } else {
            return false;
        }