Tokenizer::getWords() - Code Metrics - Inspection of "Refactor fulltext search functions and class Doku_..." - splitbrain/dokuwiki - Measure and Improve Code Quality continuously with Scrutinizer

Failed Conditions

Pull Request — master (#2943)

by Andreas

created 2020-09-10 15:40 UTC

Tokenizer::getWords() B

↳ Parent: Tokenizer

Complexity

Conditions	11
Paths	108

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	11
nc	108
nop	2
dl	0
loc	40
rs	7.25
c	0
b	0
f	0

How to fix Complexity

<?php

namespace dokuwiki\Search;

use dokuwiki\Extension\Event;
use dokuwiki\Utf8;

// set the minimum token length to use in the index
// (note, this doesn't apply to numeric tokens)
const MINWORDLENGTH = 2;

/**
 * DokuWiki Tokenizer class (Singleton)
 */
class Tokenizer
{
    /** @var Tokenizer */
    protected static $instance = null;

    /** @var array $Stopwords Words that tokenizer ignores */
    protected $Stopwords;

    /** @var int $MinWordLength  minimum token length */
    protected $MinWordLength;

    /**
     * Tokenizer constructor. Singleton, thus protected!
     */
    protected function __construct()
    {
        // set the minimum token length to use in the index
        // (note, this doesn't apply to numeric tokens)
        $this->MinWordLength = (defined('IDX_MINWORDLENGTH'))
            ? IDX_MINWORDLENGTH
            : MINWORDLENGTH;

        $this->Stopwords = $this->getStopwords();
    }

    /**
     * Get new or existing singleton instance of the Tokenizer
     *
     * @return Tokenizer
     */
    public static function getInstance()
    {
        if (is_null(static::$instance)) {
            static::$instance = new static();
        }
        return static::$instance;
    }

    /**
     * Returns words that will be ignored
     *
     * @return array                list of stop words
     *
     * @author Tom N Harris <[email protected]>
     */
    public function getStopwords()
    {
        if (!isset($this->Stopwords)) {
            global $conf;
            $swFile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
            if (file_exists($swFile)) {
                $this->Stopwords = file($swFile, FILE_IGNORE_NEW_LINES);
            } else {
                $this->Stopwords = array();
           }
        }
        return $this->Stopwords;
    }

    /**
     * Returns minimum word length to be used in the index
     */
    public function getMinWordLength()
    {
        return $this->MinWordLength;
    }

    /**
     * Split the text into words for fulltext search
     *
     * @triggers INDEXER_TEXT_PREPARE
     * This event allows plugins to modify the text before it gets tokenized.
     * Plugins intercepting this event should also intercept INDEX_VERSION_GET
     *
     * @param string    $text   plain text
     * @param bool      $wc     are wildcards allowed?
     * @return array            list of words in the text
     *
     * @author Tom N Harris <[email protected]>
     * @author Andreas Gohr <[email protected]>
     */
    public function getWords($text, $wc=false)
    {
        $wc = ($wc) ? '' : '\*';

        // prepare the text to be tokenized
        $event = new Event('INDEXER_TEXT_PREPARE', $text);
        if ($event->advise_before(true)) {
            if (preg_match('/[^0-9A-Za-z ]/u', $text)) {
                $text = Utf8\Asian::separateAsianWords($text);
            }
        }
        $event->advise_after();
        unset($event);

        $text = strtr($text,
                       array(
                           "\r" => ' ',
                           "\n" => ' ',
                           "\t" => ' ',
                           "\xC2\xAD" => '', //soft-hyphen
                       )
                     );
        if (preg_match('/[^0-9A-Za-z ]/u', $text)) {
            $text = Utf8\Clean::stripspecials($text, ' ', '\._\-:'.$wc);
        }

        $wordlist = explode(' ', $text);
        foreach ($wordlist as $i => $word) {
            $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ?
                Utf8\PhpString::strtolower($word) : strtolower($word);
        }

        foreach ($wordlist as $i => $word) {
            if ((!is_numeric($word) && strlen($word) < $this->MinWordLength)
              || array_search($word, $this->getStopwords(), true) !== false) {
                unset($wordlist[$i]);
            }
        }
        return array_values($wordlist);
    }
}


1			<?php
2
3			namespace dokuwiki\Search;
4
5			use dokuwiki\Extension\Event;
6			use dokuwiki\Utf8;
7
8			// set the minimum token length to use in the index
9			// (note, this doesn't apply to numeric tokens)
10			const MINWORDLENGTH = 2;
11
12			/**
13			* DokuWiki Tokenizer class (Singleton)
14			*/
15			class Tokenizer
16			{
17			/** @var Tokenizer */
18			protected static $instance = null;
19
20			/** @var array $Stopwords Words that tokenizer ignores */
21			protected $Stopwords;
22
23			/** @var int $MinWordLength minimum token length */
24			protected $MinWordLength;
25
26			/**
27			* Tokenizer constructor. Singleton, thus protected!
28			*/
29			protected function __construct()
30			{
31			// set the minimum token length to use in the index
32			// (note, this doesn't apply to numeric tokens)
33			$this->MinWordLength = (defined('IDX_MINWORDLENGTH'))
34			? IDX_MINWORDLENGTH
35			: MINWORDLENGTH;
36
37			$this->Stopwords = $this->getStopwords();
38			}
39
40			/**
41			* Get new or existing singleton instance of the Tokenizer
42			*
43			* @return Tokenizer
44			*/
45			public static function getInstance()
46			{
47			if (is_null(static::$instance)) {
48			static::$instance = new static();
49			}
50			return static::$instance;
51			}
52
53			/**
54			* Returns words that will be ignored
55			*
56			* @return array list of stop words
57			*
58			* @author Tom N Harris <[email protected]>
59			*/
60			public function getStopwords()
61			{
62			if (!isset($this->Stopwords)) {
63			global $conf;
64			$swFile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
65			if (file_exists($swFile)) {
66			$this->Stopwords = file($swFile, FILE_IGNORE_NEW_LINES);
67			} else {
68			$this->Stopwords = array();
69			}
70			}
71			return $this->Stopwords;
72			}
73
74			/**
75			* Returns minimum word length to be used in the index
76			*/
77			public function getMinWordLength()
78			{
79			return $this->MinWordLength;
80			}
81
82			/**
83			* Split the text into words for fulltext search
84			*
85			* @triggers INDEXER_TEXT_PREPARE
86			* This event allows plugins to modify the text before it gets tokenized.
87			* Plugins intercepting this event should also intercept INDEX_VERSION_GET
88			*
89			* @param string $text plain text
90			* @param bool $wc are wildcards allowed?
91			* @return array list of words in the text
92			*
93			* @author Tom N Harris <[email protected]>
94			* @author Andreas Gohr <[email protected]>
95			*/
96			public function getWords($text, $wc=false)
97			{
98			$wc = ($wc) ? '' : '\*';
99
100			// prepare the text to be tokenized
101			$event = new Event('INDEXER_TEXT_PREPARE', $text);
102			if ($event->advise_before(true)) {
103			if (preg_match('/[^0-9A-Za-z ]/u', $text)) {
104			$text = Utf8\Asian::separateAsianWords($text);
105			}
106			}
107			$event->advise_after();
108			unset($event);
109
110			$text = strtr($text,
111			array(
112			"\r" => ' ',
113			"\n" => ' ',
114			"\t" => ' ',
115			"\xC2\xAD" => '', //soft-hyphen
116			)
117			);
118			if (preg_match('/[^0-9A-Za-z ]/u', $text)) {
119			$text = Utf8\Clean::stripspecials($text, ' ', '\._\-:'.$wc);
120			}
121
122			$wordlist = explode(' ', $text);
123			foreach ($wordlist as $i => $word) {
124			$wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ?
125			Utf8\PhpString::strtolower($word) : strtolower($word);
126			}
127
128			foreach ($wordlist as $i => $word) {
129			if ((!is_numeric($word) && strlen($word) < $this->MinWordLength)
130			\|\| array_search($word, $this->getStopwords(), true) !== false) {
131			unset($wordlist[$i]);
132			}
133			}
134			return array_values($wordlist);
135			}
136			}
137

splitbrain / dokuwiki

Pull Request — master (#2943)

Tokenizer::getWords() B

Complexity

Size

Duplication

Importance

How to fix Complexity

Long Method

Duplication Side-by-Side

Filter issues like