FulltextIndex::clear() - Code Metrics - Inspection of "Refactor fulltext search functions and class Doku_..." - splitbrain/dokuwiki - Measure and Improve Code Quality continuously with Scrutinizer

Failed Conditions

Pull Request — master (#2943)

unknown

created 2020-11-10 10:35 UTC

FulltextIndex::clear() A

↳ Parent: FulltextIndex

Complexity

Conditions	4
Paths	8

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	4
nc	8
nop	1
dl	0
loc	17
rs	9.7
c	0
b	0
f	0

<?php

namespace dokuwiki\Search;

use dokuwiki\Search\Exception\IndexAccessException;
use dokuwiki\Search\Exception\IndexLockException;
use dokuwiki\Search\Exception\IndexWriteException;
use dokuwiki\Search\Tokenizer;
use dokuwiki\Utf8;

/**
 * Class DokuWiki Fulltext Index
 *
 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
 * @author     Andreas Gohr <[email protected]>
 * @author Tom N Harris <[email protected]>
 */
class FulltextIndex extends AbstractIndex
{
    // numeric page id to be added to or deleted from the Fulltext index
    protected $pageID;

    /**
     * FulltextIndex constructor
     *
     * @param string|int $page a page name or numeric page id
     */
    public function __construct($page = null)
    {
        if (isset($page)) {
            $this->pageID = is_int($page) ? $page : $this->getPID($page);
        }
    }

    /**
     * Measure the length of a string
     * Differs from strlen in handling of asian characters.
     *
     * @author Tom N Harris <[email protected]>
     *
     * @param string $w
     * @return int
     */
    public function wordlen($w)
    {
        $l = strlen($w);
        // If left alone, all chinese "words" will get put into w3.idx
        // So the "length" of a "word" is faked
        if (preg_match_all('/[\xE2-\xEF]/', $w, $leadbytes)) {
            foreach ($leadbytes[0] as $b) {
                $l += ord($b) - 0xE1;
            }
        }
        return $l;
    }

    /**
     * Adds the contents of a page to the fulltext index
     *
     * The added text replaces previous words for the same page.
     * An empty value erases the page.
     *
     * @param string $text the body of the page
     * @param bool $requireLock should be false only if the caller is resposible for index lock
     * @return bool  if the function completed successfully
     *
     * @throws IndexAccessException
     * @throws IndexLockException
     * @throws IndexWriteException
     * @author Andreas Gohr <[email protected]>
     * @author Tom N Harris <[email protected]>
     */
    public function addWords($text, $requireLock = true)
    {
        // load known documents
        if (!isset($this->pageID)) {
            throw new IndexAccessException('Indexer: page unknown to addWords');
        } else {
            $pid = $this->pageID;
        }
 
        if ($requireLock) $this->lock();

        $pagewords = array();
        // get word usage in page
        $words = $this->getWords($text);

        foreach (array_keys($words) as $wlen) {
            $index = $this->getIndex('i', $wlen);
            foreach ($words[$wlen] as $wid => $freq) {
                $idx = ($wid < count($index)) ? $index[$wid] : '';
                $index[$wid] = $this->updateTuple($idx, $pid, $freq);
                $pagewords[] = "{$wlen}*{$wid}";
            }
            $this->saveIndex('i', $wlen, $index);
        }

        // Remove obsolete index entries
        $pageword_idx = $this->getIndexKey('pageword', '', $pid);
        if ($pageword_idx !== '') {
            $oldwords = explode(':', $pageword_idx);
            $delwords = array_diff($oldwords, $pagewords);
            $upwords = array();
            foreach ($delwords as $word) {
                if ($word != '') {
                    list($wlen, $wid) = explode('*', $word);
                    $wid = (int)$wid;
                    $upwords[$wlen][] = $wid;
                }
            }
            foreach ($upwords as $wlen => $widx) {
                $index = $this->getIndex('i', $wlen);
                foreach ($widx as $wid) {
                    $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
                }
                $this->saveIndex('i', $wlen, $index);
            }
        }
        // Save the reverse index
        $pageword_idx = implode(':', $pagewords);
        $this->saveIndexKey('pageword', '', $pid, $pageword_idx);

        if ($requireLock) $this->unlock();
        return true;
    }

    /**
     * Split the words in a page and add them to the index
     *
     * @param string $text content of the page
     * @return array  list of word IDs and number of times used, false on errors
     *
     * @throws IndexWriteException
     * @author Andreas Gohr <[email protected]>
     * @author Christopher Smith <[email protected]>
     * @author Tom N Harris <[email protected]>
     */
    protected function getWords($text)
    {
        $tokens = Tokenizer::getWords($text);
        $tokens = array_count_values($tokens);  // count the frequency of each token

        $words = array();
        foreach ($tokens as $w => $c) {
            $l = $this->wordlen($w);
            if (isset($words[$l])) {
                $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0);
            } else {
                $words[$l] = array($w => $c);
            }
        }

        // arrive here with $words = array(wordlen => array(word => frequency))
        $word_idx_modified = false;
        $index = array();   //resulting index
        foreach (array_keys($words) as $wlen) {
            $word_idx = $this->getIndex('w', $wlen);
            foreach ($words[$wlen] as $word => $freq) {
                $word = (string)$word;
                $wid = array_search($word, $word_idx, true);
                if ($wid === false) {
                    $wid = count($word_idx);
                    $word_idx[] = $word;
                    $word_idx_modified = true;
                }
                if (!isset($index[$wlen])) {
                    $index[$wlen] = array();
                }
                $index[$wlen][$wid] = $freq;
            }
            // save back the word index
            if ($word_idx_modified) $this->saveIndex('w', $wlen, $word_idx);
        }

        return $index;
    }

    /**
     * Delete the contents of a page to the fulltext index
     *
     * @param bool $requireLock should be false only if the caller is resposible for index lock
     * @return bool  If renaming the value has been successful, false on error
     *
     * @throws IndexAccessException
     * @throws IndexLockException
     * @throws IndexWriteException
     * @author Satoshi Sahara <[email protected]>
     * @author Tom N Harris <[email protected]>
     */
    public function deleteWords($requireLock = true)
    {
        // load known documents
        if (!isset($this->pageID)) {
            throw new IndexAccessException('Indexer: page unknown to deleteWords');
        } else {
            $pid = $this->pageID;
        }

        if ($requireLock) $this->lock();

        // remove obsolete index entries
        $pageword_idx = $this->getIndexKey('pageword', '', $pid);
        if ($pageword_idx !== '') {
            $delwords = explode(':', $pageword_idx);
            $upwords = array();
            foreach ($delwords as $word) {
                if ($word != '') {
                    list($wlen, $wid) = explode('*', $word);
                    $wid = (int)$wid;
                    $upwords[$wlen][] = $wid;
                }
            }
            foreach ($upwords as $wlen => $widx) {
                $index = $this->getIndex('i', $wlen);
                foreach ($widx as $wid) {
                    $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
                }
                $this->saveIndex('i', $wlen, $index);
            }
        }
        // save the reverse index
        $this->saveIndexKey('pageword', '', $pid, '');

        if ($requireLock) $this->unlock();
        return true;
    }

    /**
     * Find pages in the fulltext index containing the words,
     *
     * The search words must be pre-tokenized, meaning only letters and
     * numbers with an optional wildcard
     *
     * The returned array will have the original tokens as key. The values
     * in the returned list is an array with the page names as keys and the
     * number of times that token appears on the page as value.
     *
     * @param array  $tokens list of words to search for
     * @return array         list of page names with usage counts
     *
     * @author Tom N Harris <[email protected]>
     * @author Andreas Gohr <[email protected]>
     */
    public function lookupWords(&$tokens)
    {
        $result = array();
        $wids = $this->getIndexWords($tokens, $result);
        if (empty($wids)) return array();
        // load known words and documents
        $page_idx = $this->getIndex('page', '');
        $docs = array();
        foreach (array_keys($wids) as $wlen) {
            $wids[$wlen] = array_unique($wids[$wlen]);
            $index = $this->getIndex('i', $wlen);
            foreach ($wids[$wlen] as $ixid) {
                if ($ixid < count($index)) {
                    $docs["{$wlen}*{$ixid}"] = $this->parseTuples($page_idx, $index[$ixid]);
                }
            }
        }
        // merge found pages into final result array
        $final = array();
        foreach ($result as $word => $res) {
            $final[$word] = array();
            foreach ($res as $wid) {
                // handle the case when ($ixid < count($index)) has been false
                // and thus $docs[$wid] hasn't been set.
                if (!isset($docs[$wid])) continue;
                $hits =& $docs[$wid];
                foreach ($hits as $hitkey => $hitcnt) {
                    // make sure the document still exists
                    if (!page_exists($hitkey, '', false)) continue;
                    if (!isset($final[$word][$hitkey])) {
                        $final[$word][$hitkey] = $hitcnt;
                    } else {
                        $final[$word][$hitkey] += $hitcnt;
                    }
                }
            }
        }
        return $final;
    }

    /**
     * Find the index ID of each search term
     *
     * The query terms should only contain valid characters, with a '*' at
     * either the beginning or end of the word (or both).
     * The $result parameter can be used to merge the index locations with
     * the appropriate query term.
     *
     * @param array  $words  The query terms.
     * @param array  $result Set to word => array("length*id" ...)
     * @return array         Set to length => array(id ...)
     *
     * @author Tom N Harris <[email protected]>
     */
    protected function getIndexWords(&$words, &$result)
    {
        $tokens = array();
        $tokenlength = array();
        $tokenwild = array();
        foreach ($words as $word) {
            $result[$word] = array();
            $caret = '^';
            $dollar = '$';
            $xword = $word;
            $wlen = $this->wordlen($word);

            // check for wildcards
            if (substr($xword, 0, 1) == '*') {
                $xword = substr($xword, 1);
                $caret = '';
                $wlen -= 1;
            }
            if (substr($xword, -1, 1) == '*') {
                $xword = substr($xword, 0, -1);
                $dollar = '';
                $wlen -= 1;
            }
            if ($wlen < Tokenizer::getMinWordLength()
                && $caret && $dollar && !is_numeric($xword)
            ) {
                continue;
            }
            if (!isset($tokens[$xword])) {
                $tokenlength[$wlen][] = $xword;
            }
            if (!$caret || !$dollar) {
                $re = $caret.preg_quote($xword, '/').$dollar;
                $tokens[$xword][] = array($word, '/'.$re.'/');
                if (!isset($tokenwild[$xword])) {
                    $tokenwild[$xword] = $wlen;
                }
            } else {
                $tokens[$xword][] = array($word, null);
            }
        }
        asort($tokenwild);
        // $tokens = array( base word => array( [ query term , regexp ] ... ) ... )
        // $tokenlength = array( base word length => base word ... )
        // $tokenwild = array( base word => base word length ... )
        $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength));
        $indexes_known = $this->getIndexLengths($length_filter);
        if (!empty($tokenwild)) sort($indexes_known);
        // get word IDs
        $wids = array();
        foreach ($indexes_known as $ixlen) {
            $word_idx = $this->getIndex('w', $ixlen);
            // handle exact search
            if (isset($tokenlength[$ixlen])) {
                foreach ($tokenlength[$ixlen] as $xword) {
                    $wid = array_search($xword, $word_idx, true);
                    if ($wid !== false) {
                        $wids[$ixlen][] = $wid;
                        foreach ($tokens[$xword] as $w)
                            $result[$w[0]][] = "{$ixlen}*{$wid}";
                    }
                }
            }
            // handle wildcard search
            foreach ($tokenwild as $xword => $wlen) {
                if ($wlen >= $ixlen) break;
                foreach ($tokens[$xword] as $w) {
                    if (is_null($w[1])) continue;
                    foreach (array_keys(preg_grep($w[1], $word_idx)) as $wid) {
                        $wids[$ixlen][] = $wid;
                        $result[$w[0]][] = "{$ixlen}*{$wid}";
                    }
                }
            }
        }
        return $wids;
    }

    /**
     * Get the word lengths that have been indexed
     *
     * Reads the index directory and returns an array of lengths
     * that there are indices for.
     *
     * @author YoBoY <[email protected]>
     *
     * @param array|int $filter
     * @return array
     */
    public function getIndexLengths($filter)
    {
        global $conf;
        $idx = array();
        if (is_array($filter)) {
            // testing if index files exist only
            $path = $conf['indexdir']."/i";
            foreach ($filter as $key => $value) {
                if (file_exists($path.$key.'.idx')) {
                    $idx[] = $key;
                }
            }
        } else {
            $lengths = $this->listIndexLengths();
            foreach ($lengths as $key => $length) {
                // keep all the values equal or superior
                if ((int)$length >= (int)$filter) {
                    $idx[] = $length;
                }
            }
        }
        return $idx;
    }

    /**
     * Get the list of lengths indexed in the wiki
     *
     * Read the index directory or a cache file and returns
     * a sorted array of lengths of the words used in the wiki.
     *
     * @author YoBoY <[email protected]>
     *
     * @return array
     */
    public function listIndexLengths()
    {
        global $conf;
        $lengthsFile = $conf['indexdir'].'/lengths.idx';

        // testing what we have to do, create a cache file or not.
        if ($conf['readdircache'] == 0) {
            $docache = false;
        } else {
            clearstatcache();
            if (file_exists($lengthsFile)
                && (time() < @filemtime($lengthsFile) + $conf['readdircache'])
            ) {
                $lengths = @file($lengthsFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
                if ($lengths !== false) {
                    $idx = array();
                    foreach ($lengths as $length) {
                        $idx[] = (int)$length;
                    }
                    return $idx;
                }
            }
            $docache = true;
        }

        if ($conf['readdircache'] == 0 || $docache) {
            $dir = @opendir($conf['indexdir']);
            if ($dir === false) return array();
            $idx = array();
            while (($f = readdir($dir)) !== false) {
                if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') {
                    $i = substr($f, 1, -4);
                    if (is_numeric($i)) $idx[] = (int)$i;
                }
            }
            closedir($dir);
            sort($idx);
            // save this in a file
            if ($docache) {
                $handle = @fopen($lengthsFile, 'w');
                @fwrite($handle, implode("\n", $idx));
                @fclose($handle);
            }
            return $idx;
        }
        return array();
    }

    /**
     * Return a list of words sorted by number of times used
     *
     * @param int       $min    bottom frequency threshold
     * @param int       $max    upper frequency limit. No limit if $max<$min
     * @param int       $minlen minimum length of words to count
     * @return array            list of words as the keys and frequency as value
     *
     * @author Tom N Harris <[email protected]>
     */
    public function histogram($min=1, $max=0, $minlen=3)
    {
        return (new MetadataIndex())->histogram($min, $max, $minlen);
    }

    /**
     * Clear the Fulltext Index
     *
     * @param bool $requireLock should be false only if the caller is resposible for index lock
     * @return bool  If the index has been cleared successfully
     * @throws Exception\IndexLockException
     */
    public function clear($requireLock = true)
    {
        global $conf;

        if ($requireLock) $this->lock();

        $lengths = $this->listIndexLengths();
        foreach ($lengths as $length) {
            @unlink($conf['indexdir'].'/i'.$length.'.idx');
            @unlink($conf['indexdir'].'/w'.$length.'.idx');
        }
        @unlink($conf['indexdir'].'/lengths.idx');
        @unlink($conf['indexdir'].'/pageword.idx');

        if ($requireLock) $this->unlock();
        return true;
    }
}


1			<?php
2
3			namespace dokuwiki\Search;
4
5			use dokuwiki\Search\Exception\IndexAccessException;
6			use dokuwiki\Search\Exception\IndexLockException;
7			use dokuwiki\Search\Exception\IndexWriteException;
8			use dokuwiki\Search\Tokenizer;
9			use dokuwiki\Utf8;
10
11			/**
12			* Class DokuWiki Fulltext Index
13			*
14			* @license GPL 2 (http://www.gnu.org/licenses/gpl.html)
15			* @author Andreas Gohr <[email protected]>
16			* @author Tom N Harris <[email protected]>
17			*/
18			class FulltextIndex extends AbstractIndex
19			{
20			// numeric page id to be added to or deleted from the Fulltext index
21			protected $pageID;
22
23			/**
24			* FulltextIndex constructor
25			*
26			* @param string\|int $page a page name or numeric page id
27			*/
28			public function __construct($page = null)
29			{
30			if (isset($page)) {
31			$this->pageID = is_int($page) ? $page : $this->getPID($page);
32			}
33			}
34
35			/**
36			* Measure the length of a string
37			* Differs from strlen in handling of asian characters.
38			*
39			* @author Tom N Harris <[email protected]>
40			*
41			* @param string $w
42			* @return int
43			*/
44			public function wordlen($w)
45			{
46			$l = strlen($w);
47			// If left alone, all chinese "words" will get put into w3.idx
48			// So the "length" of a "word" is faked
49			if (preg_match_all('/[\xE2-\xEF]/', $w, $leadbytes)) {
50			foreach ($leadbytes[0] as $b) {
51			$l += ord($b) - 0xE1;
52			}
53			}
54			return $l;
55			}
56
57			/**
58			* Adds the contents of a page to the fulltext index
59			*
60			* The added text replaces previous words for the same page.
61			* An empty value erases the page.
62			*
63			* @param string $text the body of the page
64			* @param bool $requireLock should be false only if the caller is resposible for index lock
65			* @return bool if the function completed successfully
66			*
67			* @throws IndexAccessException
68			* @throws IndexLockException
69			* @throws IndexWriteException
70			* @author Andreas Gohr <[email protected]>
71			* @author Tom N Harris <[email protected]>
72			*/
73			public function addWords($text, $requireLock = true)
74			{
75			// load known documents
76			if (!isset($this->pageID)) {
77			throw new IndexAccessException('Indexer: page unknown to addWords');
78			} else {
79			$pid = $this->pageID;
80			}
81
82			if ($requireLock) $this->lock();
83
84			$pagewords = array();
85			// get word usage in page
86			$words = $this->getWords($text);
87
88			foreach (array_keys($words) as $wlen) {
89			$index = $this->getIndex('i', $wlen);
90			foreach ($words[$wlen] as $wid => $freq) {
91			$idx = ($wid < count($index)) ? $index[$wid] : '';
92			$index[$wid] = $this->updateTuple($idx, $pid, $freq);
93			$pagewords[] = "{$wlen}*{$wid}";
94			}
95			$this->saveIndex('i', $wlen, $index);
96			}
97
98			// Remove obsolete index entries
99			$pageword_idx = $this->getIndexKey('pageword', '', $pid);
100			if ($pageword_idx !== '') {
101			$oldwords = explode(':', $pageword_idx);
102			$delwords = array_diff($oldwords, $pagewords);
103			$upwords = array();
104			foreach ($delwords as $word) {
105			if ($word != '') {
106			list($wlen, $wid) = explode('*', $word);
107			$wid = (int)$wid;
108			$upwords[$wlen][] = $wid;
109			}
110			}
111			foreach ($upwords as $wlen => $widx) {
112			$index = $this->getIndex('i', $wlen);
113			foreach ($widx as $wid) {
114			$index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
115			}
116			$this->saveIndex('i', $wlen, $index);
117			}
118			}
119			// Save the reverse index
120			$pageword_idx = implode(':', $pagewords);
121			$this->saveIndexKey('pageword', '', $pid, $pageword_idx);
122
123			if ($requireLock) $this->unlock();
124			return true;
125			}
126
127			/**
128			* Split the words in a page and add them to the index
129			*
130			* @param string $text content of the page
131			* @return array list of word IDs and number of times used, false on errors
132			*
133			* @throws IndexWriteException
134			* @author Andreas Gohr <[email protected]>
135			* @author Christopher Smith <[email protected]>
136			* @author Tom N Harris <[email protected]>
137			*/
138			protected function getWords($text)
139			{
140			$tokens = Tokenizer::getWords($text);
141			$tokens = array_count_values($tokens); // count the frequency of each token
142
143			$words = array();
144			foreach ($tokens as $w => $c) {
145			$l = $this->wordlen($w);
146			if (isset($words[$l])) {
147			$words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0);
148			} else {
149			$words[$l] = array($w => $c);
150			}
151			}
152
153			// arrive here with $words = array(wordlen => array(word => frequency))
154			$word_idx_modified = false;
155			$index = array(); //resulting index
156			foreach (array_keys($words) as $wlen) {
157			$word_idx = $this->getIndex('w', $wlen);
158			foreach ($words[$wlen] as $word => $freq) {
159			$word = (string)$word;
160			$wid = array_search($word, $word_idx, true);
161			if ($wid === false) {
162			$wid = count($word_idx);
163			$word_idx[] = $word;
164			$word_idx_modified = true;
165			}
166			if (!isset($index[$wlen])) {
167			$index[$wlen] = array();
168			}
169			$index[$wlen][$wid] = $freq;
170			}
171			// save back the word index
172			if ($word_idx_modified) $this->saveIndex('w', $wlen, $word_idx);
173			}
174
175			return $index;
176			}
177
178			/**
179			* Delete the contents of a page to the fulltext index
180			*
181			* @param bool $requireLock should be false only if the caller is resposible for index lock
182			* @return bool If renaming the value has been successful, false on error
183			*
184			* @throws IndexAccessException
185			* @throws IndexLockException
186			* @throws IndexWriteException
187			* @author Satoshi Sahara <[email protected]>
188			* @author Tom N Harris <[email protected]>
189			*/
190			public function deleteWords($requireLock = true)
191			{
192			// load known documents
193			if (!isset($this->pageID)) {
194			throw new IndexAccessException('Indexer: page unknown to deleteWords');
195			} else {
196			$pid = $this->pageID;
197			}
198
199			if ($requireLock) $this->lock();
200
201			// remove obsolete index entries
202			$pageword_idx = $this->getIndexKey('pageword', '', $pid);
203			if ($pageword_idx !== '') {
204			$delwords = explode(':', $pageword_idx);
205			$upwords = array();
206			foreach ($delwords as $word) {
207			if ($word != '') {
208			list($wlen, $wid) = explode('*', $word);
209			$wid = (int)$wid;
210			$upwords[$wlen][] = $wid;
211			}
212			}
213			foreach ($upwords as $wlen => $widx) {
214			$index = $this->getIndex('i', $wlen);
215			foreach ($widx as $wid) {
216			$index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
217			}
218			$this->saveIndex('i', $wlen, $index);
219			}
220			}
221			// save the reverse index
222			$this->saveIndexKey('pageword', '', $pid, '');
223
224			if ($requireLock) $this->unlock();
225			return true;
226			}
227
228			/**
229			* Find pages in the fulltext index containing the words,
230			*
231			* The search words must be pre-tokenized, meaning only letters and
232			* numbers with an optional wildcard
233			*
234			* The returned array will have the original tokens as key. The values
235			* in the returned list is an array with the page names as keys and the
236			* number of times that token appears on the page as value.
237			*
238			* @param array $tokens list of words to search for
239			* @return array list of page names with usage counts
240			*
241			* @author Tom N Harris <[email protected]>
242			* @author Andreas Gohr <[email protected]>
243			*/
244			public function lookupWords(&$tokens)
245			{
246			$result = array();
247			$wids = $this->getIndexWords($tokens, $result);
248			if (empty($wids)) return array();
249			// load known words and documents
250			$page_idx = $this->getIndex('page', '');
251			$docs = array();
252			foreach (array_keys($wids) as $wlen) {
253			$wids[$wlen] = array_unique($wids[$wlen]);
254			$index = $this->getIndex('i', $wlen);
255			foreach ($wids[$wlen] as $ixid) {
256			if ($ixid < count($index)) {
257			$docs["{$wlen}*{$ixid}"] = $this->parseTuples($page_idx, $index[$ixid]);
258			}
259			}
260			}
261			// merge found pages into final result array
262			$final = array();
263			foreach ($result as $word => $res) {
264			$final[$word] = array();
265			foreach ($res as $wid) {
266			// handle the case when ($ixid < count($index)) has been false
267			// and thus $docs[$wid] hasn't been set.
268			if (!isset($docs[$wid])) continue;
269			$hits =& $docs[$wid];
270			foreach ($hits as $hitkey => $hitcnt) {
271			// make sure the document still exists
272			if (!page_exists($hitkey, '', false)) continue;
273			if (!isset($final[$word][$hitkey])) {
274			$final[$word][$hitkey] = $hitcnt;
275			} else {
276			$final[$word][$hitkey] += $hitcnt;
277			}
278			}
279			}
280			}
281			return $final;
282			}
283
284			/**
285			* Find the index ID of each search term
286			*
287			* The query terms should only contain valid characters, with a '*' at
288			* either the beginning or end of the word (or both).
289			* The $result parameter can be used to merge the index locations with
290			* the appropriate query term.
291			*
292			* @param array $words The query terms.
293			* @param array $result Set to word => array("length*id" ...)
294			* @return array Set to length => array(id ...)
295			*
296			* @author Tom N Harris <[email protected]>
297			*/
298			protected function getIndexWords(&$words, &$result)
299			{
300			$tokens = array();
301			$tokenlength = array();
302			$tokenwild = array();
303			foreach ($words as $word) {
304			$result[$word] = array();
305			$caret = '^';
306			$dollar = '$';
307			$xword = $word;
308			$wlen = $this->wordlen($word);
309
310			// check for wildcards
311			if (substr($xword, 0, 1) == '*') {
312			$xword = substr($xword, 1);
313			$caret = '';
314			$wlen -= 1;
315			}
316			if (substr($xword, -1, 1) == '*') {
317			$xword = substr($xword, 0, -1);
318			$dollar = '';
319			$wlen -= 1;
320			}
321			if ($wlen < Tokenizer::getMinWordLength()
322			&& $caret && $dollar && !is_numeric($xword)
323			) {
324			continue;
325			}
326			if (!isset($tokens[$xword])) {
327			$tokenlength[$wlen][] = $xword;
328			}
329			if (!$caret \|\| !$dollar) {
330			$re = $caret.preg_quote($xword, '/').$dollar;
331			$tokens[$xword][] = array($word, '/'.$re.'/');
332			if (!isset($tokenwild[$xword])) {
333			$tokenwild[$xword] = $wlen;
334			}
335			} else {
336			$tokens[$xword][] = array($word, null);
337			}
338			}
339			asort($tokenwild);
340			// $tokens = array( base word => array( [ query term , regexp ] ... ) ... )
341			// $tokenlength = array( base word length => base word ... )
342			// $tokenwild = array( base word => base word length ... )
343			$length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength));
344			$indexes_known = $this->getIndexLengths($length_filter);
345			if (!empty($tokenwild)) sort($indexes_known);
346			// get word IDs
347			$wids = array();
348			foreach ($indexes_known as $ixlen) {
349			$word_idx = $this->getIndex('w', $ixlen);
350			// handle exact search
351			if (isset($tokenlength[$ixlen])) {
352			foreach ($tokenlength[$ixlen] as $xword) {
353			$wid = array_search($xword, $word_idx, true);
354			if ($wid !== false) {
355			$wids[$ixlen][] = $wid;
356			foreach ($tokens[$xword] as $w)
357			$result[$w[0]][] = "{$ixlen}*{$wid}";
358			}
359			}
360			}
361			// handle wildcard search
362			foreach ($tokenwild as $xword => $wlen) {
363			if ($wlen >= $ixlen) break;
364			foreach ($tokens[$xword] as $w) {
365			if (is_null($w[1])) continue;
366			foreach (array_keys(preg_grep($w[1], $word_idx)) as $wid) {
367			$wids[$ixlen][] = $wid;
368			$result[$w[0]][] = "{$ixlen}*{$wid}";
369			}
370			}
371			}
372			}
373			return $wids;
374			}
375
376			/**
377			* Get the word lengths that have been indexed
378			*
379			* Reads the index directory and returns an array of lengths
380			* that there are indices for.
381			*
382			* @author YoBoY <[email protected]>
383			*
384			* @param array\|int $filter
385			* @return array
386			*/
387			public function getIndexLengths($filter)
388			{
389			global $conf;
390			$idx = array();
391			if (is_array($filter)) {
392			// testing if index files exist only
393			$path = $conf['indexdir']."/i";
394			foreach ($filter as $key => $value) {
395			if (file_exists($path.$key.'.idx')) {
396			$idx[] = $key;
397			}
398			}
399			} else {
400			$lengths = $this->listIndexLengths();
401			foreach ($lengths as $key => $length) {
402			// keep all the values equal or superior
403			if ((int)$length >= (int)$filter) {
404			$idx[] = $length;
405			}
406			}
407			}
408			return $idx;
409			}
410
411			/**
412			* Get the list of lengths indexed in the wiki
413			*
414			* Read the index directory or a cache file and returns
415			* a sorted array of lengths of the words used in the wiki.
416			*
417			* @author YoBoY <[email protected]>
418			*
419			* @return array
420			*/
421			public function listIndexLengths()
422			{
423			global $conf;
424			$lengthsFile = $conf['indexdir'].'/lengths.idx';
425
426			// testing what we have to do, create a cache file or not.
427			if ($conf['readdircache'] == 0) {
428			$docache = false;
429			} else {
430			clearstatcache();
431			if (file_exists($lengthsFile)
432			&& (time() < @filemtime($lengthsFile) + $conf['readdircache'])
433			) {
434			$lengths = @file($lengthsFile, FILE_IGNORE_NEW_LINES \| FILE_SKIP_EMPTY_LINES);
435			if ($lengths !== false) {
436			$idx = array();
437			foreach ($lengths as $length) {
438			$idx[] = (int)$length;
439			}
440			return $idx;
441			}
442			}
443			$docache = true;
444			}
445
446			if ($conf['readdircache'] == 0 \|\| $docache) {
447			$dir = @opendir($conf['indexdir']);
448			if ($dir === false) return array();
449			$idx = array();
450			while (($f = readdir($dir)) !== false) {
451			if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') {
452			$i = substr($f, 1, -4);
453			if (is_numeric($i)) $idx[] = (int)$i;
454			}
455			}
456			closedir($dir);
457			sort($idx);
458			// save this in a file
459			if ($docache) {
460			$handle = @fopen($lengthsFile, 'w');
461			@fwrite($handle, implode("\n", $idx));
462			@fclose($handle);
463			}
464			return $idx;
465			}
466			return array();
467			}
468
469			/**
470			* Return a list of words sorted by number of times used
471			*
472			* @param int $min bottom frequency threshold
473			* @param int $max upper frequency limit. No limit if $max<$min
474			* @param int $minlen minimum length of words to count
475			* @return array list of words as the keys and frequency as value
476			*
477			* @author Tom N Harris <[email protected]>
478			*/
479			public function histogram($min=1, $max=0, $minlen=3)
480			{
481			return (new MetadataIndex())->histogram($min, $max, $minlen);
482			}
483
484			/**
485			* Clear the Fulltext Index
486			*
487			* @param bool $requireLock should be false only if the caller is resposible for index lock
488			* @return bool If the index has been cleared successfully
489			* @throws Exception\IndexLockException
490			*/
491			public function clear($requireLock = true)
492			{
493			global $conf;
494
495			if ($requireLock) $this->lock();
496
497			$lengths = $this->listIndexLengths();
498			foreach ($lengths as $length) {
499			@unlink($conf['indexdir'].'/i'.$length.'.idx');
500			@unlink($conf['indexdir'].'/w'.$length.'.idx');
501			}
502			@unlink($conf['indexdir'].'/lengths.idx');
503			@unlink($conf['indexdir'].'/pageword.idx');
504
505			if ($requireLock) $this->unlock();
506			return true;
507			}
508			}
509

splitbrain / dokuwiki

Pull Request — master (#2943)

FulltextIndex::clear() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like