RegularParser::tokenize() - Code Metrics - Inspection of "Regular parser performance improvements" - thunderer/Shortcode - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#28)

by Tomasz

created 2016-01-23 20:31 UTC

RegularParser::tokenize() B

↳ Parent: RegularParser

Complexity

Conditions	4
Paths	4

Size

Total Lines	23
Code Lines	11

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	2
CRAP Score	4

Importance

Changes	4
Bugs	1	Features	0

Metric	Value
c	4
b	1
f	0
dl	0
loc	23
ccs	2
cts	2
cp	1
rs	8.7972
cc	4
eloc	11
nc	4
nop	1
crap	4

<?php
namespace Thunder\Shortcode\Parser;

use Thunder\Shortcode\Shortcode\ParsedShortcode;
use Thunder\Shortcode\Shortcode\Shortcode;
use Thunder\Shortcode\Syntax\CommonSyntax;
use Thunder\Shortcode\Syntax\SyntaxInterface;

/**
 * @author Tomasz Kowalczyk <[email protected]>
 */
final class RegularParser implements ParserInterface
{
    private $lexerRules;
    private $tokens;
    private $tokensCount;
    private $position;
    private $backtracks;
    private $syntax;

    const TOKEN_OPEN = 1;
    const TOKEN_CLOSE = 2;
    const TOKEN_MARKER = 3;
    const TOKEN_SEPARATOR = 4;
    const TOKEN_DELIMITER = 5;
    const TOKEN_STRING = 6;
    const TOKEN_WS = 7;

    public function __construct(SyntaxInterface $syntax = null)
    {
        $this->syntax = $syntax ?: new CommonSyntax();

        $quote = function($text) { return '~^('.preg_replace('/(.)/us', '\\\\$0', $text).')~us'; };

        $this->lexerRules = array(
            self::TOKEN_OPEN => $quote($this->syntax->getOpeningTag()),
            self::TOKEN_CLOSE => $quote($this->syntax->getClosingTag()),
            self::TOKEN_MARKER => $quote($this->syntax->getClosingTagMarker()),
            self::TOKEN_SEPARATOR => $quote($this->syntax->getParameterValueSeparator()),
            self::TOKEN_DELIMITER => $quote($this->syntax->getParameterValueDelimiter()),
            self::TOKEN_WS => '~^(\s+)~us',
            self::TOKEN_STRING => '~^([\w-]+|\\\\.|.)~us',
        );
    }

    /**
     * @param string $text
     *
     * @return ParsedShortcode[]
     */
    public function parse($text)
    {
        $this->tokens = $this->tokenize($text);
        $this->backtracks = array();
        $this->position = 0;
        $this->tokensCount = count($this->tokens);

        $shortcodes = array();
        while($this->position < $this->tokensCount) {
            while($this->position < $this->tokensCount && !$this->lookahead(self::TOKEN_OPEN)) {
                $this->position++;
            }
            foreach($this->shortcode(true) ?: array() as $shortcode) {
                $shortcodes[] = $shortcode;
            }
        }

        return $shortcodes;
    }

    private function getObject($name, $parameters, $bbCode, $offset, $content, $text)
    {
        return new ParsedShortcode(new Shortcode($name, $parameters, $content, $bbCode), $text, $offset);
    }

    /* --- RULES ----------------------------------------------------------- */

    private function shortcode($isRoot)
    {
        $name = null;
        $offset = null;

        $setName = function(array $token) use(&$name) { $name = $token[1]; };
        $setOffset = function(array $token) use(&$offset) { $offset = $token[2]; };

        $isRoot && $this->beginBacktrack();
        if(!$this->match(self::TOKEN_OPEN, $setOffset, true)) { return false; }
        if(!$this->match(self::TOKEN_STRING, $setName, false)) { return false; }
        if($this->lookahead(self::TOKEN_STRING, null)) { return false; }
        if(!preg_match_all('/^[a-zA-Z0-9-]+$/', $name, $matches)) { return false; }
        $this->match(self::TOKEN_WS);
        if(false === ($bbCode = $this->bbCode())) { return false; }
        if(false === ($parameters = $this->parameters())) { return false; }

        // self-closing
        if($this->match(self::TOKEN_MARKER, null, true)) {
            if(!$this->match(self::TOKEN_CLOSE)) { return false; }

            return array($this->getObject($name, $parameters, $bbCode, $offset, null, $this->getBacktrack()));
        }

        // just-closed or with-content
        if(!$this->match(self::TOKEN_CLOSE)) { return false; }
        $this->beginBacktrack();
        list($content, $shortcodes) = $this->content($name);
        if(false === $content) {
            $this->backtrack(false);
            $text = $this->backtrack(false);
            return array_merge(array($this->getObject($name, $parameters, $bbCode, $offset, null, $text)), $shortcodes);
        }
        array_pop($this->backtracks);
        if(!$this->close($name)) { return false; }

        return array($this->getObject($name, $parameters, $bbCode, $offset, $content, $this->getBacktrack()));
    }

    private function content($name)
    {
        $content = null;
        $shortcodes = array();
        $appendContent = function(array $token) use(&$content) { $content .= $token[1]; };

        while($this->position < $this->tokensCount) {
            while($this->match(array(self::TOKEN_STRING, self::TOKEN_WS), $appendContent)) {
                continue;
            }

            $this->beginBacktrack();
            $matchedShortcodes = $this->shortcode(false);
            if(false !== $matchedShortcodes) {
                $shortcodes = array_merge($shortcodes, $matchedShortcodes);
                continue;
            }
            $this->backtrack();

            $this->beginBacktrack();
            if(false !== $this->close($name)) {
                if(null === $content) { $content = ''; }
                $this->backtrack();
                $shortcodes = array();
                break;
            }
            $this->backtrack();
            if($this->position < $this->tokensCount) {
                $shortcodes = array();
                break;
            }

            $this->match(null, $appendContent);
        }

        return array($this->position < $this->tokensCount ? $content : false, $shortcodes);
    }

    private function close($openingName)
    {
        $closingName = null;
        $setName = function(array $token) use(&$closingName) { $closingName = $token[1]; };

        if(!$this->match(self::TOKEN_OPEN, null, true)) { return false; }
        if(!$this->match(self::TOKEN_MARKER, null, true)) { return false; }
        if(!$this->match(self::TOKEN_STRING, $setName, true)) { return false; }
        if(!$this->match(self::TOKEN_CLOSE)) { return false; }

        return $openingName === $closingName;
    }

    private function bbCode()
    {
        return $this->match(self::TOKEN_SEPARATOR, null, true) ? $this->value() : null;
    }

    private function parameters()
    {
        $parameters = array();
        $setName = function(array $token) use(&$name) { $name = $token[1]; };

        while(true) {
            $name = null;
$x = 1;
$callable = function() use ($x) {
    $x = 2; // Not visible in outer scope. If you would like this, how
            // about using a different variable name than $x?
};

$callable();
var_dump($x); // integer(1)

            $this->match(self::TOKEN_WS);
            if($this->lookahead(array(self::TOKEN_MARKER, self::TOKEN_CLOSE))) { break; }
            if(!$this->match(self::TOKEN_STRING, $setName, true)) { return false; }
            if(!$this->match(self::TOKEN_SEPARATOR, null, true)) { $parameters[$name] = null; continue; }
            if(false === ($value = $this->value())) { return false; }
            $this->match(self::TOKEN_WS);

            $parameters[$name] = $value;
        }

        return $parameters;
    }

    private function value()
    {
        $value = '';
        $appendValue = function(array $token) use(&$value) { $value .= $token[1]; };

        if($this->match(self::TOKEN_DELIMITER)) {
            while($this->position < $this->tokensCount && !$this->lookahead(self::TOKEN_DELIMITER)) {
                $this->match(null, $appendValue);
            }

            return $this->match(self::TOKEN_DELIMITER) ? $value : false;
        }

        return $this->match(self::TOKEN_STRING, $appendValue) ? $value : false;
    }

    /* --- PARSER ---------------------------------------------------------- */

    private function beginBacktrack()
    {
        $this->backtracks[] = array();
    }

    private function getBacktrack()
    {
        // switch from array_map() to array_column() when dropping support for PHP <5.5
        return implode('', array_map(function(array $token) { return $token[1]; }, array_pop($this->backtracks)));
    }

    private function backtrack($modifyPosition = true)
    {
        $tokens = array_pop($this->backtracks);
        $count = count($tokens);
        if($modifyPosition) {
            $this->position -= $count;
        }

        foreach($this->backtracks as &$backtrack) {
            // array_pop() in loop is much faster than array_slice() because
            // it operates directly on the passed array
            for($i = 0; $i < $count; $i++) {
                array_pop($backtrack);
            }
        }

        return implode('', array_map(function(array $token) { return $token[1]; }, $tokens));
    }

    private function lookahead($type, $callback = null)
    {
        if($this->position >= $this->tokensCount) {
            return false;
        }

        $type = (array)$type;
        $token = $this->tokens[$this->position];
        if(!empty($type) && !in_array($token[0], $type)) {
            return false;
        }

        /** @var $callback callable */
        $callback && $callback($token);

        return true;
    }

    private function match($type, $callbacks = null, $ws = false)
    {
        if($this->position >= $this->tokensCount) {
            return false;
        }

        $type = (array)$type;
        $token = $this->tokens[$this->position];
        if(!empty($type) && !in_array($token[0], $type)) {
            return false;
        }
        foreach($this->backtracks as &$backtrack) {
            $backtrack[] = $token;
        }

        $this->position++;
        foreach((array)$callbacks as $callback) {
            $callback($token);
        }

        $ws && $this->match(self::TOKEN_WS);

        return true;
    }

    /* --- LEXER ----------------------------------------------------------- */

    private function tokenize($text)
    {
        $tokens = array();
        // performance improvement: start generating tokens after first opening
        // tag position because it's impossible to find shortcode earlier
        // WARNING: mb_strpos() hangs on PHP 5.5 when used with encoding parameter
        // $position = mb_strpos($text, $this->syntax->getOpeningTag(), 0, 'utf-8');

        // $text = mb_substr($text, $position);

        $position = 0;

        while(mb_strlen($text) > 0) {
            foreach($this->lexerRules as $token => $regex) {
                if(preg_match($regex, $text, $matches)) {
                    $tokens[] = array($token, $matches[0], $position);
                    $text = mb_substr($text, mb_strlen($matches[0]));
                    $position += mb_strlen($matches[0], 'utf-8');
                    break;
                }
            }
        }

        return $tokens;
    }
}


GitHub Access Token became invalid

Pull Request — master (#28)

RegularParser::tokenize() B

Complexity

Size

Duplication

Code Coverage

Importance

Change not visible in outer-scope

Change visible in outer-scope

1		<?php
2		namespace Thunder\Shortcode\Parser;
3
4		use Thunder\Shortcode\Shortcode\ParsedShortcode;
5		use Thunder\Shortcode\Shortcode\Shortcode;
6		use Thunder\Shortcode\Syntax\CommonSyntax;
7		use Thunder\Shortcode\Syntax\SyntaxInterface;
8
9		/**
10		* @author Tomasz Kowalczyk <[email protected]>
11		*/
12		final class RegularParser implements ParserInterface
13		{
14		private $lexerRules;
15		private $tokens;
16		private $tokensCount;
17		private $position;
18		private $backtracks;
19		private $syntax;
20
21		const TOKEN_OPEN = 1;
22		const TOKEN_CLOSE = 2;
23		const TOKEN_MARKER = 3;
24		const TOKEN_SEPARATOR = 4;
25		const TOKEN_DELIMITER = 5;
26		const TOKEN_STRING = 6;
27	4	const TOKEN_WS = 7;
28
29	4	public function __construct(SyntaxInterface $syntax = null)
30		{
31		$this->syntax = $syntax ?: new CommonSyntax();
32
33	4	$quote = function($text) { return '~^('.preg_replace('/(.)/us', '\\\\$0', $text).')~us'; };
34	4
35	4	$this->lexerRules = array(
36	4	self::TOKEN_OPEN => $quote($this->syntax->getOpeningTag()),
37	4	self::TOKEN_CLOSE => $quote($this->syntax->getClosingTag()),
38	4	self::TOKEN_MARKER => $quote($this->syntax->getClosingTagMarker()),
39	4	self::TOKEN_SEPARATOR => $quote($this->syntax->getParameterValueSeparator()),
40	4	self::TOKEN_DELIMITER => $quote($this->syntax->getParameterValueDelimiter()),
41		self::TOKEN_WS => '~^(\s+)~us',
42	4	self::TOKEN_STRING => '~^([\w-]+\|\\\\.\|.)~us',
43		);
44		}
45
46		/**
47		* @param string $text
48		*
49	36	* @return ParsedShortcode[]
50		*/
51	36	public function parse($text)
52	36	{
53		$this->tokens = $this->tokenize($text);
54	36	$this->backtracks = array();
55	36	$this->position = 0;
56	35	$this->tokensCount = count($this->tokens);
57	13
58		$shortcodes = array();
59	35	while($this->position < $this->tokensCount) {
60	28	while($this->position < $this->tokensCount && !$this->lookahead(self::TOKEN_OPEN)) {
61		$this->position++;
62		}
63		foreach($this->shortcode(true) ?: array() as $shortcode) {
64	36	$shortcodes[] = $shortcode;
65		}
66		}
67	28
68		return $shortcodes;
69	28	}
70
71		private function getObject($name, $parameters, $bbCode, $offset, $content, $text)
72		{
73		return new ParsedShortcode(new Shortcode($name, $parameters, $content, $bbCode), $text, $offset);
74	35	}
75
76	35	/* --- RULES ----------------------------------------------------------- */
77	35
78		private function shortcode($isRoot)
79		{
80		$name = null;
81		$offset = null;
82	35
83	35	$setName = function(array $token) use(&$name) { $name = $token[1]; };
84	35	$setOffset = function(array $token) use(&$offset) { $offset = $token[2]; };
85	32
86	32	$isRoot && $this->beginBacktrack();
87	31	if(!$this->match(self::TOKEN_OPEN, $setOffset, true)) { return false; }
88	31	if(!$this->match(self::TOKEN_STRING, $setName, false)) { return false; }
89	31	if($this->lookahead(self::TOKEN_STRING, null)) { return false; }
90		if(!preg_match_all('/^[a-zA-Z0-9-]+$/', $name, $matches)) { return false; }
91		$this->match(self::TOKEN_WS);
92	29	if(false === ($bbCode = $this->bbCode())) { return false; }
93	12	if(false === ($parameters = $this->parameters())) { return false; }
94
95	11	// self-closing
96		if($this->match(self::TOKEN_MARKER, null, true)) {
97		if(!$this->match(self::TOKEN_CLOSE)) { return false; }
98
99	19	return array($this->getObject($name, $parameters, $bbCode, $offset, null, $this->getBacktrack()));
100	19	}
101	19
102	12	// just-closed or with-content
103		if(!$this->match(self::TOKEN_CLOSE)) { return false; }
104	12	$this->beginBacktrack();
105		list($content, $shortcodes) = $this->content($name);
106	8	if(false === $content) {
107	8	$this->backtrack(false);
108		$text = $this->backtrack(false);
109	8	return array_merge(array($this->getObject($name, $parameters, $bbCode, $offset, null, $text)), $shortcodes);
110		}
111		array_pop($this->backtracks);
112	19	if(!$this->close($name)) { return false; }
113
114	19	return array($this->getObject($name, $parameters, $bbCode, $offset, $content, $this->getBacktrack()));
115		}
116
117	19	private function content($name)
118	11	{
119	11	$content = null;
120		$shortcodes = array();
121		$appendContent = function(array $token) use(&$content) { $content .= $token[1]; };
122	11
123	11	while($this->position < $this->tokensCount) {
124	4	while($this->match(array(self::TOKEN_STRING, self::TOKEN_WS), $appendContent)) {
125	4	continue;
126		}
127	10
128		$this->beginBacktrack();
129	10	$matchedShortcodes = $this->shortcode(false);
130	10	if(false !== $matchedShortcodes) {
131	8	$shortcodes = array_merge($shortcodes, $matchedShortcodes);
132	8	continue;
133	8	}
134		$this->backtrack();
135	3
136		$this->beginBacktrack();
137	3	if(false !== $this->close($name)) {
138		if(null === $content) { $content = ''; }
139		$this->backtrack();
140	19	$shortcodes = array();
141		break;
142		}
143	10	$this->backtrack();
144		if($this->position < $this->tokensCount) {
145	10	$shortcodes = array();
146		break;
147		}
148	10
149	8	$this->match(null, $appendContent);
150	8	}
151	8
152		return array($this->position < $this->tokensCount ? $content : false, $shortcodes);
153	8	}
154
155		private function close($openingName)
156	31	{
157		$closingName = null;
158	31	$setName = function(array $token) use(&$closingName) { $closingName = $token[1]; };
159
160		if(!$this->match(self::TOKEN_OPEN, null, true)) { return false; }
161	31	if(!$this->match(self::TOKEN_MARKER, null, true)) { return false; }
162		if(!$this->match(self::TOKEN_STRING, $setName, true)) { return false; }
163	31	if(!$this->match(self::TOKEN_CLOSE)) { return false; }
164
165		return $openingName === $closingName;
166	31	}
167	31
168		private function bbCode()
169	31	{
170	31	return $this->match(self::TOKEN_SEPARATOR, null, true) ? $this->value() : null;
171	22	}
172	21
173	20	private function parameters()
174	19	{
175		$parameters = array();
176	19	$setName = function(array $token) use(&$name) { $name = $token[1]; };
177
178		while(true) {
179	29	$name = null;
		0 ignored issues – show Bug introduced 2016-01-20 03:55 UTC by Report Bug Copy Issue Report Consider using a different name than the imported variable `$name`, or did you forget to import by reference? It seems like you are assigning to a variable which was imported through a `use` statement which was not imported by reference. For clarity, we suggest to use a different name or import by reference depending on whether you would like to have the change visibile in outer-scope. Change not visible in outer-scope $x = 1; $callable = function() use ($x) { $x = 2; // Not visible in outer scope. If you would like this, how // about using a different variable name than $x? }; $callable(); var_dump($x); // integer(1) Change visible in outer-scope $x = 1; $callable = function() use (&$x) { $x = 2; }; $callable(); var_dump($x); // integer(2) Loading history...
180
181		$this->match(self::TOKEN_WS);
182	21	if($this->lookahead(array(self::TOKEN_MARKER, self::TOKEN_CLOSE))) { break; }
183		if(!$this->match(self::TOKEN_STRING, $setName, true)) { return false; }
184	21	if(!$this->match(self::TOKEN_SEPARATOR, null, true)) { $parameters[$name] = null; continue; }
185		if(false === ($value = $this->value())) { return false; }
186		$this->match(self::TOKEN_WS);
187	21
188	18	$parameters[$name] = $value;
189	18	}
190
191		return $parameters;
192	18	}
193
194		private function value()
195	8	{
196		$value = '';
197		$appendValue = function(array $token) use(&$value) { $value .= $token[1]; };
198
199		if($this->match(self::TOKEN_DELIMITER)) {
200	28	while($this->position < $this->tokensCount && !$this->lookahead(self::TOKEN_DELIMITER)) {
201		$this->match(null, $appendValue);
202	28	}
203
204		return $this->match(self::TOKEN_DELIMITER) ? $value : false;
205	35	}
206
207	35	return $this->match(self::TOKEN_STRING, $appendValue) ? $value : false;
208	35	}
209
210		/* --- PARSER ---------------------------------------------------------- */
211
212		private function beginBacktrack()
213		{
214		$this->backtracks[] = array();
215	19	}
216
217	19	private function getBacktrack()
218	11	{
219		// switch from array_map() to array_column() when dropping support for PHP <5.5
220	11	return implode('', array_map(function(array $token) { return $token[1]; }, array_pop($this->backtracks)));
221	11	}
222
223		private function backtrack($modifyPosition = true)
224	19	{
225		$tokens = array_pop($this->backtracks);
226	36	$count = count($tokens);
227		if($modifyPosition) {
228	36	$this->position -= $count;
229		}
230
231	35	foreach($this->backtracks as &$backtrack) {
232		// array_pop() in loop is much faster than array_slice() because
233	35	// it operates directly on the passed array
234	1	for($i = 0; $i < $count; $i++) {
235		array_pop($backtrack);
236		}
237	35	}
238	35
239	35	return implode('', array_map(function(array $token) { return $token[1]; }, $tokens));
240	34	}
241
242		private function lookahead($type, $callback = null)
243		{
244	35	if($this->position >= $this->tokensCount) {
245		return false;
246	35	}
247
248		$type = (array)$type;
249	35	$token = $this->tokens[$this->position];
250		if(!empty($type) && !in_array($token[0], $type)) {
251	35	return false;
252	13	}
253
254		/** @var $callback callable */
255	35	$callback && $callback($token);
256	35
257	35	return true;
258	35	}
259
260	35	private function match($type, $callbacks = null, $ws = false)
261	35	{
262		if($this->position >= $this->tokensCount) {
263		return false;
264	35	}
265	35
266	35	$type = (array)$type;
267		$token = $this->tokens[$this->position];
268		if(!empty($type) && !in_array($token[0], $type)) {
269	35	return false;
270		}
271	35	foreach($this->backtracks as &$backtrack) {
272		$backtrack[] = $token;
273		}
274
275		$this->position++;
276	36	foreach((array)$callbacks as $callback) {
277		$callback($token);
278	36	}
279	36
280		$ws && $this->match(self::TOKEN_WS);
281	36
282	35	return true;
283	35	}
284	35
285	35	/* --- LEXER ----------------------------------------------------------- */
286	35
287	35	private function tokenize($text)
288		{
289		$tokens = array();
290		// performance improvement: start generating tokens after first opening
291		// tag position because it's impossible to find shortcode earlier
292	36	// WARNING: mb_strpos() hangs on PHP 5.5 when used with encoding parameter
293		// $position = mb_strpos($text, $this->syntax->getOpeningTag(), 0, 'utf-8');
		0 ignored issues – show Unused Code Comprehensibility introduced 2016-01-23 20:33 UTC by Report Bug Copy Issue Report `60%` of this comment could be valid code. Did you maybe forget this after debugging? Sometimes obsolete code just ends up commented out instead of removed. In this case it is better to remove the code once you have checked you do not need it. The code might also have been commented out for debugging purposes. In this case it is vital that someone uncomments it again or your project may behave in very unexpected ways in production. This check looks for comments that seem to be mostly valid code and reports them. Loading history...
294		// $text = mb_substr($text, $position);
		0 ignored issues – show Unused Code Comprehensibility introduced 2016-01-23 20:33 UTC by Report Bug Copy Issue Report `54%` of this comment could be valid code. Did you maybe forget this after debugging? Sometimes obsolete code just ends up commented out instead of removed. In this case it is better to remove the code once you have checked you do not need it. The code might also have been commented out for debugging purposes. In this case it is vital that someone uncomments it again or your project may behave in very unexpected ways in production. This check looks for comments that seem to be mostly valid code and reports them. Loading history...
295		$position = 0;
296
297		while(mb_strlen($text) > 0) {
298		foreach($this->lexerRules as $token => $regex) {
299		if(preg_match($regex, $text, $matches)) {
300		$tokens[] = array($token, $matches[0], $position);
301		$text = mb_substr($text, mb_strlen($matches[0]));
302		$position += mb_strlen($matches[0], 'utf-8');
303		break;
304		}
305		}
306		}
307
308		return $tokens;
309		}
310		}
311

thunderer / Shortcode

GitHub Access Token became invalid

Pull Request — master (#28)

RegularParser::tokenize() B

Complexity

Size

Duplication

Code Coverage

Importance

Change not visible in outer-scope

Change visible in outer-scope

Duplication Side-by-Side

Filter issues like