RegularParser::tokenize() - Code Metrics - Inspection of "RegularParser improvements" - thunderer/Shortcode - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#72)

by Tomasz

created 2018-12-09 00:09 UTC

RegularParser::tokenize() A

↳ Parent: RegularParser

Complexity

Conditions	5
Paths	6

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	13
CRAP Score	5.009

Importance

Changes

Metric	Value
dl	0
loc	19
ccs	13
cts	14
cp	0.9286
rs	9.3222
c	0
b	0
f	0
cc	5
nc	6
nop	1
crap	5.009

<?php
namespace Thunder\Shortcode\Parser;

use Thunder\Shortcode\Shortcode\ParsedShortcode;
use Thunder\Shortcode\Shortcode\Shortcode;
use Thunder\Shortcode\Syntax\CommonSyntax;
use Thunder\Shortcode\Syntax\SyntaxInterface;
use Thunder\Shortcode\Utility\RegexBuilderUtility;

/**
 * @author Tomasz Kowalczyk <[email protected]>
 */
final class RegularParser implements ParserInterface
{
    private $lexerRegex;
    private $nameRegex;
    private $tokens;
    private $tokensCount;
    private $position;
    /** @var int[] */
    private $backtracks;
    private $lastBacktrack;
    private $tokenMap;

    const TOKEN_OPEN = 1;
    const TOKEN_CLOSE = 2;
    const TOKEN_MARKER = 3;
    const TOKEN_SEPARATOR = 4;
    const TOKEN_DELIMITER = 5;
    const TOKEN_STRING = 6;
    const TOKEN_WS = 7;

    public function __construct(SyntaxInterface $syntax = null)
    {
        $this->lexerRegex = $this->prepareLexer($syntax ?: new CommonSyntax());
        $this->nameRegex = '~^'.RegexBuilderUtility::buildNameRegex().'$~us';
    }

    /**
     * @param string $text
     *
     * @return ParsedShortcode[]
     */
    public function parse($text)
    {
        $this->tokens = $this->tokenize($text);
        $this->backtracks = array();
        $this->lastBacktrack = 0;
        $this->position = 0;
        $this->tokensCount = \count($this->tokens);

        $shortcodes = array();
        while($this->position < $this->tokensCount) {
            while($this->position < $this->tokensCount && false === $this->lookahead(self::TOKEN_OPEN)) {
                $this->position++;
            }
            $names = array();
            $this->beginBacktrack();
            $matches = $this->shortcode($names);
            if(\is_array($matches)) {
                foreach($matches as $shortcode) {
                    $shortcodes[] = $shortcode;
                }
            }
        }

        return $shortcodes;
    }

    private function getObject($name, $parameters, $bbCode, $offset, $content, $text)
    {
        return new ParsedShortcode(new Shortcode($name, $parameters, $content, $bbCode), $text, $offset);
    }

    /* --- RULES ----------------------------------------------------------- */

    private function shortcode(array &$names)
    {
        if(!$this->match(self::TOKEN_OPEN, false)) { return false; }
        $offset = $this->tokens[$this->position - 1][2];
        $this->match(self::TOKEN_WS, false);
        if('' === $name = $this->match(self::TOKEN_STRING, false)) { return false; }
        if($this->lookahead(self::TOKEN_STRING)) { return false; }
        if(1 !== preg_match($this->nameRegex, $name, $matches)) { return false; }
        $this->match(self::TOKEN_WS, false);
        // bbCode
        $bbCode = $this->match(self::TOKEN_SEPARATOR, true) ? $this->value() : null;
        if(false === $bbCode) { return false; }
        // parameters
        if(false === ($parameters = $this->parameters())) { return false; }

        // self-closing
        if($this->match(self::TOKEN_MARKER, true)) {
            if(!$this->match(self::TOKEN_CLOSE, false)) { return false; }

            return array($this->getObject($name, $parameters, $bbCode, $offset, null, $this->getBacktrack()));
        }

        // just-closed or with-content
        if(!$this->match(self::TOKEN_CLOSE, false)) { return false; }
        $this->beginBacktrack();
        $names[] = $name;
        list($content, $shortcodes, $closingName) = $this->content($names);
        if(null !== $closingName && $closingName !== $name) {
            array_pop($names);
            array_pop($this->backtracks);
            array_pop($this->backtracks);

            return $closingName;
        }
        if(false === $content || $closingName !== $name) {
            $this->backtrack(false);
            $text = $this->backtrack(false);

            return array_merge(array($this->getObject($name, $parameters, $bbCode, $offset, null, $text)), $shortcodes);
        }
        $content = $this->getBacktrack();
        if(!$this->close($names)) { return false; }

        return array($this->getObject($name, $parameters, $bbCode, $offset, $content, $this->getBacktrack()));
    }

    private function content(array &$names)
    {
        $content = '';
        $shortcodes = array();
        $closingName = null;

        while($this->position < $this->tokensCount) {
            while($this->position < $this->tokensCount && false === $this->lookahead(self::TOKEN_OPEN)) {

                $content .= $this->match(null, true);
            }

            $this->beginBacktrack();
            $matchedShortcodes = $this->shortcode($names);
            if(\is_string($matchedShortcodes)) {
                $closingName = $matchedShortcodes;
                break;
            }
            if(\is_array($matchedShortcodes)) {
                foreach($matchedShortcodes as $matchedShortcode) {
                    $shortcodes[] = $matchedShortcode;
                }
                continue;
            }
            $this->backtrack();

            $this->beginBacktrack();
            if(false !== ($closingName = $this->close($names))) {
                if(null === $content) { $content = ''; }
                $this->backtrack();
                $shortcodes = array();
                break;
            }
            $closingName = null;
            $this->backtrack();

            $content .= $this->match(null, false);
        }

        return array($this->position < $this->tokensCount ? $content : false, $shortcodes, $closingName);
    }

    private function close(array &$names)
    {
        if(!$this->match(self::TOKEN_OPEN, true)) { return false; }
        if(!$this->match(self::TOKEN_MARKER, true)) { return false; }
        if(!$closingName = $this->match(self::TOKEN_STRING, true)) { return false; }
        if(!$this->match(self::TOKEN_CLOSE, false)) { return false; }

        return \in_array($closingName, $names, true) ? $closingName : false;
    }

    private function parameters()
    {
        $parameters = array();

        while(true) {
            $this->match(self::TOKEN_WS, false);
            if($this->lookahead(self::TOKEN_MARKER) || $this->lookahead(self::TOKEN_CLOSE)) { break; }
            if(!$name = $this->match(self::TOKEN_STRING, true)) { return false; }
            if(!$this->match(self::TOKEN_SEPARATOR, true)) { $parameters[$name] = null; continue; }
            if(false === ($value = $this->value())) { return false; }
            $this->match(self::TOKEN_WS, false);

            $parameters[$name] = $value;
        }

        return $parameters;
    }

    private function value()
    {
        $value = '';

        if($this->match(self::TOKEN_DELIMITER, false)) {
            while($this->position < $this->tokensCount && false === $this->lookahead(self::TOKEN_DELIMITER)) {

                $value .= $this->match(null, false);
            }

            return $this->match(self::TOKEN_DELIMITER, false) ? $value : false;
        }

        if($tmp = $this->match(self::TOKEN_STRING, false)) {
            $value .= $tmp;
            while($tmp = $this->match(self::TOKEN_STRING, false)) {
                $value .= $tmp;
            }

            return $value;
        }

        return false;
    }

    /* --- PARSER ---------------------------------------------------------- */

    private function beginBacktrack()
    {
        $this->backtracks[] = $this->position;
        $this->lastBacktrack = $this->position;
    }

    private function getBacktrack()
    {
        $position = array_pop($this->backtracks);
        $backtrack = '';
        for($i = $position; $i < $this->position; $i++) {

            $backtrack .= $this->tokens[$i][1];
        }

        return $backtrack;
    }

    private function backtrack($modifyPosition = true)
    {
        $position = array_pop($this->backtracks);
        if($modifyPosition) {
            $this->position = $position;
        }

        $backtrack = '';
        for($i = $position; $i < $this->lastBacktrack; $i++) {

            $backtrack .= $this->tokens[$i][1];
        }
        $this->lastBacktrack = $position;

        return $backtrack;
    }

    private function lookahead($type)
    {
        return $this->position < $this->tokensCount && (empty($type) || $this->tokens[$this->position][0] === $type);
    }

    private function match($type, $ws)
    {
        if($this->position >= $this->tokensCount) {
            return '';
        }

        $token = $this->tokens[$this->position];
        if(!empty($type) && $token[0] !== $type) {
            return '';
        }

        $this->position++;
        if($ws && $this->position < $this->tokensCount && $this->tokens[$this->position][0] === self::TOKEN_WS) {
            $this->position++;
        }

        return $token[1];
    }

    /* --- LEXER ----------------------------------------------------------- */

    private function tokenize($text)
    {
        preg_match_all($this->lexerRegex, $text, $matches, PREG_OFFSET_CAPTURE);
        if(preg_last_error() !== PREG_NO_ERROR) {
            throw new \RuntimeException(sprintf('PCRE failure `%s`.', preg_last_error()));
        }

        $tokens = array();
        $position = 0;
        foreach($matches[0] as $match) {
            $type = isset($this->tokenMap[$match[0]])
                ? $this->tokenMap[$match[0]]
                : (ctype_space($match[0]) ? self::TOKEN_WS : self::TOKEN_STRING);
            $tokens[] = array($type, $match[0], $position);
            $position += mb_strlen($match[0], 'utf-8');
        }

        return $tokens;
    }

    private function prepareLexer(SyntaxInterface $syntax)
    {
        $this->tokenMap = array(
            $syntax->getOpeningTag() => self::TOKEN_OPEN,
            $syntax->getClosingTag() => self::TOKEN_CLOSE,
            $syntax->getClosingTagMarker() => self::TOKEN_MARKER,
            $syntax->getParameterValueSeparator() => self::TOKEN_SEPARATOR,
            $syntax->getParameterValueDelimiter() => self::TOKEN_DELIMITER,
        );

        $quote = function($text) {
            return preg_replace('/(.)/us', '\\\\$0', $text);
        };
        $symbols = array_map($quote, array_keys($this->tokenMap));

        return '~('.implode('|', $symbols).'|\s+|\\\\.|[\w-]+|.)~us';
    }
}


1			<?php
2			namespace Thunder\Shortcode\Parser;
3
4			use Thunder\Shortcode\Shortcode\ParsedShortcode;
5			use Thunder\Shortcode\Shortcode\Shortcode;
6			use Thunder\Shortcode\Syntax\CommonSyntax;
7			use Thunder\Shortcode\Syntax\SyntaxInterface;
8			use Thunder\Shortcode\Utility\RegexBuilderUtility;
9
10			/**
11			* @author Tomasz Kowalczyk <[email protected]>
12			*/
13			final class RegularParser implements ParserInterface
14			{
15			private $lexerRegex;
16			private $nameRegex;
17			private $tokens;
18			private $tokensCount;
19			private $position;
20			/** @var int[] */
21			private $backtracks;
22			private $lastBacktrack;
23			private $tokenMap;
24
25			const TOKEN_OPEN = 1;
26			const TOKEN_CLOSE = 2;
27			const TOKEN_MARKER = 3;
28			const TOKEN_SEPARATOR = 4;
29			const TOKEN_DELIMITER = 5;
30			const TOKEN_STRING = 6;
31			const TOKEN_WS = 7;
32
33	14		public function __construct(SyntaxInterface $syntax = null)
34			{
35	14		$this->lexerRegex = $this->prepareLexer($syntax ?: new CommonSyntax());
36	14		$this->nameRegex = '~^'.RegexBuilderUtility::buildNameRegex().'$~us';
37	14		}
38
39			/**
40			* @param string $text
41			*
42			* @return ParsedShortcode[]
43			*/
44	57		public function parse($text)
45			{
46	57		$this->tokens = $this->tokenize($text);
47	57		$this->backtracks = array();
48	57		$this->lastBacktrack = 0;
49	57		$this->position = 0;
50	57		$this->tokensCount = \count($this->tokens);
51
52	57		$shortcodes = array();
53	57		while($this->position < $this->tokensCount) {
54	56		while($this->position < $this->tokensCount && false === $this->lookahead(self::TOKEN_OPEN)) {
55	24		$this->position++;
56	24		}
57	56		$names = array();
58	56		$this->beginBacktrack();
59	56		$matches = $this->shortcode($names);
60	56		if(\is_array($matches)) {
61	48		foreach($matches as $shortcode) {
62	48		$shortcodes[] = $shortcode;
63	48		}
64	48		}
65	56		}
66
67	57		return $shortcodes;
68			}
69
70	48		private function getObject($name, $parameters, $bbCode, $offset, $content, $text)
71			{
72	48		return new ParsedShortcode(new Shortcode($name, $parameters, $content, $bbCode), $text, $offset);
73			}
74
75			/* --- RULES ----------------------------------------------------------- */
76
77	56		private function shortcode(array &$names)
78			{
79	56		if(!$this->match(self::TOKEN_OPEN, false)) { return false; }
80	56		$offset = $this->tokens[$this->position - 1][2];
81	56		$this->match(self::TOKEN_WS, false);
82	56		if('' === $name = $this->match(self::TOKEN_STRING, false)) { return false; }
83	53		if($this->lookahead(self::TOKEN_STRING)) { return false; }
84	53		if(1 !== preg_match($this->nameRegex, $name, $matches)) { return false; }
85	52		$this->match(self::TOKEN_WS, false);
86			// bbCode
87	52		$bbCode = $this->match(self::TOKEN_SEPARATOR, true) ? $this->value() : null;
88	52		if(false === $bbCode) { return false; }
89			// parameters
90	51		if(false === ($parameters = $this->parameters())) { return false; }
91
92			// self-closing
93	49		if($this->match(self::TOKEN_MARKER, true)) {
94	16		if(!$this->match(self::TOKEN_CLOSE, false)) { return false; }
95
96	15		return array($this->getObject($name, $parameters, $bbCode, $offset, null, $this->getBacktrack()));
97			}
98
99			// just-closed or with-content
100	37		if(!$this->match(self::TOKEN_CLOSE, false)) { return false; }
101	37		$this->beginBacktrack();
102	37		$names[] = $name;
103	37		list($content, $shortcodes, $closingName) = $this->content($names);
104	37		if(null !== $closingName && $closingName !== $name) {
105	6		array_pop($names);
106	6		array_pop($this->backtracks);
107	6		array_pop($this->backtracks);
108
109	6		return $closingName;
110			}
111	37		if(false === $content \|\| $closingName !== $name) {
112	24		$this->backtrack(false);
113	24		$text = $this->backtrack(false);
114
115	24		return array_merge(array($this->getObject($name, $parameters, $bbCode, $offset, null, $text)), $shortcodes);
116			}
117	19		$content = $this->getBacktrack();
118	19		if(!$this->close($names)) { return false; }
119
120	19		return array($this->getObject($name, $parameters, $bbCode, $offset, $content, $this->getBacktrack()));
121			}
122
123	37		private function content(array &$names)
124			{
125	37		$content = '';
126	37		$shortcodes = array();
127	37		$closingName = null;
128
129	37		while($this->position < $this->tokensCount) {
130	28	View Code Duplication	while($this->position < $this->tokensCount && false === $this->lookahead(self::TOKEN_OPEN)) {
			0 ignored issues – show Duplication introduced 2016-04-25 21:25 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
131	24		$content .= $this->match(null, true);
132	24		}
133
134	28		$this->beginBacktrack();
135	28		$matchedShortcodes = $this->shortcode($names);
136	28		if(\is_string($matchedShortcodes)) {
137	6		$closingName = $matchedShortcodes;
138	6		break;
139			}
140	28		if(\is_array($matchedShortcodes)) {
141	15		foreach($matchedShortcodes as $matchedShortcode) {
142	15		$shortcodes[] = $matchedShortcode;
143	15		}
144	15		continue;
145			}
146	22		$this->backtrack();
147
148	22		$this->beginBacktrack();
149	22		if(false !== ($closingName = $this->close($names))) {
150	19		if(null === $content) { $content = ''; }
151	19		$this->backtrack();
152	19		$shortcodes = array();
153	19		break;
154			}
155	8		$closingName = null;
156	8		$this->backtrack();
157
158	8		$content .= $this->match(null, false);
159	8		}
160
161	37		return array($this->position < $this->tokensCount ? $content : false, $shortcodes, $closingName);
162			}
163
164	22		private function close(array &$names)
165			{
166	22		if(!$this->match(self::TOKEN_OPEN, true)) { return false; }
167	20		if(!$this->match(self::TOKEN_MARKER, true)) { return false; }
168	20		if(!$closingName = $this->match(self::TOKEN_STRING, true)) { return false; }
169	20		if(!$this->match(self::TOKEN_CLOSE, false)) { return false; }
170
171	20		return \in_array($closingName, $names, true) ? $closingName : false;
172			}
173
174	51		private function parameters()
175			{
176	51		$parameters = array();
177
178	51		while(true) {
179	51		$this->match(self::TOKEN_WS, false);
180	51		if($this->lookahead(self::TOKEN_MARKER) \|\| $this->lookahead(self::TOKEN_CLOSE)) { break; }
181	27		if(!$name = $this->match(self::TOKEN_STRING, true)) { return false; }
182	26		if(!$this->match(self::TOKEN_SEPARATOR, true)) { $parameters[$name] = null; continue; }
183	25		if(false === ($value = $this->value())) { return false; }
184	24		$this->match(self::TOKEN_WS, false);
185
186	24		$parameters[$name] = $value;
187	24		}
188
189	49		return $parameters;
190			}
191
192	27		private function value()
193			{
194	27		$value = '';
195
196	27		if($this->match(self::TOKEN_DELIMITER, false)) {
197	19	View Code Duplication	while($this->position < $this->tokensCount && false === $this->lookahead(self::TOKEN_DELIMITER)) {
			0 ignored issues – show Duplication introduced 2016-04-25 21:25 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
198	19		$value .= $this->match(null, false);
199	19		}
200
201	19		return $this->match(self::TOKEN_DELIMITER, false) ? $value : false;
202			}
203
204	14		if($tmp = $this->match(self::TOKEN_STRING, false)) {
205	13		$value .= $tmp;
206	13		while($tmp = $this->match(self::TOKEN_STRING, false)) {
207	2		$value .= $tmp;
208	2		}
209
210	13		return $value;
211			}
212
213	1		return false;
214			}
215
216			/* --- PARSER ---------------------------------------------------------- */
217
218	56		private function beginBacktrack()
219			{
220	56		$this->backtracks[] = $this->position;
221	56		$this->lastBacktrack = $this->position;
222	56		}
223
224	30		private function getBacktrack()
225			{
226	30		$position = array_pop($this->backtracks);
227	30		$backtrack = '';
228	30	View Code Duplication	for($i = $position; $i < $this->position; $i++) {
			0 ignored issues – show Duplication introduced 2018-12-08 20:25 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
229	30		$backtrack .= $this->tokens[$i][1];
230	30		}
231
232	30		return $backtrack;
233			}
234
235	37		private function backtrack($modifyPosition = true)
236			{
237	37		$position = array_pop($this->backtracks);
238	37		if($modifyPosition) {
239	22		$this->position = $position;
240	22		}
241
242	37		$backtrack = '';
243	37	View Code Duplication	for($i = $position; $i < $this->lastBacktrack; $i++) {
			0 ignored issues – show Duplication introduced 2018-12-08 20:25 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
244	24		$backtrack .= $this->tokens[$i][1];
245	24		}
246	37		$this->lastBacktrack = $position;
247
248	37		return $backtrack;
249			}
250
251	56		private function lookahead($type)
252			{
253	56		return $this->position < $this->tokensCount && (empty($type) \|\| $this->tokens[$this->position][0] === $type);
254			}
255
256	56		private function match($type, $ws)
257			{
258	56		if($this->position >= $this->tokensCount) {
259	20		return '';
260			}
261
262	56		$token = $this->tokens[$this->position];
263	56		if(!empty($type) && $token[0] !== $type) {
264	56		return '';
265			}
266
267	56		$this->position++;
268	56		if($ws && $this->position < $this->tokensCount && $this->tokens[$this->position][0] === self::TOKEN_WS) {
269	17		$this->position++;
270	17		}
271
272	56		return $token[1];
273			}
274
275			/* --- LEXER ----------------------------------------------------------- */
276
277	57		private function tokenize($text)
278			{
279	57		preg_match_all($this->lexerRegex, $text, $matches, PREG_OFFSET_CAPTURE);
280	57		if(preg_last_error() !== PREG_NO_ERROR) {
281			throw new \RuntimeException(sprintf('PCRE failure `%s`.', preg_last_error()));
282			}
283
284	57		$tokens = array();
285	57		$position = 0;
286	57		foreach($matches[0] as $match) {
287	56		$type = isset($this->tokenMap[$match[0]])
288	56		? $this->tokenMap[$match[0]]
289	56		: (ctype_space($match[0]) ? self::TOKEN_WS : self::TOKEN_STRING);
290	56		$tokens[] = array($type, $match[0], $position);
291	56		$position += mb_strlen($match[0], 'utf-8');
292	57		}
293
294	57		return $tokens;
295			}
296
297	14		private function prepareLexer(SyntaxInterface $syntax)
298			{
299	14		$this->tokenMap = array(
300	14		$syntax->getOpeningTag() => self::TOKEN_OPEN,
301	14		$syntax->getClosingTag() => self::TOKEN_CLOSE,
302	14		$syntax->getClosingTagMarker() => self::TOKEN_MARKER,
303	14		$syntax->getParameterValueSeparator() => self::TOKEN_SEPARATOR,
304	14		$syntax->getParameterValueDelimiter() => self::TOKEN_DELIMITER,
305			);
306
307	14		$quote = function($text) {
308	14		return preg_replace('/(.)/us', '\\\\$0', $text);
309	14		};
310	14		$symbols = array_map($quote, array_keys($this->tokenMap));
311
312	14		return '~('.implode('\|', $symbols).'\|\s+\|\\\\.\|[\w-]+\|.)~us';
313			}
314			}
315

thunderer / Shortcode

GitHub Access Token became invalid

Pull Request — master (#72)

RegularParser::tokenize() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like