RegularParser::tokenize() - Code Metrics - Inspection of "Regular parser performance improvements" - thunderer/Shortcode - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#28)

by Tomasz

created 2016-01-24 18:50 UTC

RegularParser::tokenize() C

↳ Parent: RegularParser

Complexity

Conditions	9
Paths	9

Size

Total Lines	23
Code Lines	18

Duplication

Lines	7
Ratio	30.43 %

Code Coverage

Tests	19
CRAP Score	9.0101

Importance

Changes	5
Bugs	1	Features	0

Metric	Value
c	5
b	1
f	0
dl	7
loc	23
ccs	19
cts	20
cp	0.95
rs	5.8541
cc	9
eloc	18
nc	9
nop	1
crap	9.0101

<?php
namespace Thunder\Shortcode\Parser;

use Thunder\Shortcode\Shortcode\ParsedShortcode;
use Thunder\Shortcode\Shortcode\Shortcode;
use Thunder\Shortcode\Syntax\CommonSyntax;
use Thunder\Shortcode\Syntax\SyntaxInterface;

/**
 * @author Tomasz Kowalczyk <[email protected]>
 */
final class RegularParser implements ParserInterface
{
    private $lexerRegex;
    private $tokens;
    private $tokensCount;
    private $position;
    private $backtracks;

    const TOKEN_OPEN = 1;
    const TOKEN_CLOSE = 2;
    const TOKEN_MARKER = 3;
    const TOKEN_SEPARATOR = 4;
    const TOKEN_DELIMITER = 5;
    const TOKEN_STRING = 6;
    const TOKEN_WS = 7;

    public function __construct(SyntaxInterface $syntax = null)
    {
        $this->lexerRegex = $this->getTokenizerRegex($syntax ?: new CommonSyntax());
    }

    /**
     * @param string $text
     *
     * @return ParsedShortcode[]
     */
    public function parse($text)
    {
        $this->tokens = $this->tokenize($text);
        $this->backtracks = array();
        $this->position = 0;
        $this->tokensCount = count($this->tokens);

        $shortcodes = array();
        while($this->position < $this->tokensCount) {
            while($this->position < $this->tokensCount && !$this->lookahead(self::TOKEN_OPEN)) {
                $this->position++;
            }
            foreach($this->shortcode(true) ?: array() as $shortcode) {
                $shortcodes[] = $shortcode;
            }
        }

        return $shortcodes;
    }

    private function getObject($name, $parameters, $bbCode, $offset, $content, $text)
    {
        return new ParsedShortcode(new Shortcode($name, $parameters, $content, $bbCode), $text, $offset);
    }

    /* --- RULES ----------------------------------------------------------- */

    private function shortcode($isRoot)
    {
        $name = null;
        $offset = null;

        $setName = function(array $token) use(&$name) { $name = $token[1]; };
        $setOffset = function(array $token) use(&$offset) { $offset = $token[2]; };

        $isRoot && $this->beginBacktrack();
        if(!$this->match(self::TOKEN_OPEN, $setOffset, true)) { return false; }
        if(!$this->match(self::TOKEN_STRING, $setName, false)) { return false; }
        if($this->lookahead(self::TOKEN_STRING, null)) { return false; }
        if(!preg_match_all('/^[a-zA-Z0-9-]+$/', $name, $matches)) { return false; }
        $this->match(self::TOKEN_WS);
        if(false === ($bbCode = $this->bbCode())) { return false; }
        if(false === ($parameters = $this->parameters())) { return false; }

        // self-closing
        if($this->match(self::TOKEN_MARKER, null, true)) {
            if(!$this->match(self::TOKEN_CLOSE)) { return false; }

            return array($this->getObject($name, $parameters, $bbCode, $offset, null, $this->getBacktrack()));
        }

        // just-closed or with-content
        if(!$this->match(self::TOKEN_CLOSE)) { return false; }
        $this->beginBacktrack();
        list($content, $shortcodes) = $this->content($name);
        if(false === $content) {
            $this->backtrack(false);
            $text = $this->backtrack(false);
            return array_merge(array($this->getObject($name, $parameters, $bbCode, $offset, null, $text)), $shortcodes);
        }
        array_pop($this->backtracks);
        if(!$this->close($name)) { return false; }

        return array($this->getObject($name, $parameters, $bbCode, $offset, $content, $this->getBacktrack()));
    }

    private function content($name)
    {
        $content = null;
        $shortcodes = array();
        $appendContent = function(array $token) use(&$content) { $content .= $token[1]; };

        while($this->position < $this->tokensCount) {
            while($this->match(array(self::TOKEN_STRING, self::TOKEN_WS), $appendContent)) {
                continue;
            }

            $this->beginBacktrack();
            $matchedShortcodes = $this->shortcode(false);
            if(false !== $matchedShortcodes) {
                $shortcodes = array_merge($shortcodes, $matchedShortcodes);
                continue;
            }
            $this->backtrack();

            $this->beginBacktrack();
            if(false !== $this->close($name)) {
                if(null === $content) { $content = ''; }
                $this->backtrack();
                $shortcodes = array();
                break;
            }
            $this->backtrack();
            if($this->position < $this->tokensCount) {
                $shortcodes = array();
                break;
            }

            $this->match(null, $appendContent);
        }

        return array($this->position < $this->tokensCount ? $content : false, $shortcodes);
    }

    private function close($openingName)
    {
        $closingName = null;
        $setName = function(array $token) use(&$closingName) { $closingName = $token[1]; };

        if(!$this->match(self::TOKEN_OPEN, null, true)) { return false; }
        if(!$this->match(self::TOKEN_MARKER, null, true)) { return false; }
        if(!$this->match(self::TOKEN_STRING, $setName, true)) { return false; }
        if(!$this->match(self::TOKEN_CLOSE)) { return false; }

        return $openingName === $closingName;
    }

    private function bbCode()
    {
        return $this->match(self::TOKEN_SEPARATOR, null, true) ? $this->value() : null;
    }

    private function parameters()
    {
        $parameters = array();
        $setName = function(array $token) use(&$name) { $name = $token[1]; };

        while(true) {
            $name = null;
$x = 1;
$callable = function() use ($x) {
    $x = 2; // Not visible in outer scope. If you would like this, how
            // about using a different variable name than $x?
};

$callable();
var_dump($x); // integer(1)

            $this->match(self::TOKEN_WS);
            if($this->lookahead(array(self::TOKEN_MARKER, self::TOKEN_CLOSE))) { break; }
            if(!$this->match(self::TOKEN_STRING, $setName, true)) { return false; }
            if(!$this->match(self::TOKEN_SEPARATOR, null, true)) { $parameters[$name] = null; continue; }
            if(false === ($value = $this->value())) { return false; }
            $this->match(self::TOKEN_WS);

            $parameters[$name] = $value;
        }

        return $parameters;
    }

    private function value()
    {
        $value = '';
        $appendValue = function(array $token) use(&$value) { $value .= $token[1]; };

        if($this->match(self::TOKEN_DELIMITER)) {
            while($this->position < $this->tokensCount && !$this->lookahead(self::TOKEN_DELIMITER)) {
                $this->match(null, $appendValue);
            }

            return $this->match(self::TOKEN_DELIMITER) ? $value : false;
        }

        return $this->match(self::TOKEN_STRING, $appendValue) ? $value : false;
    }

    /* --- PARSER ---------------------------------------------------------- */

    private function beginBacktrack()
    {
        $this->backtracks[] = array();
    }

    private function getBacktrack()
    {
        // switch from array_map() to array_column() when dropping support for PHP <5.5
        return implode('', array_map(function(array $token) { return $token[1]; }, array_pop($this->backtracks)));
    }

    private function backtrack($modifyPosition = true)
    {
        $tokens = array_pop($this->backtracks);
        $count = count($tokens);
        if($modifyPosition) {
            $this->position -= $count;
        }

        foreach($this->backtracks as &$backtrack) {
            // array_pop() in loop is much faster than array_slice() because
            // it operates directly on the passed array
            for($i = 0; $i < $count; $i++) {
                array_pop($backtrack);
            }
        }

        return implode('', array_map(function(array $token) { return $token[1]; }, $tokens));
    }

    private function lookahead($type, $callback = null)
    {
        if($this->position >= $this->tokensCount) {
            return false;
        }

        $type = (array)$type;
        $token = $this->tokens[$this->position];
        if(!empty($type) && !in_array($token[0], $type)) {
            return false;
        }

        /** @var $callback callable */
        $callback && $callback($token);

        return true;
    }

    private function match($type, $callbacks = null, $ws = false)
    {
        if($this->position >= $this->tokensCount) {
            return false;
        }

        $type = (array)$type;
        $token = $this->tokens[$this->position];
        if(!empty($type) && !in_array($token[0], $type)) {
            return false;
        }
        foreach($this->backtracks as &$backtrack) {
            $backtrack[] = $token;
        }

        $this->position++;
        foreach((array)$callbacks as $callback) {
            $callback($token);
        }

        $ws && $this->match(self::TOKEN_WS);

        return true;
    }

    /* --- LEXER ----------------------------------------------------------- */

    private function tokenize($text)
    {
        preg_match_all($this->lexerRegex, $text, $matches, PREG_SET_ORDER | PREG_OFFSET_CAPTURE);
        $tokens = array();
        $position = 0;
        $type = null;
        $token = null;
        foreach($matches as $match) {
            switch(true) {
                case -1 !== $match['open'][1]: $token = $match['open'][0]; $type = self::TOKEN_OPEN; break;

                case -1 !== $match['close'][1]: $token = $match['close'][0]; $type = self::TOKEN_CLOSE; break;

                case -1 !== $match['marker'][1]: $token = $match['marker'][0]; $type = self::TOKEN_MARKER; break;

                case -1 !== $match['separator'][1]: $token = $match['separator'][0]; $type = self::TOKEN_SEPARATOR; break;

                case -1 !== $match['delimiter'][1]: $token = $match['delimiter'][0]; $type = self::TOKEN_DELIMITER; break;

                case -1 !== $match['ws'][1]: $token = $match['ws'][0]; $type = self::TOKEN_WS; break;

                case -1 !== $match['string'][1]: $token = $match['string'][0]; $type = self::TOKEN_STRING; break;

            }
            $tokens[] = array($type, $token, $position);
            $position += mb_strlen($token, 'utf-8');
        }

        return $tokens;
    }

    private function getTokenizerRegex(SyntaxInterface $syntax)
    {
        $quote = function($text, $group) {
            return '(?<'.$group.'>'.preg_replace('/(.)/us', '\\\\$0', $text).')';
        };

        $rules = array(
            $quote($syntax->getOpeningTag(), 'open'),
            $quote($syntax->getClosingTag(), 'close'),
            $quote($syntax->getClosingTagMarker(), 'marker'),
            $quote($syntax->getParameterValueSeparator(), 'separator'),
            $quote($syntax->getParameterValueDelimiter(), 'delimiter'),
            '(?<ws>\s+)',
            '(?<string>[\w-]+|\\\\.|.)',
        );

        return '~('.implode('|', $rules).')~us';
    }
}


GitHub Access Token became invalid

Pull Request — master (#28)

RegularParser::tokenize() C

Complexity

Size

Duplication

Code Coverage

Importance

Change not visible in outer-scope

Change visible in outer-scope

1			<?php
2			namespace Thunder\Shortcode\Parser;
3
4			use Thunder\Shortcode\Shortcode\ParsedShortcode;
5			use Thunder\Shortcode\Shortcode\Shortcode;
6			use Thunder\Shortcode\Syntax\CommonSyntax;
7			use Thunder\Shortcode\Syntax\SyntaxInterface;
8
9			/**
10			* @author Tomasz Kowalczyk <[email protected]>
11			*/
12			final class RegularParser implements ParserInterface
13			{
14			private $lexerRegex;
15			private $tokens;
16			private $tokensCount;
17			private $position;
18			private $backtracks;
19
20			const TOKEN_OPEN = 1;
21			const TOKEN_CLOSE = 2;
22			const TOKEN_MARKER = 3;
23			const TOKEN_SEPARATOR = 4;
24			const TOKEN_DELIMITER = 5;
25			const TOKEN_STRING = 6;
26			const TOKEN_WS = 7;
27
28	4		public function __construct(SyntaxInterface $syntax = null)
29			{
30	4		$this->lexerRegex = $this->getTokenizerRegex($syntax ?: new CommonSyntax());
31	4		}
32
33			/**
34			* @param string $text
35			*
36			* @return ParsedShortcode[]
37			*/
38	37		public function parse($text)
39			{
40	37		$this->tokens = $this->tokenize($text);
41	37		$this->backtracks = array();
42	37		$this->position = 0;
43	37		$this->tokensCount = count($this->tokens);
44
45	37		$shortcodes = array();
46	37		while($this->position < $this->tokensCount) {
47	36		while($this->position < $this->tokensCount && !$this->lookahead(self::TOKEN_OPEN)) {
48	13		$this->position++;
49	13		}
50	36		foreach($this->shortcode(true) ?: array() as $shortcode) {
51	29		$shortcodes[] = $shortcode;
52	36		}
53	36		}
54
55	37		return $shortcodes;
56			}
57
58	29		private function getObject($name, $parameters, $bbCode, $offset, $content, $text)
59			{
60	29		return new ParsedShortcode(new Shortcode($name, $parameters, $content, $bbCode), $text, $offset);
61			}
62
63			/* --- RULES ----------------------------------------------------------- */
64
65	36		private function shortcode($isRoot)
66			{
67	36		$name = null;
68	36		$offset = null;
69
70			$setName = function(array $token) use(&$name) { $name = $token[1]; };
71			$setOffset = function(array $token) use(&$offset) { $offset = $token[2]; };
72
73	36		$isRoot && $this->beginBacktrack();
74	36		if(!$this->match(self::TOKEN_OPEN, $setOffset, true)) { return false; }
75	36		if(!$this->match(self::TOKEN_STRING, $setName, false)) { return false; }
76	33		if($this->lookahead(self::TOKEN_STRING, null)) { return false; }
77	33		if(!preg_match_all('/^[a-zA-Z0-9-]+$/', $name, $matches)) { return false; }
78	32		$this->match(self::TOKEN_WS);
79	32		if(false === ($bbCode = $this->bbCode())) { return false; }
80	32		if(false === ($parameters = $this->parameters())) { return false; }
81
82			// self-closing
83	30		if($this->match(self::TOKEN_MARKER, null, true)) {
84	12		if(!$this->match(self::TOKEN_CLOSE)) { return false; }
85
86	11		return array($this->getObject($name, $parameters, $bbCode, $offset, null, $this->getBacktrack()));
87			}
88
89			// just-closed or with-content
90	20		if(!$this->match(self::TOKEN_CLOSE)) { return false; }
91	20		$this->beginBacktrack();
92	20		list($content, $shortcodes) = $this->content($name);
93	20		if(false === $content) {
94	13		$this->backtrack(false);
95	13		$text = $this->backtrack(false);
96	13		return array_merge(array($this->getObject($name, $parameters, $bbCode, $offset, null, $text)), $shortcodes);
97			}
98	8		array_pop($this->backtracks);
99	8		if(!$this->close($name)) { return false; }
100
101	8		return array($this->getObject($name, $parameters, $bbCode, $offset, $content, $this->getBacktrack()));
102			}
103
104	20		private function content($name)
105			{
106	20		$content = null;
107	20		$shortcodes = array();
108			$appendContent = function(array $token) use(&$content) { $content .= $token[1]; };
109
110	20		while($this->position < $this->tokensCount) {
111	12		while($this->match(array(self::TOKEN_STRING, self::TOKEN_WS), $appendContent)) {
112	11		continue;
113			}
114
115	12		$this->beginBacktrack();
116	12		$matchedShortcodes = $this->shortcode(false);
117	12		if(false !== $matchedShortcodes) {
118	5		$shortcodes = array_merge($shortcodes, $matchedShortcodes);
119	5		continue;
120			}
121	10		$this->backtrack();
122
123	10		$this->beginBacktrack();
124	10		if(false !== $this->close($name)) {
125	8		if(null === $content) { $content = ''; }
126	8		$this->backtrack();
127	8		$shortcodes = array();
128	8		break;
129			}
130	3		$this->backtrack();
131	3		if($this->position < $this->tokensCount) {
132			$shortcodes = array();
133			break;
134			}
135
136	3		$this->match(null, $appendContent);
137	3		}
138
139	20		return array($this->position < $this->tokensCount ? $content : false, $shortcodes);
140			}
141
142	10		private function close($openingName)
143			{
144	10		$closingName = null;
145			$setName = function(array $token) use(&$closingName) { $closingName = $token[1]; };
146
147	10		if(!$this->match(self::TOKEN_OPEN, null, true)) { return false; }
148	8		if(!$this->match(self::TOKEN_MARKER, null, true)) { return false; }
149	8		if(!$this->match(self::TOKEN_STRING, $setName, true)) { return false; }
150	8		if(!$this->match(self::TOKEN_CLOSE)) { return false; }
151
152	8		return $openingName === $closingName;
153			}
154
155	32		private function bbCode()
156			{
157	32		return $this->match(self::TOKEN_SEPARATOR, null, true) ? $this->value() : null;
158			}
159
160	32		private function parameters()
161			{
162	32		$parameters = array();
163			$setName = function(array $token) use(&$name) { $name = $token[1]; };
164
165	32		while(true) {
166	32		$name = null;
			0 ignored issues – show Bug introduced 2016-01-20 03:55 UTC by Report Bug Copy Issue Report Consider using a different name than the imported variable `$name`, or did you forget to import by reference? It seems like you are assigning to a variable which was imported through a `use` statement which was not imported by reference. For clarity, we suggest to use a different name or import by reference depending on whether you would like to have the change visibile in outer-scope. Change not visible in outer-scope $x = 1; $callable = function() use ($x) { $x = 2; // Not visible in outer scope. If you would like this, how // about using a different variable name than $x? }; $callable(); var_dump($x); // integer(1) Change visible in outer-scope $x = 1; $callable = function() use (&$x) { $x = 2; }; $callable(); var_dump($x); // integer(2) Loading history...
167
168	32		$this->match(self::TOKEN_WS);
169	32		if($this->lookahead(array(self::TOKEN_MARKER, self::TOKEN_CLOSE))) { break; }
170	22		if(!$this->match(self::TOKEN_STRING, $setName, true)) { return false; }
171	21		if(!$this->match(self::TOKEN_SEPARATOR, null, true)) { $parameters[$name] = null; continue; }
172	20		if(false === ($value = $this->value())) { return false; }
173	19		$this->match(self::TOKEN_WS);
174
175	19		$parameters[$name] = $value;
176	19		}
177
178	30		return $parameters;
179			}
180
181	21		private function value()
182			{
183	21		$value = '';
184			$appendValue = function(array $token) use(&$value) { $value .= $token[1]; };
185
186	21		if($this->match(self::TOKEN_DELIMITER)) {
187	18		while($this->position < $this->tokensCount && !$this->lookahead(self::TOKEN_DELIMITER)) {
188	18		$this->match(null, $appendValue);
189	18		}
190
191	18		return $this->match(self::TOKEN_DELIMITER) ? $value : false;
192			}
193
194	8		return $this->match(self::TOKEN_STRING, $appendValue) ? $value : false;
195			}
196
197			/* --- PARSER ---------------------------------------------------------- */
198
199	36		private function beginBacktrack()
200			{
201	36		$this->backtracks[] = array();
202	36		}
203
204			private function getBacktrack()
205			{
206			// switch from array_map() to array_column() when dropping support for PHP <5.5
207			return implode('', array_map(function(array $token) { return $token[1]; }, array_pop($this->backtracks)));
208			}
209
210	20		private function backtrack($modifyPosition = true)
211			{
212	20		$tokens = array_pop($this->backtracks);
213	20		$count = count($tokens);
214	20		if($modifyPosition) {
215	10		$this->position -= $count;
216	10		}
217
218	20		foreach($this->backtracks as &$backtrack) {
219			// array_pop() in loop is much faster than array_slice() because
220			// it operates directly on the passed array
221	20		for($i = 0; $i < $count; $i++) {
222	12		array_pop($backtrack);
223	12		}
224	20		}
225
226			return implode('', array_map(function(array $token) { return $token[1]; }, $tokens));
227			}
228
229	36		private function lookahead($type, $callback = null)
230			{
231	36		if($this->position >= $this->tokensCount) {
232	1		return false;
233			}
234
235	36		$type = (array)$type;
236	36		$token = $this->tokens[$this->position];
237	36		if(!empty($type) && !in_array($token[0], $type)) {
238	35		return false;
239			}
240
241			/** @var $callback callable */
242	36		$callback && $callback($token);
243
244	36		return true;
245			}
246
247	36		private function match($type, $callbacks = null, $ws = false)
248			{
249	36		if($this->position >= $this->tokensCount) {
250	13		return false;
251			}
252
253	36		$type = (array)$type;
254	36		$token = $this->tokens[$this->position];
255	36		if(!empty($type) && !in_array($token[0], $type)) {
256	36		return false;
257			}
258	36		foreach($this->backtracks as &$backtrack) {
259	36		$backtrack[] = $token;
260	36		}
261
262	36		$this->position++;
263	36		foreach((array)$callbacks as $callback) {
264	36		$callback($token);
265	36		}
266
267	36		$ws && $this->match(self::TOKEN_WS);
268
269	36		return true;
270			}
271
272			/* --- LEXER ----------------------------------------------------------- */
273
274	37		private function tokenize($text)
275			{
276	37		preg_match_all($this->lexerRegex, $text, $matches, PREG_SET_ORDER \| PREG_OFFSET_CAPTURE);
277	37		$tokens = array();
278	37		$position = 0;
279	37		$type = null;
280	37		$token = null;
281	37		foreach($matches as $match) {
282	36		switch(true) {
283	36	View Code Duplication	case -1 !== $match['open'][1]: $token = $match['open'][0]; $type = self::TOKEN_OPEN; break;
			0 ignored issues – show Unused Code Bug introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report The strict comparison `!==` seems to always evaluate to `true` as the types of `-1` (`integer`) and `$match['open'][1]` (`string`) can never be identical. Maybe you want to use a loose comparison `!=` instead? Loading history... Duplication introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
284	36	View Code Duplication	case -1 !== $match['close'][1]: $token = $match['close'][0]; $type = self::TOKEN_CLOSE; break;
			0 ignored issues – show Unused Code Bug introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report The strict comparison `!==` seems to always evaluate to `true` as the types of `-1` (`integer`) and `$match['close'][1]` (`string`) can never be identical. Maybe you want to use a loose comparison `!=` instead? Loading history... Duplication introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
285	35	View Code Duplication	case -1 !== $match['marker'][1]: $token = $match['marker'][0]; $type = self::TOKEN_MARKER; break;
			0 ignored issues – show Unused Code Bug introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report The strict comparison `!==` seems to always evaluate to `true` as the types of `-1` (`integer`) and `$match['marker'][1]` (`string`) can never be identical. Maybe you want to use a loose comparison `!=` instead? Loading history... Duplication introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
286	35	View Code Duplication	case -1 !== $match['separator'][1]: $token = $match['separator'][0]; $type = self::TOKEN_SEPARATOR; break;
			0 ignored issues – show Unused Code Bug introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report The strict comparison `!==` seems to always evaluate to `true` as the types of `-1` (`integer`) and `$match['separator'][1]` (`string`) can never be identical. Maybe you want to use a loose comparison `!=` instead? Loading history... Duplication introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
287	35	View Code Duplication	case -1 !== $match['delimiter'][1]: $token = $match['delimiter'][0]; $type = self::TOKEN_DELIMITER; break;
			0 ignored issues – show Unused Code Bug introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report The strict comparison `!==` seems to always evaluate to `true` as the types of `-1` (`integer`) and `$match['delimiter'][1]` (`string`) can never be identical. Maybe you want to use a loose comparison `!=` instead? Loading history... Duplication introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
288	35	View Code Duplication	case -1 !== $match['ws'][1]: $token = $match['ws'][0]; $type = self::TOKEN_WS; break;
			0 ignored issues – show Unused Code Bug introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report The strict comparison `!==` seems to always evaluate to `true` as the types of `-1` (`integer`) and `$match['ws'][1]` (`string`) can never be identical. Maybe you want to use a loose comparison `!=` instead? Loading history... Duplication introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
289	35	View Code Duplication	case -1 !== $match['string'][1]: $token = $match['string'][0]; $type = self::TOKEN_STRING; break;
			0 ignored issues – show Unused Code Bug introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report The strict comparison `!==` seems to always evaluate to `true` as the types of `-1` (`integer`) and `$match['string'][1]` (`string`) can never be identical. Maybe you want to use a loose comparison `!=` instead? Loading history... Duplication introduced 2016-01-24 18:52 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
290			}
291	36		$tokens[] = array($type, $token, $position);
292	36		$position += mb_strlen($token, 'utf-8');
293	37		}
294
295	37		return $tokens;
296			}
297
298			private function getTokenizerRegex(SyntaxInterface $syntax)
299			{
300	4		$quote = function($text, $group) {
301	4		return '(?<'.$group.'>'.preg_replace('/(.)/us', '\\\\$0', $text).')';
302	4		};
303
304			$rules = array(
305	4		$quote($syntax->getOpeningTag(), 'open'),
306	4		$quote($syntax->getClosingTag(), 'close'),
307	4		$quote($syntax->getClosingTagMarker(), 'marker'),
308	4		$quote($syntax->getParameterValueSeparator(), 'separator'),
309	4		$quote($syntax->getParameterValueDelimiter(), 'delimiter'),
310	4		'(?<ws>\s+)',
311	4		'(?<string>[\w-]+\|\\\\.\|.)',
312	4		);
313
314	4		return '~('.implode('\|', $rules).')~us';
315			}
316			}
317

thunderer / Shortcode

GitHub Access Token became invalid

Pull Request — master (#28)

RegularParser::tokenize() C

Complexity

Size

Duplication

Code Coverage

Importance

Change not visible in outer-scope

Change visible in outer-scope

Duplication Side-by-Side

Filter issues like