TokenStream::replace() - Code Metrics - Inspection of "Bumped requirements" - s9e/SourceOptimizer - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( fc1df1...d515dd )

by Josh

created 2017-12-01 02:22 UTC

TokenStream::replace() A

↳ Parent: TokenStream

Complexity

Conditions	1
Paths	1

Size

Total Lines	4
Code Lines	2

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	0
CRAP Score	2

Importance

Changes

Metric	Value
dl	0
loc	4
ccs	0
cts	3
cp	0
rs	10
c	0
b	0
f	0
cc	1
eloc	2
nc	1
nop	1
crap	2

<?php

/**
* @package   s9e\SourceOptimizer
* @copyright Copyright (c) 2014-2017 The s9e Authors
* @license   http://www.opensource.org/licenses/mit-license.php The MIT License
*/
namespace s9e\SourceOptimizer;

use ArrayAccess;
use Iterator;

class TokenStream implements ArrayAccess, Iterator
{
	/**
	* @var integer Number of tokens
	*/
	protected $cnt;

	/**
	* @var string[] List of exact triplet of tokens to exclude from minification
	*/
	protected $excludeExact = [
		// 1 - - 1 and 1 + + 1 should not become 1--1 or 1++1
		'- -',
		'+ +',
		// $a - --$b should not become $a---$b
		'- --',
		'+ ++'
	];

	/**
	* @var bool Whether the source code needs to be reparsed before this stream is handed off to a
	*           new pass
	*/
	public $needsReparsing;

	/**
	* @var integer Current token index
	*/
	protected $offset;

	/**
	* @var bool Whether some tokens have been removed
	*/
	protected $sparse;

	/**
	* @var array<array|string> List of tokens
	*/
	protected $tokens;

	/**
	* Constructor
	*
	* @param string $src Original source code
	*/
	public function __construct($src)
	{
		$this->parse($src);
	}

	/**
	* Test whether current token can be removed safely
	*
	* @return bool
	*/
	public function canRemoveCurrentToken()
	{
		$prevToken = $this->lookbehind();
		$nextToken = $this->lookahead();

		if ($prevToken === false || $nextToken === false)
		{
			return true;
		}

		if (is_array($prevToken))
		{
			if ($prevToken[0] === T_COMMENT && substr($prevToken[1], 0, 2) === '//')
			{
				return false;
			}

			$prevToken = $prevToken[1];
		}
		if (is_array($nextToken))
		{
			$nextToken = $nextToken[1];
		}

		$str = $prevToken . ' ' . $nextToken;
		if (in_array($str, $this->excludeExact, true))
		{
			return false;
		}

		$delimiters = "\t\n\r !\"#$%&'()*+,-./:;<=>?@[\\]^`{|}~";
		$prevChar = substr($prevToken, -1);
		$nextChar = $nextToken[0];

		return (strpos($delimiters, $prevChar) !== false || strpos($delimiters, $nextChar) !== false);
	}

	/**
	* Test whether current token is the given type of token
	*
	* @param  integer $tokenValue
	* @return bool
	*/
	public function is($tokenValue)
	{
		return ($this->tokens[$this->offset][0] === $tokenValue);
	}

	/**
	* Test whether current token is any of the given types of token
	*
	* @param  integer[] $tokenValues
	* @return bool
	*/
	public function isAny(array $tokenValues)
	{
		return in_array($this->tokens[$this->offset][0], $tokenValues, true);
	}

	/**
	* Test whether there's a token at given offset
	*
	* @param  integer $offset
	* @return bool
	*/
	public function offsetExists($offset)
	{
		return isset($this->tokens[$offset]);
	}

	/**
	* Return the token stored at given offset
	*
	* @param  integer $offset
	* @return array|string
	*/
	public function offsetGet($offset)
	{
		return $this->tokens[$offset];
	}

	/**
	* Replace the token stored at given offset
	*
	* @param  integer      $offset
	* @param  array|string $token
	* @return void
	*/
	public function offsetSet($offset, $token)
	{
		$this->tokens[$offset] = $token;
	}

	/**
	* Remove the token stored at given offset
	*
	* @return void
	*/
	public function offsetUnset($offset)
	{
		unset($this->tokens[$offset]);
		$this->sparse = true;
	}

	/**
	* Return the current token
	*
	* @return array|string
	*/
	public function current()
	{
		return $this->tokens[$this->offset];
	}

	/**
	* Get current token's text
	*
	* @return string
	*/
	public function currentText()
	{
		return (is_array($this->tokens[$this->offset])) ? $this->tokens[$this->offset][1] : $this->tokens[$this->offset];
	}

	/**
	* Test whether current token is noise (whitespace or comment)
	*
	* @return bool
	*/
	public function isNoise()
	{
		return in_array($this->tokens[$this->offset][0], [T_COMMENT, T_DOC_COMMENT, T_WHITESPACE], true);
	}

	/**
	* Return the offset of current token
	*
	* @return integer
	*/
	public function key()
	{
		return $this->offset;
	}

	/**
	* Peek at the next token
	*
	* @return array|string|false
	*/
	public function lookahead()
	{
		$i = $this->offset;
		while (++$i < $this->cnt)
		{
			if (isset($this->tokens[$i]))
			{
				return $this->tokens[$i];
			}
		}

		return false;
	}

	/**
	* Peek at the previous token
	*
	* @return array|string|false
	*/
	public function lookbehind()
	{
		$i = $this->offset;
		while (--$i >= 0)
		{
			if (isset($this->tokens[$i]))
			{
				return $this->tokens[$i];
			}
		}

		return false;
	}

	/**
	* Move to the next token in the stream
	*
	* @return void
	*/
	public function next()
	{
		while (++$this->offset < $this->cnt && !isset($this->tokens[$this->offset]));
	}

	/**
	* Normalize the whitespace at the end of single-line comments
	*
	* Will remove the newline at the end of single-line comments and put it either in the next token
	* if it's a T_WHITESPACE or it will insert a T_WHITESPACE otherwise.
	*
	* @return void
	*/
	protected function normalizeSingleLineComments()
	{
		$keys = [];
		foreach ($this->tokens as $k => $token)
		{
			// Remove the newline at the end of comments to put it in the next token
			if ($token[0] === T_COMMENT && $token[1][1] === '/')
			{
				$this->tokens[$k][1] = rtrim($token[1], "\n");
				$keys[] = $k + 1;
			}
		}

		foreach (array_reverse($keys) as $k)
		{
			if (isset($this->tokens[$k]) && $this->tokens[$k][0] === T_WHITESPACE)
			{
				$this->tokens[$k][1] = "\n" . $this->tokens[$k][1];
			}
			else
			{
				array_splice($this->tokens, $k, 0, [[T_WHITESPACE, "\n"]]);
			}
		}
	}

	/**
	* Parse/tokenize given PHP source
	*
	* @param  string $src
	* @return void
	*/
	protected function parse($src)
	{
		$this->tokens = token_get_all($src);
		foreach ($this->tokens as $k => &$token)
		{
			if (is_array($token))
			{
				unset($token[2]);
			}
		}
		$this->normalizeSingleLineComments();

		$this->cnt = count($this->tokens);
		$this->offset = 0;
		$this->needsReparsing = false;
		$this->sparse = false;
	}

	/**
	* Move to the previous token in the stream
	*
	* @return void
	*/
	public function previous()
	{
		while ($this->offset > 0)
		{
			--$this->offset;
			if (isset($this->tokens[$this->offset]))
			{
				break;
			}
		}
	}

	/**
	* Remove current token
	*
	* @return void
	*/
	public function remove()
	{
		$this->offsetUnset($this->offset);
	}

	/**
	* Replace current token
	*
	* @param  array|string $token Token replacement
	* @return void
	*/
	public function replace($token)
	{
		$this->offsetSet($this->offset, $token);
	}

	/**
	* Reset this stream
	*
	* @return void
	*/
	public function reset()
	{
		$this->offset = 0;
		if ($this->needsReparsing)
		{
			$this->parse($this->serialize());
		}
		elseif ($this->sparse)
		{
			$this->tokens = array_values($this->tokens);
		}
	}

	/**
	* Rewind/reset this stream
	*
	* @return void
	*/
	public function rewind()
	{
		$this->reset();
	}

	/**
	* Move the stream to given offset
	*
	* @param  integer $offset
	* @return void
	*/
	public function seek($offset)
	{
		$this->offset = $offset;
	}

	/**
	* Serialize these tokens back to source code
	*
	* @return string
	*/
	public function serialize()
	{
		$src = '';
		foreach ($this->tokens as $token)
		{
			$src .= (is_array($token)) ? $token[1] : $token;
		}

		return $src;
	}

	/**
	* Skip all whitespace, comments and docblocks starting at current offset
	*
	* @return void
	*/
	public function skipNoise()
	{
		while ($this->valid())
		{
			if (!$this->isNoise())
			{
				break;
			}
			$this->next();
		}
	}

	/**
	* Skip all whitespace starting at current offset
	*
	* @return void
	*/
	public function skipWhitespace()
	{
		while ($this->offset < $this->cnt)
		{
			if ($this->tokens[$this->offset][0] !== T_WHITESPACE)
			{
				break;
			}
			++$this->offset;
		}
	}

	/**
	* Iterate through tokens until the stream reaches a token of given value or the end of stream
	*
	* @param  integer $tokenValue The target value, e.g. T_ELSE
	* @return bool                Whether a matching token was found
	*/
	public function skipTo($tokenValue)
	{
		while (++$this->offset < $this->cnt)
		{
			if (isset($this->tokens[$this->offset][0]) && $this->tokens[$this->offset][0] === $tokenValue)
			{
				return true;
			}
		}

		return false;
	}

	/**
	* Iterate through tokens until the stream reaches given token or the end of stream
	*
	* @param  array|string $token The target token, either a string or a [tokenValue, string] pair
	* @return bool                Whether a matching token was found
	*/
	public function skipToToken($token)
	{
		while (++$this->offset < $this->cnt)
		{
			if (isset($this->tokens[$this->offset]) && $this->tokens[$this->offset] === $token)
			{
				return true;
			}
		}

		return false;
	}

	/**
	* Test whether the stream is at a valid offset
	*
	* @return bool
	*/
	public function valid()
	{
		return ($this->offset < $this->cnt);
	}
}

1			<?php
2
3			/**
4			* @package s9e\SourceOptimizer
5			* @copyright Copyright (c) 2014-2017 The s9e Authors
6			* @license http://www.opensource.org/licenses/mit-license.php The MIT License
7			*/
8			namespace s9e\SourceOptimizer;
9
10			use ArrayAccess;
11			use Iterator;
12
13			class TokenStream implements ArrayAccess, Iterator
14			{
15			/**
16			* @var integer Number of tokens
17			*/
18			protected $cnt;
19
20			/**
21			* @var string[] List of exact triplet of tokens to exclude from minification
22			*/
23			protected $excludeExact = [
24			// 1 - - 1 and 1 + + 1 should not become 1--1 or 1++1
25			'- -',
26			'+ +',
27			// $a - --$b should not become $a---$b
28			'- --',
29			'+ ++'
30			];
31
32			/**
33			* @var bool Whether the source code needs to be reparsed before this stream is handed off to a
34			* new pass
35			*/
36			public $needsReparsing;
37
38			/**
39			* @var integer Current token index
40			*/
41			protected $offset;
42
43			/**
44			* @var bool Whether some tokens have been removed
45			*/
46			protected $sparse;
47
48			/**
49			* @var array<array\|string> List of tokens
50			*/
51			protected $tokens;
52
53			/**
54			* Constructor
55			*
56			* @param string $src Original source code
57			*/
58	26		public function __construct($src)
59			{
60	26		$this->parse($src);
61	26		}
62
63			/**
64			* Test whether current token can be removed safely
65			*
66			* @return bool
67			*/
68	2		public function canRemoveCurrentToken()
69			{
70	2		$prevToken = $this->lookbehind();
71	2		$nextToken = $this->lookahead();
72
73	2		if ($prevToken === false \|\| $nextToken === false)
74			{
75			return true;
76			}
77
78	2		if (is_array($prevToken))
79			{
80	2		if ($prevToken[0] === T_COMMENT && substr($prevToken[1], 0, 2) === '//')
81			{
82	1		return false;
83			}
84
85	1		$prevToken = $prevToken[1];
86			}
87	1		if (is_array($nextToken))
88			{
89	1		$nextToken = $nextToken[1];
90			}
91
92	1		$str = $prevToken . ' ' . $nextToken;
93	1		if (in_array($str, $this->excludeExact, true))
94			{
95			return false;
96			}
97
98	1		$delimiters = "\t\n\r !\"#$%&'()*+,-./:;<=>?@[\\]^`{\|}~";
99	1		$prevChar = substr($prevToken, -1);
100	1		$nextChar = $nextToken[0];
101
102	1		return (strpos($delimiters, $prevChar) !== false \|\| strpos($delimiters, $nextChar) !== false);
103			}
104
105			/**
106			* Test whether current token is the given type of token
107			*
108			* @param integer $tokenValue
109			* @return bool
110			*/
111			public function is($tokenValue)
112			{
113			return ($this->tokens[$this->offset][0] === $tokenValue);
114			}
115
116			/**
117			* Test whether current token is any of the given types of token
118			*
119			* @param integer[] $tokenValues
120			* @return bool
121			*/
122			public function isAny(array $tokenValues)
123			{
124			return in_array($this->tokens[$this->offset][0], $tokenValues, true);
125			}
126
127			/**
128			* Test whether there's a token at given offset
129			*
130			* @param integer $offset
131			* @return bool
132			*/
133	2		public function offsetExists($offset)
134			{
135	2		return isset($this->tokens[$offset]);
136			}
137
138			/**
139			* Return the token stored at given offset
140			*
141			* @param integer $offset
142			* @return array\|string
143			*/
144	2		public function offsetGet($offset)
145			{
146	2		return $this->tokens[$offset];
147			}
148
149			/**
150			* Replace the token stored at given offset
151			*
152			* @param integer $offset
153			* @param array\|string $token
154			* @return void
155			*/
156	1		public function offsetSet($offset, $token)
157			{
158	1		$this->tokens[$offset] = $token;
159	1		}
160
161			/**
162			* Remove the token stored at given offset
163			*
164			* @return void
165			*/
166	1		public function offsetUnset($offset)
167			{
168	1		unset($this->tokens[$offset]);
169	1		$this->sparse = true;
170	1		}
171
172			/**
173			* Return the current token
174			*
175			* @return array\|string
176			*/
177	7		public function current()
178			{
179	7		return $this->tokens[$this->offset];
180			}
181
182			/**
183			* Get current token's text
184			*
185			* @return string
186			*/
187			public function currentText()
188			{
189			return (is_array($this->tokens[$this->offset])) ? $this->tokens[$this->offset][1] : $this->tokens[$this->offset];
190			}
191
192			/**
193			* Test whether current token is noise (whitespace or comment)
194			*
195			* @return bool
196			*/
197	9		public function isNoise()
198			{
199	9		return in_array($this->tokens[$this->offset][0], [T_COMMENT, T_DOC_COMMENT, T_WHITESPACE], true);
200			}
201
202			/**
203			* Return the offset of current token
204			*
205			* @return integer
206			*/
207	11		public function key()
208			{
209	11		return $this->offset;
210			}
211
212			/**
213			* Peek at the next token
214			*
215			* @return array\|string\|false
216			*/
217	2	View Code Duplication	public function lookahead()
218			{
219	2		$i = $this->offset;
220	2		while (++$i < $this->cnt)
221			{
222	2		if (isset($this->tokens[$i]))
223			{
224	2		return $this->tokens[$i];
225			}
226			}
227
228			return false;
229			}
230
231			/**
232			* Peek at the previous token
233			*
234			* @return array\|string\|false
235			*/
236	2	View Code Duplication	public function lookbehind()
237			{
238	2		$i = $this->offset;
239	2		while (--$i >= 0)
240			{
241	2		if (isset($this->tokens[$i]))
242			{
243	2		return $this->tokens[$i];
244			}
245			}
246
247			return false;
248			}
249
250			/**
251			* Move to the next token in the stream
252			*
253			* @return void
254			*/
255	7		public function next()
256			{
257	7		while (++$this->offset < $this->cnt && !isset($this->tokens[$this->offset]));
258	7		}
259
260			/**
261			* Normalize the whitespace at the end of single-line comments
262			*
263			* Will remove the newline at the end of single-line comments and put it either in the next token
264			* if it's a T_WHITESPACE or it will insert a T_WHITESPACE otherwise.
265			*
266			* @return void
267			*/
268	26		protected function normalizeSingleLineComments()
269			{
270	26		$keys = [];
271	26		foreach ($this->tokens as $k => $token)
272			{
273			// Remove the newline at the end of comments to put it in the next token
274	26		if ($token[0] === T_COMMENT && $token[1][1] === '/')
275			{
276	12		$this->tokens[$k][1] = rtrim($token[1], "\n");
277	26		$keys[] = $k + 1;
278			}
279			}
280
281	26		foreach (array_reverse($keys) as $k)
282			{
283	12		if (isset($this->tokens[$k]) && $this->tokens[$k][0] === T_WHITESPACE)
284			{
285	5		$this->tokens[$k][1] = "\n" . $this->tokens[$k][1];
286			}
287			else
288			{
289	12		array_splice($this->tokens, $k, 0, [[T_WHITESPACE, "\n"]]);
290			}
291			}
292	26		}
293
294			/**
295			* Parse/tokenize given PHP source
296			*
297			* @param string $src
298			* @return void
299			*/
300	26		protected function parse($src)
301			{
302	26		$this->tokens = token_get_all($src);
303	26		foreach ($this->tokens as $k => &$token)
304			{
305	26		if (is_array($token))
306			{
307	26		unset($token[2]);
308			}
309			}
310	26		$this->normalizeSingleLineComments();
311
312	26		$this->cnt = count($this->tokens);
313	26		$this->offset = 0;
314	26		$this->needsReparsing = false;
315	26		$this->sparse = false;
316	26		}
317
318			/**
319			* Move to the previous token in the stream
320			*
321			* @return void
322			*/
323			public function previous()
324			{
325			while ($this->offset > 0)
326			{
327			--$this->offset;
328			if (isset($this->tokens[$this->offset]))
329			{
330			break;
331			}
332			}
333			}
334
335			/**
336			* Remove current token
337			*
338			* @return void
339			*/
340			public function remove()
341			{
342			$this->offsetUnset($this->offset);
343			}
344
345			/**
346			* Replace current token
347			*
348			* @param array\|string $token Token replacement
349			* @return void
350			*/
351			public function replace($token)
352			{
353			$this->offsetSet($this->offset, $token);
354			}
355
356			/**
357			* Reset this stream
358			*
359			* @return void
360			*/
361	8		public function reset()
362			{
363	8		$this->offset = 0;
364	8		if ($this->needsReparsing)
365			{
366			$this->parse($this->serialize());
367			}
368	8		elseif ($this->sparse)
369			{
370			$this->tokens = array_values($this->tokens);
371			}
372	8		}
373
374			/**
375			* Rewind/reset this stream
376			*
377			* @return void
378			*/
379	1		public function rewind()
380			{
381	1		$this->reset();
382	1		}
383
384			/**
385			* Move the stream to given offset
386			*
387			* @param integer $offset
388			* @return void
389			*/
390	8		public function seek($offset)
391			{
392	8		$this->offset = $offset;
393	8		}
394
395			/**
396			* Serialize these tokens back to source code
397			*
398			* @return string
399			*/
400			public function serialize()
401			{
402			$src = '';
403			foreach ($this->tokens as $token)
404			{
405			$src .= (is_array($token)) ? $token[1] : $token;
406			}
407
408			return $src;
409			}
410
411			/**
412			* Skip all whitespace, comments and docblocks starting at current offset
413			*
414			* @return void
415			*/
416	5		public function skipNoise()
417			{
418	5		while ($this->valid())
419			{
420	5		if (!$this->isNoise())
421			{
422	5		break;
423			}
424	4		$this->next();
425			}
426	5		}
427
428			/**
429			* Skip all whitespace starting at current offset
430			*
431			* @return void
432			*/
433	2		public function skipWhitespace()
434			{
435	2		while ($this->offset < $this->cnt)
436			{
437	2		if ($this->tokens[$this->offset][0] !== T_WHITESPACE)
438			{
439	2		break;
440			}
441	1		++$this->offset;
442			}
443	2		}
444
445			/**
446			* Iterate through tokens until the stream reaches a token of given value or the end of stream
447			*
448			* @param integer $tokenValue The target value, e.g. T_ELSE
449			* @return bool Whether a matching token was found
450			*/
451	10	View Code Duplication	public function skipTo($tokenValue)
452			{
453	10		while (++$this->offset < $this->cnt)
454			{
455	10		if (isset($this->tokens[$this->offset][0]) && $this->tokens[$this->offset][0] === $tokenValue)
456			{
457	7		return true;
458			}
459			}
460
461	9		return false;
462			}
463
464			/**
465			* Iterate through tokens until the stream reaches given token or the end of stream
466			*
467			* @param array\|string $token The target token, either a string or a [tokenValue, string] pair
468			* @return bool Whether a matching token was found
469			*/
470	2	View Code Duplication	public function skipToToken($token)
471			{
472	2		while (++$this->offset < $this->cnt)
473			{
474	2		if (isset($this->tokens[$this->offset]) && $this->tokens[$this->offset] === $token)
475			{
476	2		return true;
477			}
478			}
479
480			return false;
481			}
482
483			/**
484			* Test whether the stream is at a valid offset
485			*
486			* @return bool
487			*/
488	8		public function valid()
489			{
490	8		return ($this->offset < $this->cnt);
491			}
492			}

s9e / SourceOptimizer

Push — master ( fc1df1...d515dd )

TokenStream::replace() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like