Tokenizer::getChar() - Code Metrics - Inspection of "Tokenizer now handles comments" - Level-2/Transphporm - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( a22740...69499d )

by Tom

created 2017-08-29 15:13 UTC

Tokenizer::getChar() A

↳ Parent: Tokenizer

Complexity

Conditions	2
Paths	2

Size

Total Lines	5
Code Lines	4

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
c	0
b	0
f	0
dl	0
loc	5
rs	9.4285
cc	2
eloc	4
nc	2
nop	1

<?php
/* @description     Transformation Style Sheets - Revolutionising PHP templating    *
 * @author          Tom Butler [email protected]                                             *
 * @copyright       2015 Tom Butler <[email protected]> | https://r.je/                      *
 * @license         http://www.opensource.org/licenses/bsd-license.php  BSD License *
 * @version         1.0                                                             */
namespace Transphporm\Parser;
class Tokenizer {
	private $str;
	const NAME = 'LITERAL';
	const STRING = 'STRING';
	const OPEN_BRACKET = 'OPEN_BRACKET';
	const CLOSE_BRACKET = 'CLOSE_BRACKET';
	const OPEN_SQUARE_BRACKET = 'SQUARE_BRACKET';
	const CLOSE_SQUARE_BRACKET = 'CLOSE_SQUARE_BRACKET';
	const CONCAT = 'CONCAT';
	const ARG = 'ARG';
	const WHITESPACE = 'WHITESPACE';
	const NEW_LINE = 'NEW_LINE';
	const DOT = 'DOT';
	const NUMERIC = 'NUMERIC';
	const EQUALS = 'EQUALS';
	const NOT = 'NOT';
	const OPEN_BRACE = 'OPEN_BRACE';
	const CLOSE_BRACE = 'CLOSE_BRACE';
	const BOOL = 'BOOL';
	const COLON = 'COLON';
	const SEMI_COLON = 'SEMI_COLON';
	const NUM_SIGN = 'NUM_SIGN';
	const GREATER_THAN = 'GREATER_THAN';
	const AT_SIGN = 'AT_SIGN';
	const SUBTRACT = 'SUBTRACT';
	const MULTIPLY = 'MULTIPLY';
	const DIVIDE = 'DIVIDE';

	private $lineNo = 1;

	private $chars = [
		'"' => self::STRING,
		'\'' => self::STRING,
		'(' => self::OPEN_BRACKET,
		')' => self::CLOSE_BRACKET,
		'[' => self::OPEN_SQUARE_BRACKET,
		']' => self::CLOSE_SQUARE_BRACKET,
		'+' => self::CONCAT,
		',' => self::ARG,
		'.' => self::DOT,
		'!' => self::NOT,
		'=' => self::EQUALS,
		'{' => self::OPEN_BRACE,
		'}' => self::CLOSE_BRACE,
		':' => self::COLON,
		';' => self::SEMI_COLON,
		'#' => self::NUM_SIGN,
		'>' => self::GREATER_THAN,
		'@' => self::AT_SIGN,
		'-' => self::SUBTRACT,
		'*' => self::MULTIPLY,
		'/' => self::DIVIDE,
		' ' => self::WHITESPACE,
		"\n" => self::NEW_LINE,
		"\r" => self::WHITESPACE,
		"\t" => self::WHITESPACE
	];

	public function __construct($str) {
		$this->str = $str;
	}

	public function getTokens($returnObj = true) {
		$tokens = [];

		for ($i = 0; $i < strlen($this->str); $i++) {
			$char = $this->identifyChar($this->str[$i]);
			$i += $this->doSingleLineComments($tokens, $char, $i);
			$i += $this->doMultiLineComments($tokens, $char, $i);
			$this->doNewLine($tokens, $char);
			$this->doSimpleTokens($tokens, $char);
			$this->doLiterals($tokens, $char, $i);
			$i += $this->doStrings($tokens, $char, $i);
			$i += $this->doBrackets($tokens, $char, $i);
			
		}
		if ($returnObj) return new Tokens($tokens);
		else return $tokens;
	}

	private function doSingleLineComments(&$tokens, $char, $i) {

		if ($char == Tokenizer::DIVIDE && isset($this->str[$i+1]) && $this->identifyChar($this->str[$i+1]) == Tokenizer::DIVIDE) {
			$pos = strpos($this->str, "\n", $i);
			return $pos ? $pos-1 : 0;
		}
	}

	private function doMultiLineComments(&$tokens, $char, $i) {

		if ($char == Tokenizer::DIVIDE && isset($this->str[$i+1]) && $this->identifyChar($this->str[$i+1]) == Tokenizer::MULTIPLY) {
			$pos = strpos($this->str, '*/', $i)+2;
			if ($this->str[$i+$pos] == "\n") $pos++;
			return $pos ? $pos : 0;
		}
	}

	private function doSimpleTokens(&$tokens, $char) {
		if (in_array($char, [Tokenizer::ARG, Tokenizer::CONCAT, Tokenizer::DOT, Tokenizer::NOT, Tokenizer::EQUALS,
			Tokenizer::COLON, Tokenizer::SEMI_COLON, Tokenizer::WHITESPACE, Tokenizer::NUM_SIGN,
			Tokenizer::GREATER_THAN, Tokenizer::AT_SIGN, Tokenizer::SUBTRACT, Tokenizer::MULTIPLY, Tokenizer::DIVIDE])) {
			$tokens[] = ['type' => $char, 'line' => $this->lineNo];
		}
	}

	private function doNewLine(&$tokens, $char) {
		if ($char == Tokenizer::NEW_LINE) {
			$this->lineNo++;
			$tokens[] = ['type' => $char, 'line' => $this->lineNo];
		}
	}

	private function isLiteral($n) {
		//Is it a normal literal character
		return isset($this->str[$n]) && ($this->identifyChar($this->str[$n]) == self::NAME
		//but a subtract can be part of a class name or a mathematical operation
				|| ($this->identifyChar($this->str[$n]) == self::SUBTRACT && !is_numeric($this->str[$n-1])));
	}

	private function doLiterals(&$tokens, $char, &$i) {
		if ($char === self::NAME) {
			$name = $this->str[$i];
			while ($this->isLiteral($i+1)) {
				$name .= $this->str[$i+1];
				$i++;
			}
			$this->processLiterals($tokens, $name);
		}
	}

	private function processLiterals(&$tokens, $name) {
		if (is_numeric($name)) $tokens[] = ['type' => self::NUMERIC, 'value' => $name];
		else if ($name == 'true') $tokens[] = ['type' => self::BOOL, 'value' => true];
		else if ($name == 'false') $tokens[] = ['type' => self::BOOL, 'value' => false];
		else $tokens[] = ['type' => self::NAME, 'value' => $name, 'line' => $this->lineNo];
	}

	private function doBrackets(&$tokens, $char, $i) {
		$types = [
			self::OPEN_BRACKET => ['(', ')'],
			self::OPEN_BRACE => ['{', '}'],
			self::OPEN_SQUARE_BRACKET => ['[', ']']
		];

		foreach ($types as $type => $brackets) {
			if ($char === $type) {
				$contents = $this->extractBrackets($i, $brackets[0], $brackets[1]);
				$tokenizer = new Tokenizer($contents);
				$tokens[] = ['type' => $type, 'value' => $tokenizer->getTokens(), 'string' => $contents, 'line' => $this->lineNo];
				return strlen($contents);
			}
		}
	}

	private function doStrings(&$tokens, $char, $i) {
		if ($char === self::STRING) {
			$string = $this->extractString($i);
			$length = strlen($string)+1;
			$string = str_replace('\\' . $this->str[$i], $this->str[$i], $string);
			$tokens[] = ['type' => self::STRING, 'value' => $string, 'line' => $this->lineNo];
			return $length;
		}
	}

	private function extractString($pos) {
		$char = $this->str[$pos];
		$end = strpos($this->str, $char, $pos+1);
		while ($end !== false && $this->str[$end-1] == '\\') $end = strpos($this->str, $char, $end+1);

		return substr($this->str, $pos+1, $end-$pos-1);
	}

	private function extractBrackets($open, $startBracket = '(', $closeBracket = ')') {
		$close = strpos($this->str, $closeBracket, $open);

		$cPos = $open+1;
		while (($cPos = strpos($this->str, $startBracket, $cPos+1)) !== false && $cPos < $close) $close = strpos($this->str, $closeBracket, $close+1);
		return substr($this->str, $open+1, $close-$open-1);
	}

	private function identifyChar($chr) {
		if (isset($this->chars[$chr])) return $this->chars[$chr];
		else return self::NAME;
	}

	private function getChar($num) {

		$chars = array_reverse($this->chars);
		if (isset($chars[$num])) return $chars[$num];
		else return false;
	}
}


1			<?php
2			/* @description Transformation Style Sheets - Revolutionising PHP templating *
3			* @author Tom Butler [email protected] *
4			* @copyright 2015 Tom Butler <[email protected]> \| https://r.je/ *
5			* @license http://www.opensource.org/licenses/bsd-license.php BSD License *
6			* @version 1.0 */
7			namespace Transphporm\Parser;
8			class Tokenizer {
9			private $str;
10			const NAME = 'LITERAL';
11			const STRING = 'STRING';
12			const OPEN_BRACKET = 'OPEN_BRACKET';
13			const CLOSE_BRACKET = 'CLOSE_BRACKET';
14			const OPEN_SQUARE_BRACKET = 'SQUARE_BRACKET';
15			const CLOSE_SQUARE_BRACKET = 'CLOSE_SQUARE_BRACKET';
16			const CONCAT = 'CONCAT';
17			const ARG = 'ARG';
18			const WHITESPACE = 'WHITESPACE';
19			const NEW_LINE = 'NEW_LINE';
20			const DOT = 'DOT';
21			const NUMERIC = 'NUMERIC';
22			const EQUALS = 'EQUALS';
23			const NOT = 'NOT';
24			const OPEN_BRACE = 'OPEN_BRACE';
25			const CLOSE_BRACE = 'CLOSE_BRACE';
26			const BOOL = 'BOOL';
27			const COLON = 'COLON';
28			const SEMI_COLON = 'SEMI_COLON';
29			const NUM_SIGN = 'NUM_SIGN';
30			const GREATER_THAN = 'GREATER_THAN';
31			const AT_SIGN = 'AT_SIGN';
32			const SUBTRACT = 'SUBTRACT';
33			const MULTIPLY = 'MULTIPLY';
34			const DIVIDE = 'DIVIDE';
35
36			private $lineNo = 1;
37
38			private $chars = [
39			'"' => self::STRING,
40			'\'' => self::STRING,
41			'(' => self::OPEN_BRACKET,
42			')' => self::CLOSE_BRACKET,
43			'[' => self::OPEN_SQUARE_BRACKET,
44			']' => self::CLOSE_SQUARE_BRACKET,
45			'+' => self::CONCAT,
46			',' => self::ARG,
47			'.' => self::DOT,
48			'!' => self::NOT,
49			'=' => self::EQUALS,
50			'{' => self::OPEN_BRACE,
51			'}' => self::CLOSE_BRACE,
52			':' => self::COLON,
53			';' => self::SEMI_COLON,
54			'#' => self::NUM_SIGN,
55			'>' => self::GREATER_THAN,
56			'@' => self::AT_SIGN,
57			'-' => self::SUBTRACT,
58			'*' => self::MULTIPLY,
59			'/' => self::DIVIDE,
60			' ' => self::WHITESPACE,
61			"\n" => self::NEW_LINE,
62			"\r" => self::WHITESPACE,
63			"\t" => self::WHITESPACE
64			];
65
66			public function __construct($str) {
67			$this->str = $str;
68			}
69
70			public function getTokens($returnObj = true) {
71			$tokens = [];
72
73			for ($i = 0; $i < strlen($this->str); $i++) {
74			$char = $this->identifyChar($this->str[$i]);
75			$i += $this->doSingleLineComments($tokens, $char, $i);
76			$i += $this->doMultiLineComments($tokens, $char, $i);
77			$this->doNewLine($tokens, $char);
78			$this->doSimpleTokens($tokens, $char);
79			$this->doLiterals($tokens, $char, $i);
80			$i += $this->doStrings($tokens, $char, $i);
81			$i += $this->doBrackets($tokens, $char, $i);
82
83			}
84			if ($returnObj) return new Tokens($tokens);
85			else return $tokens;
86			}
87
88			private function doSingleLineComments(&$tokens, $char, $i) {
			0 ignored issues – show Unused Code introduced 2017-08-29 15:16 UTC by Report Bug Copy Issue Report The parameter `$tokens` is not used and could be removed. This check looks from parameters that have been defined for a function or method, but which are not used in the method body. Loading history...
89			if ($char == Tokenizer::DIVIDE && isset($this->str[$i+1]) && $this->identifyChar($this->str[$i+1]) == Tokenizer::DIVIDE) {
90			$pos = strpos($this->str, "\n", $i);
91			return $pos ? $pos-1 : 0;
92			}
93			}
94
95			private function doMultiLineComments(&$tokens, $char, $i) {
			0 ignored issues – show Unused Code introduced 2017-08-29 15:16 UTC by Report Bug Copy Issue Report The parameter `$tokens` is not used and could be removed. This check looks from parameters that have been defined for a function or method, but which are not used in the method body. Loading history...
96			if ($char == Tokenizer::DIVIDE && isset($this->str[$i+1]) && $this->identifyChar($this->str[$i+1]) == Tokenizer::MULTIPLY) {
97			$pos = strpos($this->str, '*/', $i)+2;
98			if ($this->str[$i+$pos] == "\n") $pos++;
99			return $pos ? $pos : 0;
100			}
101			}
102
103			private function doSimpleTokens(&$tokens, $char) {
104			if (in_array($char, [Tokenizer::ARG, Tokenizer::CONCAT, Tokenizer::DOT, Tokenizer::NOT, Tokenizer::EQUALS,
105			Tokenizer::COLON, Tokenizer::SEMI_COLON, Tokenizer::WHITESPACE, Tokenizer::NUM_SIGN,
106			Tokenizer::GREATER_THAN, Tokenizer::AT_SIGN, Tokenizer::SUBTRACT, Tokenizer::MULTIPLY, Tokenizer::DIVIDE])) {
107			$tokens[] = ['type' => $char, 'line' => $this->lineNo];
108			}
109			}
110
111			private function doNewLine(&$tokens, $char) {
112			if ($char == Tokenizer::NEW_LINE) {
113			$this->lineNo++;
114			$tokens[] = ['type' => $char, 'line' => $this->lineNo];
115			}
116			}
117
118			private function isLiteral($n) {
119			//Is it a normal literal character
120			return isset($this->str[$n]) && ($this->identifyChar($this->str[$n]) == self::NAME
121			//but a subtract can be part of a class name or a mathematical operation
122			\|\| ($this->identifyChar($this->str[$n]) == self::SUBTRACT && !is_numeric($this->str[$n-1])));
123			}
124
125			private function doLiterals(&$tokens, $char, &$i) {
126			if ($char === self::NAME) {
127			$name = $this->str[$i];
128			while ($this->isLiteral($i+1)) {
129			$name .= $this->str[$i+1];
130			$i++;
131			}
132			$this->processLiterals($tokens, $name);
133			}
134			}
135
136			private function processLiterals(&$tokens, $name) {
137			if (is_numeric($name)) $tokens[] = ['type' => self::NUMERIC, 'value' => $name];
138			else if ($name == 'true') $tokens[] = ['type' => self::BOOL, 'value' => true];
139			else if ($name == 'false') $tokens[] = ['type' => self::BOOL, 'value' => false];
140			else $tokens[] = ['type' => self::NAME, 'value' => $name, 'line' => $this->lineNo];
141			}
142
143			private function doBrackets(&$tokens, $char, $i) {
144			$types = [
145			self::OPEN_BRACKET => ['(', ')'],
146			self::OPEN_BRACE => ['{', '}'],
147			self::OPEN_SQUARE_BRACKET => ['[', ']']
148			];
149
150			foreach ($types as $type => $brackets) {
151			if ($char === $type) {
152			$contents = $this->extractBrackets($i, $brackets[0], $brackets[1]);
153			$tokenizer = new Tokenizer($contents);
154			$tokens[] = ['type' => $type, 'value' => $tokenizer->getTokens(), 'string' => $contents, 'line' => $this->lineNo];
155			return strlen($contents);
156			}
157			}
158			}
159
160			private function doStrings(&$tokens, $char, $i) {
161			if ($char === self::STRING) {
162			$string = $this->extractString($i);
163			$length = strlen($string)+1;
164			$string = str_replace('\\' . $this->str[$i], $this->str[$i], $string);
165			$tokens[] = ['type' => self::STRING, 'value' => $string, 'line' => $this->lineNo];
166			return $length;
167			}
168			}
169
170			private function extractString($pos) {
171			$char = $this->str[$pos];
172			$end = strpos($this->str, $char, $pos+1);
173			while ($end !== false && $this->str[$end-1] == '\\') $end = strpos($this->str, $char, $end+1);
174
175			return substr($this->str, $pos+1, $end-$pos-1);
176			}
177
178			private function extractBrackets($open, $startBracket = '(', $closeBracket = ')') {
179			$close = strpos($this->str, $closeBracket, $open);
180
181			$cPos = $open+1;
182			while (($cPos = strpos($this->str, $startBracket, $cPos+1)) !== false && $cPos < $close) $close = strpos($this->str, $closeBracket, $close+1);
183			return substr($this->str, $open+1, $close-$open-1);
184			}
185
186			private function identifyChar($chr) {
187			if (isset($this->chars[$chr])) return $this->chars[$chr];
188			else return self::NAME;
189			}
190
191			private function getChar($num) {
			0 ignored issues – show Unused Code introduced 2017-08-29 15:16 UTC by Report Bug Copy Issue Report This method is not used, and could be removed. Loading history...
192			$chars = array_reverse($this->chars);
193			if (isset($chars[$num])) return $chars[$num];
194			else return false;
195			}
196			}
197

Level-2 / Transphporm

Push — master ( a22740...69499d )

Tokenizer::getChar() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like