|
1
|
|
|
<?php |
|
2
|
|
|
namespace Transphporm\Parser\Tokenizer; |
|
3
|
|
|
use \Transphporm\Parser\Tokenizer; |
|
4
|
|
|
use \Transphporm\Parser\Tokens; |
|
5
|
|
|
|
|
6
|
|
|
class BasicChars implements \Transphporm\Parser\Tokenize { |
|
7
|
|
|
|
|
8
|
|
|
public function tokenize(TokenizedString $str, Tokens $tokens) { |
|
9
|
|
|
$this->newLine($str, $tokens); |
|
10
|
|
|
$this->whitespace($str, $tokens); |
|
11
|
|
|
$this->simpleTokens($str, $tokens); |
|
12
|
|
|
} |
|
13
|
|
|
|
|
14
|
|
|
public function whitespace(TokenizedString $str, Tokens $tokens) { |
|
15
|
|
|
//Combine whitespace, this increases performance across the board: Anywhere tokens are iterated over, whitespace is only looped once 8 spaces of indentation = 1 iteration |
|
16
|
|
|
$char = $str->identifyChar(); |
|
17
|
|
|
if ($char === Tokenizer::WHITESPACE) { |
|
18
|
|
|
$last = $tokens->end(); |
|
19
|
|
|
if ($last['type'] !== Tokenizer::WHITESPACE) { |
|
20
|
|
|
$tokens->add(['type' => $char]); |
|
21
|
|
|
} |
|
22
|
|
|
} |
|
23
|
|
|
} |
|
24
|
|
|
|
|
25
|
|
|
private function newLine(TokenizedString $str, Tokens $tokens) { |
|
26
|
|
|
if ($str->identifyChar() == Tokenizer::NEW_LINE) { |
|
27
|
|
|
$tokens->add(['type' => Tokenizer::WHITESPACE, 'line' => $str->newLine()]); |
|
28
|
|
|
} |
|
29
|
|
|
} |
|
30
|
|
|
|
|
31
|
|
|
|
|
32
|
|
|
private function simpleTokens($str, $tokens) { |
|
33
|
|
|
$char = $str->identifyChar(); |
|
34
|
|
|
if (in_array($char, [Tokenizer::ARG, Tokenizer::CONCAT, Tokenizer::DOT, Tokenizer::NOT, Tokenizer::EQUALS, |
|
35
|
|
|
Tokenizer::COLON, Tokenizer::SEMI_COLON, Tokenizer::NUM_SIGN, |
|
36
|
|
|
Tokenizer::GREATER_THAN, Tokenizer::LOWER_THAN, Tokenizer::AT_SIGN, Tokenizer::SUBTRACT, Tokenizer::MULTIPLY, Tokenizer::DIVIDE])) { |
|
37
|
|
|
$tokens->add(['type' => $char, 'line' => $str->lineNo()]); |
|
38
|
|
|
} |
|
39
|
|
|
} |
|
40
|
|
|
|
|
41
|
|
|
} |
Level-2 /
Transphporm
