Mailcode_Parser_Statement_Tokenizer::tokenize() - Code Metrics - Inspection of "Merge pull request #15 from Mistralys/named-parame..." - Mistralys/mailcode - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 466241...d4e985 )

by Sebastian

created 2023-09-06 12:21 UTC

Mailcode_Parser_Statement_Tokenizer::tokenize() A

↳ Parent: Mailcode_Parser_Statement_Tokenizer

Complexity

Conditions	2
Paths	2

Size

Total Lines	16
Code Lines	9

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	2
eloc	9
c	1
b	0
f	0
nc	2
nop	1
dl	0
loc	16
rs	9.9666

<?php
/**
 * File containing the {@see Mailcode_Parser_Statement_Tokenizer} class.
 *
 * @package Mailcode
 * @subpackage Parser
 * @see Mailcode_Parser_Statement_Tokenizer
 */

declare(strict_types=1);

namespace Mailcode;

use AppUtils\ClassHelper;
use AppUtils\ClassHelper\BaseClassHelperException;
use Mailcode\Parser\Statement\Tokenizer\EventHandler;
use Mailcode\Parser\Statement\Tokenizer\SpecialChars;

/**
 * Mailcode statement tokenizer: parses a mailcode statement
 * into its logical parts.
 *
 * @package Mailcode
 * @subpackage Parser
 * @author Sebastian Mordziol <[email protected]>
 */
class Mailcode_Parser_Statement_Tokenizer
{
    public const ERROR_TOKENIZE_METHOD_MISSING = 49801;
    public const ERROR_INVALID_TOKEN_CREATED = 49802;
    public const ERROR_INVALID_TOKEN_CLASS = 49803;
    public const ERROR_TARGET_INSERT_TOKEN_NOT_FOUND = 49804;

    /**
     * @var string[]
     */
    protected array $tokenClasses = array(
        Mailcode_Parser_Statement_Tokenizer_Process_LegacySyntaxConversion::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Variables::class,
        Mailcode_Parser_Statement_Tokenizer_Process_NormalizeQuotes::class,
        Mailcode_Parser_Statement_Tokenizer_Process_EncodeSpecialChars::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Keywords::class,
        // Must be before named parameters to exclude equal signs in strings
        Mailcode_Parser_Statement_Tokenizer_Process_StringLiterals::class,
        // Must be before numbers, because named parameters can contain numbers
        Mailcode_Parser_Statement_Tokenizer_Process_NamedParameters::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Numbers::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Operands::class,
        Mailcode_Parser_Statement_Tokenizer_Process_ExtractTokens::class,
        // Must be at the end when all tokens have been determined
        Mailcode_Parser_Statement_Tokenizer_Process_SetNames::class,
    );
    
   /**
    * @var Mailcode_Parser_Statement
    */
    protected Mailcode_Parser_Statement $statement;
    
   /**
    * @var string
    */
    protected string $tokenized = '';
    
    /**
     * @var Mailcode_Parser_Statement_Tokenizer_Token[]
     */
    protected array $tokensOrdered = array();
    
   /**
    * @var string[]
    */
    protected static array $ids = array();

    /**
     * @var callable[]
     */
    protected array $changeHandlers = array();

    private EventHandler $eventHandler;

    public function __construct(Mailcode_Parser_Statement $statement)
    {
        $this->statement = $statement;
        $this->eventHandler = new EventHandler($this);

        $this->tokenize($statement->getStatementString());
    }

    public function getSourceCommand() : ?Mailcode_Commands_Command
    {
        return $this->statement->getSourceCommand();
    }

   /**
    * Retrieves all tokens detected in the statement string, in 
    * the order they were found.
    * 
    * @return Mailcode_Parser_Statement_Tokenizer_Token[]
    */
    public function getTokens() : array
    {
        return $this->tokensOrdered;
    }

    public function hasTokens() : bool
    {
        return !empty($this->tokensOrdered);
    }
    
   /**
    * Whether there were any unknown tokens in the statement.
    * 
    * @return bool
    */
    public function hasUnknown() : bool
    {
        $unknown = $this->getUnknown();
        
        return !empty($unknown);
    }
    
   /**
    * Retrieves all unknown content tokens, if any.
    * 
    * @return Mailcode_Parser_Statement_Tokenizer_Token_Unknown[]
    */
    public function getUnknown() : array
    {
        $result = array();
        
        foreach($this->tokensOrdered as $token)
        {
            if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Unknown)
            {
                $result[] = $token;
            }
        }
        
        return $result;
    }
    
    public function getFirstUnknown() : ?Mailcode_Parser_Statement_Tokenizer_Token_Unknown
    {
        $unknown = $this->getUnknown();
        
        if(!empty($unknown))
        {
            return array_shift($unknown);
        }
        
        return null;
    }
    
    public function getNormalized() : string
    {
        $parts = array();

        foreach($this->tokensOrdered as $token)
        {
            $string = $token->getNormalized();
            
            if($string === '') {
                continue;
            }

            // Only add spaces between tokens if they require spacing
            if($token->hasSpacing()) {
                $string .= ' ';
            }

            $parts[] = $string;
        }
        
        return trim(implode('', $parts));
    }

    /**
     * Goes through all tokenization processors, in the order that
     * they are defined in the tokenCategories property. This filters
     * the statement string, and extracts the tokens contained within.
     *
     * @param string $statement
     *
     * @throws Mailcode_Parser_Exception
     *
     * @see Mailcode_Parser_Statement_Tokenizer_Process
     */
    protected function tokenize(string $statement) : void
    {
        $statement = trim($statement);
        $tokens = array();

        foreach($this->tokenClasses as $tokenClass)
        {
            $processor = $this->createProcessor($tokenClass, $statement, $tokens);
            $processor->process();

            $statement = $processor->getStatement();
            $tokens = $processor->getTokens();
        }

        $this->tokenized = $statement;
        $this->tokensOrdered = $tokens;
    }

    /**
     * @param string $className
     * @param string $statement
     * @param Mailcode_Parser_Statement_Tokenizer_Token[] $tokens
     * @return Mailcode_Parser_Statement_Tokenizer_Process
     * @throws Mailcode_Parser_Exception
     */
    protected function createProcessor(string $className, string $statement, array $tokens) : Mailcode_Parser_Statement_Tokenizer_Process
    {
        $instance = new $className($this, $statement, $tokens);

        if($instance instanceof Mailcode_Parser_Statement_Tokenizer_Process)
        {
            return $instance;
        }

        throw new Mailcode_Parser_Exception(
            'Unknown statement token.',
            sprintf(
                'The tokenize class [%s] does not extend the base process class.',
                $className
            ),
            self::ERROR_TOKENIZE_METHOD_MISSING
        );
    }

    /**
     * @param string $type
     * @param string $matchedText
     * @param mixed $subject
     * @return Mailcode_Parser_Statement_Tokenizer_Token
     */
    public function createToken(string $type, string $matchedText, $subject=null) : Mailcode_Parser_Statement_Tokenizer_Token
    {
        $tokenID = $this->generateID();

        $class = Mailcode_Parser_Statement_Tokenizer_Token::class.'_'.$type;

        $token = new $class($tokenID, $matchedText, $subject, $this->getSourceCommand());

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token)
        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token class',
            sprintf(
                'The class [%s] does not extend the base token class.',
                get_class($token)
            ),
            self::ERROR_INVALID_TOKEN_CLASS
        );
    }

    private function createVariable(Mailcode_Variables_Variable $variable) : Mailcode_Parser_Statement_Tokenizer_Token_Variable
    {
        return ClassHelper::requireObjectInstanceOf(
            Mailcode_Parser_Statement_Tokenizer_Token_Variable::class,
            $this->createToken('Variable', dollarize($variable->getFullName()), $variable)
        );
    }

    private function createKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
    {
        $name = rtrim($name, ':').':';

        $token = $this->createToken('Keyword', $name);

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Keyword)

        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token created',
            '',
            self::ERROR_INVALID_TOKEN_CREATED
        );
    }

    public function appendKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
    {
        $token = $this->createKeyword($name);

        $this->appendToken($token);

        return $token;
    }

    private function createStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
    {
        $token = $this->createToken('StringLiteral', SpecialChars::encodeAll($text));

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral)

        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token created',
            '',
            self::ERROR_INVALID_TOKEN_CREATED
        );
    }

    private function createNumber(string $number) : Mailcode_Parser_Statement_Tokenizer_Token_Number
    {
        $token = $this->createToken('Number', $number);

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Number)

        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token created',
            '',
            self::ERROR_INVALID_TOKEN_CREATED
        );
    }

    public function appendStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
    {
        $token = $this->createStringLiteral($text);

        $this->appendToken($token);

        return $token;
    }

    public function appendNumber(string $number) : Mailcode_Parser_Statement_Tokenizer_Token_Number
    {
        $token = $this->createNumber($number);

        $this->appendToken($token);

        return $token;
    }

    public function prependStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
    {
        $token = $this->createStringLiteral($text);

        $this->prependToken($token);

        return $token;
    }

    public function removeToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : Mailcode_Parser_Statement_Tokenizer
    {
        $name = $this->findNameToken($token);
        if($name !== null) {
            $this->removeToken($name);
        }

        $keep = array();
        $tokenID = $token->getID();
        $removed = false;

        foreach ($this->tokensOrdered as $checkToken)
        {
            if($checkToken->getID() === $tokenID)
            {
                $removed = true;
                continue;
            }

            $keep[] = $checkToken;
        }

        $this->tokensOrdered = $keep;

        if($removed)
        {
            $this->eventHandler->handleTokenRemoved($token);
        }

        return $this;
    }

    /**
     * @param Mailcode_Parser_Statement_Tokenizer_Token $token
     * @return $this
     */
    protected function appendToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
    {
        $this->tokensOrdered[] = $token;

        $this->eventHandler->handleTokenAppended($token);

        return $this;
    }

    /**
     * @param Mailcode_Parser_Statement_Tokenizer_Token $token
     * @return $this
     */
    protected function prependToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
    {
        array_unshift($this->tokensOrdered, $token);

        $this->eventHandler->handleTokenPrepended($token);

        return $this;
    }
    
   /**
    * Generates a unique alphabet-based ID without numbers
    * to use as token name, to avoid conflicts with the
    * numbers detection.
    *
    * @return string
    */
    protected function generateID() : string
    {
        static $alphas;

        if(!isset($alphas))
        {
            $alphas = range('A', 'Z');
        }

        $amount = 12;

        $result = '';

        for($i=0; $i < $amount; $i++)
        {
            $result .= $alphas[array_rand($alphas)];
        }

        if(!in_array($result, self::$ids))
        {
            self::$ids[] = $result;
            return $result;
        }

        return $this->generateID();
    }

    /**
     * @param callable $callback
     */
    public function onTokensChanged(callable $callback) : void
    {
        $this->changeHandlers[] = $callback;
    }

    /**
     * @return EventHandler
     */
    public function getEventHandler() : EventHandler
    {
        return $this->eventHandler;
    }

    public function appendVariable(Mailcode_Variables_Variable $variable) : Mailcode_Parser_Statement_Tokenizer_Token_Variable
    {
        $token = $this->createVariable($variable);
        $this->appendToken($token);
        return $token;
    }

    public function findNameToken(Mailcode_Parser_Statement_Tokenizer_Token $targetToken) : ?Mailcode_Parser_Statement_Tokenizer_Token_ParamName
    {
        $targetID = $targetToken->getID();

        foreach($this->tokensOrdered as $idx => $token)
        {
            if($token->getID() === $targetID)
            {
                $prev = $this->tokensOrdered[$idx-1] ?? null;
                if($prev instanceof Mailcode_Parser_Statement_Tokenizer_Token_ParamName)
                {
                    return $prev;
                }
            }
        }

        return null;
    }

    /**
     * Injects a parameter name token into the statement, before
     * the target token. Existing parameter names are replaced.
     *
     * @param Mailcode_Parser_Statement_Tokenizer_Token $targetToken
     * @param string $name
     * @return Mailcode_Parser_Statement_Tokenizer_Token_ParamName
     *
     * @throws Mailcode_Parser_Exception {@see self::ERROR_TARGET_INSERT_TOKEN_NOT_FOUND}
     * @throws BaseClassHelperException
     */
    public function injectParamName(Mailcode_Parser_Statement_Tokenizer_Token $targetToken, string $name) : Mailcode_Parser_Statement_Tokenizer_Token_ParamName
    {
        $existing = $this->findNameToken($targetToken);
        if($existing) {
            $this->removeToken($existing);
        }

        $nameToken = ClassHelper::requireObjectInstanceOf(
            Mailcode_Parser_Statement_Tokenizer_Token_ParamName::class,
            $this->createToken('ParamName', $name.'=')
        );

        $this->insertBefore($targetToken, $nameToken);
        $targetToken->registerNameToken($nameToken);

        return $nameToken;
    }

    public function insertBefore(Mailcode_Parser_Statement_Tokenizer_Token $targetToken, Mailcode_Parser_Statement_Tokenizer_Token $newToken) : self
    {
        $targetID = $targetToken->getID();
        $tokens = array();
        $found = false;

        foreach($this->tokensOrdered as $token)
        {
            if($token->getID() === $targetID)
            {
                $tokens[] = $newToken;
                $found = true;
            }

            $tokens[] = $token;
        }

        if($found) {
            $this->tokensOrdered = $tokens;

            return $this;
        }

        throw new Mailcode_Parser_Exception(
            'Could not find target token for insertion.',
            sprintf(
                'The token [%s] was not found in the statement [%s].',
                $targetToken->getNormalized(),
                $this->getNormalized()
            ),
            self::ERROR_TARGET_INSERT_TOKEN_NOT_FOUND
        );
    }

    private function dumpTokens() : void

    {
        echo PHP_EOL;
        echo 'Statement: ['.$this->getNormalized().']'.PHP_EOL;
        echo 'Tokens:'.PHP_EOL;

        foreach($this->tokensOrdered as $idx => $token)
        {
            echo '- #'.$idx.' '.$token->getID().''.PHP_EOL;
            echo '     Type: '.$token->getTypeID().PHP_EOL;
            echo '     Normalized: ['.$token->getNormalized().']'.PHP_EOL;
        }
    }
}


1			<?php
2			/**
3			* File containing the {@see Mailcode_Parser_Statement_Tokenizer} class.
4			*
5			* @package Mailcode
6			* @subpackage Parser
7			* @see Mailcode_Parser_Statement_Tokenizer
8			*/
9
10			declare(strict_types=1);
11
12			namespace Mailcode;
13
14			use AppUtils\ClassHelper;
15			use AppUtils\ClassHelper\BaseClassHelperException;
16			use Mailcode\Parser\Statement\Tokenizer\EventHandler;
17			use Mailcode\Parser\Statement\Tokenizer\SpecialChars;
18
19			/**
20			* Mailcode statement tokenizer: parses a mailcode statement
21			* into its logical parts.
22			*
23			* @package Mailcode
24			* @subpackage Parser
25			* @author Sebastian Mordziol <[email protected]>
26			*/
27			class Mailcode_Parser_Statement_Tokenizer
28			{
29			public const ERROR_TOKENIZE_METHOD_MISSING = 49801;
30			public const ERROR_INVALID_TOKEN_CREATED = 49802;
31			public const ERROR_INVALID_TOKEN_CLASS = 49803;
32			public const ERROR_TARGET_INSERT_TOKEN_NOT_FOUND = 49804;
33
34			/**
35			* @var string[]
36			*/
37			protected array $tokenClasses = array(
38			Mailcode_Parser_Statement_Tokenizer_Process_LegacySyntaxConversion::class,
39			Mailcode_Parser_Statement_Tokenizer_Process_Variables::class,
40			Mailcode_Parser_Statement_Tokenizer_Process_NormalizeQuotes::class,
41			Mailcode_Parser_Statement_Tokenizer_Process_EncodeSpecialChars::class,
42			Mailcode_Parser_Statement_Tokenizer_Process_Keywords::class,
43			// Must be before named parameters to exclude equal signs in strings
44			Mailcode_Parser_Statement_Tokenizer_Process_StringLiterals::class,
45			// Must be before numbers, because named parameters can contain numbers
46			Mailcode_Parser_Statement_Tokenizer_Process_NamedParameters::class,
47			Mailcode_Parser_Statement_Tokenizer_Process_Numbers::class,
48			Mailcode_Parser_Statement_Tokenizer_Process_Operands::class,
49			Mailcode_Parser_Statement_Tokenizer_Process_ExtractTokens::class,
50			// Must be at the end when all tokens have been determined
51			Mailcode_Parser_Statement_Tokenizer_Process_SetNames::class,
52			);
53
54			/**
55			* @var Mailcode_Parser_Statement
56			*/
57			protected Mailcode_Parser_Statement $statement;
58
59			/**
60			* @var string
61			*/
62			protected string $tokenized = '';
63
64			/**
65			* @var Mailcode_Parser_Statement_Tokenizer_Token[]
66			*/
67			protected array $tokensOrdered = array();
68
69			/**
70			* @var string[]
71			*/
72			protected static array $ids = array();
73
74			/**
75			* @var callable[]
76			*/
77			protected array $changeHandlers = array();
78
79			private EventHandler $eventHandler;
80
81			public function __construct(Mailcode_Parser_Statement $statement)
82			{
83			$this->statement = $statement;
84			$this->eventHandler = new EventHandler($this);
85
86			$this->tokenize($statement->getStatementString());
87			}
88
89			public function getSourceCommand() : ?Mailcode_Commands_Command
90			{
91			return $this->statement->getSourceCommand();
92			}
93
94			/**
95			* Retrieves all tokens detected in the statement string, in
96			* the order they were found.
97			*
98			* @return Mailcode_Parser_Statement_Tokenizer_Token[]
99			*/
100			public function getTokens() : array
101			{
102			return $this->tokensOrdered;
103			}
104
105			public function hasTokens() : bool
106			{
107			return !empty($this->tokensOrdered);
108			}
109
110			/**
111			* Whether there were any unknown tokens in the statement.
112			*
113			* @return bool
114			*/
115			public function hasUnknown() : bool
116			{
117			$unknown = $this->getUnknown();
118
119			return !empty($unknown);
120			}
121
122			/**
123			* Retrieves all unknown content tokens, if any.
124			*
125			* @return Mailcode_Parser_Statement_Tokenizer_Token_Unknown[]
126			*/
127			public function getUnknown() : array
128			{
129			$result = array();
130
131			foreach($this->tokensOrdered as $token)
132			{
133			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Unknown)
134			{
135			$result[] = $token;
136			}
137			}
138
139			return $result;
140			}
141
142			public function getFirstUnknown() : ?Mailcode_Parser_Statement_Tokenizer_Token_Unknown
143			{
144			$unknown = $this->getUnknown();
145
146			if(!empty($unknown))
147			{
148			return array_shift($unknown);
149			}
150
151			return null;
152			}
153
154			public function getNormalized() : string
155			{
156			$parts = array();
157
158			foreach($this->tokensOrdered as $token)
159			{
160			$string = $token->getNormalized();
161
162			if($string === '') {
163			continue;
164			}
165
166			// Only add spaces between tokens if they require spacing
167			if($token->hasSpacing()) {
168			$string .= ' ';
169			}
170
171			$parts[] = $string;
172			}
173
174			return trim(implode('', $parts));
175			}
176
177			/**
178			* Goes through all tokenization processors, in the order that
179			* they are defined in the tokenCategories property. This filters
180			* the statement string, and extracts the tokens contained within.
181			*
182			* @param string $statement
183			*
184			* @throws Mailcode_Parser_Exception
185			*
186			* @see Mailcode_Parser_Statement_Tokenizer_Process
187			*/
188			protected function tokenize(string $statement) : void
189			{
190			$statement = trim($statement);
191			$tokens = array();
192
193			foreach($this->tokenClasses as $tokenClass)
194			{
195			$processor = $this->createProcessor($tokenClass, $statement, $tokens);
196			$processor->process();
197
198			$statement = $processor->getStatement();
199			$tokens = $processor->getTokens();
200			}
201
202			$this->tokenized = $statement;
203			$this->tokensOrdered = $tokens;
204			}
205
206			/**
207			* @param string $className
208			* @param string $statement
209			* @param Mailcode_Parser_Statement_Tokenizer_Token[] $tokens
210			* @return Mailcode_Parser_Statement_Tokenizer_Process
211			* @throws Mailcode_Parser_Exception
212			*/
213			protected function createProcessor(string $className, string $statement, array $tokens) : Mailcode_Parser_Statement_Tokenizer_Process
214			{
215			$instance = new $className($this, $statement, $tokens);
216
217			if($instance instanceof Mailcode_Parser_Statement_Tokenizer_Process)
218			{
219			return $instance;
220			}
221
222			throw new Mailcode_Parser_Exception(
223			'Unknown statement token.',
224			sprintf(
225			'The tokenize class [%s] does not extend the base process class.',
226			$className
227			),
228			self::ERROR_TOKENIZE_METHOD_MISSING
229			);
230			}
231
232			/**
233			* @param string $type
234			* @param string $matchedText
235			* @param mixed $subject
236			* @return Mailcode_Parser_Statement_Tokenizer_Token
237			*/
238			public function createToken(string $type, string $matchedText, $subject=null) : Mailcode_Parser_Statement_Tokenizer_Token
239			{
240			$tokenID = $this->generateID();
241
242			$class = Mailcode_Parser_Statement_Tokenizer_Token::class.'_'.$type;
243
244			$token = new $class($tokenID, $matchedText, $subject, $this->getSourceCommand());
245
246			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token)
247			{
248			return $token;
249			}
250
251			throw new Mailcode_Parser_Exception(
252			'Invalid token class',
253			sprintf(
254			'The class [%s] does not extend the base token class.',
255			get_class($token)
256			),
257			self::ERROR_INVALID_TOKEN_CLASS
258			);
259			}
260
261			private function createVariable(Mailcode_Variables_Variable $variable) : Mailcode_Parser_Statement_Tokenizer_Token_Variable
262			{
263			return ClassHelper::requireObjectInstanceOf(
264			Mailcode_Parser_Statement_Tokenizer_Token_Variable::class,
265			$this->createToken('Variable', dollarize($variable->getFullName()), $variable)
266			);
267			}
268
269			private function createKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
270			{
271			$name = rtrim($name, ':').':';
272
273			$token = $this->createToken('Keyword', $name);
274
275			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Keyword)
			0 ignored issues – show introduced 2022-02-24 11:17 UTC by Report Bug Copy Issue Report `$token` is always a sub-type of `Mailcode\Mailcode_Parser...Tokenizer_Token_Keyword`. Loading history...
276			{
277			return $token;
278			}
279
280			throw new Mailcode_Parser_Exception(
281			'Invalid token created',
282			'',
283			self::ERROR_INVALID_TOKEN_CREATED
284			);
285			}
286
287			public function appendKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
288			{
289			$token = $this->createKeyword($name);
290
291			$this->appendToken($token);
292
293			return $token;
294			}
295
296			private function createStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
297			{
298			$token = $this->createToken('StringLiteral', SpecialChars::encodeAll($text));
299
300			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral)
			0 ignored issues – show introduced 2022-03-01 17:44 UTC by Report Bug Copy Issue Report `$token` is always a sub-type of `Mailcode\Mailcode_Parser...zer_Token_StringLiteral`. Loading history...
301			{
302			return $token;
303			}
304
305			throw new Mailcode_Parser_Exception(
306			'Invalid token created',
307			'',
308			self::ERROR_INVALID_TOKEN_CREATED
309			);
310			}
311
312			private function createNumber(string $number) : Mailcode_Parser_Statement_Tokenizer_Token_Number
313			{
314			$token = $this->createToken('Number', $number);
315
316			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Number)
			0 ignored issues – show introduced 2023-09-06 12:18 UTC by Report Bug Copy Issue Report `$token` is always a sub-type of `Mailcode\Mailcode_Parser..._Tokenizer_Token_Number`. Loading history...
317			{
318			return $token;
319			}
320
321			throw new Mailcode_Parser_Exception(
322			'Invalid token created',
323			'',
324			self::ERROR_INVALID_TOKEN_CREATED
325			);
326			}
327
328			public function appendStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
329			{
330			$token = $this->createStringLiteral($text);
331
332			$this->appendToken($token);
333
334			return $token;
335			}
336
337			public function appendNumber(string $number) : Mailcode_Parser_Statement_Tokenizer_Token_Number
338			{
339			$token = $this->createNumber($number);
340
341			$this->appendToken($token);
342
343			return $token;
344			}
345
346			public function prependStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
347			{
348			$token = $this->createStringLiteral($text);
349
350			$this->prependToken($token);
351
352			return $token;
353			}
354
355			public function removeToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : Mailcode_Parser_Statement_Tokenizer
356			{
357			$name = $this->findNameToken($token);
358			if($name !== null) {
359			$this->removeToken($name);
360			}
361
362			$keep = array();
363			$tokenID = $token->getID();
364			$removed = false;
365
366			foreach ($this->tokensOrdered as $checkToken)
367			{
368			if($checkToken->getID() === $tokenID)
369			{
370			$removed = true;
371			continue;
372			}
373
374			$keep[] = $checkToken;
375			}
376
377			$this->tokensOrdered = $keep;
378
379			if($removed)
380			{
381			$this->eventHandler->handleTokenRemoved($token);
382			}
383
384			return $this;
385			}
386
387			/**
388			* @param Mailcode_Parser_Statement_Tokenizer_Token $token
389			* @return $this
390			*/
391			protected function appendToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
392			{
393			$this->tokensOrdered[] = $token;
394
395			$this->eventHandler->handleTokenAppended($token);
396
397			return $this;
398			}
399
400			/**
401			* @param Mailcode_Parser_Statement_Tokenizer_Token $token
402			* @return $this
403			*/
404			protected function prependToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
405			{
406			array_unshift($this->tokensOrdered, $token);
407
408			$this->eventHandler->handleTokenPrepended($token);
409
410			return $this;
411			}
412
413			/**
414			* Generates a unique alphabet-based ID without numbers
415			* to use as token name, to avoid conflicts with the
416			* numbers detection.
417			*
418			* @return string
419			*/
420			protected function generateID() : string
421			{
422			static $alphas;
423
424			if(!isset($alphas))
425			{
426			$alphas = range('A', 'Z');
427			}
428
429			$amount = 12;
430
431			$result = '';
432
433			for($i=0; $i < $amount; $i++)
434			{
435			$result .= $alphas[array_rand($alphas)];
436			}
437
438			if(!in_array($result, self::$ids))
439			{
440			self::$ids[] = $result;
441			return $result;
442			}
443
444			return $this->generateID();
445			}
446
447			/**
448			* @param callable $callback
449			*/
450			public function onTokensChanged(callable $callback) : void
451			{
452			$this->changeHandlers[] = $callback;
453			}
454
455			/**
456			* @return EventHandler
457			*/
458			public function getEventHandler() : EventHandler
459			{
460			return $this->eventHandler;
461			}
462
463			public function appendVariable(Mailcode_Variables_Variable $variable) : Mailcode_Parser_Statement_Tokenizer_Token_Variable
464			{
465			$token = $this->createVariable($variable);
466			$this->appendToken($token);
467			return $token;
468			}
469
470			public function findNameToken(Mailcode_Parser_Statement_Tokenizer_Token $targetToken) : ?Mailcode_Parser_Statement_Tokenizer_Token_ParamName
471			{
472			$targetID = $targetToken->getID();
473
474			foreach($this->tokensOrdered as $idx => $token)
475			{
476			if($token->getID() === $targetID)
477			{
478			$prev = $this->tokensOrdered[$idx-1] ?? null;
479			if($prev instanceof Mailcode_Parser_Statement_Tokenizer_Token_ParamName)
480			{
481			return $prev;
482			}
483			}
484			}
485
486			return null;
487			}
488
489			/**
490			* Injects a parameter name token into the statement, before
491			* the target token. Existing parameter names are replaced.
492			*
493			* @param Mailcode_Parser_Statement_Tokenizer_Token $targetToken
494			* @param string $name
495			* @return Mailcode_Parser_Statement_Tokenizer_Token_ParamName
496			*
497			* @throws Mailcode_Parser_Exception {@see self::ERROR_TARGET_INSERT_TOKEN_NOT_FOUND}
498			* @throws BaseClassHelperException
499			*/
500			public function injectParamName(Mailcode_Parser_Statement_Tokenizer_Token $targetToken, string $name) : Mailcode_Parser_Statement_Tokenizer_Token_ParamName
501			{
502			$existing = $this->findNameToken($targetToken);
503			if($existing) {
504			$this->removeToken($existing);
505			}
506
507			$nameToken = ClassHelper::requireObjectInstanceOf(
508			Mailcode_Parser_Statement_Tokenizer_Token_ParamName::class,
509			$this->createToken('ParamName', $name.'=')
510			);
511
512			$this->insertBefore($targetToken, $nameToken);
513			$targetToken->registerNameToken($nameToken);
514
515			return $nameToken;
516			}
517
518			public function insertBefore(Mailcode_Parser_Statement_Tokenizer_Token $targetToken, Mailcode_Parser_Statement_Tokenizer_Token $newToken) : self
519			{
520			$targetID = $targetToken->getID();
521			$tokens = array();
522			$found = false;
523
524			foreach($this->tokensOrdered as $token)
525			{
526			if($token->getID() === $targetID)
527			{
528			$tokens[] = $newToken;
529			$found = true;
530			}
531
532			$tokens[] = $token;
533			}
534
535			if($found) {
536			$this->tokensOrdered = $tokens;
537
538			return $this;
539			}
540
541			throw new Mailcode_Parser_Exception(
542			'Could not find target token for insertion.',
543			sprintf(
544			'The token [%s] was not found in the statement [%s].',
545			$targetToken->getNormalized(),
546			$this->getNormalized()
547			),
548			self::ERROR_TARGET_INSERT_TOKEN_NOT_FOUND
549			);
550			}
551
552			private function dumpTokens() : void
			0 ignored issues – show Unused Code introduced 2023-09-06 12:18 UTC by Report Bug Copy Issue Report The method `dumpTokens()` is not used, and could be removed. This check looks for private methods that have been defined, but are not used inside the class. Loading history...
553			{
554			echo PHP_EOL;
555			echo 'Statement: ['.$this->getNormalized().']'.PHP_EOL;
556			echo 'Tokens:'.PHP_EOL;
557
558			foreach($this->tokensOrdered as $idx => $token)
559			{
560			echo '- #'.$idx.' '.$token->getID().''.PHP_EOL;
561			echo ' Type: '.$token->getTypeID().PHP_EOL;
562			echo ' Normalized: ['.$token->getNormalized().']'.PHP_EOL;
563			}
564			}
565			}
566

Mistralys / mailcode

Push — master ( 466241...d4e985 )

Mailcode_Parser_Statement_Tokenizer::tokenize() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like