Mailcode_Parser_Statement_Tokenizer::triggerTokensChanged() - Code Metrics - Inspection of "Added changes." - Mistralys/mailcode - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 0f91e9...e0eb75 )

by Sebastian

created 2022-03-03 09:53 UTC

triggerTokensChanged() A

↳ Parent: Mailcode_Parser_Statement_Tokenizer

Complexity

Conditions	2
Paths	2

Size

Total Lines	5
Code Lines	2

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	2
eloc	2
c	0
b	0
f	0
nc	2
nop	0
dl	0
loc	5
rs	10

<?php
/**
 * File containing the {@see Mailcode_Parser_Statement_Tokenizer} class.
 *
 * @package Mailcode
 * @subpackage Parser
 * @see Mailcode_Parser_Statement_Tokenizer
 */

declare(strict_types=1);

namespace Mailcode;

use Mailcode\Parser\Statement\Tokenizer\EventHandler;
use Mailcode\Parser\Statement\Tokenizer\SpecialChars;

/**
 * Mailcode statement tokenizer: parses a mailcode statement
 * into its logical parts.
 *
 * @package Mailcode
 * @subpackage Parser
 * @author Sebastian Mordziol <[email protected]>
 */
class Mailcode_Parser_Statement_Tokenizer
{
    public const ERROR_TOKENIZE_METHOD_MISSING = 49801;
    public const ERROR_INVALID_TOKEN_CREATED = 49802;
    public const ERROR_INVALID_TOKEN_CLASS = 49803;

    /**
     * @var string[]
     */
    protected array $tokenClasses = array(
        Mailcode_Parser_Statement_Tokenizer_Process_Variables::class,
        Mailcode_Parser_Statement_Tokenizer_Process_NormalizeQuotes::class,
        Mailcode_Parser_Statement_Tokenizer_Process_EncodeSpecialChars::class,
        Mailcode_Parser_Statement_Tokenizer_Process_StringLiterals::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Keywords::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Numbers::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Operands::class,
        Mailcode_Parser_Statement_Tokenizer_Process_ExtractTokens::class
    );
    
   /**
    * @var Mailcode_Parser_Statement
    */
    protected Mailcode_Parser_Statement $statement;
    
   /**
    * @var string
    */
    protected string $tokenized = '';
    
    /**
     * @var Mailcode_Parser_Statement_Tokenizer_Token[]
     */
    protected array $tokensOrdered = array();
    
   /**
    * @var string[]
    */
    protected static array $ids = array();

    /**
     * @var callable[]
     */
    protected array $changeHandlers = array();

    private EventHandler $eventHandler;

    public function __construct(Mailcode_Parser_Statement $statement)
    {
        $this->statement = $statement;
        $this->eventHandler = new EventHandler($this);

        $this->tokenize($statement->getStatementString());
    }

    public function getSourceCommand() : ?Mailcode_Commands_Command
    {
        return $this->statement->getSourceCommand();
    }

   /**
    * Retrieves all tokens detected in the statement string, in 
    * the order they were found.
    * 
    * @return Mailcode_Parser_Statement_Tokenizer_Token[]
    */
    public function getTokens() : array
    {
        return $this->tokensOrdered;
    }

    public function hasTokens() : bool
    {
        return !empty($this->tokensOrdered);
    }
    
   /**
    * Whether there were any unknown tokens in the statement.
    * 
    * @return bool
    */
    public function hasUnknown() : bool
    {
        $unknown = $this->getUnknown();
        
        return !empty($unknown);
    }
    
   /**
    * Retrieves all unknown content tokens, if any.
    * 
    * @return Mailcode_Parser_Statement_Tokenizer_Token_Unknown[]
    */
    public function getUnknown() : array
    {
        $result = array();
        
        foreach($this->tokensOrdered as $token)
        {
            if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Unknown)
            {
                $result[] = $token;
            }
        }
        
        return $result;
    }
    
    public function getFirstUnknown() : ?Mailcode_Parser_Statement_Tokenizer_Token_Unknown
    {
        $unknown = $this->getUnknown();
        
        if(!empty($unknown))
        {
            return array_shift($unknown);
        }
        
        return null;
    }
    
    public function getNormalized() : string
    {
        $parts = array();

        foreach($this->tokensOrdered as $token)
        {
            $string = $token->getNormalized();
            
            if($string !== '')
            {
                $parts[] = $string;
            }
        }
        
        return implode(' ', $parts);
    }

    /**
     * Goes through all tokenization processors, in the order that
     * they are defined in the tokenCategories property. This filters
     * the statement string, and extracts the tokens contained within.
     *
     * @param string $statement
     *
     * @throws Mailcode_Parser_Exception
     *
     * @see Mailcode_Parser_Statement_Tokenizer_Process
     */
    protected function tokenize(string $statement) : void
    {
        $statement = trim($statement);
        $tokens = array();

        foreach($this->tokenClasses as $tokenClass)
        {
            $processor = $this->createProcessor($tokenClass, $statement, $tokens);
            $processor->process();

            $statement = $processor->getStatement();
            $tokens = $processor->getTokens();
        }

        $this->tokenized = $statement;
        $this->tokensOrdered = $tokens;
    }

    /**
     * @param string $className
     * @param string $statement
     * @param Mailcode_Parser_Statement_Tokenizer_Token[] $tokens
     * @return Mailcode_Parser_Statement_Tokenizer_Process
     * @throws Mailcode_Parser_Exception
     */
    protected function createProcessor(string $className, string $statement, array $tokens) : Mailcode_Parser_Statement_Tokenizer_Process
    {
        $instance = new $className($this, $statement, $tokens);

        if($instance instanceof Mailcode_Parser_Statement_Tokenizer_Process)
        {
            return $instance;
        }

        throw new Mailcode_Parser_Exception(
            'Unknown statement token.',
            sprintf(
                'The tokenize class [%s] does not extend the base process class.',
                $className
            ),
            self::ERROR_TOKENIZE_METHOD_MISSING
        );
    }

    /**
     * @param string $type
     * @param string $matchedText
     * @param mixed $subject
     * @return Mailcode_Parser_Statement_Tokenizer_Token
     */
    public function createToken(string $type, string $matchedText, $subject=null) : Mailcode_Parser_Statement_Tokenizer_Token
    {
        $tokenID = $this->generateID();

        $class = Mailcode_Parser_Statement_Tokenizer_Token::class.'_'.$type;

        $token = new $class($tokenID, $matchedText, $subject, $this->getSourceCommand());

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token)
        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token class',
            sprintf(
                'The class [%s] does not extend the base token class.',
                get_class($token)
            ),
            self::ERROR_INVALID_TOKEN_CLASS
        );
    }

    private function createKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
    {
        $name = rtrim($name, ':').':';

        $token = $this->createToken('Keyword', $name);

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Keyword)

        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token created',
            '',
            self::ERROR_INVALID_TOKEN_CREATED
        );
    }

    public function appendKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
    {
        $token = $this->createKeyword($name);

        $this->appendToken($token);

        return $token;
    }

    private function createStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
    {
        $token = $this->createToken('StringLiteral', SpecialChars::encodeAll($text));

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral)

        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token created',
            '',
            self::ERROR_INVALID_TOKEN_CREATED
        );
    }

    public function appendStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
    {
        $token = $this->createStringLiteral($text);

        $this->appendToken($token);

        return $token;
    }

    public function prependStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
    {
        $token = $this->createStringLiteral($text);

        $this->prependToken($token);

        return $token;
    }

    public function removeToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : Mailcode_Parser_Statement_Tokenizer
    {
        $keep = array();
        $tokenID = $token->getID();
        $removed = false;

        foreach ($this->tokensOrdered as $checkToken)
        {
            if($checkToken->getID() === $tokenID)
            {
                $removed = true;
                continue;
            }

            $keep[] = $checkToken;
        }

        $this->tokensOrdered = $keep;

        if($removed)
        {
            $this->eventHandler->handleTokenRemoved($token);
        }

        return $this;
    }

    /**
     * @param Mailcode_Parser_Statement_Tokenizer_Token $token
     * @return $this
     */
    protected function appendToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
    {
        $this->tokensOrdered[] = $token;

        $this->eventHandler->handleTokenAppended($token);

        return $this;
    }

    /**
     * @param Mailcode_Parser_Statement_Tokenizer_Token $token
     * @return $this
     */
    protected function prependToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
    {
        array_unshift($this->tokensOrdered, $token);

        $this->eventHandler->handleTokenPrepended($token);

        return $this;
    }
    
   /**
    * Generates a unique alphabet-based ID without numbers
    * to use as token name, to avoid conflicts with the
    * numbers detection.
    *
    * @return string
    */
    protected function generateID() : string
    {
        static $alphas;

        if(!isset($alphas))
        {
            $alphas = range('A', 'Z');
        }

        $amount = 12;

        $result = '';

        for($i=0; $i < $amount; $i++)
        {
            $result .= $alphas[array_rand($alphas)];
        }

        if(!in_array($result, self::$ids))
        {
            self::$ids[] = $result;
            return $result;
        }

        return $this->generateID();
    }

    /**
     * @param callable $callback
     */
    public function onTokensChanged(callable $callback) : void
    {
        $this->changeHandlers[] = $callback;
    }

    /**
     * @return EventHandler
     */
    public function getEventHandler() : EventHandler
    {
        return $this->eventHandler;
    }
}


1			<?php
2			/**
3			* File containing the {@see Mailcode_Parser_Statement_Tokenizer} class.
4			*
5			* @package Mailcode
6			* @subpackage Parser
7			* @see Mailcode_Parser_Statement_Tokenizer
8			*/
9
10			declare(strict_types=1);
11
12			namespace Mailcode;
13
14			use Mailcode\Parser\Statement\Tokenizer\EventHandler;
15			use Mailcode\Parser\Statement\Tokenizer\SpecialChars;
16
17			/**
18			* Mailcode statement tokenizer: parses a mailcode statement
19			* into its logical parts.
20			*
21			* @package Mailcode
22			* @subpackage Parser
23			* @author Sebastian Mordziol <[email protected]>
24			*/
25			class Mailcode_Parser_Statement_Tokenizer
26			{
27			public const ERROR_TOKENIZE_METHOD_MISSING = 49801;
28			public const ERROR_INVALID_TOKEN_CREATED = 49802;
29			public const ERROR_INVALID_TOKEN_CLASS = 49803;
30
31			/**
32			* @var string[]
33			*/
34			protected array $tokenClasses = array(
35			Mailcode_Parser_Statement_Tokenizer_Process_Variables::class,
36			Mailcode_Parser_Statement_Tokenizer_Process_NormalizeQuotes::class,
37			Mailcode_Parser_Statement_Tokenizer_Process_EncodeSpecialChars::class,
38			Mailcode_Parser_Statement_Tokenizer_Process_StringLiterals::class,
39			Mailcode_Parser_Statement_Tokenizer_Process_Keywords::class,
40			Mailcode_Parser_Statement_Tokenizer_Process_Numbers::class,
41			Mailcode_Parser_Statement_Tokenizer_Process_Operands::class,
42			Mailcode_Parser_Statement_Tokenizer_Process_ExtractTokens::class
43			);
44
45			/**
46			* @var Mailcode_Parser_Statement
47			*/
48			protected Mailcode_Parser_Statement $statement;
49
50			/**
51			* @var string
52			*/
53			protected string $tokenized = '';
54
55			/**
56			* @var Mailcode_Parser_Statement_Tokenizer_Token[]
57			*/
58			protected array $tokensOrdered = array();
59
60			/**
61			* @var string[]
62			*/
63			protected static array $ids = array();
64
65			/**
66			* @var callable[]
67			*/
68			protected array $changeHandlers = array();
69
70			private EventHandler $eventHandler;
71
72			public function __construct(Mailcode_Parser_Statement $statement)
73			{
74			$this->statement = $statement;
75			$this->eventHandler = new EventHandler($this);
76
77			$this->tokenize($statement->getStatementString());
78			}
79
80			public function getSourceCommand() : ?Mailcode_Commands_Command
81			{
82			return $this->statement->getSourceCommand();
83			}
84
85			/**
86			* Retrieves all tokens detected in the statement string, in
87			* the order they were found.
88			*
89			* @return Mailcode_Parser_Statement_Tokenizer_Token[]
90			*/
91			public function getTokens() : array
92			{
93			return $this->tokensOrdered;
94			}
95
96			public function hasTokens() : bool
97			{
98			return !empty($this->tokensOrdered);
99			}
100
101			/**
102			* Whether there were any unknown tokens in the statement.
103			*
104			* @return bool
105			*/
106			public function hasUnknown() : bool
107			{
108			$unknown = $this->getUnknown();
109
110			return !empty($unknown);
111			}
112
113			/**
114			* Retrieves all unknown content tokens, if any.
115			*
116			* @return Mailcode_Parser_Statement_Tokenizer_Token_Unknown[]
117			*/
118			public function getUnknown() : array
119			{
120			$result = array();
121
122			foreach($this->tokensOrdered as $token)
123			{
124			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Unknown)
125			{
126			$result[] = $token;
127			}
128			}
129
130			return $result;
131			}
132
133			public function getFirstUnknown() : ?Mailcode_Parser_Statement_Tokenizer_Token_Unknown
134			{
135			$unknown = $this->getUnknown();
136
137			if(!empty($unknown))
138			{
139			return array_shift($unknown);
140			}
141
142			return null;
143			}
144
145			public function getNormalized() : string
146			{
147			$parts = array();
148
149			foreach($this->tokensOrdered as $token)
150			{
151			$string = $token->getNormalized();
152
153			if($string !== '')
154			{
155			$parts[] = $string;
156			}
157			}
158
159			return implode(' ', $parts);
160			}
161
162			/**
163			* Goes through all tokenization processors, in the order that
164			* they are defined in the tokenCategories property. This filters
165			* the statement string, and extracts the tokens contained within.
166			*
167			* @param string $statement
168			*
169			* @throws Mailcode_Parser_Exception
170			*
171			* @see Mailcode_Parser_Statement_Tokenizer_Process
172			*/
173			protected function tokenize(string $statement) : void
174			{
175			$statement = trim($statement);
176			$tokens = array();
177
178			foreach($this->tokenClasses as $tokenClass)
179			{
180			$processor = $this->createProcessor($tokenClass, $statement, $tokens);
181			$processor->process();
182
183			$statement = $processor->getStatement();
184			$tokens = $processor->getTokens();
185			}
186
187			$this->tokenized = $statement;
188			$this->tokensOrdered = $tokens;
189			}
190
191			/**
192			* @param string $className
193			* @param string $statement
194			* @param Mailcode_Parser_Statement_Tokenizer_Token[] $tokens
195			* @return Mailcode_Parser_Statement_Tokenizer_Process
196			* @throws Mailcode_Parser_Exception
197			*/
198			protected function createProcessor(string $className, string $statement, array $tokens) : Mailcode_Parser_Statement_Tokenizer_Process
199			{
200			$instance = new $className($this, $statement, $tokens);
201
202			if($instance instanceof Mailcode_Parser_Statement_Tokenizer_Process)
203			{
204			return $instance;
205			}
206
207			throw new Mailcode_Parser_Exception(
208			'Unknown statement token.',
209			sprintf(
210			'The tokenize class [%s] does not extend the base process class.',
211			$className
212			),
213			self::ERROR_TOKENIZE_METHOD_MISSING
214			);
215			}
216
217			/**
218			* @param string $type
219			* @param string $matchedText
220			* @param mixed $subject
221			* @return Mailcode_Parser_Statement_Tokenizer_Token
222			*/
223			public function createToken(string $type, string $matchedText, $subject=null) : Mailcode_Parser_Statement_Tokenizer_Token
224			{
225			$tokenID = $this->generateID();
226
227			$class = Mailcode_Parser_Statement_Tokenizer_Token::class.'_'.$type;
228
229			$token = new $class($tokenID, $matchedText, $subject, $this->getSourceCommand());
230
231			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token)
232			{
233			return $token;
234			}
235
236			throw new Mailcode_Parser_Exception(
237			'Invalid token class',
238			sprintf(
239			'The class [%s] does not extend the base token class.',
240			get_class($token)
241			),
242			self::ERROR_INVALID_TOKEN_CLASS
243			);
244			}
245
246			private function createKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
247			{
248			$name = rtrim($name, ':').':';
249
250			$token = $this->createToken('Keyword', $name);
251
252			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Keyword)
			0 ignored issues – show introduced 2022-02-24 11:17 UTC by Report Bug Copy Issue Report `$token` is always a sub-type of `Mailcode\Mailcode_Parser...Tokenizer_Token_Keyword`. Loading history...
253			{
254			return $token;
255			}
256
257			throw new Mailcode_Parser_Exception(
258			'Invalid token created',
259			'',
260			self::ERROR_INVALID_TOKEN_CREATED
261			);
262			}
263
264			public function appendKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
265			{
266			$token = $this->createKeyword($name);
267
268			$this->appendToken($token);
269
270			return $token;
271			}
272
273			private function createStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
274			{
275			$token = $this->createToken('StringLiteral', SpecialChars::encodeAll($text));
276
277			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral)
			0 ignored issues – show introduced 2022-03-01 17:44 UTC by Report Bug Copy Issue Report `$token` is always a sub-type of `Mailcode\Mailcode_Parser...zer_Token_StringLiteral`. Loading history...
278			{
279			return $token;
280			}
281
282			throw new Mailcode_Parser_Exception(
283			'Invalid token created',
284			'',
285			self::ERROR_INVALID_TOKEN_CREATED
286			);
287			}
288
289			public function appendStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
290			{
291			$token = $this->createStringLiteral($text);
292
293			$this->appendToken($token);
294
295			return $token;
296			}
297
298			public function prependStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
299			{
300			$token = $this->createStringLiteral($text);
301
302			$this->prependToken($token);
303
304			return $token;
305			}
306
307			public function removeToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : Mailcode_Parser_Statement_Tokenizer
308			{
309			$keep = array();
310			$tokenID = $token->getID();
311			$removed = false;
312
313			foreach ($this->tokensOrdered as $checkToken)
314			{
315			if($checkToken->getID() === $tokenID)
316			{
317			$removed = true;
318			continue;
319			}
320
321			$keep[] = $checkToken;
322			}
323
324			$this->tokensOrdered = $keep;
325
326			if($removed)
327			{
328			$this->eventHandler->handleTokenRemoved($token);
329			}
330
331			return $this;
332			}
333
334			/**
335			* @param Mailcode_Parser_Statement_Tokenizer_Token $token
336			* @return $this
337			*/
338			protected function appendToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
339			{
340			$this->tokensOrdered[] = $token;
341
342			$this->eventHandler->handleTokenAppended($token);
343
344			return $this;
345			}
346
347			/**
348			* @param Mailcode_Parser_Statement_Tokenizer_Token $token
349			* @return $this
350			*/
351			protected function prependToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
352			{
353			array_unshift($this->tokensOrdered, $token);
354
355			$this->eventHandler->handleTokenPrepended($token);
356
357			return $this;
358			}
359
360			/**
361			* Generates a unique alphabet-based ID without numbers
362			* to use as token name, to avoid conflicts with the
363			* numbers detection.
364			*
365			* @return string
366			*/
367			protected function generateID() : string
368			{
369			static $alphas;
370
371			if(!isset($alphas))
372			{
373			$alphas = range('A', 'Z');
374			}
375
376			$amount = 12;
377
378			$result = '';
379
380			for($i=0; $i < $amount; $i++)
381			{
382			$result .= $alphas[array_rand($alphas)];
383			}
384
385			if(!in_array($result, self::$ids))
386			{
387			self::$ids[] = $result;
388			return $result;
389			}
390
391			return $this->generateID();
392			}
393
394			/**
395			* @param callable $callback
396			*/
397			public function onTokensChanged(callable $callback) : void
398			{
399			$this->changeHandlers[] = $callback;
400			}
401
402			/**
403			* @return EventHandler
404			*/
405			public function getEventHandler() : EventHandler
406			{
407			return $this->eventHandler;
408			}
409			}
410

Mistralys / mailcode

Push — master ( 0f91e9...e0eb75 )

triggerTokensChanged() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like