Mailcode_Parser_Statement_Tokenizer::appendStringLiteral() - Code Metrics - Inspection of "Added changes." - Mistralys/mailcode - Measure and Improve Code Quality continuously with Scrutinizer

Test Failed

Push — master ( 9ff364...742ef2 )

by Sebastian

created 2022-03-01 17:40 UTC

appendStringLiteral() A

↳ Parent: Mailcode_Parser_Statement_Tokenizer

Complexity

Conditions	2
Paths	2

Size

Total Lines	16
Code Lines	9

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	2
eloc	9
c	1
b	0
f	0
nc	2
nop	1
dl	0
loc	16
rs	9.9666

<?php
/**
 * File containing the {@see Mailcode_Parser_Statement_Tokenizer} class.
 *
 * @package Mailcode
 * @subpackage Parser
 * @see Mailcode_Parser_Statement_Tokenizer
 */

declare(strict_types=1);

namespace Mailcode;

use Mailcode\Parser\Statement\Tokenizer\SpecialChars;

/**
 * Mailcode statement tokenizer: parses a mailcode statement
 * into its logical parts.
 *
 * @package Mailcode
 * @subpackage Parser
 * @author Sebastian Mordziol <[email protected]>
 */
class Mailcode_Parser_Statement_Tokenizer
{
    public const ERROR_TOKENIZE_METHOD_MISSING = 49801;
    public const ERROR_INVALID_TOKEN_CREATED = 49802;
    public const ERROR_INVALID_TOKEN_CLASS = 49803;

    /**
     * @var string[]
     */
    protected array $tokenClasses = array(
        Mailcode_Parser_Statement_Tokenizer_Process_Variables::class,
        Mailcode_Parser_Statement_Tokenizer_Process_NormalizeQuotes::class,
        Mailcode_Parser_Statement_Tokenizer_Process_EncodeSpecialChars::class,
        Mailcode_Parser_Statement_Tokenizer_Process_StringLiterals::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Keywords::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Numbers::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Operands::class,
        Mailcode_Parser_Statement_Tokenizer_Process_ExtractTokens::class
    );
    
   /**
    * @var Mailcode_Parser_Statement
    */
    protected Mailcode_Parser_Statement $statement;
    
   /**
    * @var string
    */
    protected string $tokenized = '';
    
    /**
     * @var Mailcode_Parser_Statement_Tokenizer_Token[]
     */
    protected array $tokensOrdered = array();
    
   /**
    * @var string[]
    */
    protected static array $ids = array();

    /**
     * @var callable[]
     */
    protected array $changeHandlers = array();

    public function __construct(Mailcode_Parser_Statement $statement)
    {
        $this->statement = $statement;

        $this->tokenize($statement->getStatementString());
    }

    public function getSourceCommand() : ?Mailcode_Commands_Command
    {
        return $this->statement->getSourceCommand();
    }

   /**
    * Retrieves all tokens detected in the statement string, in 
    * the order they were found.
    * 
    * @return Mailcode_Parser_Statement_Tokenizer_Token[]
    */
    public function getTokens() : array
    {
        return $this->tokensOrdered;
    }

    public function hasTokens() : bool
    {
        return !empty($this->tokensOrdered);
    }
    
   /**
    * Whether there were any unknown tokens in the statement.
    * 
    * @return bool
    */
    public function hasUnknown() : bool
    {
        $unknown = $this->getUnknown();
        
        return !empty($unknown);
    }
    
   /**
    * Retrieves all unknown content tokens, if any.
    * 
    * @return Mailcode_Parser_Statement_Tokenizer_Token_Unknown[]
    */
    public function getUnknown() : array
    {
        $result = array();
        
        foreach($this->tokensOrdered as $token)
        {
            if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Unknown)
            {
                $result[] = $token;
            }
        }
        
        return $result;
    }
    
    public function getFirstUnknown() : ?Mailcode_Parser_Statement_Tokenizer_Token_Unknown
    {
        $unknown = $this->getUnknown();
        
        if(!empty($unknown))
        {
            return array_shift($unknown);
        }
        
        return null;
    }
    
    public function getNormalized() : string
    {
        $parts = array();
        
        foreach($this->tokensOrdered as $token)
        {
            $string = $token->getNormalized();
            
            if($string !== '')
            {
                $parts[] = $string;
            }
        }
        
        return implode(' ', $parts);
    }

    /**
     * Goes through all tokenization processors, in the order that
     * they are defined in the tokenCategories property. This filters
     * the statement string, and extracts the tokens contained within.
     *
     * @param string $statement
     *
     * @throws Mailcode_Parser_Exception
     *
     * @see Mailcode_Parser_Statement_Tokenizer_Process
     */
    protected function tokenize(string $statement) : void
    {
        $statement = trim($statement);
        $tokens = array();

        foreach($this->tokenClasses as $tokenClass)
        {
            $processor = $this->createProcessor($tokenClass, $statement, $tokens);
            $processor->process();

            $statement = $processor->getStatement();
            $tokens = $processor->getTokens();
        }

        $this->tokenized = $statement;
        $this->tokensOrdered = $tokens;
    }

    /**
     * @param string $className
     * @param string $statement
     * @param Mailcode_Parser_Statement_Tokenizer_Token[] $tokens
     * @return Mailcode_Parser_Statement_Tokenizer_Process
     * @throws Mailcode_Parser_Exception
     */
    protected function createProcessor(string $className, string $statement, array $tokens) : Mailcode_Parser_Statement_Tokenizer_Process
    {
        $instance = new $className($this, $statement, $tokens);

        if($instance instanceof Mailcode_Parser_Statement_Tokenizer_Process)
        {
            return $instance;
        }

        throw new Mailcode_Parser_Exception(
            'Unknown statement token.',
            sprintf(
                'The tokenize class [%s] does not extend the base process class.',
                $className
            ),
            self::ERROR_TOKENIZE_METHOD_MISSING
        );
    }

    /**
     * @param string $type
     * @param string $matchedText
     * @param mixed $subject
     * @return Mailcode_Parser_Statement_Tokenizer_Token
     */
    public function createToken(string $type, string $matchedText, $subject=null) : Mailcode_Parser_Statement_Tokenizer_Token
    {
        $tokenID = $this->generateID();

        $class = Mailcode_Parser_Statement_Tokenizer_Token::class.'_'.$type;

        $token = new $class($tokenID, $matchedText, $subject, $this->getSourceCommand());

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token)
        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token class',
            sprintf(
                'The class [%s] does not extend the base token class.',
                get_class($token)
            ),
            self::ERROR_INVALID_TOKEN_CLASS
        );
    }

    public function appendKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
    {
        $name = rtrim($name, ':').':';

        $token = $this->appendToken('Keyword', $name);

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Keyword)

        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token created',
            '',
            self::ERROR_INVALID_TOKEN_CREATED
        );
    }

    public function appendStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
    {
        $token = $this->appendToken(
            'StringLiteral',
            SpecialChars::encodeAll($text)
        );

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral)

        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token created',
            '',
            self::ERROR_INVALID_TOKEN_CREATED
        );
    }

    public function removeToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : Mailcode_Parser_Statement_Tokenizer
    {
        $keep = array();
        $tokenID = $token->getID();

        foreach ($this->tokensOrdered as $checkToken)
        {
            if($checkToken->getID() !== $tokenID)
            {
                $keep[] = $checkToken;
            }
        }

        $this->tokensOrdered = $keep;

        $this->triggerTokensChanged();

        return $this;
    }

    /**
     * @param string $type
     * @param string $matchedText
     * @param mixed $subject
     * @return Mailcode_Parser_Statement_Tokenizer_Token
     */
    protected function appendToken(string $type, string $matchedText, $subject=null) : Mailcode_Parser_Statement_Tokenizer_Token
    {
        $token = $this->createToken($type, $matchedText, $subject);

        $this->tokensOrdered[] = $token;

        $this->triggerTokensChanged();

        return $token;
    }
    
   /**
    * Generates a unique alphabet-based ID without numbers
    * to use as token name, to avoid conflicts with the
    * numbers detection.
    *
    * @return string
    */
    protected function generateID() : string
    {
        static $alphas;

        if(!isset($alphas))
        {
            $alphas = range('A', 'Z');
        }

        $amount = 12;

        $result = '';

        for($i=0; $i < $amount; $i++)
        {
            $result .= $alphas[array_rand($alphas)];
        }

        if(!in_array($result, self::$ids))
        {
            self::$ids[] = $result;
            return $result;
        }

        return $this->generateID();
    }

    /**
     * @param callable $callback
     */
    public function onTokensChanged(callable $callback) : void
    {
        $this->changeHandlers[] = $callback;
    }

    protected function triggerTokensChanged() : void
    {
        foreach ($this->changeHandlers as $callback)
        {
            $callback($this);
        }
    }
}


1			<?php
2			/**
3			* File containing the {@see Mailcode_Parser_Statement_Tokenizer} class.
4			*
5			* @package Mailcode
6			* @subpackage Parser
7			* @see Mailcode_Parser_Statement_Tokenizer
8			*/
9
10			declare(strict_types=1);
11
12			namespace Mailcode;
13
14			use Mailcode\Parser\Statement\Tokenizer\SpecialChars;
15
16			/**
17			* Mailcode statement tokenizer: parses a mailcode statement
18			* into its logical parts.
19			*
20			* @package Mailcode
21			* @subpackage Parser
22			* @author Sebastian Mordziol <[email protected]>
23			*/
24			class Mailcode_Parser_Statement_Tokenizer
25			{
26			public const ERROR_TOKENIZE_METHOD_MISSING = 49801;
27			public const ERROR_INVALID_TOKEN_CREATED = 49802;
28			public const ERROR_INVALID_TOKEN_CLASS = 49803;
29
30			/**
31			* @var string[]
32			*/
33			protected array $tokenClasses = array(
34			Mailcode_Parser_Statement_Tokenizer_Process_Variables::class,
35			Mailcode_Parser_Statement_Tokenizer_Process_NormalizeQuotes::class,
36			Mailcode_Parser_Statement_Tokenizer_Process_EncodeSpecialChars::class,
37			Mailcode_Parser_Statement_Tokenizer_Process_StringLiterals::class,
38			Mailcode_Parser_Statement_Tokenizer_Process_Keywords::class,
39			Mailcode_Parser_Statement_Tokenizer_Process_Numbers::class,
40			Mailcode_Parser_Statement_Tokenizer_Process_Operands::class,
41			Mailcode_Parser_Statement_Tokenizer_Process_ExtractTokens::class
42			);
43
44			/**
45			* @var Mailcode_Parser_Statement
46			*/
47			protected Mailcode_Parser_Statement $statement;
48
49			/**
50			* @var string
51			*/
52			protected string $tokenized = '';
53
54			/**
55			* @var Mailcode_Parser_Statement_Tokenizer_Token[]
56			*/
57			protected array $tokensOrdered = array();
58
59			/**
60			* @var string[]
61			*/
62			protected static array $ids = array();
63
64			/**
65			* @var callable[]
66			*/
67			protected array $changeHandlers = array();
68
69			public function __construct(Mailcode_Parser_Statement $statement)
70			{
71			$this->statement = $statement;
72
73			$this->tokenize($statement->getStatementString());
74			}
75
76			public function getSourceCommand() : ?Mailcode_Commands_Command
77			{
78			return $this->statement->getSourceCommand();
79			}
80
81			/**
82			* Retrieves all tokens detected in the statement string, in
83			* the order they were found.
84			*
85			* @return Mailcode_Parser_Statement_Tokenizer_Token[]
86			*/
87			public function getTokens() : array
88			{
89			return $this->tokensOrdered;
90			}
91
92			public function hasTokens() : bool
93			{
94			return !empty($this->tokensOrdered);
95			}
96
97			/**
98			* Whether there were any unknown tokens in the statement.
99			*
100			* @return bool
101			*/
102			public function hasUnknown() : bool
103			{
104			$unknown = $this->getUnknown();
105
106			return !empty($unknown);
107			}
108
109			/**
110			* Retrieves all unknown content tokens, if any.
111			*
112			* @return Mailcode_Parser_Statement_Tokenizer_Token_Unknown[]
113			*/
114			public function getUnknown() : array
115			{
116			$result = array();
117
118			foreach($this->tokensOrdered as $token)
119			{
120			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Unknown)
121			{
122			$result[] = $token;
123			}
124			}
125
126			return $result;
127			}
128
129			public function getFirstUnknown() : ?Mailcode_Parser_Statement_Tokenizer_Token_Unknown
130			{
131			$unknown = $this->getUnknown();
132
133			if(!empty($unknown))
134			{
135			return array_shift($unknown);
136			}
137
138			return null;
139			}
140
141			public function getNormalized() : string
142			{
143			$parts = array();
144
145			foreach($this->tokensOrdered as $token)
146			{
147			$string = $token->getNormalized();
148
149			if($string !== '')
150			{
151			$parts[] = $string;
152			}
153			}
154
155			return implode(' ', $parts);
156			}
157
158			/**
159			* Goes through all tokenization processors, in the order that
160			* they are defined in the tokenCategories property. This filters
161			* the statement string, and extracts the tokens contained within.
162			*
163			* @param string $statement
164			*
165			* @throws Mailcode_Parser_Exception
166			*
167			* @see Mailcode_Parser_Statement_Tokenizer_Process
168			*/
169			protected function tokenize(string $statement) : void
170			{
171			$statement = trim($statement);
172			$tokens = array();
173
174			foreach($this->tokenClasses as $tokenClass)
175			{
176			$processor = $this->createProcessor($tokenClass, $statement, $tokens);
177			$processor->process();
178
179			$statement = $processor->getStatement();
180			$tokens = $processor->getTokens();
181			}
182
183			$this->tokenized = $statement;
184			$this->tokensOrdered = $tokens;
185			}
186
187			/**
188			* @param string $className
189			* @param string $statement
190			* @param Mailcode_Parser_Statement_Tokenizer_Token[] $tokens
191			* @return Mailcode_Parser_Statement_Tokenizer_Process
192			* @throws Mailcode_Parser_Exception
193			*/
194			protected function createProcessor(string $className, string $statement, array $tokens) : Mailcode_Parser_Statement_Tokenizer_Process
195			{
196			$instance = new $className($this, $statement, $tokens);
197
198			if($instance instanceof Mailcode_Parser_Statement_Tokenizer_Process)
199			{
200			return $instance;
201			}
202
203			throw new Mailcode_Parser_Exception(
204			'Unknown statement token.',
205			sprintf(
206			'The tokenize class [%s] does not extend the base process class.',
207			$className
208			),
209			self::ERROR_TOKENIZE_METHOD_MISSING
210			);
211			}
212
213			/**
214			* @param string $type
215			* @param string $matchedText
216			* @param mixed $subject
217			* @return Mailcode_Parser_Statement_Tokenizer_Token
218			*/
219			public function createToken(string $type, string $matchedText, $subject=null) : Mailcode_Parser_Statement_Tokenizer_Token
220			{
221			$tokenID = $this->generateID();
222
223			$class = Mailcode_Parser_Statement_Tokenizer_Token::class.'_'.$type;
224
225			$token = new $class($tokenID, $matchedText, $subject, $this->getSourceCommand());
226
227			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token)
228			{
229			return $token;
230			}
231
232			throw new Mailcode_Parser_Exception(
233			'Invalid token class',
234			sprintf(
235			'The class [%s] does not extend the base token class.',
236			get_class($token)
237			),
238			self::ERROR_INVALID_TOKEN_CLASS
239			);
240			}
241
242			public function appendKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
243			{
244			$name = rtrim($name, ':').':';
245
246			$token = $this->appendToken('Keyword', $name);
247
248			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Keyword)
			0 ignored issues – show introduced 2022-02-24 11:17 UTC by Report Bug Copy Issue Report `$token` is always a sub-type of `Mailcode\Mailcode_Parser...Tokenizer_Token_Keyword`. Loading history...
249			{
250			return $token;
251			}
252
253			throw new Mailcode_Parser_Exception(
254			'Invalid token created',
255			'',
256			self::ERROR_INVALID_TOKEN_CREATED
257			);
258			}
259
260			public function appendStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
261			{
262			$token = $this->appendToken(
263			'StringLiteral',
264			SpecialChars::encodeAll($text)
265			);
266
267			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral)
			0 ignored issues – show introduced 2022-03-01 17:44 UTC by Report Bug Copy Issue Report `$token` is always a sub-type of `Mailcode\Mailcode_Parser...zer_Token_StringLiteral`. Loading history...
268			{
269			return $token;
270			}
271
272			throw new Mailcode_Parser_Exception(
273			'Invalid token created',
274			'',
275			self::ERROR_INVALID_TOKEN_CREATED
276			);
277			}
278
279			public function removeToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : Mailcode_Parser_Statement_Tokenizer
280			{
281			$keep = array();
282			$tokenID = $token->getID();
283
284			foreach ($this->tokensOrdered as $checkToken)
285			{
286			if($checkToken->getID() !== $tokenID)
287			{
288			$keep[] = $checkToken;
289			}
290			}
291
292			$this->tokensOrdered = $keep;
293
294			$this->triggerTokensChanged();
295
296			return $this;
297			}
298
299			/**
300			* @param string $type
301			* @param string $matchedText
302			* @param mixed $subject
303			* @return Mailcode_Parser_Statement_Tokenizer_Token
304			*/
305			protected function appendToken(string $type, string $matchedText, $subject=null) : Mailcode_Parser_Statement_Tokenizer_Token
306			{
307			$token = $this->createToken($type, $matchedText, $subject);
308
309			$this->tokensOrdered[] = $token;
310
311			$this->triggerTokensChanged();
312
313			return $token;
314			}
315
316			/**
317			* Generates a unique alphabet-based ID without numbers
318			* to use as token name, to avoid conflicts with the
319			* numbers detection.
320			*
321			* @return string
322			*/
323			protected function generateID() : string
324			{
325			static $alphas;
326
327			if(!isset($alphas))
328			{
329			$alphas = range('A', 'Z');
330			}
331
332			$amount = 12;
333
334			$result = '';
335
336			for($i=0; $i < $amount; $i++)
337			{
338			$result .= $alphas[array_rand($alphas)];
339			}
340
341			if(!in_array($result, self::$ids))
342			{
343			self::$ids[] = $result;
344			return $result;
345			}
346
347			return $this->generateID();
348			}
349
350			/**
351			* @param callable $callback
352			*/
353			public function onTokensChanged(callable $callback) : void
354			{
355			$this->changeHandlers[] = $callback;
356			}
357
358			protected function triggerTokensChanged() : void
359			{
360			foreach ($this->changeHandlers as $callback)
361			{
362			$callback($this);
363			}
364			}
365			}
366

Mistralys / mailcode

Push — master ( 9ff364...742ef2 )

appendStringLiteral() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like