Mailcode_Parser_Statement_Tokenizer::prependToken() - Code Metrics - Inspection of "Added type hints." - Mistralys/mailcode - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 7822c6...0f91e9 )

by Sebastian

created 2022-03-03 07:24 UTC

prependToken() A

↳ Parent: Mailcode_Parser_Statement_Tokenizer

Complexity

Conditions	1
Paths	1

Size

Total Lines	7
Code Lines	3

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	1
eloc	3
c	1
b	0
f	0
nc	1
nop	1
dl	0
loc	7
rs	10

<?php
/**
 * File containing the {@see Mailcode_Parser_Statement_Tokenizer} class.
 *
 * @package Mailcode
 * @subpackage Parser
 * @see Mailcode_Parser_Statement_Tokenizer
 */

declare(strict_types=1);

namespace Mailcode;

use Mailcode\Parser\Statement\Tokenizer\SpecialChars;
use phpDocumentor\Guides\RestructuredText\Exception\InvalidTableStructure;
filter:
    dependency_paths: ["lib/*"]

/**
 * Mailcode statement tokenizer: parses a mailcode statement
 * into its logical parts.
 *
 * @package Mailcode
 * @subpackage Parser
 * @author Sebastian Mordziol <[email protected]>
 */
class Mailcode_Parser_Statement_Tokenizer
{
    public const ERROR_TOKENIZE_METHOD_MISSING = 49801;
    public const ERROR_INVALID_TOKEN_CREATED = 49802;
    public const ERROR_INVALID_TOKEN_CLASS = 49803;

    /**
     * @var string[]
     */
    protected array $tokenClasses = array(
        Mailcode_Parser_Statement_Tokenizer_Process_Variables::class,
        Mailcode_Parser_Statement_Tokenizer_Process_NormalizeQuotes::class,
        Mailcode_Parser_Statement_Tokenizer_Process_EncodeSpecialChars::class,
        Mailcode_Parser_Statement_Tokenizer_Process_StringLiterals::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Keywords::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Numbers::class,
        Mailcode_Parser_Statement_Tokenizer_Process_Operands::class,
        Mailcode_Parser_Statement_Tokenizer_Process_ExtractTokens::class
    );
    
   /**
    * @var Mailcode_Parser_Statement
    */
    protected Mailcode_Parser_Statement $statement;
    
   /**
    * @var string
    */
    protected string $tokenized = '';
    
    /**
     * @var Mailcode_Parser_Statement_Tokenizer_Token[]
     */
    protected array $tokensOrdered = array();
    
   /**
    * @var string[]
    */
    protected static array $ids = array();

    /**
     * @var callable[]
     */
    protected array $changeHandlers = array();

    public function __construct(Mailcode_Parser_Statement $statement)
    {
        $this->statement = $statement;

        $this->tokenize($statement->getStatementString());
    }

    public function getSourceCommand() : ?Mailcode_Commands_Command
    {
        return $this->statement->getSourceCommand();
    }

   /**
    * Retrieves all tokens detected in the statement string, in 
    * the order they were found.
    * 
    * @return Mailcode_Parser_Statement_Tokenizer_Token[]
    */
    public function getTokens() : array
    {
        return $this->tokensOrdered;
    }

    public function hasTokens() : bool
    {
        return !empty($this->tokensOrdered);
    }
    
   /**
    * Whether there were any unknown tokens in the statement.
    * 
    * @return bool
    */
    public function hasUnknown() : bool
    {
        $unknown = $this->getUnknown();
        
        return !empty($unknown);
    }
    
   /**
    * Retrieves all unknown content tokens, if any.
    * 
    * @return Mailcode_Parser_Statement_Tokenizer_Token_Unknown[]
    */
    public function getUnknown() : array
    {
        $result = array();
        
        foreach($this->tokensOrdered as $token)
        {
            if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Unknown)
            {
                $result[] = $token;
            }
        }
        
        return $result;
    }
    
    public function getFirstUnknown() : ?Mailcode_Parser_Statement_Tokenizer_Token_Unknown
    {
        $unknown = $this->getUnknown();
        
        if(!empty($unknown))
        {
            return array_shift($unknown);
        }
        
        return null;
    }
    
    public function getNormalized() : string
    {
        $parts = array();

        foreach($this->tokensOrdered as $token)
        {
            $string = $token->getNormalized();
            
            if($string !== '')
            {
                $parts[] = $string;
            }
        }
        
        return implode(' ', $parts);
    }

    /**
     * Goes through all tokenization processors, in the order that
     * they are defined in the tokenCategories property. This filters
     * the statement string, and extracts the tokens contained within.
     *
     * @param string $statement
     *
     * @throws Mailcode_Parser_Exception
     *
     * @see Mailcode_Parser_Statement_Tokenizer_Process
     */
    protected function tokenize(string $statement) : void
    {
        $statement = trim($statement);
        $tokens = array();

        foreach($this->tokenClasses as $tokenClass)
        {
            $processor = $this->createProcessor($tokenClass, $statement, $tokens);
            $processor->process();

            $statement = $processor->getStatement();
            $tokens = $processor->getTokens();
        }

        $this->tokenized = $statement;
        $this->tokensOrdered = $tokens;
    }

    /**
     * @param string $className
     * @param string $statement
     * @param Mailcode_Parser_Statement_Tokenizer_Token[] $tokens
     * @return Mailcode_Parser_Statement_Tokenizer_Process
     * @throws Mailcode_Parser_Exception
     */
    protected function createProcessor(string $className, string $statement, array $tokens) : Mailcode_Parser_Statement_Tokenizer_Process
    {
        $instance = new $className($this, $statement, $tokens);

        if($instance instanceof Mailcode_Parser_Statement_Tokenizer_Process)
        {
            return $instance;
        }

        throw new Mailcode_Parser_Exception(
            'Unknown statement token.',
            sprintf(
                'The tokenize class [%s] does not extend the base process class.',
                $className
            ),
            self::ERROR_TOKENIZE_METHOD_MISSING
        );
    }

    /**
     * @param string $type
     * @param string $matchedText
     * @param mixed $subject
     * @return Mailcode_Parser_Statement_Tokenizer_Token
     */
    public function createToken(string $type, string $matchedText, $subject=null) : Mailcode_Parser_Statement_Tokenizer_Token
    {
        $tokenID = $this->generateID();

        $class = Mailcode_Parser_Statement_Tokenizer_Token::class.'_'.$type;

        $token = new $class($tokenID, $matchedText, $subject, $this->getSourceCommand());

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token)
        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token class',
            sprintf(
                'The class [%s] does not extend the base token class.',
                get_class($token)
            ),
            self::ERROR_INVALID_TOKEN_CLASS
        );
    }

    private function createKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
    {
        $name = rtrim($name, ':').':';

        $token = $this->createToken('Keyword', $name);

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Keyword)

        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token created',
            '',
            self::ERROR_INVALID_TOKEN_CREATED
        );
    }

    public function appendKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
    {
        $token = $this->createKeyword($name);

        $this->appendToken($token);

        return $token;
    }

    private function createStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
    {
        $token = $this->createToken('StringLiteral', SpecialChars::encodeAll($text));

        if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral)

        {
            return $token;
        }

        throw new Mailcode_Parser_Exception(
            'Invalid token created',
            '',
            self::ERROR_INVALID_TOKEN_CREATED
        );
    }

    public function appendStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
    {
        $token = $this->createStringLiteral($text);

        $this->appendToken($token);

        return $token;
    }

    public function prependStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
    {
        $token = $this->createStringLiteral($text);

        $this->prependToken($token);

        return $token;
    }

    public function removeToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : Mailcode_Parser_Statement_Tokenizer
    {
        $keep = array();
        $tokenID = $token->getID();

        foreach ($this->tokensOrdered as $checkToken)
        {
            if($checkToken->getID() !== $tokenID)
            {
                $keep[] = $checkToken;
            }
        }

        $this->tokensOrdered = $keep;

        $this->triggerTokensChanged();

        return $this;
    }

    /**
     * @param Mailcode_Parser_Statement_Tokenizer_Token $token
     * @return $this
     */
    protected function appendToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
    {
        $this->tokensOrdered[] = $token;

        $this->triggerTokensChanged();

        return $this;
    }

    /**
     * @param Mailcode_Parser_Statement_Tokenizer_Token $token
     * @return $this
     */
    protected function prependToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
    {
        array_unshift($this->tokensOrdered, $token);

        $this->triggerTokensChanged();

        return $this;
    }
    
   /**
    * Generates a unique alphabet-based ID without numbers
    * to use as token name, to avoid conflicts with the
    * numbers detection.
    *
    * @return string
    */
    protected function generateID() : string
    {
        static $alphas;

        if(!isset($alphas))
        {
            $alphas = range('A', 'Z');
        }

        $amount = 12;

        $result = '';

        for($i=0; $i < $amount; $i++)
        {
            $result .= $alphas[array_rand($alphas)];
        }

        if(!in_array($result, self::$ids))
        {
            self::$ids[] = $result;
            return $result;
        }

        return $this->generateID();
    }

    /**
     * @param callable $callback
     */
    public function onTokensChanged(callable $callback) : void
    {
        $this->changeHandlers[] = $callback;
    }

    protected function triggerTokensChanged() : void
    {
        foreach ($this->changeHandlers as $callback)
        {
            $callback($this);
        }
    }
}


1			<?php
2			/**
3			* File containing the {@see Mailcode_Parser_Statement_Tokenizer} class.
4			*
5			* @package Mailcode
6			* @subpackage Parser
7			* @see Mailcode_Parser_Statement_Tokenizer
8			*/
9
10			declare(strict_types=1);
11
12			namespace Mailcode;
13
14			use Mailcode\Parser\Statement\Tokenizer\SpecialChars;
15			use phpDocumentor\Guides\RestructuredText\Exception\InvalidTableStructure;
			0 ignored issues – show Bug introduced 2022-03-03 07:28 UTC by Report Bug Copy Issue Report The type `phpDocumentor\Guides\Res...n\InvalidTableStructure` was not found. Maybe you did not declare it correctly or list all dependencies? The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. `excluded_paths: ["lib/"]`, you can move it to the dependency path list as follows: filter: dependency_paths: ["lib/"] For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths Loading history...
16
17			/**
18			* Mailcode statement tokenizer: parses a mailcode statement
19			* into its logical parts.
20			*
21			* @package Mailcode
22			* @subpackage Parser
23			* @author Sebastian Mordziol <[email protected]>
24			*/
25			class Mailcode_Parser_Statement_Tokenizer
26			{
27			public const ERROR_TOKENIZE_METHOD_MISSING = 49801;
28			public const ERROR_INVALID_TOKEN_CREATED = 49802;
29			public const ERROR_INVALID_TOKEN_CLASS = 49803;
30
31			/**
32			* @var string[]
33			*/
34			protected array $tokenClasses = array(
35			Mailcode_Parser_Statement_Tokenizer_Process_Variables::class,
36			Mailcode_Parser_Statement_Tokenizer_Process_NormalizeQuotes::class,
37			Mailcode_Parser_Statement_Tokenizer_Process_EncodeSpecialChars::class,
38			Mailcode_Parser_Statement_Tokenizer_Process_StringLiterals::class,
39			Mailcode_Parser_Statement_Tokenizer_Process_Keywords::class,
40			Mailcode_Parser_Statement_Tokenizer_Process_Numbers::class,
41			Mailcode_Parser_Statement_Tokenizer_Process_Operands::class,
42			Mailcode_Parser_Statement_Tokenizer_Process_ExtractTokens::class
43			);
44
45			/**
46			* @var Mailcode_Parser_Statement
47			*/
48			protected Mailcode_Parser_Statement $statement;
49
50			/**
51			* @var string
52			*/
53			protected string $tokenized = '';
54
55			/**
56			* @var Mailcode_Parser_Statement_Tokenizer_Token[]
57			*/
58			protected array $tokensOrdered = array();
59
60			/**
61			* @var string[]
62			*/
63			protected static array $ids = array();
64
65			/**
66			* @var callable[]
67			*/
68			protected array $changeHandlers = array();
69
70			public function __construct(Mailcode_Parser_Statement $statement)
71			{
72			$this->statement = $statement;
73
74			$this->tokenize($statement->getStatementString());
75			}
76
77			public function getSourceCommand() : ?Mailcode_Commands_Command
78			{
79			return $this->statement->getSourceCommand();
80			}
81
82			/**
83			* Retrieves all tokens detected in the statement string, in
84			* the order they were found.
85			*
86			* @return Mailcode_Parser_Statement_Tokenizer_Token[]
87			*/
88			public function getTokens() : array
89			{
90			return $this->tokensOrdered;
91			}
92
93			public function hasTokens() : bool
94			{
95			return !empty($this->tokensOrdered);
96			}
97
98			/**
99			* Whether there were any unknown tokens in the statement.
100			*
101			* @return bool
102			*/
103			public function hasUnknown() : bool
104			{
105			$unknown = $this->getUnknown();
106
107			return !empty($unknown);
108			}
109
110			/**
111			* Retrieves all unknown content tokens, if any.
112			*
113			* @return Mailcode_Parser_Statement_Tokenizer_Token_Unknown[]
114			*/
115			public function getUnknown() : array
116			{
117			$result = array();
118
119			foreach($this->tokensOrdered as $token)
120			{
121			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Unknown)
122			{
123			$result[] = $token;
124			}
125			}
126
127			return $result;
128			}
129
130			public function getFirstUnknown() : ?Mailcode_Parser_Statement_Tokenizer_Token_Unknown
131			{
132			$unknown = $this->getUnknown();
133
134			if(!empty($unknown))
135			{
136			return array_shift($unknown);
137			}
138
139			return null;
140			}
141
142			public function getNormalized() : string
143			{
144			$parts = array();
145
146			foreach($this->tokensOrdered as $token)
147			{
148			$string = $token->getNormalized();
149
150			if($string !== '')
151			{
152			$parts[] = $string;
153			}
154			}
155
156			return implode(' ', $parts);
157			}
158
159			/**
160			* Goes through all tokenization processors, in the order that
161			* they are defined in the tokenCategories property. This filters
162			* the statement string, and extracts the tokens contained within.
163			*
164			* @param string $statement
165			*
166			* @throws Mailcode_Parser_Exception
167			*
168			* @see Mailcode_Parser_Statement_Tokenizer_Process
169			*/
170			protected function tokenize(string $statement) : void
171			{
172			$statement = trim($statement);
173			$tokens = array();
174
175			foreach($this->tokenClasses as $tokenClass)
176			{
177			$processor = $this->createProcessor($tokenClass, $statement, $tokens);
178			$processor->process();
179
180			$statement = $processor->getStatement();
181			$tokens = $processor->getTokens();
182			}
183
184			$this->tokenized = $statement;
185			$this->tokensOrdered = $tokens;
186			}
187
188			/**
189			* @param string $className
190			* @param string $statement
191			* @param Mailcode_Parser_Statement_Tokenizer_Token[] $tokens
192			* @return Mailcode_Parser_Statement_Tokenizer_Process
193			* @throws Mailcode_Parser_Exception
194			*/
195			protected function createProcessor(string $className, string $statement, array $tokens) : Mailcode_Parser_Statement_Tokenizer_Process
196			{
197			$instance = new $className($this, $statement, $tokens);
198
199			if($instance instanceof Mailcode_Parser_Statement_Tokenizer_Process)
200			{
201			return $instance;
202			}
203
204			throw new Mailcode_Parser_Exception(
205			'Unknown statement token.',
206			sprintf(
207			'The tokenize class [%s] does not extend the base process class.',
208			$className
209			),
210			self::ERROR_TOKENIZE_METHOD_MISSING
211			);
212			}
213
214			/**
215			* @param string $type
216			* @param string $matchedText
217			* @param mixed $subject
218			* @return Mailcode_Parser_Statement_Tokenizer_Token
219			*/
220			public function createToken(string $type, string $matchedText, $subject=null) : Mailcode_Parser_Statement_Tokenizer_Token
221			{
222			$tokenID = $this->generateID();
223
224			$class = Mailcode_Parser_Statement_Tokenizer_Token::class.'_'.$type;
225
226			$token = new $class($tokenID, $matchedText, $subject, $this->getSourceCommand());
227
228			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token)
229			{
230			return $token;
231			}
232
233			throw new Mailcode_Parser_Exception(
234			'Invalid token class',
235			sprintf(
236			'The class [%s] does not extend the base token class.',
237			get_class($token)
238			),
239			self::ERROR_INVALID_TOKEN_CLASS
240			);
241			}
242
243			private function createKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
244			{
245			$name = rtrim($name, ':').':';
246
247			$token = $this->createToken('Keyword', $name);
248
249			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_Keyword)
			0 ignored issues – show introduced 2022-02-24 11:17 UTC by Report Bug Copy Issue Report `$token` is always a sub-type of `Mailcode\Mailcode_Parser...Tokenizer_Token_Keyword`. Loading history...
250			{
251			return $token;
252			}
253
254			throw new Mailcode_Parser_Exception(
255			'Invalid token created',
256			'',
257			self::ERROR_INVALID_TOKEN_CREATED
258			);
259			}
260
261			public function appendKeyword(string $name) : Mailcode_Parser_Statement_Tokenizer_Token_Keyword
262			{
263			$token = $this->createKeyword($name);
264
265			$this->appendToken($token);
266
267			return $token;
268			}
269
270			private function createStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
271			{
272			$token = $this->createToken('StringLiteral', SpecialChars::encodeAll($text));
273
274			if($token instanceof Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral)
			0 ignored issues – show introduced 2022-03-01 17:44 UTC by Report Bug Copy Issue Report `$token` is always a sub-type of `Mailcode\Mailcode_Parser...zer_Token_StringLiteral`. Loading history...
275			{
276			return $token;
277			}
278
279			throw new Mailcode_Parser_Exception(
280			'Invalid token created',
281			'',
282			self::ERROR_INVALID_TOKEN_CREATED
283			);
284			}
285
286			public function appendStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
287			{
288			$token = $this->createStringLiteral($text);
289
290			$this->appendToken($token);
291
292			return $token;
293			}
294
295			public function prependStringLiteral(string $text) : Mailcode_Parser_Statement_Tokenizer_Token_StringLiteral
296			{
297			$token = $this->createStringLiteral($text);
298
299			$this->prependToken($token);
300
301			return $token;
302			}
303
304			public function removeToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : Mailcode_Parser_Statement_Tokenizer
305			{
306			$keep = array();
307			$tokenID = $token->getID();
308
309			foreach ($this->tokensOrdered as $checkToken)
310			{
311			if($checkToken->getID() !== $tokenID)
312			{
313			$keep[] = $checkToken;
314			}
315			}
316
317			$this->tokensOrdered = $keep;
318
319			$this->triggerTokensChanged();
320
321			return $this;
322			}
323
324			/**
325			* @param Mailcode_Parser_Statement_Tokenizer_Token $token
326			* @return $this
327			*/
328			protected function appendToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
329			{
330			$this->tokensOrdered[] = $token;
331
332			$this->triggerTokensChanged();
333
334			return $this;
335			}
336
337			/**
338			* @param Mailcode_Parser_Statement_Tokenizer_Token $token
339			* @return $this
340			*/
341			protected function prependToken(Mailcode_Parser_Statement_Tokenizer_Token $token) : self
342			{
343			array_unshift($this->tokensOrdered, $token);
344
345			$this->triggerTokensChanged();
346
347			return $this;
348			}
349
350			/**
351			* Generates a unique alphabet-based ID without numbers
352			* to use as token name, to avoid conflicts with the
353			* numbers detection.
354			*
355			* @return string
356			*/
357			protected function generateID() : string
358			{
359			static $alphas;
360
361			if(!isset($alphas))
362			{
363			$alphas = range('A', 'Z');
364			}
365
366			$amount = 12;
367
368			$result = '';
369
370			for($i=0; $i < $amount; $i++)
371			{
372			$result .= $alphas[array_rand($alphas)];
373			}
374
375			if(!in_array($result, self::$ids))
376			{
377			self::$ids[] = $result;
378			return $result;
379			}
380
381			return $this->generateID();
382			}
383
384			/**
385			* @param callable $callback
386			*/
387			public function onTokensChanged(callable $callback) : void
388			{
389			$this->changeHandlers[] = $callback;
390			}
391
392			protected function triggerTokensChanged() : void
393			{
394			foreach ($this->changeHandlers as $callback)
395			{
396			$callback($this);
397			}
398			}
399			}
400

Mistralys / mailcode

Push — master ( 7822c6...0f91e9 )

prependToken() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like