TranslationExtractor::splitTokensAsParams() - Code Metrics - Inspection of "Extractor" - yiisoft/translator - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#40)

by Def

created 2021-03-21 16:06 UTC

TranslationExtractor::splitTokensAsParams() B

↳ Parent: TranslationExtractor

Complexity

Conditions	8
Paths	6

Size

Total Lines	25
Code Lines	17

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	16
CRAP Score	8.013

Importance

Changes	3
Bugs	0	Features	0

Metric	Value
eloc	17
c	3
b	0
f	0
dl	0
loc	25
ccs	16
cts	17
cp	0.9412
rs	8.4444
cc	8
nc	6
nop	1
crap	8.013

<?php

declare(strict_types=1);

namespace Yiisoft\Translator;

use Yiisoft\Files\FileHelper;
use Yiisoft\Files\PathMatcher\PathMatcher;

/**
 * Extractor messages
 */
final class TranslationExtractor
{
    private string $translator = '->translate';

    /**
     * @var array<string|array{0: int, 1: string, 2: int}>
     */
    private array $translatorTokens = [];
    private int $sizeOfTranslator = 0;

    private array $skippedLines = [];
    private array $skippedLinesOfFile = [];

    private string $defaultCategory = '';

    /**
     * @var string[]
     */
    private array $only = ['**.php'];

    /**
     * @var string[]
     */
    private array $except = [
        '.svn',
        '.git',
        '.gitignore',
        '.gitkeep',
        '.hgignore',
        '.hgkeep',
        '/messages',
    ];

    private static array $commaSpare = [
        ')' => '(',
        ']' => '[',
        '}' => '{',
    ];

    /**
     * @param string $path
     * @param string[]|null $only
     * @param string[]|null $except
     *
     * @return array
     */
    public function extract(string $path, ?array $only = null, ?array $except = null): array
    {
        if (!is_dir($path)) {
            throw new \RuntimeException(sprintf('Directory "%s" does not exist.', $path));
        }

        $translatorTokens = token_get_all('<?php ' . $this->translator);
        array_shift($translatorTokens);
        $this->translatorTokens = $translatorTokens;
        $this->sizeOfTranslator = count($this->translatorTokens);


        if ($this->sizeOfTranslator < 2) {
            throw new \RuntimeException('Translator tokens cannot be shorttest 2 tokens.');
        }

        return $this->getMessageFromPath($path, $only === null ? $this->only : $only, $except === null ? $this->except : $except);
    }

    /**
     * @param string $path
     * @param string[] $only
     * @param string[] $except
     *
     * @psalm-return array<array-key|string, mixed|non-empty-list<string>>
     */
    private function getMessageFromPath(string $path, array $only, array $except): array
    {
        $messages = [];

        $files = FileHelper::findFiles($path, [
            'filter' => (new pathMatcher())->only(...$only)->except(...$except),
            'recursive' => true,
        ]);

        /**
         * @var string[] $files
         */
        foreach ($files as $file) {
            $messages = array_merge_recursive($messages, $this->extractMessagesFromFile($file));
        }

        return $messages;
    }

    /**
     * @param string $fileName
     *
     * @psalm-return array<array-key|string, mixed|non-empty-list<string>>
     */
    private function extractMessagesFromFile(string $fileName): array
    {
        $fileContent = file_get_contents($fileName);
        $tokens = token_get_all($fileContent);

        $this->skippedLinesOfFile = [];
        $messages = $this->extractMessagesFromTokens($tokens);

        if (!empty($this->skippedLinesOfFile)) {
            $this->skippedLines[$fileName] = $this->skippedLinesOfFile;
        }

        return $messages;
    }

    /**
     * @psalm-param array<string|array{0: int, 1: string, 2: int}> $tokens
     *
     * @psalm-return array<array-key|string, mixed|non-empty-list<string>>
     */
    private function extractMessagesFromTokens(array $tokens): array
    {
        $messages = $buffer = [];
        $matchedTokensCount = $pendingParenthesisCount = 0;
        $isStartedTranslator = false;

        foreach ($tokens as $tokenIndex => $token) {
            if (is_array($token) && in_array($token[0], [T_WHITESPACE, T_COMMENT])) {
                continue;
            }

            if ($isStartedTranslator) {
                if ($pendingParenthesisCount === 0 && $this->tokensEqual($token, ')')) {
                    $messages = array_merge_recursive($messages, $this->extractParametersFromTokens($buffer));
                    $isStartedTranslator = false;
                    $pendingParenthesisCount = 0;
                    $buffer = [];
                } else {
                    if ($this->tokensEqual($token, '(')) {
                        $pendingParenthesisCount++;
                    } elseif ($this->tokensEqual($token, ')')) {
                        $pendingParenthesisCount--;
                    }
                    $buffer[] = $token;
                }
            } else {
                if ($matchedTokensCount === $this->sizeOfTranslator) {
                    if ($this->tokensEqual($token, '(')) {
                        $isStartedTranslator = true;
                    }
                    $matchedTokensCount = 0;
                }

                if ($this->tokensEqual($token, $this->translatorTokens[$matchedTokensCount])) {
                    $matchedTokensCount++;
                } else {
                    $matchedTokensCount = 0;
                }
            }
        }

        return $messages;
    }

    /**
     * @param array $tokens
     * @psalm-param array<string|array{0: int, 1: string, 2: int}> $tokens
     *
     * @return array
     * @psalm-return array<array-key|string, mixed|non-empty-list<string>>
     */
    private function extractParametersFromTokens(array $tokens): array
    {
        $parameters = $this->splitTokensAsParams($tokens);

        if (!isset($parameters['id'])) {
            $this->skippedLinesOfFile[] = $tokens;
            return [];
        }

        $messages = [$parameters['category'] ?? $this->defaultCategory => [$parameters['id']]];

        // Get translation messages from parameters
        if (isset($parameters['parameters'])) {
            $messages = array_merge_recursive($messages, $this->extractMessagesFromTokens($parameters['parameters']));
        }

        return $messages;
    }

    /**
     * @psalm-param array<string|array{0: int, 1: string, 2: int}> $tokens
     *
     * @psalm-return array{category?: null|string, id?: null|string, parameters?: null|list<array{0: int, 1: string, 2: int}|string>}
     */
    private function splitTokensAsParams(array $tokens): array
    {
        $parameters = [];
        $parameterIndex = 0;
        $commaStack = [];

        foreach ($tokens as $token) {
            if (empty($commaStack) && $token === ',') {
                $parameterIndex++;
                continue;
            }
            if (is_string($token)) {
                if (in_array($token, self::$commaSpare)) {
                    $commaStack[] = $token;
                } elseif (isset(self::$commaSpare[$token])&& array_pop($commaStack) !== self::$commaSpare[$token]) {
                    return [];
                }
            }
            $parameters[$parameterIndex][] = $token;
        }

        return [
            'id' => $this->getMessageStringFromTokens($parameters[0] ?? []),
            'parameters' => $parameters[1] ?? null,
            'category' => $this->getMessageStringFromTokens($parameters[2] ?? []),
        ];
    }

    /**
     * @psalm-param array<string|array{0: int, 1: string, 2: int}> $tokens
     *
     * @return string|null
     */
    private function getMessageStringFromTokens(array $tokens): ?string
    {
        if (empty($tokens) || $tokens[0][0] !== T_CONSTANT_ENCAPSED_STRING) {
            return null;
        }

        $fullMessage = substr($tokens[0][1], 1, -1);

        $i = 1;
        $countTokens = count($tokens);
        while ($i < $countTokens && $tokens[$i] === '.') {
            if ($tokens[$i + 1][0] === T_CONSTANT_ENCAPSED_STRING) {
                $fullMessage .= substr($tokens[$i + 1][1], 1, -1);
            } elseif (in_array($tokens[$i + 1][0], [T_LNUMBER, T_DNUMBER])) {
                $fullMessage .= $tokens[$i + 1][1];
            } else {
                return null;
            }

            $i += 2;
        }

        return stripcslashes($fullMessage);
    }

    /**
     * Finds out if two PHP tokens are equal.
     *
     * @param array{0: int, 1: string, 2: int}|string $a
     * @param array{0: int, 1: string, 2: int}|string $b
     *
     * @return bool
     */
    private function tokensEqual($a, $b): bool
    {
        if (is_string($a)) {
            return $a === $b;
        }

        return $a[0] === $b[0] && $a[1] == $b[1];
    }

    public function hasSkippedLines(): bool
    {
        return !empty($this->skippedLines);
    }

    public function getSkippedLines(): array
    {
        return $this->skippedLines;
    }

    public function getDefaultCategory(): string
    {
        return $this->defaultCategory;
    }

    public function setDefaultCategory(string $defaultCategory): void
    {
        $this->defaultCategory = $defaultCategory;
    }

    public function withTranslator(string $translator): self
    {
        $new = clone $this;
        $new->translator = $translator;
        return $new;
    }
}


1		<?php
2
3		declare(strict_types=1);
4
5		namespace Yiisoft\Translator;
6
7		use Yiisoft\Files\FileHelper;
8		use Yiisoft\Files\PathMatcher\PathMatcher;
9
10		/**
11		* Extractor messages
12		*/
13		final class TranslationExtractor
14		{
15		private string $translator = '->translate';
16
17		/**
18		* @var array<string\|array{0: int, 1: string, 2: int}>
19		*/
20		private array $translatorTokens = [];
21		private int $sizeOfTranslator = 0;
22
23		private array $skippedLines = [];
24		private array $skippedLinesOfFile = [];
25
26		private string $defaultCategory = '';
27
28		/**
29		* @var string[]
30		*/
31		private array $only = ['**.php'];
32
33		/**
34		* @var string[]
35		*/
36		private array $except = [
37		'.svn',
38		'.git',
39		'.gitignore',
40		'.gitkeep',
41		'.hgignore',
42		'.hgkeep',
43		'/messages',
44		];
45
46		private static array $commaSpare = [
47		')' => '(',
48		']' => '[',
49		'}' => '{',
50		];
51
52		/**
53		* @param string $path
54		* @param string[]\|null $only
55		* @param string[]\|null $except
56		*
57		* @return array
58		*/
59	9	public function extract(string $path, ?array $only = null, ?array $except = null): array
60		{
61	9	if (!is_dir($path)) {
62	1	throw new \RuntimeException(sprintf('Directory "%s" does not exist.', $path));
63		}
64
65	8	$translatorTokens = token_get_all('<?php ' . $this->translator);
66	8	array_shift($translatorTokens);
67	8	$this->translatorTokens = $translatorTokens;
68	8	$this->sizeOfTranslator = count($this->translatorTokens);
69
70
71	8	if ($this->sizeOfTranslator < 2) {
72	1	throw new \RuntimeException('Translator tokens cannot be shorttest 2 tokens.');
73		}
74
75	7	return $this->getMessageFromPath($path, $only === null ? $this->only : $only, $except === null ? $this->except : $except);
76		}
77
78		/**
79		* @param string $path
80		* @param string[] $only
81		* @param string[] $except
82		*
83		* @psalm-return array<array-key\|string, mixed\|non-empty-list<string>>
84		*/
85	7	private function getMessageFromPath(string $path, array $only, array $except): array
86		{
87	7	$messages = [];
88
89	7	$files = FileHelper::findFiles($path, [
90	7	'filter' => (new pathMatcher())->only(...$only)->except(...$except),
91		'recursive' => true,
92		]);
93
94		/**
95		* @var string[] $files
96		*/
97	7	foreach ($files as $file) {
98	7	$messages = array_merge_recursive($messages, $this->extractMessagesFromFile($file));
99		}
100
101	7	return $messages;
102		}
103
104		/**
105		* @param string $fileName
106		*
107		* @psalm-return array<array-key\|string, mixed\|non-empty-list<string>>
108		*/
109	7	private function extractMessagesFromFile(string $fileName): array
110		{
111	7	$fileContent = file_get_contents($fileName);
112	7	$tokens = token_get_all($fileContent);
113
114	7	$this->skippedLinesOfFile = [];
115	7	$messages = $this->extractMessagesFromTokens($tokens);
116
117	7	if (!empty($this->skippedLinesOfFile)) {
118	4	$this->skippedLines[$fileName] = $this->skippedLinesOfFile;
119		}
120
121	7	return $messages;
122		}
123
124		/**
125		* @psalm-param array<string\|array{0: int, 1: string, 2: int}> $tokens
126		*
127		* @psalm-return array<array-key\|string, mixed\|non-empty-list<string>>
128		*/
129	7	private function extractMessagesFromTokens(array $tokens): array
130		{
131	7	$messages = $buffer = [];
132	7	$matchedTokensCount = $pendingParenthesisCount = 0;
133	7	$isStartedTranslator = false;
134
135	7	foreach ($tokens as $tokenIndex => $token) {
136	7	if (is_array($token) && in_array($token[0], [T_WHITESPACE, T_COMMENT])) {
137	7	continue;
138		}
139
140	7	if ($isStartedTranslator) {
141	7	if ($pendingParenthesisCount === 0 && $this->tokensEqual($token, ')')) {
142	7	$messages = array_merge_recursive($messages, $this->extractParametersFromTokens($buffer));
143	7	$isStartedTranslator = false;
144	7	$pendingParenthesisCount = 0;
145	7	$buffer = [];
146		} else {
147	7	if ($this->tokensEqual($token, '(')) {
148	7	$pendingParenthesisCount++;
149	7	} elseif ($this->tokensEqual($token, ')')) {
150	7	$pendingParenthesisCount--;
151		}
152	7	$buffer[] = $token;
153		}
154		} else {
155	7	if ($matchedTokensCount === $this->sizeOfTranslator) {
156	7	if ($this->tokensEqual($token, '(')) {
157	7	$isStartedTranslator = true;
158		}
159	7	$matchedTokensCount = 0;
160		}
161
162	7	if ($this->tokensEqual($token, $this->translatorTokens[$matchedTokensCount])) {
163	7	$matchedTokensCount++;
164		} else {
165	7	$matchedTokensCount = 0;
166		}
167		}
168		}
169
170	7	return $messages;
171		}
172
173		/**
174		* @param array $tokens
175		* @psalm-param array<string\|array{0: int, 1: string, 2: int}> $tokens
176		*
177		* @return array
178		* @psalm-return array<array-key\|string, mixed\|non-empty-list<string>>
179		*/
180	7	private function extractParametersFromTokens(array $tokens): array
181		{
182	7	$parameters = $this->splitTokensAsParams($tokens);
183
184	7	if (!isset($parameters['id'])) {
185	4	$this->skippedLinesOfFile[] = $tokens;
186	4	return [];
187		}
188
189	5	$messages = [$parameters['category'] ?? $this->defaultCategory => [$parameters['id']]];
190
191		// Get translation messages from parameters
192	5	if (isset($parameters['parameters'])) {
193	4	$messages = array_merge_recursive($messages, $this->extractMessagesFromTokens($parameters['parameters']));
194		}
195
196	5	return $messages;
197		}
198
199		/**
200		* @psalm-param array<string\|array{0: int, 1: string, 2: int}> $tokens
201		*
202		* @psalm-return array{category?: null\|string, id?: null\|string, parameters?: null\|list<array{0: int, 1: string, 2: int}\|string>}
203		*/
204	7	private function splitTokensAsParams(array $tokens): array
205		{
206	7	$parameters = [];
207	7	$parameterIndex = 0;
208	7	$commaStack = [];
209
210	7	foreach ($tokens as $token) {
211	7	if (empty($commaStack) && $token === ',') {
212	6	$parameterIndex++;
213	6	continue;
214		}
215	7	if (is_string($token)) {
216	6	if (in_array($token, self::$commaSpare)) {
217	6	$commaStack[] = $token;
218	6	} elseif (isset(self::$commaSpare[$token])&& array_pop($commaStack) !== self::$commaSpare[$token]) {
219		return [];
220		}
221		}
222	7	$parameters[$parameterIndex][] = $token;
223		}
224
225		return [
226	7	'id' => $this->getMessageStringFromTokens($parameters[0] ?? []),
227	7	'parameters' => $parameters[1] ?? null,
228	7	'category' => $this->getMessageStringFromTokens($parameters[2] ?? []),
229		];
230		}
231
232		/**
233		* @psalm-param array<string\|array{0: int, 1: string, 2: int}> $tokens
234		*
235		* @return string\|null
236		*/
237	7	private function getMessageStringFromTokens(array $tokens): ?string
238		{
239	7	if (empty($tokens) \|\| $tokens[0][0] !== T_CONSTANT_ENCAPSED_STRING) {
240	6	return null;
241		}
242
243	7	$fullMessage = substr($tokens[0][1], 1, -1);
244
245	7	$i = 1;
246	7	$countTokens = count($tokens);
247	7	while ($i < $countTokens && $tokens[$i] === '.') {
248	6	if ($tokens[$i + 1][0] === T_CONSTANT_ENCAPSED_STRING) {
249	6	$fullMessage .= substr($tokens[$i + 1][1], 1, -1);
250	4	} elseif (in_array($tokens[$i + 1][0], [T_LNUMBER, T_DNUMBER])) {
251	4	$fullMessage .= $tokens[$i + 1][1];
252		} else {
253	3	return null;
254		}
255
256	6	$i += 2;
257		}
258
259	5	return stripcslashes($fullMessage);
260		}
261
262		/**
263		* Finds out if two PHP tokens are equal.
264		*
265		* @param array{0: int, 1: string, 2: int}\|string $a
266		* @param array{0: int, 1: string, 2: int}\|string $b
267		*
268		* @return bool
269		*/
270	7	private function tokensEqual($a, $b): bool
271		{
272	7	if (is_string($a)) {
273	7	return $a === $b;
274		}
275
276	7	return $a[0] === $b[0] && $a[1] == $b[1];
277		}
278
279	7	public function hasSkippedLines(): bool
280		{
281	7	return !empty($this->skippedLines);
282		}
283
284	4	public function getSkippedLines(): array
285		{
286	4	return $this->skippedLines;
287		}
288
289	2	public function getDefaultCategory(): string
290		{
291	2	return $this->defaultCategory;
292		}
293
294	6	public function setDefaultCategory(string $defaultCategory): void
295		{
296	6	$this->defaultCategory = $defaultCategory;
297	6	}
298
299	2	public function withTranslator(string $translator): self
300		{
301	2	$new = clone $this;
302	2	$new->translator = $translator;
303	2	return $new;
304		}
305		}
306

yiisoft / translator

Pull Request — master (#40)

TranslationExtractor::splitTokensAsParams() B

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like