Language::_tokens() - Code Metrics - Inspection of "Embedding languages in languages in languages now..." - kadet1090/KeyLighter - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 5ee129...947530 )

by Kacper

created 2016-03-09 18:43 UTC

Language::_tokens() C

↳ Parent: Language

Complexity

Conditions	8
Paths	28

Size

Total Lines	34
Code Lines	17

Duplication

Lines	0
Ratio	0 %

Importance

Changes	5
Bugs	0	Features	0

Metric	Value
c	5
b	0
f	0
dl	0
loc	34
rs	5.3846
cc	8
eloc	17
nc	28
nop	3

<?php
/**
 * Highlighter
 *
 * Copyright (C) 2015, Some right reserved.
 *
 * @author Kacper "Kadet" Donat <[email protected]>
 *
 * Contact with author:
 * Xmpp: [email protected]
 * E-mail: [email protected]
 *
 * From Kadet with love.
 */

namespace Kadet\Highlighter\Language;

use Kadet\Highlighter\Matcher\WholeMatcher;
use Kadet\Highlighter\Parser\LanguageToken;
use Kadet\Highlighter\Parser\Rule;
use Kadet\Highlighter\Parser\Token;
use Kadet\Highlighter\Parser\TokenFactory;
use Kadet\Highlighter\Parser\TokenIterator;
use Kadet\Highlighter\Utils\ArrayHelper;

/**
 * Class Language
 *
 * @package Kadet\Highlighter\Language
 */
abstract class Language
{
    /**
     * Tokenizer rules
     *
     * @var Rule[]
     */
    private $_rules;

    /**
     * @var array
     */
    private $_options = [];

    /**
     * Language constructor.
     *
     * @param array $options
     */
    public function __construct(array $options = []) {
        $this->_options  = array_merge([
            'embedded' => [],
        ], $options);
        $this->_rules    = $this->getRules();
    }

    /**
     * Tokenization rules definition
     *
     * @return array
     */
    public abstract function getRules();


    /**
     * Parses source and removes wrong tokens.
     *
     * @param TokenIterator|string $tokens
     *
     * @return TokenIterator
     */
    public function parse($tokens = null, $additional = [])
    {
        if (is_string($tokens)) {
            $tokens = $this->tokenize($tokens, $additional);
        } elseif(!$tokens instanceof TokenIterator) {
            throw new \InvalidArgumentException('$tokens must be string or TokenIterator');
        }

        $start = $tokens->current();

        /** @var Token[] $result */
        $result = [$start];

        $context = [];
        $all = [];

        /** @var Token $token */
        for($tokens->next(); $tokens->valid(); $tokens->next()) {
            $token = $tokens->current();

            if (!$token->isValid($this, $context)) {
                continue;
            }

            if ($token->isStart()) {
                if ($token instanceof LanguageToken) {
                    /** @noinspection PhpUndefinedMethodInspection bug */
                    $result = array_merge(
                        $result,
                        $token->getLanguage()->parse($tokens)->getTokens()
                    );
                } else {
                    $all[spl_object_hash($token)] = $result[] = $token;
                    $context[spl_object_hash($token)] = $token->name;
                }
            } else {
                $start = $token->getStart();

                if ($token instanceof LanguageToken && $token->getRule()->getLanguage() === $this) {
                    $result[0]->setEnd($token);

                    if($result[0]->getRule()->postProcess) {
                        $source = substr($tokens->getSource(), $result[0]->pos, $result[0]->getLength());

                        $tokens = $this->tokenize($source, $result, $result[0]->pos);
                        $result = $this->parse($tokens)->getTokens();
                    }

                    # closing unclosed tokens
                    foreach(array_reverse($context) as $hash => $name) {
                        $result[$hash]->setEnd(new Token([$name, 'pos' => $token->pos]));
                    }

                    $result[] = $token;
                    break;
                } else {
                    if ($start !== null) {
                        unset($context[spl_object_hash($start)]);
                    } else {
                        /** @noinspection PhpUnusedParameterInspection */
                        $start = ArrayHelper::find(array_reverse($context), function ($k, $v) use ($token) {
                            return $v === $token->name;
                        });

                        if ($start !== false) {
                            $token->setStart($all[$start]);
                            unset($context[$start]);
                        }
                    }

                    $result[] = $token;
                }
            }
        }

        return new TokenIterator($result, $tokens->getSource());
    }

    /**
     * Tokenize source
     *
     * @param $source
     *
     * @return array
     */
    private function _tokens($source, $additional = [], $offset = 0)
    {
        $this->_rules['language.' . $this->getIdentifier()] = $this->getOpenClose();

        $result = [];
        foreach ($this->_rules as $name => $rules) {
            if (!is_array($rules)) {
                $rules = [$rules];
            }

            /** @var Rule $rule */
            foreach ($rules as $rule) {
                if($rule->getLanguage() === false) {
                    $rule->setLanguage($this);
                }

                $rule->factory->setBase($name);
                $result = array_merge($result, $rule->match($source));
            }
        }

        foreach($this->getEmbedded() as $language) {
            $result = array_merge($result, $language->_tokens($source));
        }

        // Array map would be cool, but is a lot slower
        if($offset) {
            foreach ($result as $item) {
                $item->pos += $offset;
            }
        }

        return array_merge($result, $additional);
    }

    public function tokenize($source, $additional = [], $offset = 0)
    {
        $iterator = new TokenIterator($this->_tokens($source, $additional, $offset), $source);
        $iterator->sort();
        $iterator->rewind();
        return $iterator;
    }

    /**
     * Unique language identifier, for example 'php'
     *
     * @return string
     */
    public abstract function getIdentifier();


    /**
     * Language range Rule(s)
     *
     * @return Rule|Rule[]
     */
    public function getOpenClose()
    {
        return new Rule(
            new WholeMatcher(), [
                'priority' => 1000,
                'factory'  => new TokenFactory('Kadet\\Highlighter\\Parser\\LanguageToken'),
                'inject'   => $this,
                'language' => null
            ]
        );
    }

    /**
     * @return Language[]
     */
    public function getEmbedded() {
        return $this->_options['embedded'];
    }

    /**
     * @param Language $lang
     */
    public function embed(Language $lang) {
        $this->_options['embedded'][] = $lang;
    }
}


1			<?php
2			/**
3			* Highlighter
4			*
5			* Copyright (C) 2015, Some right reserved.
6			*
7			* @author Kacper "Kadet" Donat <[email protected]>
8			*
9			* Contact with author:
10			* Xmpp: [email protected]
11			* E-mail: [email protected]
12			*
13			* From Kadet with love.
14			*/
15
16			namespace Kadet\Highlighter\Language;
17
18			use Kadet\Highlighter\Matcher\WholeMatcher;
19			use Kadet\Highlighter\Parser\LanguageToken;
20			use Kadet\Highlighter\Parser\Rule;
21			use Kadet\Highlighter\Parser\Token;
22			use Kadet\Highlighter\Parser\TokenFactory;
23			use Kadet\Highlighter\Parser\TokenIterator;
24			use Kadet\Highlighter\Utils\ArrayHelper;
25
26			/**
27			* Class Language
28			*
29			* @package Kadet\Highlighter\Language
30			*/
31			abstract class Language
32			{
33			/**
34			* Tokenizer rules
35			*
36			* @var Rule[]
37			*/
38			private $_rules;
39
40			/**
41			* @var array
42			*/
43			private $_options = [];
44
45			/**
46			* Language constructor.
47			*
48			* @param array $options
49			*/
50			public function __construct(array $options = []) {
51			$this->_options = array_merge([
52			'embedded' => [],
53			], $options);
54			$this->_rules = $this->getRules();
55			}
56
57			/**
58			* Tokenization rules definition
59			*
60			* @return array
61			*/
62			public abstract function getRules();
			0 ignored issues – show Coding Style introduced 2016-03-06 00:35 UTC by Report Bug Copy Issue Report The abstract declaration must precede the visibility declaration Loading history...
63
64			/**
65			* Parses source and removes wrong tokens.
66			*
67			* @param TokenIterator\|string $tokens
68			*
69			* @return TokenIterator
70			*/
71			public function parse($tokens = null, $additional = [])
72			{
73			if (is_string($tokens)) {
74			$tokens = $this->tokenize($tokens, $additional);
75			} elseif(!$tokens instanceof TokenIterator) {
76			throw new \InvalidArgumentException('$tokens must be string or TokenIterator');
77			}
78
79			$start = $tokens->current();
80
81			/** @var Token[] $result */
82			$result = [$start];
83
84			$context = [];
85			$all = [];
86
87			/** @var Token $token */
88			for($tokens->next(); $tokens->valid(); $tokens->next()) {
89			$token = $tokens->current();
90
91			if (!$token->isValid($this, $context)) {
92			continue;
93			}
94
95			if ($token->isStart()) {
96			if ($token instanceof LanguageToken) {
97			/** @noinspection PhpUndefinedMethodInspection bug */
98			$result = array_merge(
99			$result,
100			$token->getLanguage()->parse($tokens)->getTokens()
101			);
102			} else {
103			$all[spl_object_hash($token)] = $result[] = $token;
104			$context[spl_object_hash($token)] = $token->name;
105			}
106			} else {
107			$start = $token->getStart();
108
109			if ($token instanceof LanguageToken && $token->getRule()->getLanguage() === $this) {
110			$result[0]->setEnd($token);
111
112			if($result[0]->getRule()->postProcess) {
113			$source = substr($tokens->getSource(), $result[0]->pos, $result[0]->getLength());
114
115			$tokens = $this->tokenize($source, $result, $result[0]->pos);
116			$result = $this->parse($tokens)->getTokens();
117			}
118
119			# closing unclosed tokens
120			foreach(array_reverse($context) as $hash => $name) {
121			$result[$hash]->setEnd(new Token([$name, 'pos' => $token->pos]));
122			}
123
124			$result[] = $token;
125			break;
126			} else {
127			if ($start !== null) {
128			unset($context[spl_object_hash($start)]);
129			} else {
130			/** @noinspection PhpUnusedParameterInspection */
131			$start = ArrayHelper::find(array_reverse($context), function ($k, $v) use ($token) {
132			return $v === $token->name;
133			});
134
135			if ($start !== false) {
136			$token->setStart($all[$start]);
137			unset($context[$start]);
138			}
139			}
140
141			$result[] = $token;
142			}
143			}
144			}
145
146			return new TokenIterator($result, $tokens->getSource());
147			}
148
149			/**
150			* Tokenize source
151			*
152			* @param $source
153			*
154			* @return array
155			*/
156			private function _tokens($source, $additional = [], $offset = 0)
157			{
158			$this->_rules['language.' . $this->getIdentifier()] = $this->getOpenClose();
159
160			$result = [];
161			foreach ($this->_rules as $name => $rules) {
162			if (!is_array($rules)) {
163			$rules = [$rules];
164			}
165
166			/** @var Rule $rule */
167			foreach ($rules as $rule) {
168			if($rule->getLanguage() === false) {
169			$rule->setLanguage($this);
170			}
171
172			$rule->factory->setBase($name);
173			$result = array_merge($result, $rule->match($source));
174			}
175			}
176
177			foreach($this->getEmbedded() as $language) {
178			$result = array_merge($result, $language->_tokens($source));
179			}
180
181			// Array map would be cool, but is a lot slower
182			if($offset) {
183			foreach ($result as $item) {
184			$item->pos += $offset;
185			}
186			}
187
188			return array_merge($result, $additional);
189			}
190
191			public function tokenize($source, $additional = [], $offset = 0)
192			{
193			$iterator = new TokenIterator($this->_tokens($source, $additional, $offset), $source);
194			$iterator->sort();
195			$iterator->rewind();
196			return $iterator;
197			}
198
199			/**
200			* Unique language identifier, for example 'php'
201			*
202			* @return string
203			*/
204			public abstract function getIdentifier();
			0 ignored issues – show Coding Style introduced 2016-03-06 00:35 UTC by Report Bug Copy Issue Report The abstract declaration must precede the visibility declaration Loading history...
205
206			/**
207			* Language range Rule(s)
208			*
209			* @return Rule\|Rule[]
210			*/
211			public function getOpenClose()
212			{
213			return new Rule(
214			new WholeMatcher(), [
215			'priority' => 1000,
216			'factory' => new TokenFactory('Kadet\\Highlighter\\Parser\\LanguageToken'),
217			'inject' => $this,
218			'language' => null
219			]
220			);
221			}
222
223			/**
224			* @return Language[]
225			*/
226			public function getEmbedded() {
227			return $this->_options['embedded'];
228			}
229
230			/**
231			* @param Language $lang
232			*/
233			public function embed(Language $lang) {
234			$this->_options['embedded'][] = $lang;
235			}
236			}
237

kadet1090 / KeyLighter

Push — master ( 5ee129...947530 )

Language::_tokens() C

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like