TokenStream::getSymbol() - Code Metrics - Inspection of "Scrutinizer fixes" - graze/morphism - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#60)

by Burhan

created 2019-03-30 14:36 UTC

TokenStream::getSymbol() A

↳ Parent: Project

Complexity

Conditions	2
Paths	2

Size

Total Lines	10
Code Lines	5

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	4
CRAP Score	2.032

Importance

Changes

Metric	Value
cc	2
eloc	5
nc	2
nop	2
dl	0
loc	10
ccs	4
cts	5
cp	0.8
crap	2.032
rs	10
c	0
b	0
f	0

<?php
namespace Graze\Morphism\Parse;

use LogicException;
use RuntimeException;

class TokenStream
{
    /** @var string */
    private $path;
    /** @var string */
    private $text;
    /** @var int */
    private $len;
    /** @var int */
    private $offset = 0;
    /** @var bool */
    private $inConditional = false;
    /** @var array */
    private $memo = [];

    /** @var array */
    private static $skipTokenTypes = [
        Token::CONDITIONAL_START => true,
        Token::CONDITIONAL_END   => true,
        Token::COMMENT           => true,
        Token::WHITESPACE        => true,
    ];

    private function __construct()
    {
    }

    // TODO - this is a hack that needs to be refactored

    // perhaps supply a LineStream interface that is satisfied
    // by a FileLineStream or ConnectionLineStream for example?
    /**
     * @param string $text
     * @param string $label
     * @return TokenStream
     */
    public static function newFromText($text, $label)
    {
        $stream = new self;
        $stream->path = $label;
        $stream->text = $text;
        $stream->len = strlen($text);
        return $stream;
    }

    /**
     * @param string $path
     * @return TokenStream
     */
    public static function newFromFile($path)
    {
        $text = @file_get_contents($path);
        if ($text === false) {
            throw new RuntimeException("$path: could not open file");
        }
        $stream = new self;
        $stream->path = $path;
        $stream->text = $text;
        $stream->len = strlen($text);
        return $stream;
    }

    /**
     * @return Token|mixed
     */
    private function nextTokenRaw()
    {
        $startOffset = $this->offset;
        if (isset($this->memo[$startOffset])) {
            $entry = $this->memo[$startOffset];
            $this->offset = $entry[0];
            $this->inConditional = $entry[1];
            return $entry[2];
        }

        if ($this->offset >= $this->len) {
            $token = new Token(Token::EOF);
            $this->offset = $this->len;
        } else {
            list($token, $offset) = $this->getNextTokenRaw($this->text, $this->offset);
            $this->offset = $offset;
        }

        $this->memo[$startOffset] = [
            $this->offset,
            $this->inConditional,
            $token,
        ];

        return $token;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getNextTokenRaw($text, $offset)
    {
        // currently unsupported:
        //
        //      charset prefixed strings, e.g. _utf8'wingwang'
        //      temporal literals, e.g. DATE'2014-07-08'
        //      the null literal, i.e. \N

        switch ($text[$offset]) {
            case " ":
            case "\n":
            case "\r":
            case "\t":
                $n = strspn($text, " \n\r\t", $offset);
                return [
                    new Token(Token::WHITESPACE, substr($text, $offset, $n)),
                    $offset + $n
                ];

            case '#':
                return
                    $this->getComment($text, $offset);

            case '.':
            case '+':
                return
                    $this->getNumber($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '-':
                return
                    $this->getComment($text, $offset) ?:
                    $this->getNumber($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '*':
                return
                    $this->getConditionEnd($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '/':
                return
                    $this->getConditionalStart($text, $offset) ?:
                    $this->getMultilineComment($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '0':
                // Handle hex if needed
                if (isset($text[$offset+1]) && $text[$offset+1] === 'x') {
                    return
                        $this->getHex($text, $offset) ?:
                        $this->getIdentifier($text, $offset);
                }
                // Handle non-hex leading zero.
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                return
                    $this->getNumber($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case '"':
            case "'":
                return
                    $this->getString($text, $offset);

            case '`':
                return
                    $this->getQuotedIdentifier($text, $offset);

            case 'B':
            case 'b':
                return
                    $this->getBin($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case 'X':
            case 'x':
                return
                    $this->getHex($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case '$':
            case '_':
            case 'A':
            case 'a':
            case 'C':
            case 'c':
            case 'D':
            case 'd':
            case 'E':
            case 'e':
            case 'F':
            case 'f':
            case 'G':
            case 'g':
            case 'H':
            case 'h':
            case 'I':
            case 'i':
            case 'J':
            case 'j':
            case 'K':
            case 'k':
            case 'L':
            case 'l':
            case 'M':
            case 'm':
            case 'N':
            case 'n':
            case 'O':
            case 'o':
            case 'P':
            case 'p':
            case 'Q':
            case 'q':
            case 'R':
            case 'r':
            case 'S':
            case 's':
            case 'T':
            case 't':
            case 'U':
            case 'u':
            case 'V':
            case 'v':
            case 'W':
            case 'w':
            case 'Y':
            case 'y':
            case 'Z':
            case 'z':
                return
                    $this->getIdentifier($text, $offset);

            case '!':
            case '%':
            case '&':
            case '(':
            case ')':
            case ',':
            case ':':
            case ';':
            case '<':
            case '=':
            case '>':
            case '@':
            case '^':
            case '|':
            case '~':
                return
                    $this->getSpecialSymbol($text, $offset);

            case '?':
            case '[':
            case '\\':
            case ']':
            case '{':
            case '}':
            default:
                $ch = $text[$offset];
                throw new LogicException("Lexer is confused by char '$ch' ord " . ord($ch));
        }
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getQuotedIdentifier($text, $offset)
    {
        if (preg_match('/`((?:[^`]|``)*)`()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset)) {
            $token = Token::fromIdentifier($pregMatch[1][0]);
            return [
                $token,
                $pregMatch[2][1]
            ];
        }
        throw new RuntimeException("Unterminated identifier: $text");
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getSpecialSymbol($text, $offset)
    {
        // TODO - should probably be a new token type 'variable' for @ and @@

        preg_match('/\A(?:<=|>=|<>|!=|:=|@@|&&|\|\||[=~!@%^&();:,<>|])()/xms', substr($text, $offset, 2), $pregMatch, PREG_OFFSET_CAPTURE);
        return [
            new Token(Token::SYMBOL, $pregMatch[0][0]),
            $offset + $pregMatch[1][1]
        ];
    }

    /**
     * Get the start of a conditional comment.
     *
     * https://dev.mysql.com/doc/refman/5.7/en/comments.html
     *
     * Does not support optimiser hints. See examples below.
     *
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getConditionalStart($text, $offset)
    {
        // Example conditional comments which can't be displayed in the docblock because they clash:
        // - /*! MySQL-specific code */ (execute the given code)
        // - /*!12345 MySQL-specific code */ (execute the given code only if the version matches)
        // Unsupported:
        // - SELECT /*+ BKA(t1) */ FROM ... ;

        if (// 10 comes from allowing for the /*! sequence, a MySQL version number, and a space
            preg_match('_\A/\*!([0-9]*)\s_ms', substr($text, $offset, 10)) &&
            preg_match('_/\*!([0-9]*)\s_ms', $text, $pregMatch, 0, $offset)
        ) {
            $this->inConditional = true;
            return [
                new Token(Token::CONDITIONAL_START, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * Get the end of a conditional comment. See _getConditionStart() for details.
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getConditionEnd($text, $offset)
    {
        if (substr($text, $offset, 2) === '*/') {
            if (!$this->inConditional) {
                throw new RuntimeException("Unexpected '*/'");
            }
            $this->inConditional = false;
            return [
                new Token(Token::CONDITIONAL_END),
                $offset + 2
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getMultilineComment($text, $offset)
    {
        if (substr($text, $offset, 2) === '/*') {
            $pos = strpos($text, '*/', $offset);
            if ($pos !== false) {
                return [
                    new Token(Token::COMMENT, substr($text, $offset, $pos - $offset + 2)),
                    $pos + 2
                ];
            }
            throw new RuntimeException("Unterminated '/*'");
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getComment($text, $offset)
    {
        if (preg_match('/\A(?:#|--\s)/ms', substr($text, $offset, 3))) {
            $pos = strpos($text, "\n", $offset);
            if ($pos !== false) {
                return [
                    new Token(Token::COMMENT, substr($text, $offset, $pos - $offset)),
                    $pos + 1
                ];
            }
            return [
                new Token(Token::COMMENT, $text),
                strlen($text)
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getString($text, $offset)
    {
        $quote = $text[$offset];
        if (preg_match(
            '/' .
            $quote .
            '(' .
                '(?:' .
                    '[^\\\\' . $quote . ']' .   // not \ or "
                    '|\\\\.' .                  // escaped quotearacter
                    '|' .  $quote . $quote .    // ""
                ')*' .
            ')' .
            $quote .
            '/ms',
            $text,
            $pregMatch,
            0,
            $offset
        )
        ) {
            $token = Token::fromString($pregMatch[1], $quote);
            return [
                $token,
                $offset + strlen($pregMatch[0])
            ];
        }
        throw new RuntimeException("Unterminated string $quote...$quote");
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getNumber($text, $offset)
    {
        if (preg_match('/\A[-+]?[.]?[0-9]/ms', substr($text, $offset, 3)) &&
            preg_match('/[-+]?(?:[0-9]+(?:[.][0-9]*)?|[.][0-9]+)(?:[eE][-+]?[0-9]+)?/ms', $text, $pregMatch, 0, $offset)
        ) {
            return [
                new Token(Token::NUMBER, $pregMatch[0]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * Parse a hex string of the form "0x<hex digits>" or "x'<hex digits>'".
     *
     * https://dev.mysql.com/doc/refman/5.7/en/hexadecimal-literals.html
     *
     * - Only an even number of digits is valid.
     * - Case insensitive for hex digits.
     * - Case insensitive 'x' in quoted notation.
     * - Case sensitive 'x' for leading zero notation.
     *
     * Valid examples:
     * - x'BA5EBA11'
     * - x'decea5ed'
     * - X'5eed'
     * - 0xb01dface
     * - 0xBADC0DED
     *
     * Invalid examples
     * - x'00f' (odd number of digits)
     * - x'gg'  (invalid hex character)
     * - 0XFFFF (upper case 'x')
     *
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getHex($text, $offset)
    {
        $pregMatch = [];

        $matchesLeadingZeroNotation = function ($text, $offset, &$pregMatch) {
            return
                preg_match('/\A0x([0-9a-fA-F]*)/ms', $text, $pregMatch, 0, $offset);
        };

        $matchesXQuotedNotation = function ($text, $offset, &$pregMatch) {
            return
                preg_match('/\Ax\'[0-9a-f\']/ims', substr($text, $offset, 3)) &&
                preg_match('/x\'([0-9a-f]*)\'/ims', $text, $pregMatch, 0, $offset);
        };

        if ($matchesLeadingZeroNotation($text, $offset, $pregMatch) ||
            $matchesXQuotedNotation($text, $offset, $pregMatch)) {
            if (strlen($pregMatch[1]) % 2 != 0) {
                throw new RuntimeException("Invalid hex literal");
            }
            return [
                new Token(Token::HEX, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getBin($text, $offset)
    {
        if (preg_match('/\Ab\'[01\']/ms', substr($text, $offset, 3)) &&
            preg_match('/b\'([01]*)\'/ms', $text, $pregMatch, 0, $offset)
        ) {
            return [
                new Token(Token::BIN, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getIdentifier($text, $offset)
    {
        preg_match('/[a-zA-Z0-9$_]+()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset);
        return [
            new Token(Token::IDENTIFIER, $pregMatch[0][0]),
            $pregMatch[1][1]
        ];
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getSymbol($text, $offset)
    {
        if (preg_match('/\A(?:[-+*.\/])/xms', substr($text, $offset, 2), $pregMatch)) {
            return [
                new Token(Token::SYMBOL, $pregMatch[0]),
                $offset + strlen($pregMatch[0])
            ];
        }

        return [];
    }

    /**
     * @return Token|mixed
     */
    public function nextToken()
    {
        while (true) {
            $token = $this->nextTokenRaw();
            if (!isset(self::$skipTokenTypes[$token->type])) {
                return $token;
            }
        }

        return null;
    }

    /**
     * @return object
     */
    public function getMark()
    {
        return (object)[
            'offset'        => $this->offset,
            'inConditional' => $this->inConditional,
        ];
    }

    /**
     * @param mixed $mark
     */
    public function rewind($mark)
    {
        $this->offset        = $mark->offset;
        $this->inConditional = $mark->inConditional;
    }

    /**
     * This function will consume the requested content from the stream without trying to parse and tokenise it.
     * It is used by {@see peek()}.
     *
     * @param mixed $spec
     * @return bool
     */
    public function consume($spec)
    {
        // inline getMark()
        $markOffset        = $this->offset;
        $markInConditional = $this->inConditional;

        if (is_string($spec)) {
            foreach (explode(' ', $spec) as $text) {
                $token = $this->nextToken();
                // inline $token->eq(...)

                if (strcasecmp($token->text, $text) !== 0 ||
                    $token->type !== Token::IDENTIFIER
                ) {
                    // inline rewind()
                    $this->offset        = $markOffset;
                    $this->inConditional = $markInConditional;
                    return false;
                }
            }
        } else {
            foreach ($spec as $match) {
                list($type, $text) = $match;
                $token = $this->nextToken();
                // inline $token->eq(...)

                if (strcasecmp($token->text, $text) !== 0 ||
                    $token->type !== $type
                ) {
                    // inline rewind()
                    $this->offset        = $markOffset;
                    $this->inConditional = $markInConditional;
                    return false;
                }
            }
        }

        return true;
    }

    /**
     * @param mixed $spec
     * @return bool
     */
    public function peek($spec)
    {
        // inline getMark()
        $markOffset        = $this->offset;
        $markInConditional = $this->inConditional;

        $result = $this->consume($spec);

        // inline rewind()
        $this->offset        = $markOffset;
        $this->inConditional = $markInConditional;

        return $result;
    }

    /**
     * @param string $type
     * @param string $text
     * @return string
     */
    public function expect($type, $text = null)
    {
        $token = $this->nextToken();
        if (!$token->eq($type, $text)) {
            throw new RuntimeException("Expected '$text'");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectName()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::IDENTIFIER) {
            throw new RuntimeException("Expected identifier");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectOpenParen()
    {
        return $this->expect(Token::SYMBOL, '(');
    }

    /**
     * @return string
     */
    public function expectCloseParen()
    {
        return $this->expect(Token::SYMBOL, ')');
    }

    /**
     * @return int
     */
    public function expectNumber()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::NUMBER) {
            throw new RuntimeException("Expected number");
        }
        return 1 * $token->text;
    }

    /**
     * @return string
     */
    public function expectString()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::STRING) {
            throw new RuntimeException("Expected string");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectStringExtended()
    {
        $token = $this->nextToken();
        switch ($token->type) {
            case Token::STRING:
                return $token->text;
            case Token::HEX:
                return $token->asString();
            case Token::BIN:
                return $token->asString();
            default:
                throw new RuntimeException("Expected string");
        }
    }

    /**
     * Provides context for error messages.
     *
     * For example, given this invalid table definition ...
     *
     *     CREATE TABLE `foo` (
     *         `a` bar DEFAULT NULL
     *     );
     *
     * ... this function will produce something like this:
     *
     *     schema/morphism test/foo.sql, line 2: unknown datatype 'bar'
     *     1: CREATE TABLE `foo` (
     *     2:   `a` bar<<HERE>> DEFAULT NULL
     *
     * @param string $message
     * @return string
     */
    public function contextualise($message)
    {
        $preContextLines = 4;
        $postContextLines = 0;

        // get position of eol strictly before offset
        $prevEolPos = strrpos($this->text, "\n", $this->offset - strlen($this->text) - 1);
        $prevEolPos = ($prevEolPos === false) ? -1 : $prevEolPos;

        // get position of eol on or after offset
        $nextEolPos = strpos($this->text, "\n", $this->offset);
        $nextEolPos = ($nextEolPos === false) ? strlen($this->text) : $nextEolPos;

        // count number of newlines up to but not including offset
        $lineNo = substr_count($this->text, "\n", 0, $this->offset);
        $lines = explode("\n", $this->text);

        $contextLines = array_slice(
            $lines,
            max(0, $lineNo - $preContextLines),
            min($lineNo, $preContextLines),
            true // preserve keys
        );
        $contextLines += [
            $lineNo =>
                substr($this->text, $prevEolPos + 1, $this->offset - ($prevEolPos + 1)) .
                "<<HERE>>".
                substr($this->text, $this->offset, $nextEolPos - $this->offset)
        ];
        $contextLines += array_slice(
            $lines,
            $lineNo + 1,
            $postContextLines,
            true // preserve keys
        );

        $context = '';
        $width = strlen($lineNo + 1 + $postContextLines);
        foreach ($contextLines as $i => $line) {
            $context .= sprintf("\n%{$width}d: %s", $i + 1, $line);
        }

        return sprintf("%s, line %d: %s%s", $this->path, $lineNo + 1, $message, $context);
    }
}


1		<?php
2		namespace Graze\Morphism\Parse;
3
4		use LogicException;
5		use RuntimeException;
6
7		class TokenStream
8		{
9		/** @var string */
10		private $path;
11		/** @var string */
12		private $text;
13		/** @var int */
14		private $len;
15		/** @var int */
16		private $offset = 0;
17		/** @var bool */
18		private $inConditional = false;
19		/** @var array */
20		private $memo = [];
21
22		/** @var array */
23		private static $skipTokenTypes = [
24		Token::CONDITIONAL_START => true,
25		Token::CONDITIONAL_END => true,
26		Token::COMMENT => true,
27		Token::WHITESPACE => true,
28		];
29
30	598	private function __construct()
31		{
32	598	}
33
34		// TODO - this is a hack that needs to be refactored
		0 ignored issues – show Coding Style Best Practice introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report Comments for TODO tasks are often forgotten in the code; it might be better to use a dedicated issue tracker. Loading history...
35		// perhaps supply a LineStream interface that is satisfied
36		// by a FileLineStream or ConnectionLineStream for example?
37		/**
38		* @param string $text
39		* @param string $label
40		* @return TokenStream
41		*/
42	1	public static function newFromText($text, $label)
43		{
44	1	$stream = new self;
45	1	$stream->path = $label;
46	1	$stream->text = $text;
47	1	$stream->len = strlen($text);
48	1	return $stream;
49		}
50
51		/**
52		* @param string $path
53		* @return TokenStream
54		*/
55	598	public static function newFromFile($path)
56		{
57	598	$text = @file_get_contents($path);
58	598	if ($text === false) {
59	1	throw new RuntimeException("$path: could not open file");
60		}
61	597	$stream = new self;
62	597	$stream->path = $path;
63	597	$stream->text = $text;
64	597	$stream->len = strlen($text);
65	597	return $stream;
66		}
67
68		/**
69		* @return Token\|mixed
70		*/
71	594	private function nextTokenRaw()
72		{
73	594	$startOffset = $this->offset;
74	594	if (isset($this->memo[$startOffset])) {
75	389	$entry = $this->memo[$startOffset];
76	389	$this->offset = $entry[0];
77	389	$this->inConditional = $entry[1];
78	389	return $entry[2];
79		}
80
81	594	if ($this->offset >= $this->len) {
82	349	$token = new Token(Token::EOF);
83	349	$this->offset = $this->len;
84		} else {
85	591	list($token, $offset) = $this->getNextTokenRaw($this->text, $this->offset);
86	579	$this->offset = $offset;
87		}
88
89	582	$this->memo[$startOffset] = [
90	582	$this->offset,
91	582	$this->inConditional,
92	582	$token,
93		];
94
95	582	return $token;
96		}
97
98		/**
99		* @param string $text
100		* @param int $offset
101		* @return array\|null
102		*/
103	591	private function getNextTokenRaw($text, $offset)
104		{
105		// currently unsupported:
106		//
107		// charset prefixed strings, e.g. _utf8'wingwang'
108		// temporal literals, e.g. DATE'2014-07-08'
109		// the null literal, i.e. \N
110
111	591	switch ($text[$offset]) {
112	591	case " ":
113	591	case "\n":
114	591	case "\r":
115	591	case "\t":
116	387	$n = strspn($text, " \n\r\t", $offset);
117		return [
118	387	new Token(Token::WHITESPACE, substr($text, $offset, $n)),
119	387	$offset + $n
120		];
121
122	591	case '#':
123		return
124	2	$this->getComment($text, $offset);
125
126	590	case '.':
127	589	case '+':
128		return
129	5	$this->getNumber($text, $offset) ?:
130	5	$this->getSymbol($text, $offset);
131
132	586	case '-':
133		return
134	4	$this->getComment($text, $offset) ?:
135	2	$this->getNumber($text, $offset) ?:
136	4	$this->getSymbol($text, $offset);
137
138	584	case '*':
139		return
140	4	$this->getConditionEnd($text, $offset) ?:
141	3	$this->getSymbol($text, $offset);
142
143	582	case '/':
144		return
145	8	$this->getConditionalStart($text, $offset) ?:
146	4	$this->getMultilineComment($text, $offset) ?:
147	7	$this->getSymbol($text, $offset);
148
149	580	case '0':
150		// Handle hex if needed
151	22	if (isset($text[$offset+1]) && $text[$offset+1] === 'x') {
152		return
153	4	$this->getHex($text, $offset) ?:
154	3	$this->getIdentifier($text, $offset);
155		}
156		// Handle non-hex leading zero.
157	575	case '1':
158	556	case '2':
159	556	case '3':
160	556	case '4':
161	555	case '5':
162	555	case '6':
163	555	case '7':
164	555	case '8':
165	555	case '9':
166		return
167	137	$this->getNumber($text, $offset) ?:
168	137	$this->getIdentifier($text, $offset);
169
170	555	case '"':
171	550	case "'":
172		return
173	57	$this->getString($text, $offset);
174
175	542	case '`':
176		return
177	8	$this->getQuotedIdentifier($text, $offset);
178
179	536	case 'B':
180	536	case 'b':
181		return
182	52	$this->getBin($text, $offset) ?:
183	52	$this->getIdentifier($text, $offset);
184
185	531	case 'X':
186	530	case 'x':
187		return
188	294	$this->getHex($text, $offset) ?:
189	294	$this->getIdentifier($text, $offset);
190
191	517	case '$':
192	515	case '_':
193	512	case 'A':
194	512	case 'a':
195	500	case 'C':
196	500	case 'c':
197	481	case 'D':
198	481	case 'd':
199	461	case 'E':
200	461	case 'e':
201	461	case 'F':
202	461	case 'f':
203	450	case 'G':
204	450	case 'g':
205	450	case 'H':
206	450	case 'h':
207	450	case 'I':
208	450	case 'i':
209	434	case 'J':
210	434	case 'j':
211	434	case 'K':
212	434	case 'k':
213	432	case 'L':
214	432	case 'l':
215	426	case 'M':
216	426	case 'm':
217	421	case 'N':
218	400	case 'n':
219	395	case 'O':
220	395	case 'o':
221	395	case 'P':
222	395	case 'p':
223	391	case 'Q':
224	391	case 'q':
225	391	case 'R':
226	391	case 'r':
227	390	case 'S':
228	390	case 's':
229	382	case 'T':
230	382	case 't':
231	350	case 'U':
232	349	case 'u':
233	320	case 'V':
234	320	case 'v':
235	314	case 'W':
236	314	case 'w':
237	314	case 'Y':
238	314	case 'y':
239	304	case 'Z':
240	304	case 'z':
241		return
242	489	$this->getIdentifier($text, $offset);
243
244	300	case '!':
245	299	case '%':
246	298	case '&':
247	297	case '(':
248	293	case ')':
249	218	case ',':
250	173	case ':':
251	172	case ';':
252	100	case '<':
253	98	case '=':
254	12	case '>':
255	11	case '@':
256	9	case '^':
257	9	case '\|':
258	7	case '~':
259		return
260	293	$this->getSpecialSymbol($text, $offset);
261
262	7	case '?':
263	6	case '[':
264	5	case '\\':
265	4	case ']':
266	3	case '{':
267	2	case '}':
268		default:
269	7	$ch = $text[$offset];
270	7	throw new LogicException("Lexer is confused by char '$ch' ord " . ord($ch));
271		}
272		}
273
274		/**
275		* @param string $text
276		* @param int $offset
277		* @return array
278		*/
279	8	private function getQuotedIdentifier($text, $offset)
280		{
281	8	if (preg_match('/`((?:[^`]\|``)*)`()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset)) {
282	7	$token = Token::fromIdentifier($pregMatch[1][0]);
283		return [
284	7	$token,
285	7	$pregMatch[2][1]
286		];
287		}
288	1	throw new RuntimeException("Unterminated identifier: $text");
289		}
290
291		/**
292		* @param string $text
293		* @param int $offset
294		* @return array
295		*/
296	293	private function getSpecialSymbol($text, $offset)
297		{
298		// TODO - should probably be a new token type 'variable' for @ and @@
		0 ignored issues – show Coding Style Best Practice introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report Comments for TODO tasks are often forgotten in the code; it might be better to use a dedicated issue tracker. Loading history...
299	293	preg_match('/\A(?:<=\|>=\|<>\|!=\|:=\|@@\|&&\|\\|\\|\|[=~!@%^&();:,<>\|])()/xms', substr($text, $offset, 2), $pregMatch, PREG_OFFSET_CAPTURE);
300		return [
301	293	new Token(Token::SYMBOL, $pregMatch[0][0]),
302	293	$offset + $pregMatch[1][1]
303		];
304		}
305
306		/**
307		* Get the start of a conditional comment.
308		*
309		* https://dev.mysql.com/doc/refman/5.7/en/comments.html
310		*
311		* Does not support optimiser hints. See examples below.
312		*
313		* @param string $text
314		* @param int $offset
315		* @return array\|null
316		*/
317	8	private function getConditionalStart($text, $offset)
318		{
319		// Example conditional comments which can't be displayed in the docblock because they clash:
320		// - /! MySQL-specific code / (execute the given code)
321		// - /!12345 MySQL-specific code / (execute the given code only if the version matches)
322		// Unsupported:
323		// - SELECT /+ BKA(t1) / FROM ... ;
324
325		if (// 10 comes from allowing for the /*! sequence, a MySQL version number, and a space
326	8	preg_match('_\A/\!([0-9])\s_ms', substr($text, $offset, 10)) &&
327	8	preg_match('_/\!([0-9])\s_ms', $text, $pregMatch, 0, $offset)
328		) {
329	4	$this->inConditional = true;
330		return [
331	4	new Token(Token::CONDITIONAL_START, $pregMatch[1]),
332	4	$offset + strlen($pregMatch[0])
333		];
334		}
335	4	return null;
336		}
337
338		/**
339		* Get the end of a conditional comment. See _getConditionStart() for details.
340		* @param string $text
341		* @param int $offset
342		* @return array\|null
343		*/
344	4	private function getConditionEnd($text, $offset)
345		{
346	4	if (substr($text, $offset, 2) === '*/') {
347	3	if (!$this->inConditional) {
348	1	throw new RuntimeException("Unexpected '*/'");
349		}
350	2	$this->inConditional = false;
351		return [
352	2	new Token(Token::CONDITIONAL_END),
353	2	$offset + 2
354		];
355		}
356	1	return null;
357		}
358
359		/**
360		* @param string $text
361		* @param int $offset
362		* @return array\|null
363		*/
364	4	private function getMultilineComment($text, $offset)
365		{
366	4	if (substr($text, $offset, 2) === '/*') {
367	3	$pos = strpos($text, '*/', $offset);
368	3	if ($pos !== false) {
369		return [
370	2	new Token(Token::COMMENT, substr($text, $offset, $pos - $offset + 2)),
371	2	$pos + 2
372		];
373		}
374	1	throw new RuntimeException("Unterminated '/*'");
375		}
376	1	return null;
377		}
378
379		/**
380		* @param string $text
381		* @param int $offset
382		* @return array\|null
383		*/
384	6	private function getComment($text, $offset)
385		{
386	6	if (preg_match('/\A(?:#\|--\s)/ms', substr($text, $offset, 3))) {
387	4	$pos = strpos($text, "\n", $offset);
388	4	if ($pos !== false) {
389		return [
390	3	new Token(Token::COMMENT, substr($text, $offset, $pos - $offset)),
391	3	$pos + 1
392		];
393		}
394		return [
395	1	new Token(Token::COMMENT, $text),
396	1	strlen($text)
397		];
398		}
399	2	return null;
400		}
401
402		/**
403		* @param string $text
404		* @param int $offset
405		* @return array
406		*/
407	57	private function getString($text, $offset)
408		{
409	57	$quote = $text[$offset];
410	57	if (preg_match(
411		'/' .
412	57	$quote .
413	57	'(' .
414	57	'(?:' .
415	57	'[^\\\\' . $quote . ']' . // not \ or "
416	57	'\|\\\\.' . // escaped quotearacter
417	57	'\|' . $quote . $quote . // ""
418	57	')*' .
419	57	')' .
420	57	$quote .
421	57	'/ms',
422	57	$text,
423	57	$pregMatch,
424	57	0,
425	57	$offset
426		)
427		) {
428	56	$token = Token::fromString($pregMatch[1], $quote);
429		return [
430	56	$token,
431	56	$offset + strlen($pregMatch[0])
432		];
433		}
434	1	throw new RuntimeException("Unterminated string $quote...$quote");
435		}
436
437		/**
438		* @param string $text
439		* @param int $offset
440		* @return array\|null
441		*/
442	144	private function getNumber($text, $offset)
443		{
444	144	if (preg_match('/\A[-+]?[.]?[0-9]/ms', substr($text, $offset, 3)) &&
445	144	preg_match('/[-+]?(?:[0-9]+(?:[.][0-9]*)?\|[.][0-9]+)(?:[eE][-+]?[0-9]+)?/ms', $text, $pregMatch, 0, $offset)
446		) {
447		return [
448	141	new Token(Token::NUMBER, $pregMatch[0]),
449	141	$offset + strlen($pregMatch[0])
450		];
451		}
452	3	return null;
453		}
454
455		/**
456		* Parse a hex string of the form "0x<hex digits>" or "x'<hex digits>'".
457		*
458		* https://dev.mysql.com/doc/refman/5.7/en/hexadecimal-literals.html
459		*
460		* - Only an even number of digits is valid.
461		* - Case insensitive for hex digits.
462		* - Case insensitive 'x' in quoted notation.
463		* - Case sensitive 'x' for leading zero notation.
464		*
465		* Valid examples:
466		* - x'BA5EBA11'
467		* - x'decea5ed'
468		* - X'5eed'
469		* - 0xb01dface
470		* - 0xBADC0DED
471		*
472		* Invalid examples
473		* - x'00f' (odd number of digits)
474		* - x'gg' (invalid hex character)
475		* - 0XFFFF (upper case 'x')
476		*
477		* @param string $text
478		* @param int $offset
479		* @return array\|null
480		*/
481	298	private function getHex($text, $offset)
482		{
483	298	$pregMatch = [];
484
485		$matchesLeadingZeroNotation = function ($text, $offset, &$pregMatch) {
486		return
487	298	preg_match('/\A0x([0-9a-fA-F]*)/ms', $text, $pregMatch, 0, $offset);
488	298	};
489
490	298	$matchesXQuotedNotation = function ($text, $offset, &$pregMatch) {
491		return
492	294	preg_match('/\Ax\'[0-9a-f\']/ims', substr($text, $offset, 3)) &&
493	294	preg_match('/x\'([0-9a-f]*)\'/ims', $text, $pregMatch, 0, $offset);
494	298	};
495
496	298	if ($matchesLeadingZeroNotation($text, $offset, $pregMatch) \|\|
497	298	$matchesXQuotedNotation($text, $offset, $pregMatch)) {
498	11	if (strlen($pregMatch[1]) % 2 != 0) {
499	1	throw new RuntimeException("Invalid hex literal");
500		}
501		return [
502	10	new Token(Token::HEX, $pregMatch[1]),
503	10	$offset + strlen($pregMatch[0])
504		];
505		}
506	288	return null;
507		}
508
509		/**
510		* @param string $text
511		* @param int $offset
512		* @return array\|null
513		*/
514	52	private function getBin($text, $offset)
515		{
516	52	if (preg_match('/\Ab\'[01\']/ms', substr($text, $offset, 3)) &&
517	52	preg_match('/b\'([01]*)\'/ms', $text, $pregMatch, 0, $offset)
518		) {
519		return [
520	5	new Token(Token::BIN, $pregMatch[1]),
521	5	$offset + strlen($pregMatch[0])
522		];
523		}
524	48	return null;
525		}
526
527		/**
528		* @param string $text
529		* @param int $offset
530		* @return array
531		*/
532	502	private function getIdentifier($text, $offset)
533		{
534	502	preg_match('/[a-zA-Z0-9$_]+()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset);
535		return [
536	502	new Token(Token::IDENTIFIER, $pregMatch[0][0]),
537	502	$pregMatch[1][1]
538		];
539		}
540
541		/**
542		* @param string $text
543		* @param int $offset
544		* @return array
545		*/
546	5	private function getSymbol($text, $offset)
547		{
548	5	if (preg_match('/\A(?:[-+*.\/])/xms', substr($text, $offset, 2), $pregMatch)) {
549		return [
550	5	new Token(Token::SYMBOL, $pregMatch[0]),
551	5	$offset + strlen($pregMatch[0])
552		];
553		}
554
555		return [];
556		}
557
558		/**
559		* @return Token\|mixed
560		*/
561	594	public function nextToken()
562		{
563	594	while (true) {
564	594	$token = $this->nextTokenRaw();
565	582	if (!isset(self::$skipTokenTypes[$token->type])) {
566	582	return $token;
567		}
568		}
569
570		return null;
571		}
572
573		/**
574		* @return object
575		*/
576	452	public function getMark()
577		{
578		return (object)[
579	452	'offset' => $this->offset,
580	452	'inConditional' => $this->inConditional,
581		];
582		}
583
584		/**
585		* @param mixed $mark
586		*/
587	430	public function rewind($mark)
588		{
589	430	$this->offset = $mark->offset;
590	430	$this->inConditional = $mark->inConditional;
591	430	}
592
593		/**
594		* This function will consume the requested content from the stream without trying to parse and tokenise it.
595		* It is used by {@see peek()}.
596		*
597		* @param mixed $spec
598		* @return bool
599		*/
600	472	public function consume($spec)
601		{
602		// inline getMark()
603	472	$markOffset = $this->offset;
604	472	$markInConditional = $this->inConditional;
605
606	472	if (is_string($spec)) {
607	363	foreach (explode(' ', $spec) as $text) {
608	363	$token = $this->nextToken();
609		// inline $token->eq(...)
		0 ignored issues – show Unused Code Comprehensibility introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report `56%` of this comment could be valid code. Did you maybe forget this after debugging? Sometimes obsolete code just ends up commented out instead of removed. In this case it is better to remove the code once you have checked you do not need it. The code might also have been commented out for debugging purposes. In this case it is vital that someone uncomments it again or your project may behave in very unexpected ways in production. This check looks for comments that seem to be mostly valid code and reports them. Loading history...
610	363	if (strcasecmp($token->text, $text) !== 0 \|\|
611	363	$token->type !== Token::IDENTIFIER
612		) {
613		// inline rewind()
614	357	$this->offset = $markOffset;
615	357	$this->inConditional = $markInConditional;
616	363	return false;
617		}
618		}
619		} else {
620	429	foreach ($spec as $match) {
621	429	list($type, $text) = $match;
622	429	$token = $this->nextToken();
623		// inline $token->eq(...)
		0 ignored issues – show Unused Code Comprehensibility introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report `56%` of this comment could be valid code. Did you maybe forget this after debugging? Sometimes obsolete code just ends up commented out instead of removed. In this case it is better to remove the code once you have checked you do not need it. The code might also have been commented out for debugging purposes. In this case it is vital that someone uncomments it again or your project may behave in very unexpected ways in production. This check looks for comments that seem to be mostly valid code and reports them. Loading history...
624	429	if (strcasecmp($token->text, $text) !== 0 \|\|
625	429	$token->type !== $type
626		) {
627		// inline rewind()
628	325	$this->offset = $markOffset;
629	325	$this->inConditional = $markInConditional;
630	429	return false;
631		}
632		}
633		}
634
635	308	return true;
636		}
637
638		/**
639		* @param mixed $spec
640		* @return bool
641		*/
642	7	public function peek($spec)
643		{
644		// inline getMark()
645	7	$markOffset = $this->offset;
646	7	$markInConditional = $this->inConditional;
647
648	7	$result = $this->consume($spec);
649
650		// inline rewind()
651	7	$this->offset = $markOffset;
652	7	$this->inConditional = $markInConditional;
653
654	7	return $result;
655		}
656
657		/**
658		* @param string $type
659		* @param string $text
660		* @return string
661		*/
662	201	public function expect($type, $text = null)
663		{
664	201	$token = $this->nextToken();
665	201	if (!$token->eq($type, $text)) {
666	3	throw new RuntimeException("Expected '$text'");
667		}
668	198	return $token->text;
669		}
670
671		/**
672		* @return string
673		*/
674	368	public function expectName()
675		{
676	368	$token = $this->nextToken();
677	368	if ($token->type !== Token::IDENTIFIER) {
678	2	throw new RuntimeException("Expected identifier");
679		}
680	367	return $token->text;
681		}
682
683		/**
684		* @return string
685		*/
686	104	public function expectOpenParen()
687		{
688	104	return $this->expect(Token::SYMBOL, '(');
689		}
690
691		/**
692		* @return string
693		*/
694	49	public function expectCloseParen()
695		{
696	49	return $this->expect(Token::SYMBOL, ')');
697		}
698
699		/**
700		* @return int
701		*/
702	85	public function expectNumber()
703		{
704	85	$token = $this->nextToken();
705	85	if ($token->type !== Token::NUMBER) {
706	1	throw new RuntimeException("Expected number");
707		}
708	84	return 1 * $token->text;
709		}
710
711		/**
712		* @return string
713		*/
714	21	public function expectString()
715		{
716	21	$token = $this->nextToken();
717	21	if ($token->type !== Token::STRING) {
718	1	throw new RuntimeException("Expected string");
719		}
720	20	return $token->text;
721		}
722
723		/**
724		* @return string
725		*/
726	21	public function expectStringExtended()
727		{
728	21	$token = $this->nextToken();
729	21	switch ($token->type) {
730	21	case Token::STRING:
731	16	return $token->text;
732	5	case Token::HEX:
733	3	return $token->asString();
734	2	case Token::BIN:
735	1	return $token->asString();
736		default:
737	1	throw new RuntimeException("Expected string");
738		}
739		}
740
741		/**
742		* Provides context for error messages.
743		*
744		* For example, given this invalid table definition ...
745		*
746		* CREATE TABLE `foo` (
747		* `a` bar DEFAULT NULL
748		* );
749		*
750		* ... this function will produce something like this:
751		*
752		* schema/morphism test/foo.sql, line 2: unknown datatype 'bar'
753		* 1: CREATE TABLE `foo` (
754		* 2: `a` bar<<HERE>> DEFAULT NULL
755		*
756		* @param string $message
757		* @return string
758		*/
759	1	public function contextualise($message)
760		{
761	1	$preContextLines = 4;
762	1	$postContextLines = 0;
763
764		// get position of eol strictly before offset
765	1	$prevEolPos = strrpos($this->text, "\n", $this->offset - strlen($this->text) - 1);
766	1	$prevEolPos = ($prevEolPos === false) ? -1 : $prevEolPos;
767
768		// get position of eol on or after offset
769	1	$nextEolPos = strpos($this->text, "\n", $this->offset);
770	1	$nextEolPos = ($nextEolPos === false) ? strlen($this->text) : $nextEolPos;
771
772		// count number of newlines up to but not including offset
773	1	$lineNo = substr_count($this->text, "\n", 0, $this->offset);
774	1	$lines = explode("\n", $this->text);
775
776	1	$contextLines = array_slice(
777	1	$lines,
778	1	max(0, $lineNo - $preContextLines),
779	1	min($lineNo, $preContextLines),
780	1	true // preserve keys
781		);
782		$contextLines += [
783		$lineNo =>
784	1	substr($this->text, $prevEolPos + 1, $this->offset - ($prevEolPos + 1)) .
785	1	"<<HERE>>".
786	1	substr($this->text, $this->offset, $nextEolPos - $this->offset)
787		];
788	1	$contextLines += array_slice(
789	1	$lines,
790	1	$lineNo + 1,
791	1	$postContextLines,
792	1	true // preserve keys
793		);
794
795	1	$context = '';
796	1	$width = strlen($lineNo + 1 + $postContextLines);
797	1	foreach ($contextLines as $i => $line) {
798	1	$context .= sprintf("\n%{$width}d: %s", $i + 1, $line);
799		}
800
801	1	return sprintf("%s, line %d: %s%s", $this->path, $lineNo + 1, $message, $context);
802		}
803		}
804

graze / morphism

GitHub Access Token became invalid

Pull Request — master (#60)

TokenStream::getSymbol() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like