TokenStream::getNumber() - Code Metrics - Inspection of "Implement foreign keys" - graze/morphism - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#61)

by Brendan

created 2019-07-30 04:50 UTC

TokenStream::getNumber() A

↳ Parent: Project

Complexity

Conditions	3
Paths	2

Size

Total Lines	11
Code Lines	6

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	5
CRAP Score	3.0416

Importance

Changes

Metric	Value
cc	3
eloc	6
nc	2
nop	2
dl	0
loc	11
ccs	5
cts	6
cp	0.8333
crap	3.0416
rs	10
c	0
b	0
f	0

<?php
namespace Graze\Morphism\Parse;

use LogicException;
use RuntimeException;

class TokenStream
{
    /** @var string */
    private $path;
    /** @var string */
    private $text;
    /** @var int */
    private $len;
    /** @var int */
    private $offset = 0;
    /** @var bool */
    private $inConditional = false;
    /** @var array */
    private $memo = [];

    /** @var array */
    private static $skipTokenTypes = [
        Token::CONDITIONAL_START => true,
        Token::CONDITIONAL_END   => true,
        Token::COMMENT           => true,
        Token::WHITESPACE        => true,
    ];

    private function __construct()
    {
    }

    // TODO - this is a hack that needs to be refactored
    // perhaps supply a LineStream interface that is satisfied
    // by a FileLineStream or ConnectionLineStream for example?
    /**
     * @param string $text
     * @param string $label
     * @return TokenStream
     */
    public static function newFromText($text, $label)
    {
        $stream = new self;
        $stream->path = $label;
        $stream->text = $text;
        $stream->len = strlen($text);
        return $stream;
    }

    /**
     * @param string $path
     * @return TokenStream
     */
    public static function newFromFile($path)
    {
        $text = @file_get_contents($path);
        if ($text === false) {
            throw new RuntimeException("$path: could not open file");
        }
        $stream = new self;
        $stream->path = $path;
        $stream->text = $text;
        $stream->len = strlen($text);
        return $stream;
    }

    /**
     * @return Token|mixed
     */
    private function nextTokenRaw()
    {
        $startOffset = $this->offset;
        if (isset($this->memo[$startOffset])) {
            $entry = $this->memo[$startOffset];
            $this->offset = $entry[0];
            $this->inConditional = $entry[1];
            return $entry[2];
        }

        if ($this->offset >= $this->len) {
            $token = new Token(Token::EOF);
            $this->offset = $this->len;
        } else {
            list($token, $offset) = $this->getNextTokenRaw($this->text, $this->offset);
            $this->offset = $offset;
        }

        $this->memo[$startOffset] = [
            $this->offset,
            $this->inConditional,
            $token,
        ];

        return $token;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getNextTokenRaw($text, $offset)
    {
        // currently unsupported:
        //
        //      charset prefixed strings, e.g. _utf8'wingwang'
        //      temporal literals, e.g. DATE'2014-07-08'
        //      the null literal, i.e. \N

        switch ($text[$offset]) {
            case " ":
            case "\n":
            case "\r":
            case "\t":
                $n = strspn($text, " \n\r\t", $offset);
                return [
                    new Token(Token::WHITESPACE, substr($text, $offset, $n)),
                    $offset + $n
                ];

            case '#':
                return
                    $this->getComment($text, $offset);

            case '.':
            case '+':
                return
                    $this->getNumber($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '-':
                return
                    $this->getComment($text, $offset) ?:
                    $this->getNumber($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '*':
                return
                    $this->getConditionEnd($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '/':
                return
                    $this->getConditionalStart($text, $offset) ?:
                    $this->getMultilineComment($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '0':
                // Handle hex if needed
                if (isset($text[$offset+1]) && $text[$offset+1] === 'x') {
                    return
                        $this->getHex($text, $offset) ?:
                        $this->getIdentifier($text, $offset);
                }
                // Handle non-hex leading zero.
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                return
                    $this->getNumber($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case '"':
            case "'":
                return
                    $this->getString($text, $offset);

            case '`':
                return
                    $this->getQuotedIdentifier($text, $offset);

            case 'B':
            case 'b':
                return
                    $this->getBin($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case 'X':
            case 'x':
                return
                    $this->getHex($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case '$':
            case '_':
            case 'A':
            case 'a':
            case 'C':
            case 'c':
            case 'D':
            case 'd':
            case 'E':
            case 'e':
            case 'F':
            case 'f':
            case 'G':
            case 'g':
            case 'H':
            case 'h':
            case 'I':
            case 'i':
            case 'J':
            case 'j':
            case 'K':
            case 'k':
            case 'L':
            case 'l':
            case 'M':
            case 'm':
            case 'N':
            case 'n':
            case 'O':
            case 'o':
            case 'P':
            case 'p':
            case 'Q':
            case 'q':
            case 'R':
            case 'r':
            case 'S':
            case 's':
            case 'T':
            case 't':
            case 'U':
            case 'u':
            case 'V':
            case 'v':
            case 'W':
            case 'w':
            case 'Y':
            case 'y':
            case 'Z':
            case 'z':
                return
                    $this->getIdentifier($text, $offset);

            case '!':
            case '%':
            case '&':
            case '(':
            case ')':
            case ',':
            case ':':
            case ';':
            case '<':
            case '=':
            case '>':
            case '@':
            case '^':
            case '|':
            case '~':
                return
                    $this->getSpecialSymbol($text, $offset);

            case '?':
            case '[':
            case '\\':
            case ']':
            case '{':
            case '}':
            default:
                $ch = $text[$offset];
                throw new LogicException("Lexer is confused by char '$ch' ord " . ord($ch));
        }
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getQuotedIdentifier($text, $offset)
    {
        if (preg_match('/`((?:[^`]|``)*)`()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset)) {
            $token = Token::fromIdentifier($pregMatch[1][0]);
            return [
                $token,
                $pregMatch[2][1]
            ];
        }
        throw new RuntimeException("Unterminated identifier: $text");
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getSpecialSymbol($text, $offset)
    {
        // TODO - should probably be a new token type 'variable' for @ and @@
        preg_match('/\A(?:<=|>=|<>|!=|:=|@@|&&|\|\||[=~!@%^&();:,<>|])()/xms', substr($text, $offset, 2), $pregMatch, PREG_OFFSET_CAPTURE);
        return [
            new Token(Token::SYMBOL, $pregMatch[0][0]),
            $offset + $pregMatch[1][1]
        ];
    }

    /**
     * Get the start of a conditional comment.
     *
     * https://dev.mysql.com/doc/refman/5.7/en/comments.html
     *
     * Does not support optimiser hints. See examples below.
     *
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getConditionalStart($text, $offset)
    {
        // Example conditional comments which can't be displayed in the docblock because they clash:
        // - /*! MySQL-specific code */ (execute the given code)
        // - /*!12345 MySQL-specific code */ (execute the given code only if the version matches)
        // Unsupported:
        // - SELECT /*+ BKA(t1) */ FROM ... ;

        if (// 10 comes from allowing for the /*! sequence, a MySQL version number, and a space
            preg_match('_\A/\*!([0-9]*)\s_ms', substr($text, $offset, 10)) &&
            preg_match('_/\*!([0-9]*)\s_ms', $text, $pregMatch, 0, $offset)
        ) {
            $this->inConditional = true;
            return [
                new Token(Token::CONDITIONAL_START, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * Get the end of a conditional comment. See _getConditionStart() for details.
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getConditionEnd($text, $offset)
    {
        if (substr($text, $offset, 2) === '*/') {
            if (!$this->inConditional) {
                throw new RuntimeException("Unexpected '*/'");
            }
            $this->inConditional = false;
            return [
                new Token(Token::CONDITIONAL_END),
                $offset + 2
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getMultilineComment($text, $offset)
    {
        if (substr($text, $offset, 2) === '/*') {
            $pos = strpos($text, '*/', $offset);
            if ($pos !== false) {
                return [
                    new Token(Token::COMMENT, substr($text, $offset, $pos - $offset + 2)),
                    $pos + 2
                ];
            }
            throw new RuntimeException("Unterminated '/*'");
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getComment($text, $offset)
    {
        if (preg_match('/\A(?:#|--\s)/ms', substr($text, $offset, 3))) {
            $pos = strpos($text, "\n", $offset);
            if ($pos !== false) {
                return [
                    new Token(Token::COMMENT, substr($text, $offset, $pos - $offset)),
                    $pos + 1
                ];
            }
            return [
                new Token(Token::COMMENT, $text),
                strlen($text)
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getString($text, $offset)
    {
        $quote = $text[$offset];
        if (preg_match(
            '/' .
            $quote .
            '(' .
                '(?:' .
                    '[^\\\\' . $quote . ']' .   // not \ or "
                    '|\\\\.' .                  // escaped quotearacter
                    '|' .  $quote . $quote .    // ""
                ')*' .
            ')' .
            $quote .
            '/ms',
            $text,
            $pregMatch,
            0,
            $offset
        )
        ) {
            $token = Token::fromString($pregMatch[1], $quote);
            return [
                $token,
                $offset + strlen($pregMatch[0])
            ];
        }
        throw new RuntimeException("Unterminated string $quote...$quote");
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getNumber($text, $offset)
    {
        if (preg_match('/\A[-+]?[.]?[0-9]/ms', substr($text, $offset, 3)) &&
            preg_match('/[-+]?(?:[0-9]+(?:[.][0-9]*)?|[.][0-9]+)(?:[eE][-+]?[0-9]+)?/ms', $text, $pregMatch, 0, $offset)
        ) {
            return [
                new Token(Token::NUMBER, $pregMatch[0]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * Parse a hex string of the form "0x<hex digits>" or "x'<hex digits>'".
     *
     * https://dev.mysql.com/doc/refman/5.7/en/hexadecimal-literals.html
     *
     * - Only an even number of digits is valid.
     * - Case insensitive for hex digits.
     * - Case insensitive 'x' in quoted notation.
     * - Case sensitive 'x' for leading zero notation.
     *
     * Valid examples:
     * - x'BA5EBA11'
     * - x'decea5ed'
     * - X'5eed'
     * - 0xb01dface
     * - 0xBADC0DED
     *
     * Invalid examples
     * - x'00f' (odd number of digits)
     * - x'gg'  (invalid hex character)
     * - 0XFFFF (upper case 'x')
     *
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getHex($text, $offset)
    {
        $pregMatch = [];

        $matchesLeadingZeroNotation = function ($text, $offset, &$pregMatch) {
            return
                preg_match('/\A0x([0-9a-fA-F]*)/ms', $text, $pregMatch, 0, $offset);
        };

        $matchesXQuotedNotation = function ($text, $offset, &$pregMatch) {
            return
                preg_match('/\Ax\'[0-9a-f\']/ims', substr($text, $offset, 3)) &&
                preg_match('/x\'([0-9a-f]*)\'/ims', $text, $pregMatch, 0, $offset);
        };

        if ($matchesLeadingZeroNotation($text, $offset, $pregMatch) ||
            $matchesXQuotedNotation($text, $offset, $pregMatch)) {
            if (strlen($pregMatch[1]) % 2 != 0) {
                throw new RuntimeException("Invalid hex literal");
            }
            return [
                new Token(Token::HEX, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getBin($text, $offset)
    {
        if (preg_match('/\Ab\'[01\']/ms', substr($text, $offset, 3)) &&
            preg_match('/b\'([01]*)\'/ms', $text, $pregMatch, 0, $offset)
        ) {
            return [
                new Token(Token::BIN, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getIdentifier($text, $offset)
    {
        preg_match('/[a-zA-Z0-9$_]+()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset);
        return [
            new Token(Token::IDENTIFIER, $pregMatch[0][0]),
            $pregMatch[1][1]
        ];
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getSymbol($text, $offset)
    {
        if (preg_match('/\A(?:[-+*.\/])/xms', substr($text, $offset, 2), $pregMatch)) {
            return [
                new Token(Token::SYMBOL, $pregMatch[0]),
                $offset + strlen($pregMatch[0])
            ];
        }

        return [];
    }

    /**
     * @return Token|mixed
     */
    public function nextToken()
    {
        while (true) {
            $token = $this->nextTokenRaw();
            if (!isset(self::$skipTokenTypes[$token->type])) {
                return $token;
            }
        }

        return null;
    }

    /**
     * @return object
     */
    public function getMark()
    {
        return (object)[
            'offset'        => $this->offset,
            'inConditional' => $this->inConditional,
        ];
    }

    /**
     * @param mixed $mark
     */
    public function rewind($mark)
    {
        $this->offset        = $mark->offset;
        $this->inConditional = $mark->inConditional;
    }

    /**
     * This function will consume the requested content from the stream without trying to parse and tokenise it.
     * It is used by {@see peek()}.
     *
     * @param mixed $spec
     * @return bool
     */
    public function consume($spec)
    {
        // inline getMark()
        $markOffset        = $this->offset;
        $markInConditional = $this->inConditional;

        if (is_string($spec)) {
            foreach (explode(' ', $spec) as $text) {
                $token = $this->nextToken();
                // inline $token->eq(...)
                if (strcasecmp($token->text, $text) !== 0 ||
                    $token->type !== Token::IDENTIFIER
                ) {
                    // inline rewind()
                    $this->offset        = $markOffset;
                    $this->inConditional = $markInConditional;
                    return false;
                }
            }
        } else {
            foreach ($spec as $match) {
                list($type, $text) = $match;
                $token = $this->nextToken();
                // inline $token->eq(...)
                if (strcasecmp($token->text, $text) !== 0 ||
                    $token->type !== $type
                ) {
                    // inline rewind()
                    $this->offset        = $markOffset;
                    $this->inConditional = $markInConditional;
                    return false;
                }
            }
        }

        return true;
    }

    /**
     * @param mixed $spec
     * @return bool
     */
    public function peek($spec)
    {
        // inline getMark()
        $markOffset        = $this->offset;
        $markInConditional = $this->inConditional;

        $result = $this->consume($spec);

        // inline rewind()
        $this->offset        = $markOffset;
        $this->inConditional = $markInConditional;

        return $result;
    }

    /**
     * @param string $type
     * @param string $text
     * @return string
     */
    public function expect($type, $text = null)
    {
        $token = $this->nextToken();
        if (!$token->eq($type, $text)) {
            throw new RuntimeException("Expected '$text'");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectName()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::IDENTIFIER) {
            throw new RuntimeException("Expected identifier");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectOpenParen()
    {
        return $this->expect(Token::SYMBOL, '(');
    }

    /**
     * @return string
     */
    public function expectCloseParen()
    {
        return $this->expect(Token::SYMBOL, ')');
    }

    /**
     * @return int
     */
    public function expectNumber()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::NUMBER) {
            throw new RuntimeException("Expected number");
        }
        return 1 * $token->text;
    }

    /**
     * @return string
     */
    public function expectString()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::STRING) {
            throw new RuntimeException("Expected string");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectStringExtended()
    {
        $token = $this->nextToken();
        switch ($token->type) {
            case Token::STRING:
                return $token->text;
            case Token::HEX:
                return $token->asString();
            case Token::BIN:
                return $token->asString();
            default:
                throw new RuntimeException("Expected string");
        }
    }

    /**
     * Provides context for error messages.
     *
     * For example, given this invalid table definition ...
     *
     *     CREATE TABLE `foo` (
     *         `a` bar DEFAULT NULL
     *     );
     *
     * ... this function will produce something like this:
     *
     *     schema/morphism test/foo.sql, line 2: unknown datatype 'bar'
     *     1: CREATE TABLE `foo` (
     *     2:   `a` bar<<HERE>> DEFAULT NULL
     *
     * @param string $message
     * @return string
     */
    public function contextualise($message)
    {
        $preContextLines = 4;
        $postContextLines = 0;

        // get position of eol strictly before offset
        $prevEolPos = strrpos($this->text, "\n", $this->offset - strlen($this->text) - 1);
        $prevEolPos = ($prevEolPos === false) ? -1 : $prevEolPos;

        // get position of eol on or after offset
        $nextEolPos = strpos($this->text, "\n", $this->offset);
        $nextEolPos = ($nextEolPos === false) ? strlen($this->text) : $nextEolPos;

        // count number of newlines up to but not including offset
        $lineNo = substr_count($this->text, "\n", 0, $this->offset);
        $lines = explode("\n", $this->text);

        $contextLines = array_slice(
            $lines,
            max(0, $lineNo - $preContextLines),
            min($lineNo, $preContextLines),
            true // preserve keys
        );
        $contextLines += [
            $lineNo =>
                substr($this->text, $prevEolPos + 1, $this->offset - ($prevEolPos + 1)) .
                "<<HERE>>".
                substr($this->text, $this->offset, $nextEolPos - $this->offset)
        ];
        $contextLines += array_slice(
            $lines,
            $lineNo + 1,
            $postContextLines,
            true // preserve keys
        );

        $context = '';
        $width = strlen($lineNo + 1 + $postContextLines);
        foreach ($contextLines as $i => $line) {
            $context .= sprintf("\n%{$width}d: %s", $i + 1, $line);
        }

        return sprintf("%s, line %d: %s%s", $this->path, $lineNo + 1, $message, $context);
    }
}


1		<?php
2		namespace Graze\Morphism\Parse;
3
4		use LogicException;
5		use RuntimeException;
6
7		class TokenStream
8		{
9		/** @var string */
10		private $path;
11		/** @var string */
12		private $text;
13		/** @var int */
14		private $len;
15		/** @var int */
16		private $offset = 0;
17		/** @var bool */
18		private $inConditional = false;
19		/** @var array */
20		private $memo = [];
21
22		/** @var array */
23		private static $skipTokenTypes = [
24		Token::CONDITIONAL_START => true,
25		Token::CONDITIONAL_END => true,
26		Token::COMMENT => true,
27		Token::WHITESPACE => true,
28		];
29
30	314	private function __construct()
31		{
32	314	}
33
34		// TODO - this is a hack that needs to be refactored
35		// perhaps supply a LineStream interface that is satisfied
36		// by a FileLineStream or ConnectionLineStream for example?
37		/**
38		* @param string $text
39		* @param string $label
40		* @return TokenStream
41		*/
42		public static function newFromText($text, $label)
43		{
44		$stream = new self;
45		$stream->path = $label;
46		$stream->text = $text;
47		$stream->len = strlen($text);
48		return $stream;
49		}
50
51		/**
52		* @param string $path
53		* @return TokenStream
54		*/
55	314	public static function newFromFile($path)
56		{
57	314	$text = @file_get_contents($path);
58	314	if ($text === false) {
59		throw new RuntimeException("$path: could not open file");
60		}
61	314	$stream = new self;
62	314	$stream->path = $path;
63	314	$stream->text = $text;
64	314	$stream->len = strlen($text);
65	314	return $stream;
66		}
67
68		/**
69		* @return Token\|mixed
70		*/
71	314	private function nextTokenRaw()
72		{
73	314	$startOffset = $this->offset;
74	314	if (isset($this->memo[$startOffset])) {
75	303	$entry = $this->memo[$startOffset];
76	303	$this->offset = $entry[0];
77	303	$this->inConditional = $entry[1];
78	303	return $entry[2];
79		}
80
81	314	if ($this->offset >= $this->len) {
82	207	$token = new Token(Token::EOF);
83	207	$this->offset = $this->len;
84		} else {
85	314	list($token, $offset) = $this->getNextTokenRaw($this->text, $this->offset);
86	314	$this->offset = $offset;
87		}
88
89	314	$this->memo[$startOffset] = [
90	314	$this->offset,
91	314	$this->inConditional,
92	314	$token,
93		];
94
95	314	return $token;
96		}
97
98		/**
99		* @param string $text
100		* @param int $offset
101		* @return array\|null
102		*/
103	314	private function getNextTokenRaw($text, $offset)
104		{
105		// currently unsupported:
106		//
107		// charset prefixed strings, e.g. _utf8'wingwang'
108		// temporal literals, e.g. DATE'2014-07-08'
109		// the null literal, i.e. \N
110
111	314	switch ($text[$offset]) {
112	314	case " ":
113	314	case "\n":
114	314	case "\r":
115	314	case "\t":
116	311	$n = strspn($text, " \n\r\t", $offset);
117		return [
118	311	new Token(Token::WHITESPACE, substr($text, $offset, $n)),
119	311	$offset + $n
120		];
121
122	314	case '#':
123		return
124		$this->getComment($text, $offset);
125
126	314	case '.':
127	314	case '+':
128		return
129		$this->getNumber($text, $offset) ?:
130		$this->getSymbol($text, $offset);
131
132	314	case '-':
133		return
134		$this->getComment($text, $offset) ?:
135		$this->getNumber($text, $offset) ?:
136		$this->getSymbol($text, $offset);
137
138	314	case '*':
139		return
140		$this->getConditionEnd($text, $offset) ?:
141		$this->getSymbol($text, $offset);
142
143	314	case '/':
144		return
145		$this->getConditionalStart($text, $offset) ?:
146		$this->getMultilineComment($text, $offset) ?:
147		$this->getSymbol($text, $offset);
148
149	314	case '0':
150		// Handle hex if needed
151	6	if (isset($text[$offset+1]) && $text[$offset+1] === 'x') {
152		return
153		$this->getHex($text, $offset) ?:
154		$this->getIdentifier($text, $offset);
155		}
156		// Handle non-hex leading zero.
157	314	case '1':
158	314	case '2':
159	314	case '3':
160	314	case '4':
161	314	case '5':
162	314	case '6':
163	314	case '7':
164	314	case '8':
165	314	case '9':
166		return
167	70	$this->getNumber($text, $offset) ?:
168	70	$this->getIdentifier($text, $offset);
169
170	314	case '"':
171	314	case "'":
172		return
173	25	$this->getString($text, $offset);
174
175	314	case '`':
176		return
177	5	$this->getQuotedIdentifier($text, $offset);
178
179	314	case 'B':
180	314	case 'b':
181		return
182	38	$this->getBin($text, $offset) ?:
183	38	$this->getIdentifier($text, $offset);
184
185	314	case 'X':
186	314	case 'x':
187		return
188	279	$this->getHex($text, $offset) ?:
189	279	$this->getIdentifier($text, $offset);
190
191	307	case '$':
192	307	case '_':
193	307	case 'A':
194	307	case 'a':
195	307	case 'C':
196	307	case 'c':
197	303	case 'D':
198	303	case 'd':
199	285	case 'E':
200	285	case 'e':
201	285	case 'F':
202	285	case 'f':
203	279	case 'G':
204	279	case 'g':
205	279	case 'H':
206	279	case 'h':
207	279	case 'I':
208	279	case 'i':
209	263	case 'J':
210	263	case 'j':
211	263	case 'K':
212	263	case 'k':
213	261	case 'L':
214	261	case 'l':
215	257	case 'M':
216	257	case 'm':
217	252	case 'N':
218	231	case 'n':
219	226	case 'O':
220	226	case 'o':
221	226	case 'P':
222	226	case 'p':
223	223	case 'Q':
224	223	case 'q':
225	223	case 'R':
226	223	case 'r':
227	222	case 'S':
228	222	case 's':
229	217	case 'T':
230	217	case 't':
231	192	case 'U':
232	192	case 'u':
233	167	case 'V':
234	167	case 'v':
235	161	case 'W':
236	161	case 'w':
237	161	case 'Y':
238	161	case 'y':
239	151	case 'Z':
240	151	case 'z':
241		return
242	305	$this->getIdentifier($text, $offset);
243
244	147	case '!':
245	147	case '%':
246	147	case '&':
247	147	case '(':
248	144	case ')':
249	109	case ',':
250	73	case ':':
251	73	case ';':
252	6	case '<':
253	6	case '=':
254	1	case '>':
255	1	case '@':
256	1	case '^':
257	1	case '\|':
258		case '~':
259		return
260	147	$this->getSpecialSymbol($text, $offset);
261
262		case '?':
263		case '[':
264		case '\\':
265		case ']':
266		case '{':
267		case '}':
268		default:
269		$ch = $text[$offset];
270		throw new LogicException("Lexer is confused by char '$ch' ord " . ord($ch));
271		}
272		}
273
274		/**
275		* @param string $text
276		* @param int $offset
277		* @return array
278		*/
279	5	private function getQuotedIdentifier($text, $offset)
280		{
281	5	if (preg_match('/`((?:[^`]\|``)*)`()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset)) {
282	5	$token = Token::fromIdentifier($pregMatch[1][0]);
283		return [
284	5	$token,
285	5	$pregMatch[2][1]
286		];
287		}
288		throw new RuntimeException("Unterminated identifier: $text");
289		}
290
291		/**
292		* @param string $text
293		* @param int $offset
294		* @return array
295		*/
296	147	private function getSpecialSymbol($text, $offset)
297		{
298		// TODO - should probably be a new token type 'variable' for @ and @@
299	147	preg_match('/\A(?:<=\|>=\|<>\|!=\|:=\|@@\|&&\|\\|\\|\|[=~!@%^&();:,<>\|])()/xms', substr($text, $offset, 2), $pregMatch, PREG_OFFSET_CAPTURE);
300		return [
301	147	new Token(Token::SYMBOL, $pregMatch[0][0]),
302	147	$offset + $pregMatch[1][1]
303		];
304		}
305
306		/**
307		* Get the start of a conditional comment.
308		*
309		* https://dev.mysql.com/doc/refman/5.7/en/comments.html
310		*
311		* Does not support optimiser hints. See examples below.
312		*
313		* @param string $text
314		* @param int $offset
315		* @return array\|null
316		*/
317		private function getConditionalStart($text, $offset)
318		{
319		// Example conditional comments which can't be displayed in the docblock because they clash:
320		// - /! MySQL-specific code / (execute the given code)
321		// - /!12345 MySQL-specific code / (execute the given code only if the version matches)
322		// Unsupported:
323		// - SELECT /+ BKA(t1) / FROM ... ;
324
325		if (// 10 comes from allowing for the /*! sequence, a MySQL version number, and a space
326		preg_match('_\A/\!([0-9])\s_ms', substr($text, $offset, 10)) &&
327		preg_match('_/\!([0-9])\s_ms', $text, $pregMatch, 0, $offset)
328		) {
329		$this->inConditional = true;
330		return [
331		new Token(Token::CONDITIONAL_START, $pregMatch[1]),
332		$offset + strlen($pregMatch[0])
333		];
334		}
335		return null;
336		}
337
338		/**
339		* Get the end of a conditional comment. See _getConditionStart() for details.
340		* @param string $text
341		* @param int $offset
342		* @return array\|null
343		*/
344		private function getConditionEnd($text, $offset)
345		{
346		if (substr($text, $offset, 2) === '*/') {
347		if (!$this->inConditional) {
348		throw new RuntimeException("Unexpected '*/'");
349		}
350		$this->inConditional = false;
351		return [
352		new Token(Token::CONDITIONAL_END),
353		$offset + 2
354		];
355		}
356		return null;
357		}
358
359		/**
360		* @param string $text
361		* @param int $offset
362		* @return array\|null
363		*/
364		private function getMultilineComment($text, $offset)
365		{
366		if (substr($text, $offset, 2) === '/*') {
367		$pos = strpos($text, '*/', $offset);
368		if ($pos !== false) {
369		return [
370		new Token(Token::COMMENT, substr($text, $offset, $pos - $offset + 2)),
371		$pos + 2
372		];
373		}
374		throw new RuntimeException("Unterminated '/*'");
375		}
376		return null;
377		}
378
379		/**
380		* @param string $text
381		* @param int $offset
382		* @return array\|null
383		*/
384		private function getComment($text, $offset)
385		{
386		if (preg_match('/\A(?:#\|--\s)/ms', substr($text, $offset, 3))) {
387		$pos = strpos($text, "\n", $offset);
388		if ($pos !== false) {
389		return [
390		new Token(Token::COMMENT, substr($text, $offset, $pos - $offset)),
391		$pos + 1
392		];
393		}
394		return [
395		new Token(Token::COMMENT, $text),
396		strlen($text)
397		];
398		}
399		return null;
400		}
401
402		/**
403		* @param string $text
404		* @param int $offset
405		* @return array
406		*/
407	25	private function getString($text, $offset)
408		{
409	25	$quote = $text[$offset];
410	25	if (preg_match(
411		'/' .
412	25	$quote .
413	25	'(' .
414	25	'(?:' .
415	25	'[^\\\\' . $quote . ']' . // not \ or "
416	25	'\|\\\\.' . // escaped quotearacter
417	25	'\|' . $quote . $quote . // ""
418	25	')*' .
419	25	')' .
420	25	$quote .
421	25	'/ms',
422	25	$text,
423	25	$pregMatch,
424	25	0,
425	25	$offset
426		)
427		) {
428	25	$token = Token::fromString($pregMatch[1], $quote);
429		return [
430	25	$token,
431	25	$offset + strlen($pregMatch[0])
432		];
433		}
434		throw new RuntimeException("Unterminated string $quote...$quote");
435		}
436
437		/**
438		* @param string $text
439		* @param int $offset
440		* @return array\|null
441		*/
442	70	private function getNumber($text, $offset)
443		{
444	70	if (preg_match('/\A[-+]?[.]?[0-9]/ms', substr($text, $offset, 3)) &&
445	70	preg_match('/[-+]?(?:[0-9]+(?:[.][0-9]*)?\|[.][0-9]+)(?:[eE][-+]?[0-9]+)?/ms', $text, $pregMatch, 0, $offset)
446		) {
447		return [
448	70	new Token(Token::NUMBER, $pregMatch[0]),
449	70	$offset + strlen($pregMatch[0])
450		];
451		}
452		return null;
453		}
454
455		/**
456		* Parse a hex string of the form "0x<hex digits>" or "x'<hex digits>'".
457		*
458		* https://dev.mysql.com/doc/refman/5.7/en/hexadecimal-literals.html
459		*
460		* - Only an even number of digits is valid.
461		* - Case insensitive for hex digits.
462		* - Case insensitive 'x' in quoted notation.
463		* - Case sensitive 'x' for leading zero notation.
464		*
465		* Valid examples:
466		* - x'BA5EBA11'
467		* - x'decea5ed'
468		* - X'5eed'
469		* - 0xb01dface
470		* - 0xBADC0DED
471		*
472		* Invalid examples
473		* - x'00f' (odd number of digits)
474		* - x'gg' (invalid hex character)
475		* - 0XFFFF (upper case 'x')
476		*
477		* @param string $text
478		* @param int $offset
479		* @return array\|null
480		*/
481	279	private function getHex($text, $offset)
482		{
483	279	$pregMatch = [];
484
485		$matchesLeadingZeroNotation = function ($text, $offset, &$pregMatch) {
486		return
487	279	preg_match('/\A0x([0-9a-fA-F]*)/ms', $text, $pregMatch, 0, $offset);
488	279	};
489
490		$matchesXQuotedNotation = function ($text, $offset, &$pregMatch) {
491		return
492	279	preg_match('/\Ax\'[0-9a-f\']/ims', substr($text, $offset, 3)) &&
493	279	preg_match('/x\'([0-9a-f]*)\'/ims', $text, $pregMatch, 0, $offset);
494	279	};
495
496	279	if ($matchesLeadingZeroNotation($text, $offset, $pregMatch) \|\|
497	279	$matchesXQuotedNotation($text, $offset, $pregMatch)) {
498	1	if (strlen($pregMatch[1]) % 2 != 0) {
499		throw new RuntimeException("Invalid hex literal");
500		}
501		return [
502	1	new Token(Token::HEX, $pregMatch[1]),
503	1	$offset + strlen($pregMatch[0])
504		];
505		}
506	279	return null;
507		}
508
509		/**
510		* @param string $text
511		* @param int $offset
512		* @return array\|null
513		*/
514	38	private function getBin($text, $offset)
515		{
516	38	if (preg_match('/\Ab\'[01\']/ms', substr($text, $offset, 3)) &&
517	38	preg_match('/b\'([01]*)\'/ms', $text, $pregMatch, 0, $offset)
518		) {
519		return [
520	1	new Token(Token::BIN, $pregMatch[1]),
521	1	$offset + strlen($pregMatch[0])
522		];
523		}
524	38	return null;
525		}
526
527		/**
528		* @param string $text
529		* @param int $offset
530		* @return array
531		*/
532	314	private function getIdentifier($text, $offset)
533		{
534	314	preg_match('/[a-zA-Z0-9$_]+()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset);
535		return [
536	314	new Token(Token::IDENTIFIER, $pregMatch[0][0]),
537	314	$pregMatch[1][1]
538		];
539		}
540
541		/**
542		* @param string $text
543		* @param int $offset
544		* @return array
545		*/
546		private function getSymbol($text, $offset)
547		{
548		if (preg_match('/\A(?:[-+*.\/])/xms', substr($text, $offset, 2), $pregMatch)) {
549		return [
550		new Token(Token::SYMBOL, $pregMatch[0]),
551		$offset + strlen($pregMatch[0])
552		];
553		}
554
555		return [];
556		}
557
558		/**
559		* @return Token\|mixed
560		*/
561	314	public function nextToken()
562		{
563	314	while (true) {
564	314	$token = $this->nextTokenRaw();
565	314	if (!isset(self::$skipTokenTypes[$token->type])) {
566	314	return $token;
567		}
568		}
569
570		return null;
571		}
572
573		/**
574		* @return object
575		*/
576	295	public function getMark()
577		{
578		return (object)[
579	295	'offset' => $this->offset,
580	295	'inConditional' => $this->inConditional,
581		];
582		}
583
584		/**
585		* @param mixed $mark
586		*/
587	292	public function rewind($mark)
588		{
589	292	$this->offset = $mark->offset;
590	292	$this->inConditional = $mark->inConditional;
591	292	}
592
593		/**
594		* This function will consume the requested content from the stream without trying to parse and tokenise it.
595		* It is used by {@see peek()}.
596		*
597		* @param mixed $spec
598		* @return bool
599		*/
600	308	public function consume($spec)
601		{
602		// inline getMark()
603	308	$markOffset = $this->offset;
604	308	$markInConditional = $this->inConditional;
605
606	308	if (is_string($spec)) {
607	298	foreach (explode(' ', $spec) as $text) {
608	298	$token = $this->nextToken();
609		// inline $token->eq(...)
610	298	if (strcasecmp($token->text, $text) !== 0 \|\|
611	298	$token->type !== Token::IDENTIFIER
612		) {
613		// inline rewind()
614	296	$this->offset = $markOffset;
615	296	$this->inConditional = $markInConditional;
616	298	return false;
617		}
618		}
619		} else {
620	276	foreach ($spec as $match) {
621	276	list($type, $text) = $match;
622	276	$token = $this->nextToken();
623		// inline $token->eq(...)
624	276	if (strcasecmp($token->text, $text) !== 0 \|\|
625	276	$token->type !== $type
626		) {
627		// inline rewind()
628	251	$this->offset = $markOffset;
629	251	$this->inConditional = $markInConditional;
630	276	return false;
631		}
632		}
633		}
634
635	181	return true;
636		}
637
638		/**
639		* @param mixed $spec
640		* @return bool
641		*/
642		public function peek($spec)
643		{
644		// inline getMark()
645		$markOffset = $this->offset;
646		$markInConditional = $this->inConditional;
647
648		$result = $this->consume($spec);
649
650		// inline rewind()
651		$this->offset = $markOffset;
652		$this->inConditional = $markInConditional;
653
654		return $result;
655		}
656
657		/**
658		* @param string $type
659		* @param string $text
660		* @return string
661		*/
662	137	public function expect($type, $text = null)
663		{
664	137	$token = $this->nextToken();
665	137	if (!$token->eq($type, $text)) {
666		throw new RuntimeException("Expected '$text'");
667		}
668	137	return $token->text;
669		}
670
671		/**
672		* @return string
673		*/
674	312	public function expectName()
675		{
676	312	$token = $this->nextToken();
677	312	if ($token->type !== Token::IDENTIFIER) {
678	1	throw new RuntimeException("Expected identifier");
679		}
680	312	return $token->text;
681		}
682
683		/**
684		* @return string
685		*/
686	95	public function expectOpenParen()
687		{
688	95	return $this->expect(Token::SYMBOL, '(');
689		}
690
691		/**
692		* @return string
693		*/
694	50	public function expectCloseParen()
695		{
696	50	return $this->expect(Token::SYMBOL, ')');
697		}
698
699		/**
700		* @return int
701		*/
702	52	public function expectNumber()
703		{
704	52	$token = $this->nextToken();
705	52	if ($token->type !== Token::NUMBER) {
706		throw new RuntimeException("Expected number");
707		}
708	52	return 1 * $token->text;
709		}
710
711		/**
712		* @return string
713		*/
714	1	public function expectString()
715		{
716	1	$token = $this->nextToken();
717	1	if ($token->type !== Token::STRING) {
718		throw new RuntimeException("Expected string");
719		}
720	1	return $token->text;
721		}
722
723		/**
724		* @return string
725		*/
726	15	public function expectStringExtended()
727		{
728	15	$token = $this->nextToken();
729	15	switch ($token->type) {
730	15	case Token::STRING:
731	15	return $token->text;
732		case Token::HEX:
733		return $token->asString();
734		case Token::BIN:
735		return $token->asString();
736		default:
737		throw new RuntimeException("Expected string");
738		}
739		}
740
741		/**
742		* Provides context for error messages.
743		*
744		* For example, given this invalid table definition ...
745		*
746		* CREATE TABLE `foo` (
747		* `a` bar DEFAULT NULL
748		* );
749		*
750		* ... this function will produce something like this:
751		*
752		* schema/morphism test/foo.sql, line 2: unknown datatype 'bar'
753		* 1: CREATE TABLE `foo` (
754		* 2: `a` bar<<HERE>> DEFAULT NULL
755		*
756		* @param string $message
757		* @return string
758		*/
759		public function contextualise($message)
760		{
761		$preContextLines = 4;
762		$postContextLines = 0;
763
764		// get position of eol strictly before offset
765		$prevEolPos = strrpos($this->text, "\n", $this->offset - strlen($this->text) - 1);
766		$prevEolPos = ($prevEolPos === false) ? -1 : $prevEolPos;
767
768		// get position of eol on or after offset
769		$nextEolPos = strpos($this->text, "\n", $this->offset);
770		$nextEolPos = ($nextEolPos === false) ? strlen($this->text) : $nextEolPos;
771
772		// count number of newlines up to but not including offset
773		$lineNo = substr_count($this->text, "\n", 0, $this->offset);
774		$lines = explode("\n", $this->text);
775
776		$contextLines = array_slice(
777		$lines,
778		max(0, $lineNo - $preContextLines),
779		min($lineNo, $preContextLines),
780		true // preserve keys
781		);
782		$contextLines += [
783		$lineNo =>
784		substr($this->text, $prevEolPos + 1, $this->offset - ($prevEolPos + 1)) .
785		"<<HERE>>".
786		substr($this->text, $this->offset, $nextEolPos - $this->offset)
787		];
788		$contextLines += array_slice(
789		$lines,
790		$lineNo + 1,
791		$postContextLines,
792		true // preserve keys
793		);
794
795		$context = '';
796		$width = strlen($lineNo + 1 + $postContextLines);
797		foreach ($contextLines as $i => $line) {
798		$context .= sprintf("\n%{$width}d: %s", $i + 1, $line);
799		}
800
801		return sprintf("%s, line %d: %s%s", $this->path, $lineNo + 1, $message, $context);
802		}
803		}
804

graze / morphism

GitHub Access Token became invalid

Pull Request — master (#61)

TokenStream::getNumber() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like