TokenStream::getHex() - Code Metrics - Inspection of "docker image for docker-hub" - graze/morphism - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#49)

by Harry

created 2018-07-13 08:39 UTC

TokenStream::getHex() A

↳ Parent: Project

Complexity

Conditions	5
Paths	3

Size

Total Lines	26
Code Lines	16

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	15
CRAP Score	5

Importance

Changes

Metric	Value
cc	5
eloc	16
nc	3
nop	2
dl	0
loc	26
rs	9.4222
c	0
b	0
f	0
ccs	15
cts	15
cp	1
crap	5

<?php
namespace Graze\Morphism\Parse;

use LogicException;
use RuntimeException;

class TokenStream
{
    /** @var string */
    private $path;
    /** @var string */
    private $text;
    /** @var int */
    private $len;
    /** @var int */
    private $offset = 0;
    /** @var bool */
    private $inConditional = false;
    /** @var array */
    private $memo = [];

    /** @var array */
    private static $skipTokenTypes = [
        Token::CONDITIONAL_START => true,
        Token::CONDITIONAL_END   => true,
        Token::COMMENT           => true,
        Token::WHITESPACE        => true,
    ];

    private function __construct()
    {
    }

    // TODO - this is a hack that needs to be refactored

    // perhaps supply a LineStream interface that is satisfied
    // by a FileLineStream or ConnectionLineStream for example?
    /**
     * @param string $text
     * @param string $label
     * @return TokenStream
     */
    public static function newFromText($text, $label)
    {
        $stream = new self;
        $stream->path = $label;
        $stream->text = $text;
        $stream->len = strlen($text);
        return $stream;
    }

    /**
     * @param string $path
     * @return TokenStream
     */
    public static function newFromFile($path)
    {
        $text = @file_get_contents($path);
        if ($text === false) {
            throw new RuntimeException("$path: could not open file");
        }
        $stream = new self;
        $stream->path = $path;
        $stream->text = $text;
        $stream->len = strlen($text);
        return $stream;
    }

    /**
     * @return Token|mixed
     */
    private function nextTokenRaw()
    {
        $startOffset = $this->offset;
        if (isset($this->memo[$startOffset])) {
            $entry = $this->memo[$startOffset];
            $this->offset = $entry[0];
            $this->inConditional = $entry[1];
            return $entry[2];
        }

        if ($this->offset >= $this->len) {
            $token = new Token(Token::EOF);
            $this->offset = $this->len;
        } else {
            list($token, $offset) = $this->getNextTokenRaw($this->text, $this->offset);
            $this->offset = $offset;
        }

        $this->memo[$startOffset] = [
            $this->offset,
            $this->inConditional,
            $token,
        ];

        return $token;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getNextTokenRaw($text, $offset)
    {
        // currently unsupported:
        //
        //      charset prefixed strings, e.g. _utf8'wingwang'
        //      temporal literals, e.g. DATE'2014-07-08'
        //      the null literal, i.e. \N

        switch ($text[$offset]) {
            case " ":
            case "\n":
            case "\r":
            case "\t":
                $n = strspn($text, " \n\r\t", $offset);
                return [
                    new Token(Token::WHITESPACE, substr($text, $offset, $n)),
                    $offset + $n
                ];

            case '#':
                return
                    $this->getComment($text, $offset);

            case '.':
            case '+':
                return
                    $this->getNumber($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '-':
                return
                    $this->getComment($text, $offset) ?:
                    $this->getNumber($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '*':
                return
                    $this->getConditionEnd($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '/':
                return
                    $this->getConditionalStart($text, $offset) ?:
                    $this->getMultilineComment($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '0':
                // Handle hex if needed
                if (isset($text[$offset+1]) && $text[$offset+1] === 'x') {
                    return
                        $this->getHex($text, $offset) ?:
                        $this->getIdentifier($text, $offset);
                }
                // Handle non-hex leading zero.
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                return
                    $this->getNumber($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case '"':
            case "'":
                return
                    $this->getString($text, $offset);

            case '`':
                return
                    $this->getQuotedIdentifier($text, $offset);

            case 'B':
            case 'b':
                return
                    $this->getBin($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case 'X':
            case 'x':
                return
                    $this->getHex($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case '$':
            case '_':
            case 'A':
            case 'a':
            case 'C':
            case 'c':
            case 'D':
            case 'd':
            case 'E':
            case 'e':
            case 'F':
            case 'f':
            case 'G':
            case 'g':
            case 'H':
            case 'h':
            case 'I':
            case 'i':
            case 'J':
            case 'j':
            case 'K':
            case 'k':
            case 'L':
            case 'l':
            case 'M':
            case 'm':
            case 'N':
            case 'n':
            case 'O':
            case 'o':
            case 'P':
            case 'p':
            case 'Q':
            case 'q':
            case 'R':
            case 'r':
            case 'S':
            case 's':
            case 'T':
            case 't':
            case 'U':
            case 'u':
            case 'V':
            case 'v':
            case 'W':
            case 'w':
            case 'Y':
            case 'y':
            case 'Z':
            case 'z':
                return
                    $this->getIdentifier($text, $offset);

            case '!':
            case '%':
            case '&':
            case '(':
            case ')':
            case ',':
            case ':':
            case ';':
            case '<':
            case '=':
            case '>':
            case '@':
            case '^':
            case '|':
            case '~':
                return
                    $this->getSpecialSymbol($text, $offset);

            case '?':
            case '[':
            case '\\':
            case ']':
            case '{':
            case '}':
            default:
                $ch = $text[$offset];
                throw new LogicException("Lexer is confused by char '$ch' ord " . ord($ch));
        }
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getQuotedIdentifier($text, $offset)
    {
        if (preg_match('/`((?:[^`]|``)*)`()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset)) {
            $token = Token::fromIdentifier($pregMatch[1][0]);
            return [
                $token,
                $pregMatch[2][1]
            ];
        }
        throw new RuntimeException("Unterminated identifier: $text");
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getSpecialSymbol($text, $offset)
    {
        // TODO - should probably be a new token type 'variable' for @ and @@

        preg_match('/\A(?:<=|>=|<>|!=|:=|@@|&&|\|\||[=~!@%^&();:,<>|])()/xms', substr($text, $offset, 2), $pregMatch, PREG_OFFSET_CAPTURE);
        return [
            new Token(Token::SYMBOL, $pregMatch[0][0]),
            $offset + $pregMatch[1][1]
        ];
    }

    /**
     * Get the start of a conditional comment.
     *
     * https://dev.mysql.com/doc/refman/5.7/en/comments.html
     *
     * Does not support optimiser hints. See examples below.
     *
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getConditionalStart($text, $offset)
    {
        // Example conditional comments which can't be displayed in the docblock because they clash:
        // - /*! MySQL-specific code */ (execute the given code)
        // - /*!12345 MySQL-specific code */ (execute the given code only if the version matches)
        // Unsupported:
        // - SELECT /*+ BKA(t1) */ FROM ... ;

        if (// 10 comes from allowing for the /*! sequence, a MySQL version number, and a space
            preg_match('_\A/\*!([0-9]*)\s_ms', substr($text, $offset, 10)) &&
            preg_match('_/\*!([0-9]*)\s_ms', $text, $pregMatch, 0, $offset)
        ) {
            $this->inConditional = true;
            return [
                new Token(Token::CONDITIONAL_START, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * Get the end of a conditional comment. See _getConditionStart() for details.
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getConditionEnd($text, $offset)
    {
        if (substr($text, $offset, 2) === '*/') {
            if (!$this->inConditional) {
                throw new RuntimeException("Unexpected '*/'");
            }
            $this->inConditional = false;
            return [
                new Token(Token::CONDITIONAL_END),
                $offset + 2
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getMultilineComment($text, $offset)
    {
        if (substr($text, $offset, 2) === '/*') {
            $pos = strpos($text, '*/', $offset);
            if ($pos !== false) {
                return [
                    new Token(Token::COMMENT, substr($text, $offset, $pos - $offset + 2)),
                    $pos + 2
                ];
            }
            throw new RuntimeException("Unterminated '/*'");
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getComment($text, $offset)
    {
        if (preg_match('/\A(?:#|--\s)/ms', substr($text, $offset, 3))) {
            $pos = strpos($text, "\n", $offset);
            if ($pos !== false) {
                return [
                    new Token(Token::COMMENT, substr($text, $offset, $pos - $offset)),
                    $pos + 1
                ];
            }
            return [
                new Token(Token::COMMENT, $text),
                strlen($text)
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getString($text, $offset)
    {
        $quote = $text[$offset];
        if (preg_match(
            '/' .
            $quote .
            '(' .
                '(?:' .
                    '[^\\\\' . $quote . ']' .   // not \ or "
                    '|\\\\.' .                  // escaped quotearacter
                    '|' .  $quote . $quote .    // ""
                ')*' .
            ')' .
            $quote .
            '/ms',
            $text,
            $pregMatch,
            0,
            $offset
        )
        ) {
            $token = Token::fromString($pregMatch[1], $quote);
            return [
                $token,
                $offset + strlen($pregMatch[0])
            ];
        }
        throw new RuntimeException("Unterminated string $quote...$quote");
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getNumber($text, $offset)
    {
        if (preg_match('/\A[-+]?[.]?[0-9]/ms', substr($text, $offset, 3)) &&
            preg_match('/[-+]?(?:[0-9]+(?:[.][0-9]*)?|[.][0-9]+)(?:[eE][-+]?[0-9]+)?/ms', $text, $pregMatch, 0, $offset)
        ) {
            return [
                new Token(Token::NUMBER, $pregMatch[0]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * Parse a hex string of the form "0x<hex digits>" or "x'<hex digits>'".
     *
     * https://dev.mysql.com/doc/refman/5.7/en/hexadecimal-literals.html
     *
     * - Only an even number of digits is valid.
     * - Case insensitive for hex digits.
     * - Case insensitive 'x' in quoted notation.
     * - Case sensitive 'x' for leading zero notation.
     *
     * Valid examples:
     * - x'BA5EBA11'
     * - x'decea5ed'
     * - X'5eed'
     * - 0xb01dface
     * - 0xBADC0DED
     *
     * Invalid examples
     * - x'00f' (odd number of digits)
     * - x'gg'  (invalid hex character)
     * - 0XFFFF (upper case 'x')
     *
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getHex($text, $offset)
    {
        $pregMatch = [];

        $matchesLeadingZeroNotation = function ($text, $offset, &$pregMatch) {
            return
                preg_match('/\A0x([0-9a-fA-F]*)/ms', $text, $pregMatch, 0, $offset);
        };

        $matchesXQuotedNotation = function ($text, $offset, &$pregMatch) {
            return
                preg_match('/\Ax\'[0-9a-f\']/ims', substr($text, $offset, 3)) &&
                preg_match('/x\'([0-9a-f]*)\'/ims', $text, $pregMatch, 0, $offset);
        };

        if ($matchesLeadingZeroNotation($text, $offset, $pregMatch) ||
            $matchesXQuotedNotation($text, $offset, $pregMatch)) {
            if (strlen($pregMatch[1]) % 2 != 0) {
                throw new RuntimeException("Invalid hex literal");
            }
            return [
                new Token(Token::HEX, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getBin($text, $offset)
    {
        if (preg_match('/\Ab\'[01\']/ms', substr($text, $offset, 3)) &&
            preg_match('/b\'([01]*)\'/ms', $text, $pregMatch, 0, $offset)
        ) {
            return [
                new Token(Token::BIN, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getIdentifier($text, $offset)
    {
        preg_match('/[a-zA-Z0-9$_]+()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset);
        return [
            new Token(Token::IDENTIFIER, $pregMatch[0][0]),
            $pregMatch[1][1]
        ];
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getSymbol($text, $offset)
    {
        if (preg_match('/\A(?:[-+*.\/])/xms', substr($text, $offset, 2), $pregMatch)) {
            return [
                new Token(Token::SYMBOL, $pregMatch[0]),
                $offset + strlen($pregMatch[0])
            ];
        }
    }

    /**
     * @return Token|mixed
     */
    public function nextToken()
    {
        while (true) {
            $token = $this->nextTokenRaw();
            if (!isset(self::$skipTokenTypes[$token->type])) {
                return $token;
            }
        }
    }

    /**
     * @return object
     */
    public function getMark()
    {
        return (object)[
            'offset'        => $this->offset,
            'inConditional' => $this->inConditional,
        ];
    }

    /**
     * @param mixed $mark
     */
    public function rewind($mark)
    {
        $this->offset        = $mark->offset;
        $this->inConditional = $mark->inConditional;
    }

    /**
     * This function will consume the requested content from the stream without trying to parse and tokenise it.
     * It is used by {@see peek()}.
     *
     * @param mixed $spec
     * @return bool
     */
    public function consume($spec)
    {
        // inline getMark()
        $markOffset        = $this->offset;
        $markInConditional = $this->inConditional;

        if (is_string($spec)) {
            foreach (explode(' ', $spec) as $text) {
                $token = $this->nextToken();
                // inline $token->eq(...)

                if (strcasecmp($token->text, $text) !== 0 ||
                    $token->type !== Token::IDENTIFIER
                ) {
                    // inline rewind()
                    $this->offset        = $markOffset;
                    $this->inConditional = $markInConditional;
                    return false;
                }
            }
        } else {
            foreach ($spec as $match) {
                list($type, $text) = $match;
                $token = $this->nextToken();
                // inline $token->eq(...)

                if (strcasecmp($token->text, $text) !== 0 ||
                    $token->type !== $type
                ) {
                    // inline rewind()
                    $this->offset        = $markOffset;
                    $this->inConditional = $markInConditional;
                    return false;
                }
            }
        }

        return true;
    }

    /**
     * @param mixed $spec
     * @return bool
     */
    public function peek($spec)
    {
        // inline getMark()
        $markOffset        = $this->offset;
        $markInConditional = $this->inConditional;

        $result = $this->consume($spec);

        // inline rewind()
        $this->offset        = $markOffset;
        $this->inConditional = $markInConditional;

        return $result;
    }

    /**
     * @param string $type
     * @param string $text
     * @return string
     */
    public function expect($type, $text = null)
    {
        $token = $this->nextToken();
        if (!$token->eq($type, $text)) {
            throw new RuntimeException("Expected '$text'");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectName()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::IDENTIFIER) {
            throw new RuntimeException("Expected identifier");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectOpenParen()
    {
        return $this->expect(Token::SYMBOL, '(');
    }

    /**
     * @return string
     */
    public function expectCloseParen()
    {
        return $this->expect(Token::SYMBOL, ')');
    }

    /**
     * @return int
     */
    public function expectNumber()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::NUMBER) {
            throw new RuntimeException("Expected number");
        }
        return 0 + $token->text;
    }

    /**
     * @return string
     */
    public function expectString()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::STRING) {
            throw new RuntimeException("Expected string");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectStringExtended()
    {
        $token = $this->nextToken();
        switch ($token->type) {
            case Token::STRING:
                return $token->text;
            case Token::HEX:
                return $token->asString();
            case Token::BIN:
                return $token->asString();
            default:
                throw new RuntimeException("Expected string");
        }
    }

    /**
     * Provides context for error messages.
     *
     * For example, given this invalid table definition ...
     *
     *     CREATE TABLE `foo` (
     *         `a` bar DEFAULT NULL
     *     );
     *
     * ... this function will produce something like this:
     *
     *     schema/morphism test/foo.sql, line 2: unknown datatype 'bar'
     *     1: CREATE TABLE `foo` (
     *     2:   `a` bar<<HERE>> DEFAULT NULL
     *
     * @param string $message
     * @return string
     */
    public function contextualise($message)
    {
        $preContextLines = 4;
        $postContextLines = 0;

        // get position of eol strictly before offset
        $prevEolPos = strrpos($this->text, "\n", $this->offset - strlen($this->text) - 1);
        $prevEolPos = ($prevEolPos === false) ? -1 : $prevEolPos;

        // get position of eol on or after offset
        $nextEolPos = strpos($this->text, "\n", $this->offset);
        $nextEolPos = ($nextEolPos === false) ? strlen($this->text) : $nextEolPos;

        // count number of newlines up to but not including offset
        $lineNo = substr_count($this->text, "\n", 0, $this->offset);
        $lines = explode("\n", $this->text);

        $contextLines = array_slice(
            $lines,
            max(0, $lineNo - $preContextLines),
            min($lineNo, $preContextLines),
            true // preserve keys
        );
        $contextLines += [
            $lineNo =>
                substr($this->text, $prevEolPos + 1, $this->offset - ($prevEolPos + 1)) .
                "<<HERE>>".
                substr($this->text, $this->offset, $nextEolPos - $this->offset)
        ];
        $contextLines += array_slice(
            $lines,
            $lineNo + 1,
            $postContextLines,
            true // preserve keys
        );

        $context = '';
        $width = strlen($lineNo + 1 + $postContextLines);
        foreach ($contextLines as $i => $line) {
            $context .= sprintf("\n%{$width}d: %s", $i + 1, $line);
        }

        return sprintf("%s, line %d: %s%s", $this->path, $lineNo + 1, $message, $context);
    }
}


1		<?php
2		namespace Graze\Morphism\Parse;
3
4		use LogicException;
5		use RuntimeException;
6
7		class TokenStream
8		{
9		/** @var string */
10		private $path;
11		/** @var string */
12		private $text;
13		/** @var int */
14		private $len;
15		/** @var int */
16		private $offset = 0;
17		/** @var bool */
18		private $inConditional = false;
19		/** @var array */
20		private $memo = [];
21
22		/** @var array */
23		private static $skipTokenTypes = [
24		Token::CONDITIONAL_START => true,
25		Token::CONDITIONAL_END => true,
26		Token::COMMENT => true,
27		Token::WHITESPACE => true,
28		];
29
30	590	private function __construct()
31		{
32	590	}
33
34		// TODO - this is a hack that needs to be refactored
		0 ignored issues – show Coding Style Best Practice introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report Comments for TODO tasks are often forgotten in the code; it might be better to use a dedicated issue tracker. Loading history...
35		// perhaps supply a LineStream interface that is satisfied
36		// by a FileLineStream or ConnectionLineStream for example?
37		/**
38		* @param string $text
39		* @param string $label
40		* @return TokenStream
41		*/
42	1	public static function newFromText($text, $label)
43		{
44	1	$stream = new self;
45	1	$stream->path = $label;
46	1	$stream->text = $text;
47	1	$stream->len = strlen($text);
48	1	return $stream;
49		}
50
51		/**
52		* @param string $path
53		* @return TokenStream
54		*/
55	590	public static function newFromFile($path)
56		{
57	590	$text = @file_get_contents($path);
58	590	if ($text === false) {
59	1	throw new RuntimeException("$path: could not open file");
60		}
61	589	$stream = new self;
62	589	$stream->path = $path;
63	589	$stream->text = $text;
64	589	$stream->len = strlen($text);
65	589	return $stream;
66		}
67
68		/**
69		* @return Token\|mixed
70		*/
71	586	private function nextTokenRaw()
72		{
73	586	$startOffset = $this->offset;
74	586	if (isset($this->memo[$startOffset])) {
75	382	$entry = $this->memo[$startOffset];
76	382	$this->offset = $entry[0];
77	382	$this->inConditional = $entry[1];
78	382	return $entry[2];
79		}
80
81	586	if ($this->offset >= $this->len) {
82	342	$token = new Token(Token::EOF);
83	342	$this->offset = $this->len;
84		} else {
85	583	list($token, $offset) = $this->getNextTokenRaw($this->text, $this->offset);
86	571	$this->offset = $offset;
87		}
88
89	574	$this->memo[$startOffset] = [
90	574	$this->offset,
91	574	$this->inConditional,
92	574	$token,
93		];
94
95	574	return $token;
96		}
97
98		/**
99		* @param string $text
100		* @param int $offset
101		* @return array\|null
102		*/
103	583	private function getNextTokenRaw($text, $offset)
104		{
105		// currently unsupported:
106		//
107		// charset prefixed strings, e.g. _utf8'wingwang'
108		// temporal literals, e.g. DATE'2014-07-08'
109		// the null literal, i.e. \N
110
111	583	switch ($text[$offset]) {
112	583	case " ":
113	583	case "\n":
114	583	case "\r":
115	583	case "\t":
116	380	$n = strspn($text, " \n\r\t", $offset);
117		return [
118	380	new Token(Token::WHITESPACE, substr($text, $offset, $n)),
119	380	$offset + $n
120		];
121
122	583	case '#':
123		return
124	2	$this->getComment($text, $offset);
125
126	582	case '.':
127	581	case '+':
128		return
129	5	$this->getNumber($text, $offset) ?:
130	5	$this->getSymbol($text, $offset);
131
132	578	case '-':
133		return
134	4	$this->getComment($text, $offset) ?:
135	2	$this->getNumber($text, $offset) ?:
136	4	$this->getSymbol($text, $offset);
137
138	576	case '*':
139		return
140	4	$this->getConditionEnd($text, $offset) ?:
141	3	$this->getSymbol($text, $offset);
142
143	574	case '/':
144		return
145	8	$this->getConditionalStart($text, $offset) ?:
146	4	$this->getMultilineComment($text, $offset) ?:
147	7	$this->getSymbol($text, $offset);
148
149	572	case '0':
150		// Handle hex if needed
151	22	if (isset($text[$offset+1]) && $text[$offset+1] === 'x') {
152		return
153	4	$this->getHex($text, $offset) ?:
154	3	$this->getIdentifier($text, $offset);
155		}
156		// Handle non-hex leading zero.
157	567	case '1':
158	548	case '2':
159	548	case '3':
160	548	case '4':
161	547	case '5':
162	547	case '6':
163	547	case '7':
164	547	case '8':
165	547	case '9':
166		return
167	137	$this->getNumber($text, $offset) ?:
168	137	$this->getIdentifier($text, $offset);
169
170	547	case '"':
171	542	case "'":
172		return
173	57	$this->getString($text, $offset);
174
175	534	case '`':
176		return
177	8	$this->getQuotedIdentifier($text, $offset);
178
179	528	case 'B':
180	528	case 'b':
181		return
182	51	$this->getBin($text, $offset) ?:
183	51	$this->getIdentifier($text, $offset);
184
185	523	case 'X':
186	522	case 'x':
187		return
188	294	$this->getHex($text, $offset) ?:
189	294	$this->getIdentifier($text, $offset);
190
191	509	case '$':
192	507	case '_':
193	504	case 'A':
194	504	case 'a':
195	492	case 'C':
196	492	case 'c':
197	473	case 'D':
198	473	case 'd':
199	453	case 'E':
200	453	case 'e':
201	453	case 'F':
202	453	case 'f':
203	444	case 'G':
204	444	case 'g':
205	444	case 'H':
206	444	case 'h':
207	444	case 'I':
208	444	case 'i':
209	428	case 'J':
210	428	case 'j':
211	428	case 'K':
212	428	case 'k':
213	426	case 'L':
214	426	case 'l':
215	420	case 'M':
216	420	case 'm':
217	415	case 'N':
218	394	case 'n':
219	390	case 'O':
220	390	case 'o':
221	390	case 'P':
222	390	case 'p':
223	386	case 'Q':
224	386	case 'q':
225	386	case 'R':
226	386	case 'r':
227	385	case 'S':
228	385	case 's':
229	377	case 'T':
230	377	case 't':
231	345	case 'U':
232	344	case 'u':
233	315	case 'V':
234	315	case 'v':
235	309	case 'W':
236	309	case 'w':
237	309	case 'Y':
238	309	case 'y':
239	299	case 'Z':
240	299	case 'z':
241		return
242	481	$this->getIdentifier($text, $offset);
243
244	295	case '!':
245	294	case '%':
246	293	case '&':
247	292	case '(':
248	288	case ')':
249	213	case ',':
250	168	case ':':
251	167	case ';':
252	95	case '<':
253	93	case '=':
254	12	case '>':
255	11	case '@':
256	9	case '^':
257	9	case '\|':
258	7	case '~':
259		return
260	288	$this->getSpecialSymbol($text, $offset);
261
262	7	case '?':
263	6	case '[':
264	5	case '\\':
265	4	case ']':
266	3	case '{':
267	2	case '}':
268		default:
269	7	$ch = $text[$offset];
270	7	throw new LogicException("Lexer is confused by char '$ch' ord " . ord($ch));
271		}
272		}
273
274		/**
275		* @param string $text
276		* @param int $offset
277		* @return array
278		*/
279	8	private function getQuotedIdentifier($text, $offset)
280		{
281	8	if (preg_match('/`((?:[^`]\|``)*)`()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset)) {
282	7	$token = Token::fromIdentifier($pregMatch[1][0]);
283		return [
284	7	$token,
285	7	$pregMatch[2][1]
286		];
287		}
288	1	throw new RuntimeException("Unterminated identifier: $text");
289		}
290
291		/**
292		* @param string $text
293		* @param int $offset
294		* @return array
295		*/
296	288	private function getSpecialSymbol($text, $offset)
297		{
298		// TODO - should probably be a new token type 'variable' for @ and @@
		0 ignored issues – show Coding Style Best Practice introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report Comments for TODO tasks are often forgotten in the code; it might be better to use a dedicated issue tracker. Loading history...
299	288	preg_match('/\A(?:<=\|>=\|<>\|!=\|:=\|@@\|&&\|\\|\\|\|[=~!@%^&();:,<>\|])()/xms', substr($text, $offset, 2), $pregMatch, PREG_OFFSET_CAPTURE);
300		return [
301	288	new Token(Token::SYMBOL, $pregMatch[0][0]),
302	288	$offset + $pregMatch[1][1]
303		];
304		}
305
306		/**
307		* Get the start of a conditional comment.
308		*
309		* https://dev.mysql.com/doc/refman/5.7/en/comments.html
310		*
311		* Does not support optimiser hints. See examples below.
312		*
313		* @param string $text
314		* @param int $offset
315		* @return array\|null
316		*/
317	8	private function getConditionalStart($text, $offset)
318		{
319		// Example conditional comments which can't be displayed in the docblock because they clash:
320		// - /! MySQL-specific code / (execute the given code)
321		// - /!12345 MySQL-specific code / (execute the given code only if the version matches)
322		// Unsupported:
323		// - SELECT /+ BKA(t1) / FROM ... ;
324
325		if (// 10 comes from allowing for the /*! sequence, a MySQL version number, and a space
326	8	preg_match('_\A/\!([0-9])\s_ms', substr($text, $offset, 10)) &&
327	8	preg_match('_/\!([0-9])\s_ms', $text, $pregMatch, 0, $offset)
328		) {
329	4	$this->inConditional = true;
330		return [
331	4	new Token(Token::CONDITIONAL_START, $pregMatch[1]),
332	4	$offset + strlen($pregMatch[0])
333		];
334		}
335	4	return null;
336		}
337
338		/**
339		* Get the end of a conditional comment. See _getConditionStart() for details.
340		* @param string $text
341		* @param int $offset
342		* @return array\|null
343		*/
344	4	private function getConditionEnd($text, $offset)
345		{
346	4	if (substr($text, $offset, 2) === '*/') {
347	3	if (!$this->inConditional) {
348	1	throw new RuntimeException("Unexpected '*/'");
349		}
350	2	$this->inConditional = false;
351		return [
352	2	new Token(Token::CONDITIONAL_END),
353	2	$offset + 2
354		];
355		}
356	1	return null;
357		}
358
359		/**
360		* @param string $text
361		* @param int $offset
362		* @return array\|null
363		*/
364	4	private function getMultilineComment($text, $offset)
365		{
366	4	if (substr($text, $offset, 2) === '/*') {
367	3	$pos = strpos($text, '*/', $offset);
368	3	if ($pos !== false) {
369		return [
370	2	new Token(Token::COMMENT, substr($text, $offset, $pos - $offset + 2)),
371	2	$pos + 2
372		];
373		}
374	1	throw new RuntimeException("Unterminated '/*'");
375		}
376	1	return null;
377		}
378
379		/**
380		* @param string $text
381		* @param int $offset
382		* @return array\|null
383		*/
384	6	private function getComment($text, $offset)
385		{
386	6	if (preg_match('/\A(?:#\|--\s)/ms', substr($text, $offset, 3))) {
387	4	$pos = strpos($text, "\n", $offset);
388	4	if ($pos !== false) {
389		return [
390	3	new Token(Token::COMMENT, substr($text, $offset, $pos - $offset)),
391	3	$pos + 1
392		];
393		}
394		return [
395	1	new Token(Token::COMMENT, $text),
396	1	strlen($text)
397		];
398		}
399	2	return null;
400		}
401
402		/**
403		* @param string $text
404		* @param int $offset
405		* @return array
406		*/
407	57	private function getString($text, $offset)
408		{
409	57	$quote = $text[$offset];
410	57	if (preg_match(
411		'/' .
412	57	$quote .
413	57	'(' .
414	57	'(?:' .
415	57	'[^\\\\' . $quote . ']' . // not \ or "
416	57	'\|\\\\.' . // escaped quotearacter
417	57	'\|' . $quote . $quote . // ""
418	57	')*' .
419	57	')' .
420	57	$quote .
421	57	'/ms',
422	57	$text,
423	57	$pregMatch,
424	57	0,
425	57	$offset
426		)
427		) {
428	56	$token = Token::fromString($pregMatch[1], $quote);
429		return [
430	56	$token,
431	56	$offset + strlen($pregMatch[0])
432		];
433		}
434	1	throw new RuntimeException("Unterminated string $quote...$quote");
435		}
436
437		/**
438		* @param string $text
439		* @param int $offset
440		* @return array\|null
441		*/
442	144	private function getNumber($text, $offset)
443		{
444	144	if (preg_match('/\A[-+]?[.]?[0-9]/ms', substr($text, $offset, 3)) &&
445	144	preg_match('/[-+]?(?:[0-9]+(?:[.][0-9]*)?\|[.][0-9]+)(?:[eE][-+]?[0-9]+)?/ms', $text, $pregMatch, 0, $offset)
446		) {
447		return [
448	141	new Token(Token::NUMBER, $pregMatch[0]),
449	141	$offset + strlen($pregMatch[0])
450		];
451		}
452	3	return null;
453		}
454
455		/**
456		* Parse a hex string of the form "0x<hex digits>" or "x'<hex digits>'".
457		*
458		* https://dev.mysql.com/doc/refman/5.7/en/hexadecimal-literals.html
459		*
460		* - Only an even number of digits is valid.
461		* - Case insensitive for hex digits.
462		* - Case insensitive 'x' in quoted notation.
463		* - Case sensitive 'x' for leading zero notation.
464		*
465		* Valid examples:
466		* - x'BA5EBA11'
467		* - x'decea5ed'
468		* - X'5eed'
469		* - 0xb01dface
470		* - 0xBADC0DED
471		*
472		* Invalid examples
473		* - x'00f' (odd number of digits)
474		* - x'gg' (invalid hex character)
475		* - 0XFFFF (upper case 'x')
476		*
477		* @param string $text
478		* @param int $offset
479		* @return array\|null
480		*/
481	298	private function getHex($text, $offset)
482		{
483	298	$pregMatch = [];
484
485		$matchesLeadingZeroNotation = function ($text, $offset, &$pregMatch) {
486		return
487	298	preg_match('/\A0x([0-9a-fA-F]*)/ms', $text, $pregMatch, 0, $offset);
488	298	};
489
490	298	$matchesXQuotedNotation = function ($text, $offset, &$pregMatch) {
491		return
492	294	preg_match('/\Ax\'[0-9a-f\']/ims', substr($text, $offset, 3)) &&
493	294	preg_match('/x\'([0-9a-f]*)\'/ims', $text, $pregMatch, 0, $offset);
494	298	};
495
496	298	if ($matchesLeadingZeroNotation($text, $offset, $pregMatch) \|\|
497	298	$matchesXQuotedNotation($text, $offset, $pregMatch)) {
498	11	if (strlen($pregMatch[1]) % 2 != 0) {
499	1	throw new RuntimeException("Invalid hex literal");
500		}
501		return [
502	10	new Token(Token::HEX, $pregMatch[1]),
503	10	$offset + strlen($pregMatch[0])
504		];
505		}
506	288	return null;
507		}
508
509		/**
510		* @param string $text
511		* @param int $offset
512		* @return array\|null
513		*/
514	51	private function getBin($text, $offset)
515		{
516	51	if (preg_match('/\Ab\'[01\']/ms', substr($text, $offset, 3)) &&
517	51	preg_match('/b\'([01]*)\'/ms', $text, $pregMatch, 0, $offset)
518		) {
519		return [
520	5	new Token(Token::BIN, $pregMatch[1]),
521	5	$offset + strlen($pregMatch[0])
522		];
523		}
524	47	return null;
525		}
526
527		/**
528		* @param string $text
529		* @param int $offset
530		* @return array
531		*/
532	494	private function getIdentifier($text, $offset)
533		{
534	494	preg_match('/[a-zA-Z0-9$_]+()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset);
535		return [
536	494	new Token(Token::IDENTIFIER, $pregMatch[0][0]),
537	494	$pregMatch[1][1]
538		];
539		}
540
541		/**
542		* @param string $text
543		* @param int $offset
544		* @return array
545		*/
546	5	private function getSymbol($text, $offset)
547		{
548	5	if (preg_match('/\A(?:[-+*.\/])/xms', substr($text, $offset, 2), $pregMatch)) {
549		return [
550	5	new Token(Token::SYMBOL, $pregMatch[0]),
551	5	$offset + strlen($pregMatch[0])
552		];
553		}
554		}
555
556		/**
557		* @return Token\|mixed
558		*/
559	586	public function nextToken()
560		{
561	586	while (true) {
562	586	$token = $this->nextTokenRaw();
563	574	if (!isset(self::$skipTokenTypes[$token->type])) {
564	574	return $token;
565		}
566		}
567		}
568
569		/**
570		* @return object
571		*/
572	452	public function getMark()
573		{
574		return (object)[
575	452	'offset' => $this->offset,
576	452	'inConditional' => $this->inConditional,
577		];
578		}
579
580		/**
581		* @param mixed $mark
582		*/
583	430	public function rewind($mark)
584		{
585	430	$this->offset = $mark->offset;
586	430	$this->inConditional = $mark->inConditional;
587	430	}
588
589		/**
590		* This function will consume the requested content from the stream without trying to parse and tokenise it.
591		* It is used by {@see peek()}.
592		*
593		* @param mixed $spec
594		* @return bool
595		*/
596	464	public function consume($spec)
597		{
598		// inline getMark()
599	464	$markOffset = $this->offset;
600	464	$markInConditional = $this->inConditional;
601
602	464	if (is_string($spec)) {
603	355	foreach (explode(' ', $spec) as $text) {
604	355	$token = $this->nextToken();
605		// inline $token->eq(...)
		0 ignored issues – show Unused Code Comprehensibility introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report `56%` of this comment could be valid code. Did you maybe forget this after debugging? Sometimes obsolete code just ends up commented out instead of removed. In this case it is better to remove the code once you have checked you do not need it. The code might also have been commented out for debugging purposes. In this case it is vital that someone uncomments it again or your project may behave in very unexpected ways in production. This check looks for comments that seem to be mostly valid code and reports them. Loading history...
606	355	if (strcasecmp($token->text, $text) !== 0 \|\|
607	355	$token->type !== Token::IDENTIFIER
608		) {
609		// inline rewind()
610	349	$this->offset = $markOffset;
611	349	$this->inConditional = $markInConditional;
612	355	return false;
613		}
614		}
615		} else {
616	424	foreach ($spec as $match) {
617	424	list($type, $text) = $match;
618	424	$token = $this->nextToken();
619		// inline $token->eq(...)
		0 ignored issues – show Unused Code Comprehensibility introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report `56%` of this comment could be valid code. Did you maybe forget this after debugging? Sometimes obsolete code just ends up commented out instead of removed. In this case it is better to remove the code once you have checked you do not need it. The code might also have been commented out for debugging purposes. In this case it is vital that someone uncomments it again or your project may behave in very unexpected ways in production. This check looks for comments that seem to be mostly valid code and reports them. Loading history...
620	424	if (strcasecmp($token->text, $text) !== 0 \|\|
621	424	$token->type !== $type
622		) {
623		// inline rewind()
624	325	$this->offset = $markOffset;
625	325	$this->inConditional = $markInConditional;
626	424	return false;
627		}
628		}
629		}
630
631	301	return true;
632		}
633
634		/**
635		* @param mixed $spec
636		* @return bool
637		*/
638	7	public function peek($spec)
639		{
640		// inline getMark()
641	7	$markOffset = $this->offset;
642	7	$markInConditional = $this->inConditional;
643
644	7	$result = $this->consume($spec);
645
646		// inline rewind()
647	7	$this->offset = $markOffset;
648	7	$this->inConditional = $markInConditional;
649
650	7	return $result;
651		}
652
653		/**
654		* @param string $type
655		* @param string $text
656		* @return string
657		*/
658	201	public function expect($type, $text = null)
659		{
660	201	$token = $this->nextToken();
661	201	if (!$token->eq($type, $text)) {
662	3	throw new RuntimeException("Expected '$text'");
663		}
664	198	return $token->text;
665		}
666
667		/**
668		* @return string
669		*/
670	361	public function expectName()
671		{
672	361	$token = $this->nextToken();
673	361	if ($token->type !== Token::IDENTIFIER) {
674	2	throw new RuntimeException("Expected identifier");
675		}
676	360	return $token->text;
677		}
678
679		/**
680		* @return string
681		*/
682	104	public function expectOpenParen()
683		{
684	104	return $this->expect(Token::SYMBOL, '(');
685		}
686
687		/**
688		* @return string
689		*/
690	49	public function expectCloseParen()
691		{
692	49	return $this->expect(Token::SYMBOL, ')');
693		}
694
695		/**
696		* @return int
697		*/
698	85	public function expectNumber()
699		{
700	85	$token = $this->nextToken();
701	85	if ($token->type !== Token::NUMBER) {
702	1	throw new RuntimeException("Expected number");
703		}
704	84	return 0 + $token->text;
705		}
706
707		/**
708		* @return string
709		*/
710	21	public function expectString()
711		{
712	21	$token = $this->nextToken();
713	21	if ($token->type !== Token::STRING) {
714	1	throw new RuntimeException("Expected string");
715		}
716	20	return $token->text;
717		}
718
719		/**
720		* @return string
721		*/
722	21	public function expectStringExtended()
723		{
724	21	$token = $this->nextToken();
725	21	switch ($token->type) {
726	21	case Token::STRING:
727	16	return $token->text;
728	5	case Token::HEX:
729	3	return $token->asString();
730	2	case Token::BIN:
731	1	return $token->asString();
732		default:
733	1	throw new RuntimeException("Expected string");
734		}
735		}
736
737		/**
738		* Provides context for error messages.
739		*
740		* For example, given this invalid table definition ...
741		*
742		* CREATE TABLE `foo` (
743		* `a` bar DEFAULT NULL
744		* );
745		*
746		* ... this function will produce something like this:
747		*
748		* schema/morphism test/foo.sql, line 2: unknown datatype 'bar'
749		* 1: CREATE TABLE `foo` (
750		* 2: `a` bar<<HERE>> DEFAULT NULL
751		*
752		* @param string $message
753		* @return string
754		*/
755	1	public function contextualise($message)
756		{
757	1	$preContextLines = 4;
758	1	$postContextLines = 0;
759
760		// get position of eol strictly before offset
761	1	$prevEolPos = strrpos($this->text, "\n", $this->offset - strlen($this->text) - 1);
762	1	$prevEolPos = ($prevEolPos === false) ? -1 : $prevEolPos;
763
764		// get position of eol on or after offset
765	1	$nextEolPos = strpos($this->text, "\n", $this->offset);
766	1	$nextEolPos = ($nextEolPos === false) ? strlen($this->text) : $nextEolPos;
767
768		// count number of newlines up to but not including offset
769	1	$lineNo = substr_count($this->text, "\n", 0, $this->offset);
770	1	$lines = explode("\n", $this->text);
771
772	1	$contextLines = array_slice(
773	1	$lines,
774	1	max(0, $lineNo - $preContextLines),
775	1	min($lineNo, $preContextLines),
776	1	true // preserve keys
777		);
778		$contextLines += [
779		$lineNo =>
780	1	substr($this->text, $prevEolPos + 1, $this->offset - ($prevEolPos + 1)) .
781	1	"<<HERE>>".
782	1	substr($this->text, $this->offset, $nextEolPos - $this->offset)
783		];
784	1	$contextLines += array_slice(
785	1	$lines,
786	1	$lineNo + 1,
787	1	$postContextLines,
788	1	true // preserve keys
789		);
790
791	1	$context = '';
792	1	$width = strlen($lineNo + 1 + $postContextLines);
793	1	foreach ($contextLines as $i => $line) {
794	1	$context .= sprintf("\n%{$width}d: %s", $i + 1, $line);
795		}
796
797	1	return sprintf("%s, line %d: %s%s", $this->path, $lineNo + 1, $message, $context);
798		}
799		}
800

graze / morphism

GitHub Access Token became invalid

Pull Request — master (#49)

TokenStream::getHex() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like