TokenStream::getString() - Code Metrics - Inspection of "Scrutinizer fixes" - graze/morphism - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#60)

by Burhan

created 2019-03-30 14:19 UTC

TokenStream::getString() A

↳ Parent: Project

Complexity

Conditions	2
Paths	2

Size

Total Lines	28
Code Lines	21

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	20
CRAP Score	2.0004

Importance

Changes

Metric	Value
cc	2
eloc	21
nc	2
nop	2
dl	0
loc	28
ccs	20
cts	21
cp	0.9524
crap	2.0004
rs	9.584
c	0
b	0
f	0

<?php
namespace Graze\Morphism\Parse;

use LogicException;
use RuntimeException;

class TokenStream
{
    /** @var string */
    private $path;
    /** @var string */
    private $text;
    /** @var int */
    private $len;
    /** @var int */
    private $offset = 0;
    /** @var bool */
    private $inConditional = false;
    /** @var array */
    private $memo = [];

    /** @var array */
    private static $skipTokenTypes = [
        Token::CONDITIONAL_START => true,
        Token::CONDITIONAL_END   => true,
        Token::COMMENT           => true,
        Token::WHITESPACE        => true,
    ];

    private function __construct()
    {
    }

    // TODO - this is a hack that needs to be refactored

    // perhaps supply a LineStream interface that is satisfied
    // by a FileLineStream or ConnectionLineStream for example?
    /**
     * @param string $text
     * @param string $label
     * @return TokenStream
     */
    public static function newFromText($text, $label)
    {
        $stream = new self;
        $stream->path = $label;
        $stream->text = $text;
        $stream->len = strlen($text);
        return $stream;
    }

    /**
     * @param string $path
     * @return TokenStream
     */
    public static function newFromFile($path)
    {
        $text = @file_get_contents($path);
        if ($text === false) {
            throw new RuntimeException("$path: could not open file");
        }
        $stream = new self;
        $stream->path = $path;
        $stream->text = $text;
        $stream->len = strlen($text);
        return $stream;
    }

    /**
     * @return Token|mixed
     */
    private function nextTokenRaw()
    {
        $startOffset = $this->offset;
        if (isset($this->memo[$startOffset])) {
            $entry = $this->memo[$startOffset];
            $this->offset = $entry[0];
            $this->inConditional = $entry[1];
            return $entry[2];
        }

        if ($this->offset >= $this->len) {
            $token = new Token(Token::EOF);
            $this->offset = $this->len;
        } else {
            list($token, $offset) = $this->getNextTokenRaw($this->text, $this->offset);
            $this->offset = $offset;
        }

        $this->memo[$startOffset] = [
            $this->offset,
            $this->inConditional,
            $token,
        ];

        return $token;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getNextTokenRaw($text, $offset)
    {
        // currently unsupported:
        //
        //      charset prefixed strings, e.g. _utf8'wingwang'
        //      temporal literals, e.g. DATE'2014-07-08'
        //      the null literal, i.e. \N

        switch ($text[$offset]) {
            case " ":
            case "\n":
            case "\r":
            case "\t":
                $n = strspn($text, " \n\r\t", $offset);
                return [
                    new Token(Token::WHITESPACE, substr($text, $offset, $n)),
                    $offset + $n
                ];

            case '#':
                return
                    $this->getComment($text, $offset);

            case '.':
            case '+':
                return
                    $this->getNumber($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '-':
                return
                    $this->getComment($text, $offset) ?:
                    $this->getNumber($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '*':
                return
                    $this->getConditionEnd($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '/':
                return
                    $this->getConditionalStart($text, $offset) ?:
                    $this->getMultilineComment($text, $offset) ?:
                    $this->getSymbol($text, $offset);

            case '0':
                // Handle hex if needed
                if (isset($text[$offset+1]) && $text[$offset+1] === 'x') {
                    return
                        $this->getHex($text, $offset) ?:
                        $this->getIdentifier($text, $offset);
                }
                // Handle non-hex leading zero.
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                return
                    $this->getNumber($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case '"':
            case "'":
                return
                    $this->getString($text, $offset);

            case '`':
                return
                    $this->getQuotedIdentifier($text, $offset);

            case 'B':
            case 'b':
                return
                    $this->getBin($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case 'X':
            case 'x':
                return
                    $this->getHex($text, $offset) ?:
                    $this->getIdentifier($text, $offset);

            case '$':
            case '_':
            case 'A':
            case 'a':
            case 'C':
            case 'c':
            case 'D':
            case 'd':
            case 'E':
            case 'e':
            case 'F':
            case 'f':
            case 'G':
            case 'g':
            case 'H':
            case 'h':
            case 'I':
            case 'i':
            case 'J':
            case 'j':
            case 'K':
            case 'k':
            case 'L':
            case 'l':
            case 'M':
            case 'm':
            case 'N':
            case 'n':
            case 'O':
            case 'o':
            case 'P':
            case 'p':
            case 'Q':
            case 'q':
            case 'R':
            case 'r':
            case 'S':
            case 's':
            case 'T':
            case 't':
            case 'U':
            case 'u':
            case 'V':
            case 'v':
            case 'W':
            case 'w':
            case 'Y':
            case 'y':
            case 'Z':
            case 'z':
                return
                    $this->getIdentifier($text, $offset);

            case '!':
            case '%':
            case '&':
            case '(':
            case ')':
            case ',':
            case ':':
            case ';':
            case '<':
            case '=':
            case '>':
            case '@':
            case '^':
            case '|':
            case '~':
                return
                    $this->getSpecialSymbol($text, $offset);

            case '?':
            case '[':
            case '\\':
            case ']':
            case '{':
            case '}':
            default:
                $ch = $text[$offset];
                throw new LogicException("Lexer is confused by char '$ch' ord " . ord($ch));
        }
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getQuotedIdentifier($text, $offset)
    {
        if (preg_match('/`((?:[^`]|``)*)`()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset)) {
            $token = Token::fromIdentifier($pregMatch[1][0]);
            return [
                $token,
                $pregMatch[2][1]
            ];
        }
        throw new RuntimeException("Unterminated identifier: $text");
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getSpecialSymbol($text, $offset)
    {
        // TODO - should probably be a new token type 'variable' for @ and @@

        preg_match('/\A(?:<=|>=|<>|!=|:=|@@|&&|\|\||[=~!@%^&();:,<>|])()/xms', substr($text, $offset, 2), $pregMatch, PREG_OFFSET_CAPTURE);
        return [
            new Token(Token::SYMBOL, $pregMatch[0][0]),
            $offset + $pregMatch[1][1]
        ];
    }

    /**
     * Get the start of a conditional comment.
     *
     * https://dev.mysql.com/doc/refman/5.7/en/comments.html
     *
     * Does not support optimiser hints. See examples below.
     *
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getConditionalStart($text, $offset)
    {
        // Example conditional comments which can't be displayed in the docblock because they clash:
        // - /*! MySQL-specific code */ (execute the given code)
        // - /*!12345 MySQL-specific code */ (execute the given code only if the version matches)
        // Unsupported:
        // - SELECT /*+ BKA(t1) */ FROM ... ;

        if (// 10 comes from allowing for the /*! sequence, a MySQL version number, and a space
            preg_match('_\A/\*!([0-9]*)\s_ms', substr($text, $offset, 10)) &&
            preg_match('_/\*!([0-9]*)\s_ms', $text, $pregMatch, 0, $offset)
        ) {
            $this->inConditional = true;
            return [
                new Token(Token::CONDITIONAL_START, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * Get the end of a conditional comment. See _getConditionStart() for details.
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getConditionEnd($text, $offset)
    {
        if (substr($text, $offset, 2) === '*/') {
            if (!$this->inConditional) {
                throw new RuntimeException("Unexpected '*/'");
            }
            $this->inConditional = false;
            return [
                new Token(Token::CONDITIONAL_END),
                $offset + 2
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getMultilineComment($text, $offset)
    {
        if (substr($text, $offset, 2) === '/*') {
            $pos = strpos($text, '*/', $offset);
            if ($pos !== false) {
                return [
                    new Token(Token::COMMENT, substr($text, $offset, $pos - $offset + 2)),
                    $pos + 2
                ];
            }
            throw new RuntimeException("Unterminated '/*'");
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getComment($text, $offset)
    {
        if (preg_match('/\A(?:#|--\s)/ms', substr($text, $offset, 3))) {
            $pos = strpos($text, "\n", $offset);
            if ($pos !== false) {
                return [
                    new Token(Token::COMMENT, substr($text, $offset, $pos - $offset)),
                    $pos + 1
                ];
            }
            return [
                new Token(Token::COMMENT, $text),
                strlen($text)
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getString($text, $offset)
    {
        $quote = $text[$offset];
        if (preg_match(
            '/' .
            $quote .
            '(' .
                '(?:' .
                    '[^\\\\' . $quote . ']' .   // not \ or "
                    '|\\\\.' .                  // escaped quotearacter
                    '|' .  $quote . $quote .    // ""
                ')*' .
            ')' .
            $quote .
            '/ms',
            $text,
            $pregMatch,
            0,
            $offset
        )
        ) {
            $token = Token::fromString($pregMatch[1], $quote);
            return [
                $token,
                $offset + strlen($pregMatch[0])
            ];
        }
        throw new RuntimeException("Unterminated string $quote...$quote");
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getNumber($text, $offset)
    {
        if (preg_match('/\A[-+]?[.]?[0-9]/ms', substr($text, $offset, 3)) &&
            preg_match('/[-+]?(?:[0-9]+(?:[.][0-9]*)?|[.][0-9]+)(?:[eE][-+]?[0-9]+)?/ms', $text, $pregMatch, 0, $offset)
        ) {
            return [
                new Token(Token::NUMBER, $pregMatch[0]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * Parse a hex string of the form "0x<hex digits>" or "x'<hex digits>'".
     *
     * https://dev.mysql.com/doc/refman/5.7/en/hexadecimal-literals.html
     *
     * - Only an even number of digits is valid.
     * - Case insensitive for hex digits.
     * - Case insensitive 'x' in quoted notation.
     * - Case sensitive 'x' for leading zero notation.
     *
     * Valid examples:
     * - x'BA5EBA11'
     * - x'decea5ed'
     * - X'5eed'
     * - 0xb01dface
     * - 0xBADC0DED
     *
     * Invalid examples
     * - x'00f' (odd number of digits)
     * - x'gg'  (invalid hex character)
     * - 0XFFFF (upper case 'x')
     *
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getHex($text, $offset)
    {
        $pregMatch = [];

        $matchesLeadingZeroNotation = function ($text, $offset, &$pregMatch) {
            return
                preg_match('/\A0x([0-9a-fA-F]*)/ms', $text, $pregMatch, 0, $offset);
        };

        $matchesXQuotedNotation = function ($text, $offset, &$pregMatch) {
            return
                preg_match('/\Ax\'[0-9a-f\']/ims', substr($text, $offset, 3)) &&
                preg_match('/x\'([0-9a-f]*)\'/ims', $text, $pregMatch, 0, $offset);
        };

        if ($matchesLeadingZeroNotation($text, $offset, $pregMatch) ||
            $matchesXQuotedNotation($text, $offset, $pregMatch)) {
            if (strlen($pregMatch[1]) % 2 != 0) {
                throw new RuntimeException("Invalid hex literal");
            }
            return [
                new Token(Token::HEX, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array|null
     */
    private function getBin($text, $offset)
    {
        if (preg_match('/\Ab\'[01\']/ms', substr($text, $offset, 3)) &&
            preg_match('/b\'([01]*)\'/ms', $text, $pregMatch, 0, $offset)
        ) {
            return [
                new Token(Token::BIN, $pregMatch[1]),
                $offset + strlen($pregMatch[0])
            ];
        }
        return null;
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getIdentifier($text, $offset)
    {
        preg_match('/[a-zA-Z0-9$_]+()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset);
        return [
            new Token(Token::IDENTIFIER, $pregMatch[0][0]),
            $pregMatch[1][1]
        ];
    }

    /**
     * @param string $text
     * @param int $offset
     * @return array
     */
    private function getSymbol($text, $offset)
    {
        if (preg_match('/\A(?:[-+*.\/])/xms', substr($text, $offset, 2), $pregMatch)) {
            return [
                new Token(Token::SYMBOL, $pregMatch[0]),
                $offset + strlen($pregMatch[0])
            ];
        }

        return [];
    }

    /**
     * @return Token|mixed
     */
    public function nextToken()
    {
        while (true) {
            $token = $this->nextTokenRaw();
            if (!isset(self::$skipTokenTypes[$token->type])) {
                return $token;
            }
        }

        return null;
    }

    /**
     * @return object
     */
    public function getMark()
    {
        return (object)[
            'offset'        => $this->offset,
            'inConditional' => $this->inConditional,
        ];
    }

    /**
     * @param mixed $mark
     */
    public function rewind($mark)
    {
        $this->offset        = $mark->offset;
        $this->inConditional = $mark->inConditional;
    }

    /**
     * This function will consume the requested content from the stream without trying to parse and tokenise it.
     * It is used by {@see peek()}.
     *
     * @param mixed $spec
     * @return bool
     */
    public function consume($spec)
    {
        // inline getMark()
        $markOffset        = $this->offset;
        $markInConditional = $this->inConditional;

        if (is_string($spec)) {
            foreach (explode(' ', $spec) as $text) {
                $token = $this->nextToken();
                // inline $token->eq(...)

                if (strcasecmp($token->text, $text) !== 0 ||
                    $token->type !== Token::IDENTIFIER
                ) {
                    // inline rewind()
                    $this->offset        = $markOffset;
                    $this->inConditional = $markInConditional;
                    return false;
                }
            }
        } else {
            foreach ($spec as $match) {
                list($type, $text) = $match;
                $token = $this->nextToken();
                // inline $token->eq(...)

                if (strcasecmp($token->text, $text) !== 0 ||
                    $token->type !== $type
                ) {
                    // inline rewind()
                    $this->offset        = $markOffset;
                    $this->inConditional = $markInConditional;
                    return false;
                }
            }
        }

        return true;
    }

    /**
     * @param mixed $spec
     * @return bool
     */
    public function peek($spec)
    {
        // inline getMark()
        $markOffset        = $this->offset;
        $markInConditional = $this->inConditional;

        $result = $this->consume($spec);

        // inline rewind()
        $this->offset        = $markOffset;
        $this->inConditional = $markInConditional;

        return $result;
    }

    /**
     * @param string $type
     * @param string $text
     * @return string
     */
    public function expect($type, $text = null)
    {
        $token = $this->nextToken();
        if (!$token->eq($type, $text)) {
            throw new RuntimeException("Expected '$text'");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectName()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::IDENTIFIER) {
            throw new RuntimeException("Expected identifier");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectOpenParen()
    {
        return $this->expect(Token::SYMBOL, '(');
    }

    /**
     * @return string
     */
    public function expectCloseParen()
    {
        return $this->expect(Token::SYMBOL, ')');
    }

    /**
     * @return int
     */
    public function expectNumber()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::NUMBER) {
            throw new RuntimeException("Expected number");
        }
        return 1 * $token->text;
    }

    /**
     * @return string
     */
    public function expectString()
    {
        $token = $this->nextToken();
        if ($token->type !== Token::STRING) {
            throw new RuntimeException("Expected string");
        }
        return $token->text;
    }

    /**
     * @return string
     */
    public function expectStringExtended()
    {
        $token = $this->nextToken();
        switch ($token->type) {
            case Token::STRING:
                return $token->text;
            case Token::HEX:
                return $token->asString();
            case Token::BIN:
                return $token->asString();
            default:
                throw new RuntimeException("Expected string");
        }
    }

    /**
     * Provides context for error messages.
     *
     * For example, given this invalid table definition ...
     *
     *     CREATE TABLE `foo` (
     *         `a` bar DEFAULT NULL
     *     );
     *
     * ... this function will produce something like this:
     *
     *     schema/morphism test/foo.sql, line 2: unknown datatype 'bar'
     *     1: CREATE TABLE `foo` (
     *     2:   `a` bar<<HERE>> DEFAULT NULL
     *
     * @param string $message
     * @return string
     */
    public function contextualise($message)
    {
        $preContextLines = 4;
        $postContextLines = 0;

        // get position of eol strictly before offset
        $prevEolPos = strrpos($this->text, "\n", $this->offset - strlen($this->text) - 1);
        $prevEolPos = ($prevEolPos === false) ? -1 : $prevEolPos;

        // get position of eol on or after offset
        $nextEolPos = strpos($this->text, "\n", $this->offset);
        $nextEolPos = ($nextEolPos === false) ? strlen($this->text) : $nextEolPos;

        // count number of newlines up to but not including offset
        $lineNo = substr_count($this->text, "\n", 0, $this->offset);
        $lines = explode("\n", $this->text);

        $contextLines = array_slice(
            $lines,
            max(0, $lineNo - $preContextLines),
            min($lineNo, $preContextLines),
            true // preserve keys
        );
        $contextLines += [
            $lineNo =>
                substr($this->text, $prevEolPos + 1, $this->offset - ($prevEolPos + 1)) .
                "<<HERE>>".
                substr($this->text, $this->offset, $nextEolPos - $this->offset)
        ];
        $contextLines += array_slice(
            $lines,
            $lineNo + 1,
            $postContextLines,
            true // preserve keys
        );

        $context = '';
        $width = strlen($lineNo + 1 + $postContextLines);
        foreach ($contextLines as $i => $line) {
            $context .= sprintf("\n%{$width}d: %s", $i + 1, $line);
        }

        return sprintf("%s, line %d: %s%s", $this->path, $lineNo + 1, $message, $context);
    }
}


1		<?php
2		namespace Graze\Morphism\Parse;
3
4		use LogicException;
5		use RuntimeException;
6
7		class TokenStream
8		{
9		/** @var string */
10		private $path;
11		/** @var string */
12		private $text;
13		/** @var int */
14		private $len;
15		/** @var int */
16		private $offset = 0;
17		/** @var bool */
18		private $inConditional = false;
19		/** @var array */
20		private $memo = [];
21
22		/** @var array */
23		private static $skipTokenTypes = [
24		Token::CONDITIONAL_START => true,
25		Token::CONDITIONAL_END => true,
26		Token::COMMENT => true,
27		Token::WHITESPACE => true,
28		];
29
30	231	private function __construct()
31		{
32	231	}
33
34		// TODO - this is a hack that needs to be refactored
		0 ignored issues – show Coding Style Best Practice introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report Comments for TODO tasks are often forgotten in the code; it might be better to use a dedicated issue tracker. Loading history...
35		// perhaps supply a LineStream interface that is satisfied
36		// by a FileLineStream or ConnectionLineStream for example?
37		/**
38		* @param string $text
39		* @param string $label
40		* @return TokenStream
41		*/
42		public static function newFromText($text, $label)
43		{
44		$stream = new self;
45		$stream->path = $label;
46		$stream->text = $text;
47		$stream->len = strlen($text);
48		return $stream;
49		}
50
51		/**
52		* @param string $path
53		* @return TokenStream
54		*/
55	231	public static function newFromFile($path)
56		{
57	231	$text = @file_get_contents($path);
58	231	if ($text === false) {
59		throw new RuntimeException("$path: could not open file");
60		}
61	231	$stream = new self;
62	231	$stream->path = $path;
63	231	$stream->text = $text;
64	231	$stream->len = strlen($text);
65	231	return $stream;
66		}
67
68		/**
69		* @return Token\|mixed
70		*/
71	231	private function nextTokenRaw()
72		{
73	231	$startOffset = $this->offset;
74	231	if (isset($this->memo[$startOffset])) {
75	221	$entry = $this->memo[$startOffset];
76	221	$this->offset = $entry[0];
77	221	$this->inConditional = $entry[1];
78	221	return $entry[2];
79		}
80
81	231	if ($this->offset >= $this->len) {
82	207	$token = new Token(Token::EOF);
83	207	$this->offset = $this->len;
84		} else {
85	231	list($token, $offset) = $this->getNextTokenRaw($this->text, $this->offset);
86	231	$this->offset = $offset;
87		}
88
89	231	$this->memo[$startOffset] = [
90	231	$this->offset,
91	231	$this->inConditional,
92	231	$token,
93		];
94
95	231	return $token;
96		}
97
98		/**
99		* @param string $text
100		* @param int $offset
101		* @return array\|null
102		*/
103	231	private function getNextTokenRaw($text, $offset)
104		{
105		// currently unsupported:
106		//
107		// charset prefixed strings, e.g. _utf8'wingwang'
108		// temporal literals, e.g. DATE'2014-07-08'
109		// the null literal, i.e. \N
110
111	231	switch ($text[$offset]) {
112	231	case " ":
113	231	case "\n":
114	231	case "\r":
115	231	case "\t":
116	229	$n = strspn($text, " \n\r\t", $offset);
117		return [
118	229	new Token(Token::WHITESPACE, substr($text, $offset, $n)),
119	229	$offset + $n
120		];
121
122	231	case '#':
123		return
124		$this->getComment($text, $offset);
125
126	231	case '.':
127	231	case '+':
128		return
129		$this->getNumber($text, $offset) ?:
130		$this->getSymbol($text, $offset);
131
132	231	case '-':
133		return
134		$this->getComment($text, $offset) ?:
135		$this->getNumber($text, $offset) ?:
136		$this->getSymbol($text, $offset);
137
138	231	case '*':
139		return
140		$this->getConditionEnd($text, $offset) ?:
141		$this->getSymbol($text, $offset);
142
143	231	case '/':
144		return
145		$this->getConditionalStart($text, $offset) ?:
146		$this->getMultilineComment($text, $offset) ?:
147		$this->getSymbol($text, $offset);
148
149	231	case '0':
150		// Handle hex if needed
151	5	if (isset($text[$offset+1]) && $text[$offset+1] === 'x') {
152		return
153		$this->getHex($text, $offset) ?:
154		$this->getIdentifier($text, $offset);
155		}
156		// Handle non-hex leading zero.
157	231	case '1':
158	231	case '2':
159	231	case '3':
160	231	case '4':
161	231	case '5':
162	231	case '6':
163	231	case '7':
164	231	case '8':
165	231	case '9':
166		return
167	61	$this->getNumber($text, $offset) ?:
168	61	$this->getIdentifier($text, $offset);
169
170	231	case '"':
171	231	case "'":
172		return
173	22	$this->getString($text, $offset);
174
175	231	case '`':
176		return
177		$this->getQuotedIdentifier($text, $offset);
178
179	231	case 'B':
180	231	case 'b':
181		return
182	21	$this->getBin($text, $offset) ?:
183	21	$this->getIdentifier($text, $offset);
184
185	231	case 'X':
186	231	case 'x':
187		return
188	222	$this->getHex($text, $offset) ?:
189	222	$this->getIdentifier($text, $offset);
190
191	224	case '$':
192	224	case '_':
193	224	case 'A':
194	224	case 'a':
195	224	case 'C':
196	224	case 'c':
197	220	case 'D':
198	220	case 'd':
199	202	case 'E':
200	202	case 'e':
201	202	case 'F':
202	202	case 'f':
203	197	case 'G':
204	197	case 'g':
205	197	case 'H':
206	197	case 'h':
207	197	case 'I':
208	197	case 'i':
209	181	case 'J':
210	181	case 'j':
211	181	case 'K':
212	181	case 'k':
213	179	case 'L':
214	179	case 'l':
215	175	case 'M':
216	175	case 'm':
217	170	case 'N':
218	149	case 'n':
219	144	case 'O':
220	144	case 'o':
221	144	case 'P':
222	144	case 'p':
223	141	case 'Q':
224	141	case 'q':
225	141	case 'R':
226	141	case 'r':
227	140	case 'S':
228	140	case 's':
229	135	case 'T':
230	135	case 't':
231	110	case 'U':
232	110	case 'u':
233	85	case 'V':
234	85	case 'v':
235	79	case 'W':
236	79	case 'w':
237	79	case 'Y':
238	79	case 'y':
239	69	case 'Z':
240	69	case 'z':
241		return
242	222	$this->getIdentifier($text, $offset);
243
244	65	case '!':
245	65	case '%':
246	65	case '&':
247	65	case '(':
248	63	case ')':
249	31	case ',':
250	6	case ':':
251	6	case ';':
252	6	case '<':
253	6	case '=':
254	1	case '>':
255	1	case '@':
256	1	case '^':
257	1	case '\|':
258		case '~':
259		return
260	65	$this->getSpecialSymbol($text, $offset);
261
262		case '?':
263		case '[':
264		case '\\':
265		case ']':
266		case '{':
267		case '}':
268		default:
269		$ch = $text[$offset];
270		throw new LogicException("Lexer is confused by char '$ch' ord " . ord($ch));
271		}
272		}
273
274		/**
275		* @param string $text
276		* @param int $offset
277		* @return array
278		*/
279		private function getQuotedIdentifier($text, $offset)
280		{
281		if (preg_match('/`((?:[^`]\|``)*)`()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset)) {
282		$token = Token::fromIdentifier($pregMatch[1][0]);
283		return [
284		$token,
285		$pregMatch[2][1]
286		];
287		}
288		throw new RuntimeException("Unterminated identifier: $text");
289		}
290
291		/**
292		* @param string $text
293		* @param int $offset
294		* @return array
295		*/
296	65	private function getSpecialSymbol($text, $offset)
297		{
298		// TODO - should probably be a new token type 'variable' for @ and @@
		0 ignored issues – show Coding Style Best Practice introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report Comments for TODO tasks are often forgotten in the code; it might be better to use a dedicated issue tracker. Loading history...
299	65	preg_match('/\A(?:<=\|>=\|<>\|!=\|:=\|@@\|&&\|\\|\\|\|[=~!@%^&();:,<>\|])()/xms', substr($text, $offset, 2), $pregMatch, PREG_OFFSET_CAPTURE);
300		return [
301	65	new Token(Token::SYMBOL, $pregMatch[0][0]),
302	65	$offset + $pregMatch[1][1]
303		];
304		}
305
306		/**
307		* Get the start of a conditional comment.
308		*
309		* https://dev.mysql.com/doc/refman/5.7/en/comments.html
310		*
311		* Does not support optimiser hints. See examples below.
312		*
313		* @param string $text
314		* @param int $offset
315		* @return array\|null
316		*/
317		private function getConditionalStart($text, $offset)
318		{
319		// Example conditional comments which can't be displayed in the docblock because they clash:
320		// - /! MySQL-specific code / (execute the given code)
321		// - /!12345 MySQL-specific code / (execute the given code only if the version matches)
322		// Unsupported:
323		// - SELECT /+ BKA(t1) / FROM ... ;
324
325		if (// 10 comes from allowing for the /*! sequence, a MySQL version number, and a space
326		preg_match('_\A/\!([0-9])\s_ms', substr($text, $offset, 10)) &&
327		preg_match('_/\!([0-9])\s_ms', $text, $pregMatch, 0, $offset)
328		) {
329		$this->inConditional = true;
330		return [
331		new Token(Token::CONDITIONAL_START, $pregMatch[1]),
332		$offset + strlen($pregMatch[0])
333		];
334		}
335		return null;
336		}
337
338		/**
339		* Get the end of a conditional comment. See _getConditionStart() for details.
340		* @param string $text
341		* @param int $offset
342		* @return array\|null
343		*/
344		private function getConditionEnd($text, $offset)
345		{
346		if (substr($text, $offset, 2) === '*/') {
347		if (!$this->inConditional) {
348		throw new RuntimeException("Unexpected '*/'");
349		}
350		$this->inConditional = false;
351		return [
352		new Token(Token::CONDITIONAL_END),
353		$offset + 2
354		];
355		}
356		return null;
357		}
358
359		/**
360		* @param string $text
361		* @param int $offset
362		* @return array\|null
363		*/
364		private function getMultilineComment($text, $offset)
365		{
366		if (substr($text, $offset, 2) === '/*') {
367		$pos = strpos($text, '*/', $offset);
368		if ($pos !== false) {
369		return [
370		new Token(Token::COMMENT, substr($text, $offset, $pos - $offset + 2)),
371		$pos + 2
372		];
373		}
374		throw new RuntimeException("Unterminated '/*'");
375		}
376		return null;
377		}
378
379		/**
380		* @param string $text
381		* @param int $offset
382		* @return array\|null
383		*/
384		private function getComment($text, $offset)
385		{
386		if (preg_match('/\A(?:#\|--\s)/ms', substr($text, $offset, 3))) {
387		$pos = strpos($text, "\n", $offset);
388		if ($pos !== false) {
389		return [
390		new Token(Token::COMMENT, substr($text, $offset, $pos - $offset)),
391		$pos + 1
392		];
393		}
394		return [
395		new Token(Token::COMMENT, $text),
396		strlen($text)
397		];
398		}
399		return null;
400		}
401
402		/**
403		* @param string $text
404		* @param int $offset
405		* @return array
406		*/
407	22	private function getString($text, $offset)
408		{
409	22	$quote = $text[$offset];
410	22	if (preg_match(
411		'/' .
412	22	$quote .
413	22	'(' .
414	22	'(?:' .
415	22	'[^\\\\' . $quote . ']' . // not \ or "
416	22	'\|\\\\.' . // escaped quotearacter
417	22	'\|' . $quote . $quote . // ""
418	22	')*' .
419	22	')' .
420	22	$quote .
421	22	'/ms',
422	22	$text,
423	22	$pregMatch,
424	22	0,
425	22	$offset
426		)
427		) {
428	22	$token = Token::fromString($pregMatch[1], $quote);
429		return [
430	22	$token,
431	22	$offset + strlen($pregMatch[0])
432		];
433		}
434		throw new RuntimeException("Unterminated string $quote...$quote");
435		}
436
437		/**
438		* @param string $text
439		* @param int $offset
440		* @return array\|null
441		*/
442	61	private function getNumber($text, $offset)
443		{
444	61	if (preg_match('/\A[-+]?[.]?[0-9]/ms', substr($text, $offset, 3)) &&
445	61	preg_match('/[-+]?(?:[0-9]+(?:[.][0-9]*)?\|[.][0-9]+)(?:[eE][-+]?[0-9]+)?/ms', $text, $pregMatch, 0, $offset)
446		) {
447		return [
448	61	new Token(Token::NUMBER, $pregMatch[0]),
449	61	$offset + strlen($pregMatch[0])
450		];
451		}
452		return null;
453		}
454
455		/**
456		* Parse a hex string of the form "0x<hex digits>" or "x'<hex digits>'".
457		*
458		* https://dev.mysql.com/doc/refman/5.7/en/hexadecimal-literals.html
459		*
460		* - Only an even number of digits is valid.
461		* - Case insensitive for hex digits.
462		* - Case insensitive 'x' in quoted notation.
463		* - Case sensitive 'x' for leading zero notation.
464		*
465		* Valid examples:
466		* - x'BA5EBA11'
467		* - x'decea5ed'
468		* - X'5eed'
469		* - 0xb01dface
470		* - 0xBADC0DED
471		*
472		* Invalid examples
473		* - x'00f' (odd number of digits)
474		* - x'gg' (invalid hex character)
475		* - 0XFFFF (upper case 'x')
476		*
477		* @param string $text
478		* @param int $offset
479		* @return array\|null
480		*/
481	222	private function getHex($text, $offset)
482		{
483	222	$pregMatch = [];
484
485		$matchesLeadingZeroNotation = function ($text, $offset, &$pregMatch) {
486		return
487	222	preg_match('/\A0x([0-9a-fA-F]*)/ms', $text, $pregMatch, 0, $offset);
488	222	};
489
490	222	$matchesXQuotedNotation = function ($text, $offset, &$pregMatch) {
491		return
492	222	preg_match('/\Ax\'[0-9a-f\']/ims', substr($text, $offset, 3)) &&
493	222	preg_match('/x\'([0-9a-f]*)\'/ims', $text, $pregMatch, 0, $offset);
494	222	};
495
496	222	if ($matchesLeadingZeroNotation($text, $offset, $pregMatch) \|\|
497	222	$matchesXQuotedNotation($text, $offset, $pregMatch)) {
498	1	if (strlen($pregMatch[1]) % 2 != 0) {
499		throw new RuntimeException("Invalid hex literal");
500		}
501		return [
502	1	new Token(Token::HEX, $pregMatch[1]),
503	1	$offset + strlen($pregMatch[0])
504		];
505		}
506	222	return null;
507		}
508
509		/**
510		* @param string $text
511		* @param int $offset
512		* @return array\|null
513		*/
514	21	private function getBin($text, $offset)
515		{
516	21	if (preg_match('/\Ab\'[01\']/ms', substr($text, $offset, 3)) &&
517	21	preg_match('/b\'([01]*)\'/ms', $text, $pregMatch, 0, $offset)
518		) {
519		return [
520	1	new Token(Token::BIN, $pregMatch[1]),
521	1	$offset + strlen($pregMatch[0])
522		];
523		}
524	21	return null;
525		}
526
527		/**
528		* @param string $text
529		* @param int $offset
530		* @return array
531		*/
532	231	private function getIdentifier($text, $offset)
533		{
534	231	preg_match('/[a-zA-Z0-9$_]+()/ms', $text, $pregMatch, PREG_OFFSET_CAPTURE, $offset);
535		return [
536	231	new Token(Token::IDENTIFIER, $pregMatch[0][0]),
537	231	$pregMatch[1][1]
538		];
539		}
540
541		/**
542		* @param string $text
543		* @param int $offset
544		* @return array
545		*/
546		private function getSymbol($text, $offset)
547		{
548		if (preg_match('/\A(?:[-+*.\/])/xms', substr($text, $offset, 2), $pregMatch)) {
549		return [
550		new Token(Token::SYMBOL, $pregMatch[0]),
551		$offset + strlen($pregMatch[0])
552		];
553		}
554
555		return [];
556		}
557
558		/**
559		* @return Token\|mixed
560		*/
561	231	public function nextToken()
562		{
563	231	while (true) {
564	231	$token = $this->nextTokenRaw();
565	231	if (!isset(self::$skipTokenTypes[$token->type])) {
566	231	return $token;
567		}
568		}
569
570		return null;
571		}
572
573		/**
574		* @return object
575		*/
576	216	public function getMark()
577		{
578		return (object)[
579	216	'offset' => $this->offset,
580	216	'inConditional' => $this->inConditional,
581		];
582		}
583
584		/**
585		* @param mixed $mark
586		*/
587	213	public function rewind($mark)
588		{
589	213	$this->offset = $mark->offset;
590	213	$this->inConditional = $mark->inConditional;
591	213	}
592
593		/**
594		* This function will consume the requested content from the stream without trying to parse and tokenise it.
595		* It is used by {@see peek()}.
596		*
597		* @param mixed $spec
598		* @return bool
599		*/
600	225	public function consume($spec)
601		{
602		// inline getMark()
603	225	$markOffset = $this->offset;
604	225	$markInConditional = $this->inConditional;
605
606	225	if (is_string($spec)) {
607	215	foreach (explode(' ', $spec) as $text) {
608	215	$token = $this->nextToken();
609		// inline $token->eq(...)
		0 ignored issues – show Unused Code Comprehensibility introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report `56%` of this comment could be valid code. Did you maybe forget this after debugging? Sometimes obsolete code just ends up commented out instead of removed. In this case it is better to remove the code once you have checked you do not need it. The code might also have been commented out for debugging purposes. In this case it is vital that someone uncomments it again or your project may behave in very unexpected ways in production. This check looks for comments that seem to be mostly valid code and reports them. Loading history...
610	215	if (strcasecmp($token->text, $text) !== 0 \|\|
611	215	$token->type !== Token::IDENTIFIER
612		) {
613		// inline rewind()
614	213	$this->offset = $markOffset;
615	213	$this->inConditional = $markInConditional;
616	215	return false;
617		}
618		}
619		} else {
620	202	foreach ($spec as $match) {
621	202	list($type, $text) = $match;
622	202	$token = $this->nextToken();
623		// inline $token->eq(...)
		0 ignored issues – show Unused Code Comprehensibility introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report `56%` of this comment could be valid code. Did you maybe forget this after debugging? Sometimes obsolete code just ends up commented out instead of removed. In this case it is better to remove the code once you have checked you do not need it. The code might also have been commented out for debugging purposes. In this case it is vital that someone uncomments it again or your project may behave in very unexpected ways in production. This check looks for comments that seem to be mostly valid code and reports them. Loading history...
624	202	if (strcasecmp($token->text, $text) !== 0 \|\|
625	202	$token->type !== $type
626		) {
627		// inline rewind()
628	181	$this->offset = $markOffset;
629	181	$this->inConditional = $markInConditional;
630	202	return false;
631		}
632		}
633		}
634
635	99	return true;
636		}
637
638		/**
639		* @param mixed $spec
640		* @return bool
641		*/
642		public function peek($spec)
643		{
644		// inline getMark()
645		$markOffset = $this->offset;
646		$markInConditional = $this->inConditional;
647
648		$result = $this->consume($spec);
649
650		// inline rewind()
651		$this->offset = $markOffset;
652		$this->inConditional = $markInConditional;
653
654		return $result;
655		}
656
657		/**
658		* @param string $type
659		* @param string $text
660		* @return string
661		*/
662	55	public function expect($type, $text = null)
663		{
664	55	$token = $this->nextToken();
665	55	if (!$token->eq($type, $text)) {
666		throw new RuntimeException("Expected '$text'");
667		}
668	55	return $token->text;
669		}
670
671		/**
672		* @return string
673		*/
674	230	public function expectName()
675		{
676	230	$token = $this->nextToken();
677	230	if ($token->type !== Token::IDENTIFIER) {
678		throw new RuntimeException("Expected identifier");
679		}
680	230	return $token->text;
681		}
682
683		/**
684		* @return string
685		*/
686	13	public function expectOpenParen()
687		{
688	13	return $this->expect(Token::SYMBOL, '(');
689		}
690
691		/**
692		* @return string
693		*/
694	42	public function expectCloseParen()
695		{
696	42	return $this->expect(Token::SYMBOL, ')');
697		}
698
699		/**
700		* @return int
701		*/
702	44	public function expectNumber()
703		{
704	44	$token = $this->nextToken();
705	44	if ($token->type !== Token::NUMBER) {
706		throw new RuntimeException("Expected number");
707		}
708	44	return 1 * $token->text;
709		}
710
711		/**
712		* @return string
713		*/
714	1	public function expectString()
715		{
716	1	$token = $this->nextToken();
717	1	if ($token->type !== Token::STRING) {
718		throw new RuntimeException("Expected string");
719		}
720	1	return $token->text;
721		}
722
723		/**
724		* @return string
725		*/
726	13	public function expectStringExtended()
727		{
728	13	$token = $this->nextToken();
729	13	switch ($token->type) {
730	13	case Token::STRING:
731	13	return $token->text;
732		case Token::HEX:
733		return $token->asString();
734		case Token::BIN:
735		return $token->asString();
736		default:
737		throw new RuntimeException("Expected string");
738		}
739		}
740
741		/**
742		* Provides context for error messages.
743		*
744		* For example, given this invalid table definition ...
745		*
746		* CREATE TABLE `foo` (
747		* `a` bar DEFAULT NULL
748		* );
749		*
750		* ... this function will produce something like this:
751		*
752		* schema/morphism test/foo.sql, line 2: unknown datatype 'bar'
753		* 1: CREATE TABLE `foo` (
754		* 2: `a` bar<<HERE>> DEFAULT NULL
755		*
756		* @param string $message
757		* @return string
758		*/
759		public function contextualise($message)
760		{
761		$preContextLines = 4;
762		$postContextLines = 0;
763
764		// get position of eol strictly before offset
765		$prevEolPos = strrpos($this->text, "\n", $this->offset - strlen($this->text) - 1);
766		$prevEolPos = ($prevEolPos === false) ? -1 : $prevEolPos;
767
768		// get position of eol on or after offset
769		$nextEolPos = strpos($this->text, "\n", $this->offset);
770		$nextEolPos = ($nextEolPos === false) ? strlen($this->text) : $nextEolPos;
771
772		// count number of newlines up to but not including offset
773		$lineNo = substr_count($this->text, "\n", 0, $this->offset);
774		$lines = explode("\n", $this->text);
775
776		$contextLines = array_slice(
777		$lines,
778		max(0, $lineNo - $preContextLines),
779		min($lineNo, $preContextLines),
780		true // preserve keys
781		);
782		$contextLines += [
783		$lineNo =>
784		substr($this->text, $prevEolPos + 1, $this->offset - ($prevEolPos + 1)) .
785		"<<HERE>>".
786		substr($this->text, $this->offset, $nextEolPos - $this->offset)
787		];
788		$contextLines += array_slice(
789		$lines,
790		$lineNo + 1,
791		$postContextLines,
792		true // preserve keys
793		);
794
795		$context = '';
796		$width = strlen($lineNo + 1 + $postContextLines);
797		foreach ($contextLines as $i => $line) {
798		$context .= sprintf("\n%{$width}d: %s", $i + 1, $line);
799		}
800
801		return sprintf("%s, line %d: %s%s", $this->path, $lineNo + 1, $message, $context);
802		}
803		}
804

graze / morphism

GitHub Access Token became invalid

Pull Request — master (#60)

TokenStream::getString() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like