TokenStreamTest::testNewFromText() - Code Metrics - Inspection of "Code coverage for Graze\Morphism\Parse\TokenStream" - graze/morphism - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#42)

by Burhan

created 2018-01-01 18:15 UTC

TokenStreamTest::testNewFromText() A

↳ Parent: Project

Complexity

Conditions	1
Paths	1

Size

Total Lines	5
Code Lines	3

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
dl	0
loc	5
rs	9.4285
c	0
b	0
f	0
cc	1
eloc	3
nc	1
nop	0

<?php

namespace Graze\Morphism\Parse;

use Graze\Morphism\Test\Parse\TestCase;
use RuntimeException;

class TokenStreamTest extends TestCase
{
    public function testNewFromText()
    {
        $stream = TokenStream::newFromText('', '');
        $this->assertThat($stream, $this->isInstanceOf(__NAMESPACE__ . '\TokenStream'));
    }

    public function testNewFromFile()
    {
        $stream = TokenStream::newFromFile("/dev/null");
        $this->assertThat($stream, $this->isInstanceOf(__NAMESPACE__ . '\TokenStream'));
    }

    /** @expectedException \Exception */
    public function testNewFromFileNotFound()
    {
        TokenStream::newFromFile(dirname(__FILE__) . "/file_not_found");
    }

    /**
     * @param string $expectedType
     * @param mixed $expectedValue
     * @param string $token
     */
    public function assertTokenEq($expectedType, $expectedValue, $token)
    {
        $this->assertTrue(
            $token->eq($expectedType, $expectedValue),

            "expected {$expectedType}[{$expectedValue}], but got " . $token->toDebugString()

        );
    }

    /**
     * @dataProvider nextTokenProvider
     * @param string $text
     * @param string $expectedType
     * @param mixed $expectedNextTokenValue
     */
    public function testNextToken($text, $expectedType, $expectedNextTokenValue)

    {
        $stream = $this->makeStream($text);
        $token = $stream->nextToken();
        $this->assertTokenEq($expectedType, $expectedNextTokenValue, $token);
    }

    /**
     * @return array
     */
    public function nextTokenProvider()
    {
        $sq = "'";
        $dq = '"';
        $bq = '`';
        $bs = "\\";

        return [
            [ '',          Token::EOF,    ''          ],

            // numbers
            [ '1',         Token::NUMBER, '1'         ],
            [ '123',       Token::NUMBER, '123'       ],
            [ '123.45',    Token::NUMBER, '123.45'    ],
            [ '.45',       Token::NUMBER, '.45'       ],
            [ '123.',      Token::NUMBER, '123.'      ],
            [ '-123',      Token::NUMBER, '-123'      ],
            [ '+123',      Token::NUMBER, '+123'      ],
            [ '1E23',      Token::NUMBER, '1E23'      ],
            [ '1e23',      Token::NUMBER, '1e23'      ],
            [ '1e+23',     Token::NUMBER, '1e+23'     ],
            [ '1e-23',     Token::NUMBER, '1e-23'     ],
            [ '+1.23e-17', Token::NUMBER, '+1.23e-17' ],

            // whitespace
            [ " 1",  Token::NUMBER, 1],
            [ "\t1", Token::NUMBER, 1],
            [ "\n1", Token::NUMBER, 1],

            // comments
            [ "/*comment*/1",   Token::NUMBER, '1'],
            [ "/**/1",          Token::NUMBER, '1'],
            [ "-- comment\n1",  Token::NUMBER, '1'],
            [ "--\n1",          Token::NUMBER, '1'],
            [ "#comment\n1",    Token::NUMBER, '1'],
            [ "#comment",       Token::EOF,    null],

            // conditional comments
            [ "/*! 12345*/",      Token::NUMBER, '12345'],
            [ "/*!12345 45678*/", Token::NUMBER, '45678'],

            // double quoted strings
            [ "{$dq}{$dq}",                     Token::STRING, ''],
            [ "{$dq}hello world{$dq}",          Token::STRING, 'hello world'],
            [ "{$dq}hello{$dq}{$dq}world{$dq}", Token::STRING, "hello{$dq}world"],     // "" => "
            [ "{$dq}hello{$bs}{$bs}world{$dq}", Token::STRING, "hello{$bs}world"],     // \\ => \
            [ "{$dq}hello{$bs}{$dq}world{$dq}", Token::STRING, "hello{$dq}world"],     // \" => "

            // single quoted strings
            [ "{$sq}{$sq}",                     Token::STRING, ''],
            [ "{$sq}hello{$sq}",                Token::STRING, 'hello'],
            [ "{$sq}hello{$sq}{$sq}world{$sq}", Token::STRING, "hello{$sq}world"],     // '' => '
            [ "{$sq}hello{$bs}{$bs}world{$sq}", Token::STRING, "hello{$bs}world"],     // \\ => \
            [ "{$sq}hello{$bs}{$sq}world{$sq}", Token::STRING, "hello{$sq}world"],     // \' => '

            // backquoted identifiers
            [ "{$bq}{$bq}",                     Token::IDENTIFIER, ''],
            [ "{$bq}hello{$bq}",                Token::IDENTIFIER, 'hello'],
            [ "{$bq}hello{$bq}{$bq}world{$bq}", Token::IDENTIFIER, "hello{$bq}world"],      // `` => `
            [ "{$bq}hello{$bs}{$bs}world{$bq}", Token::IDENTIFIER, "hello{$bs}${bs}world"], // \\ => \\
            [ "{$bq}hello{$bs}nworld{$bq}",     Token::IDENTIFIER, "hello{$bs}nworld"],     // \n => \n

            // hex literals
            [ "x''",                    Token::HEX, "" ],
            [ "x'00'",                  Token::HEX, "00" ],
            [ "x'0123456789abcdef'",    Token::HEX, "0123456789abcdef" ],
            [ "x'0123456789ABCDEF'",    Token::HEX, "0123456789ABCDEF" ],
            [ "0x0123456789abcdef",     Token::HEX, "0123456789abcdef" ],
            [ "0x0123456789ABCDEF",     Token::HEX, "0123456789ABCDEF" ],

            // binary literals
            [ "b''",            Token::BIN, "" ],
            [ "b'0'",           Token::BIN, "0" ],
            [ "b'00011011'",    Token::BIN, "00011011" ],

            // Invalid hex and binary literals - these should probably be failures, tbh.
            [ "x'GGG'",         Token::IDENTIFIER,  'x'],
            [ '0Xabc',          Token::NUMBER,      '0'],
            [ "b'2'",           Token::IDENTIFIER,  'b'],

            // unquoted identifiers
       //   [ '1_',       Token::IDENTIFIER, '1_' ],     // TODO - make this pass

            [ '_',        Token::IDENTIFIER, '_' ],
            [ '$',        Token::IDENTIFIER, '$' ],
            [ 'a',        Token::IDENTIFIER, 'a' ],
            [ 'abc',      Token::IDENTIFIER, 'abc' ],
            [ 'abc123',   Token::IDENTIFIER, 'abc123' ],
            [ '_abc',     Token::IDENTIFIER, '_abc' ],
            [ '_123',     Token::IDENTIFIER, '_123' ],
            [ '$_123abc', Token::IDENTIFIER, '$_123abc' ],

            // symbols
            [ "<=_", Token::SYMBOL, "<=" ],
            [ ">=_", Token::SYMBOL, ">=" ],
            [ "<>_", Token::SYMBOL, "<>" ],
            [ "!=_", Token::SYMBOL, "!=" ],
            [ ":=_", Token::SYMBOL, ":=" ],
            [ "&&_", Token::SYMBOL, "&&" ],
            [ "||_", Token::SYMBOL, "||" ],
            [ "@@_", Token::SYMBOL, "@@" ],
            [ "@_",  Token::SYMBOL, "@" ],
            [ "+_",  Token::SYMBOL, "+"  ],
            [ "-_",  Token::SYMBOL, "-"  ],
            [ "*_",  Token::SYMBOL, "*"  ],
            [ "/_",  Token::SYMBOL, "/"  ],
            [ "%_",  Token::SYMBOL, "%"  ],
        ];
    }

    /**
     * @param string $text
     * @dataProvider provideBadLogicNextToken
     * @expectedException LogicException
     */
    public function testBadLogicNextToken($text)
    {
        $stream = $this->makeStream($text);
        $token = $stream->nextToken();
$myVar = 'Value';
$higher = false;

if (rand(1, 6) > 3) {
    $higher = true;
} else {
    $higher = false;
}
    }

    /**
     * @return array
     */
    public function provideBadLogicNextToken()
    {
        return [
            // All of these are explicitly not valid and will result in a "Lexer is confused by ..." message.
            ['?'],
            ['['],
            [']'],
            ['\\'],
            ['{'],
            ['}'],
            // This item covers the fall through value for any characters not explicitly listed
            [chr(0)],
        ];
    }

    /**
     * @param string $text
     * @dataProvider provideBadRuntimeNextToken
     * @expectedException RuntimeException
     */
    public function testBadRuntimeNextToken($text)
    {
        $stream = $this->makeStream($text);
        $token = $stream->nextToken();
$myVar = 'Value';
$higher = false;

if (rand(1, 6) > 3) {
    $higher = true;
} else {
    $higher = false;
}
    }

    /**
     * @return array
     */
    public function provideBadRuntimeNextToken()
    {
        return [
            // Unterminated quoted identifier
            ['`foo'],
            // Unterminated '/*'
            ['/*'],
            // Unexpected end of comment
            ['*/'],
            // Unterminated string
            ["'foo"],
            // Invalid hex literal (not an even number of digits)
            ['0xaaa'],
        ];
    }

    public function testRewind()
    {
        $stream = $this->makeStream("create table t (x int, y int)");
        $stream->nextToken();
        $mark = $stream->getMark();
        $token11 = $stream->nextToken();
        $token12 = $stream->nextToken();
        $stream->rewind($mark);
        $token21 = $stream->nextToken();
        $token22 = $stream->nextToken();

        $this->assertTokenEq($token21->type, $token21->text, $token11);
        $this->assertTokenEq($token22->type, $token22->text, $token12);
    }

    /**
     * @dataProvider consumeProvider
     * @param string $text
     * @param mixed $spec
     * @param bool $success
     * @param string $type
     * @param string $value
     */
    public function testConsume($text, $spec, $success, $type, $value)

    {
        $stream = $this->makeStream($text);
        $this->assertSame(
            (bool)$success,
            (bool)$stream->consume($spec),
            "consume did not return " . ($success ? 'true' : 'false')
        );
        $token = $stream->nextToken();
        $this->assertTokenEq($type, $value, $token);
    }

    /**
     * @return array
     */
    public function consumeProvider()
    {
        return [
            ['create table t', 'create',          true,  Token::IDENTIFIER, 'table'],
            ['create table t', 'create table',    true,  Token::IDENTIFIER, 't'],
            ['create table t', 'drop',            false, Token::IDENTIFIER, 'create'],
            ['create table t', 'drop table',      false, Token::IDENTIFIER, 'create'],
            ['create table t', 'create database', false, Token::IDENTIFIER, 'create'],
            ['= "test"',       [[Token::SYMBOL, '=']], true,  Token::STRING, 'test'],
            ['= "test"',       [[Token::NUMBER, 1  ]], false, Token::SYMBOL, '='],
            ['();',            [[Token::SYMBOL, '('],
                                [Token::SYMBOL, ')']], true,  Token::SYMBOL, ';'],
        ];
    }

    /**
     * @dataProvider peekProvider
     * @param string $text
     * @param mixed $spec
     * @param bool $success
     * @param string $type
     * @param string $value
     */
    public function testPeek($text, $spec, $success, $type, $value)

    {
        $stream = $this->makeStream($text);
        $this->assertSame(
            (bool)$success,
            (bool)$stream->peek($spec),
            "peek did not return " . ($success ? 'true' : 'false')
        );
        $token = $stream->nextToken();
        $this->assertTokenEq($type, $value, $token);
    }

    /**
     * @return array
     */
    public function peekProvider()
    {
        return [
            ['create table t', 'create',          true,  Token::IDENTIFIER, 'create'],
            ['create table t', 'create table',    true,  Token::IDENTIFIER, 'create'],
            ['create table t', 'drop',            false, Token::IDENTIFIER, 'create'],
            ['create table t', 'drop table',      false, Token::IDENTIFIER, 'create'],
            ['create table t', 'create database', false, Token::IDENTIFIER, 'create'],
            ['= "test"',       [[Token::SYMBOL, '=']], true,  Token::SYMBOL, '='],
            ['();',            [[Token::SYMBOL, '('],
                                [Token::SYMBOL, ')']], true,  Token::SYMBOL, '('],
        ];
    }

    public function testExpectSucc()
    {
        $stream = $this->makeStream('create table t');
        $stream->expect(Token::IDENTIFIER, 'create');
    }

    /** @expectedException \Exception */
    public function testExpectFail()
    {
        $stream = $this->makeStream('create table t');
        $stream->expect(Token::IDENTIFIER, 'drop');
    }

    /**
     * @param string $func
     * @param string $token
     * @param mixed $expected
     * @param bool $throwsException
     * @dataProvider provideExpectedTokenType
     */
    public function testExpectedTokenType($func, $token, $expected, $throwsException)
    {
        $stream = $this->makeStream($token);
        if ($throwsException) {
            $this->setExpectedException(RuntimeException::class);
            $stream->$func();
        } else {
            $result = $stream->$func();
            $this->assertEquals($expected, $result);
        }
    }

    /**
     * @return array
     */
    public function provideExpectedTokenType()
    {
        return [
            // [ function name, token, expected value, should it throw a RuntimeException? ]

            [ 'expectCloseParen',   ')',    ')',    false],
            [ 'expectCloseParen',   'a',    null,   true],

            [ 'expectOpenParen',    '(',    '(',    false],
            [ 'expectOpenParen',    'a',    null,   true],

            [ 'expectName',         'foo',  'foo',  false],
            [ 'expectName',         '1',    null,   true],

            [ 'expectNumber',       '1',    1,      false],
            [ 'expectNumber',       'a',    null,   true],

            // An embedded string
            [ 'expectString',       "'a'",  "a",    false],
            [ 'expectString',       'a',    null,   true],

            [ 'expectStringExtended',       "'a'",  "a",    false],
            [ 'expectStringExtended',       "x'68656c6c6f21'",      'hello!',   false],
            [ 'expectStringExtended',       "X'68656c6c6f21'",      'hello!',   false],
            [ 'expectStringExtended',       '0x68656c6c6f21',      'hello!',   false],
            [ 'expectStringExtended',       "b'0111111000100011'",  '~#',       false],
            [ 'expectStringExtended',       'a',    null,   true],

        ];
    }

    /**
     * @param string $conditionalComment
     * @param array $tokenTypes
     * @dataProvider conditionalCommentProvider
     */
    public function testConditionalComment($conditionalComment, array $tokenTypes)
    {
        $stream = $this->makeStream($conditionalComment);
        $token = $stream->nextToken();
        $expectedType = current($tokenTypes);

        while ($token->type != Token::EOF) {
            $this->assertEquals($expectedType, $token->type);

            $token = $stream->nextToken();
            $expectedType = next($tokenTypes);
        }
    }

    /**
     * @return array
     */
    public function conditionalCommentProvider()
    {
        return [
            // [ conditional comment, list of expected token types ]
            [
                '/*! abcde fghij */',
                [
                    Token::IDENTIFIER,
                    Token::IDENTIFIER,
                ],
            ],
            [
                '/*!12345 fghij */',
                [
                    Token::IDENTIFIER,
                ],
            ],
        ];
    }

    public function testContextualise()
    {
        $sql = <<<EOF
CREATE TABLE `foo` (
    `a` bar DEFAULT NULL
);
EOF;
        $expected = <<<EOF
data://text/plain;base64,Q1JFQVRFIFRBQkxFIGBmb29gICgKICAgIGBhYCBiYXIgREVGQVVMVCBOVUxMCik7, line 2: unknown datatype 'bar'
1: CREATE TABLE `foo` (
2:     `a` bar<<HERE>> DEFAULT NULL
EOF;

        $stream = $this->makeStream($sql);

        // Use reflection to set the internal offset to the place where error is.
        $reflection = new \ReflectionClass($stream);
        $property = $reflection->getProperty('offset');
        $property->setAccessible(true);
        $property->setValue($stream, 32);

        $message = $stream->contextualise("unknown datatype 'bar'");
        $this->assertEquals($expected, $message);
    }
}


1		<?php
2
3		namespace Graze\Morphism\Parse;
4
5		use Graze\Morphism\Test\Parse\TestCase;
6		use RuntimeException;
7
8		class TokenStreamTest extends TestCase
9		{
10		public function testNewFromText()
11		{
12		$stream = TokenStream::newFromText('', '');
13		$this->assertThat($stream, $this->isInstanceOf(__NAMESPACE__ . '\TokenStream'));
14		}
15
16		public function testNewFromFile()
17		{
18		$stream = TokenStream::newFromFile("/dev/null");
19		$this->assertThat($stream, $this->isInstanceOf(__NAMESPACE__ . '\TokenStream'));
20		}
21
22		/** @expectedException \Exception */
23		public function testNewFromFileNotFound()
24		{
25		TokenStream::newFromFile(dirname(__FILE__) . "/file_not_found");
26		}
27
28		/**
29		* @param string $expectedType
30		* @param mixed $expectedValue
31		* @param string $token
32		*/
33		public function assertTokenEq($expectedType, $expectedValue, $token)
34		{
35		$this->assertTrue(
36		$token->eq($expectedType, $expectedValue),
		0 ignored issues – show Bug introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report The method `eq` cannot be called on `$token` (of type `string`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
37		"expected {$expectedType}[{$expectedValue}], but got " . $token->toDebugString()
		0 ignored issues – show Bug introduced 2017-07-07 13:35 UTC by Report Bug Copy Issue Report The method `toDebugString` cannot be called on `$token` (of type `string`). Methods can only be called on objects. This check looks for methods being called on variables that have been inferred to never be objects. Loading history...
38		);
39		}
40
41		/**
42		* @dataProvider nextTokenProvider
43		* @param string $text
44		* @param string $expectedType
45		* @param mixed $expectedNextTokenValue
46		*/
47		public function testNextToken($text, $expectedType, $expectedNextTokenValue)
		0 ignored issues – show Comprehensibility Naming introduced 2018-01-01 18:16 UTC by Report Bug Copy Issue Report The variable name `$expectedNextTokenValue` exceeds the maximum configured length of `20`. Very long variable names usually make code harder to read. It is therefore recommended not to make variable names too verbose. Loading history...
48		{
49		$stream = $this->makeStream($text);
50		$token = $stream->nextToken();
51		$this->assertTokenEq($expectedType, $expectedNextTokenValue, $token);
52		}
53
54		/**
55		* @return array
56		*/
57		public function nextTokenProvider()
58		{
59		$sq = "'";
60		$dq = '"';
61		$bq = '`';
62		$bs = "\\";
63
64		return [
65		[ '', Token::EOF, '' ],
66
67		// numbers
68		[ '1', Token::NUMBER, '1' ],
69		[ '123', Token::NUMBER, '123' ],
70		[ '123.45', Token::NUMBER, '123.45' ],
71		[ '.45', Token::NUMBER, '.45' ],
72		[ '123.', Token::NUMBER, '123.' ],
73		[ '-123', Token::NUMBER, '-123' ],
74		[ '+123', Token::NUMBER, '+123' ],
75		[ '1E23', Token::NUMBER, '1E23' ],
76		[ '1e23', Token::NUMBER, '1e23' ],
77		[ '1e+23', Token::NUMBER, '1e+23' ],
78		[ '1e-23', Token::NUMBER, '1e-23' ],
79		[ '+1.23e-17', Token::NUMBER, '+1.23e-17' ],
80
81		// whitespace
82		[ " 1", Token::NUMBER, 1],
83		[ "\t1", Token::NUMBER, 1],
84		[ "\n1", Token::NUMBER, 1],
85
86		// comments
87		[ "/comment/1", Token::NUMBER, '1'],
88		[ "/**/1", Token::NUMBER, '1'],
89		[ "-- comment\n1", Token::NUMBER, '1'],
90		[ "--\n1", Token::NUMBER, '1'],
91		[ "#comment\n1", Token::NUMBER, '1'],
92		[ "#comment", Token::EOF, null],
93
94		// conditional comments
95		[ "/! 12345/", Token::NUMBER, '12345'],
96		[ "/!12345 45678/", Token::NUMBER, '45678'],
97
98		// double quoted strings
99		[ "{$dq}{$dq}", Token::STRING, ''],
100		[ "{$dq}hello world{$dq}", Token::STRING, 'hello world'],
101		[ "{$dq}hello{$dq}{$dq}world{$dq}", Token::STRING, "hello{$dq}world"], // "" => "
102		[ "{$dq}hello{$bs}{$bs}world{$dq}", Token::STRING, "hello{$bs}world"], // \\ => \
103		[ "{$dq}hello{$bs}{$dq}world{$dq}", Token::STRING, "hello{$dq}world"], // \" => "
104
105		// single quoted strings
106		[ "{$sq}{$sq}", Token::STRING, ''],
107		[ "{$sq}hello{$sq}", Token::STRING, 'hello'],
108		[ "{$sq}hello{$sq}{$sq}world{$sq}", Token::STRING, "hello{$sq}world"], // '' => '
109		[ "{$sq}hello{$bs}{$bs}world{$sq}", Token::STRING, "hello{$bs}world"], // \\ => \
110		[ "{$sq}hello{$bs}{$sq}world{$sq}", Token::STRING, "hello{$sq}world"], // \' => '
111
112		// backquoted identifiers
113		[ "{$bq}{$bq}", Token::IDENTIFIER, ''],
114		[ "{$bq}hello{$bq}", Token::IDENTIFIER, 'hello'],
115		[ "{$bq}hello{$bq}{$bq}world{$bq}", Token::IDENTIFIER, "hello{$bq}world"], // `` => `
116		[ "{$bq}hello{$bs}{$bs}world{$bq}", Token::IDENTIFIER, "hello{$bs}${bs}world"], // \\ => \\
117		[ "{$bq}hello{$bs}nworld{$bq}", Token::IDENTIFIER, "hello{$bs}nworld"], // \n => \n
118
119		// hex literals
120		[ "x''", Token::HEX, "" ],
121		[ "x'00'", Token::HEX, "00" ],
122		[ "x'0123456789abcdef'", Token::HEX, "0123456789abcdef" ],
123		[ "x'0123456789ABCDEF'", Token::HEX, "0123456789ABCDEF" ],
124		[ "0x0123456789abcdef", Token::HEX, "0123456789abcdef" ],
125		[ "0x0123456789ABCDEF", Token::HEX, "0123456789ABCDEF" ],
126
127		// binary literals
128		[ "b''", Token::BIN, "" ],
129		[ "b'0'", Token::BIN, "0" ],
130		[ "b'00011011'", Token::BIN, "00011011" ],
131
132		// Invalid hex and binary literals - these should probably be failures, tbh.
133		[ "x'GGG'", Token::IDENTIFIER, 'x'],
134		[ '0Xabc', Token::NUMBER, '0'],
135		[ "b'2'", Token::IDENTIFIER, 'b'],
136
137		// unquoted identifiers
138		// [ '1_', Token::IDENTIFIER, '1_' ], // TODO - make this pass
		0 ignored issues – show Coding Style Best Practice introduced 2017-10-25 15:33 UTC by Report Bug Copy Issue Report Comments for TODO tasks are often forgotten in the code; it might be better to use a dedicated issue tracker. Loading history... Unused Code Comprehensibility introduced 2017-10-25 15:33 UTC by Report Bug Copy Issue Report `50%` of this comment could be valid code. Did you maybe forget this after debugging? Sometimes obsolete code just ends up commented out instead of removed. In this case it is better to remove the code once you have checked you do not need it. The code might also have been commented out for debugging purposes. In this case it is vital that someone uncomments it again or your project may behave in very unexpected ways in production. This check looks for comments that seem to be mostly valid code and reports them. Loading history...
139		[ '_', Token::IDENTIFIER, '_' ],
140		[ '$', Token::IDENTIFIER, '$' ],
141		[ 'a', Token::IDENTIFIER, 'a' ],
142		[ 'abc', Token::IDENTIFIER, 'abc' ],
143		[ 'abc123', Token::IDENTIFIER, 'abc123' ],
144		[ '_abc', Token::IDENTIFIER, '_abc' ],
145		[ '_123', Token::IDENTIFIER, '_123' ],
146		[ '$_123abc', Token::IDENTIFIER, '$_123abc' ],
147
148		// symbols
149		[ "<=_", Token::SYMBOL, "<=" ],
150		[ ">=_", Token::SYMBOL, ">=" ],
151		[ "<>_", Token::SYMBOL, "<>" ],
152		[ "!=_", Token::SYMBOL, "!=" ],
153		[ ":=_", Token::SYMBOL, ":=" ],
154		[ "&&_", Token::SYMBOL, "&&" ],
155		[ "\|\|_", Token::SYMBOL, "\|\|" ],
156		[ "@@_", Token::SYMBOL, "@@" ],
157		[ "@_", Token::SYMBOL, "@" ],
158		[ "+_", Token::SYMBOL, "+" ],
159		[ "-_", Token::SYMBOL, "-" ],
160		[ "_", Token::SYMBOL, "" ],
161		[ "/_", Token::SYMBOL, "/" ],
162		[ "%_", Token::SYMBOL, "%" ],
163		];
164		}
165
166		/**
167		* @param string $text
168		* @dataProvider provideBadLogicNextToken
169		* @expectedException LogicException
170		*/
171		public function testBadLogicNextToken($text)
172		{
173		$stream = $this->makeStream($text);
174		$token = $stream->nextToken();
		0 ignored issues – show Unused Code introduced 2018-01-01 18:16 UTC by Report Bug Copy Issue Report `$token` is not used, you could remove the assignment. This check looks for variable assignements that are either overwritten by other assignments or where the variable is not used subsequently. $myVar = 'Value'; $higher = false; if (rand(1, 6) > 3) { $higher = true; } else { $higher = false; } Both the `$myVar` assignment in line 1 and the `$higher` assignment in line 2 are dead. The first because `$myVar` is never used and the second because `$higher` is always overwritten for every possible time line. Loading history...
175		}
176
177		/**
178		* @return array
179		*/
180		public function provideBadLogicNextToken()
181		{
182		return [
183		// All of these are explicitly not valid and will result in a "Lexer is confused by ..." message.
184		['?'],
185		['['],
186		[']'],
187		['\\'],
188		['{'],
189		['}'],
190		// This item covers the fall through value for any characters not explicitly listed
191		[chr(0)],
192		];
193		}
194
195		/**
196		* @param string $text
197		* @dataProvider provideBadRuntimeNextToken
198		* @expectedException RuntimeException
199		*/
200		public function testBadRuntimeNextToken($text)
201		{
202		$stream = $this->makeStream($text);
203		$token = $stream->nextToken();
		0 ignored issues – show Unused Code introduced 2018-01-01 18:16 UTC by Report Bug Copy Issue Report `$token` is not used, you could remove the assignment. This check looks for variable assignements that are either overwritten by other assignments or where the variable is not used subsequently. $myVar = 'Value'; $higher = false; if (rand(1, 6) > 3) { $higher = true; } else { $higher = false; } Both the `$myVar` assignment in line 1 and the `$higher` assignment in line 2 are dead. The first because `$myVar` is never used and the second because `$higher` is always overwritten for every possible time line. Loading history...
204		}
205
206		/**
207		* @return array
208		*/
209		public function provideBadRuntimeNextToken()
210		{
211		return [
212		// Unterminated quoted identifier
213		['`foo'],
214		// Unterminated '/*'
215		['/*'],
216		// Unexpected end of comment
217		['*/'],
218		// Unterminated string
219		["'foo"],
220		// Invalid hex literal (not an even number of digits)
221		['0xaaa'],
222		];
223		}
224
225		public function testRewind()
226		{
227		$stream = $this->makeStream("create table t (x int, y int)");
228		$stream->nextToken();
229		$mark = $stream->getMark();
230		$token11 = $stream->nextToken();
231		$token12 = $stream->nextToken();
232		$stream->rewind($mark);
233		$token21 = $stream->nextToken();
234		$token22 = $stream->nextToken();
235
236		$this->assertTokenEq($token21->type, $token21->text, $token11);
237		$this->assertTokenEq($token22->type, $token22->text, $token12);
238		}
239
240		/**
241		* @dataProvider consumeProvider
242		* @param string $text
243		* @param mixed $spec
244		* @param bool $success
245		* @param string $type
246		* @param string $value
247		*/
248	View Code Duplication	public function testConsume($text, $spec, $success, $type, $value)
		0 ignored issues – show Duplication introduced 2017-08-01 20:34 UTC by Report Bug Copy Issue Report This method seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
249		{
250		$stream = $this->makeStream($text);
251		$this->assertSame(
252		(bool)$success,
253		(bool)$stream->consume($spec),
254		"consume did not return " . ($success ? 'true' : 'false')
255		);
256		$token = $stream->nextToken();
257		$this->assertTokenEq($type, $value, $token);
258		}
259
260		/**
261		* @return array
262		*/
263		public function consumeProvider()
264		{
265		return [
266		['create table t', 'create', true, Token::IDENTIFIER, 'table'],
267		['create table t', 'create table', true, Token::IDENTIFIER, 't'],
268		['create table t', 'drop', false, Token::IDENTIFIER, 'create'],
269		['create table t', 'drop table', false, Token::IDENTIFIER, 'create'],
270		['create table t', 'create database', false, Token::IDENTIFIER, 'create'],
271		['= "test"', [[Token::SYMBOL, '=']], true, Token::STRING, 'test'],
272		['= "test"', [[Token::NUMBER, 1 ]], false, Token::SYMBOL, '='],
273		['();', [[Token::SYMBOL, '('],
274		[Token::SYMBOL, ')']], true, Token::SYMBOL, ';'],
275		];
276		}
277
278		/**
279		* @dataProvider peekProvider
280		* @param string $text
281		* @param mixed $spec
282		* @param bool $success
283		* @param string $type
284		* @param string $value
285		*/
286	View Code Duplication	public function testPeek($text, $spec, $success, $type, $value)
		0 ignored issues – show Duplication introduced 2017-08-01 20:34 UTC by Report Bug Copy Issue Report This method seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
287		{
288		$stream = $this->makeStream($text);
289		$this->assertSame(
290		(bool)$success,
291		(bool)$stream->peek($spec),
292		"peek did not return " . ($success ? 'true' : 'false')
293		);
294		$token = $stream->nextToken();
295		$this->assertTokenEq($type, $value, $token);
296		}
297
298		/**
299		* @return array
300		*/
301		public function peekProvider()
302		{
303		return [
304		['create table t', 'create', true, Token::IDENTIFIER, 'create'],
305		['create table t', 'create table', true, Token::IDENTIFIER, 'create'],
306		['create table t', 'drop', false, Token::IDENTIFIER, 'create'],
307		['create table t', 'drop table', false, Token::IDENTIFIER, 'create'],
308		['create table t', 'create database', false, Token::IDENTIFIER, 'create'],
309		['= "test"', [[Token::SYMBOL, '=']], true, Token::SYMBOL, '='],
310		['();', [[Token::SYMBOL, '('],
311		[Token::SYMBOL, ')']], true, Token::SYMBOL, '('],
312		];
313		}
314
315		public function testExpectSucc()
316		{
317		$stream = $this->makeStream('create table t');
318		$stream->expect(Token::IDENTIFIER, 'create');
319		}
320
321		/** @expectedException \Exception */
322		public function testExpectFail()
323		{
324		$stream = $this->makeStream('create table t');
325		$stream->expect(Token::IDENTIFIER, 'drop');
326		}
327
328		/**
329		* @param string $func
330		* @param string $token
331		* @param mixed $expected
332		* @param bool $throwsException
333		* @dataProvider provideExpectedTokenType
334		*/
335		public function testExpectedTokenType($func, $token, $expected, $throwsException)
336		{
337		$stream = $this->makeStream($token);
338		if ($throwsException) {
339		$this->setExpectedException(RuntimeException::class);
340		$stream->$func();
341		} else {
342		$result = $stream->$func();
343		$this->assertEquals($expected, $result);
344		}
345		}
346
347		/**
348		* @return array
349		*/
350		public function provideExpectedTokenType()
351		{
352		return [
353		// [ function name, token, expected value, should it throw a RuntimeException? ]
354
355		[ 'expectCloseParen', ')', ')', false],
356		[ 'expectCloseParen', 'a', null, true],
357
358		[ 'expectOpenParen', '(', '(', false],
359		[ 'expectOpenParen', 'a', null, true],
360
361		[ 'expectName', 'foo', 'foo', false],
362		[ 'expectName', '1', null, true],
363
364		[ 'expectNumber', '1', 1, false],
365		[ 'expectNumber', 'a', null, true],
366
367		// An embedded string
368		[ 'expectString', "'a'", "a", false],
369		[ 'expectString', 'a', null, true],
370
371		[ 'expectStringExtended', "'a'", "a", false],
372		[ 'expectStringExtended', "x'68656c6c6f21'", 'hello!', false],
373		[ 'expectStringExtended', "X'68656c6c6f21'", 'hello!', false],
374		[ 'expectStringExtended', '0x68656c6c6f21', 'hello!', false],
375		[ 'expectStringExtended', "b'0111111000100011'", '~#', false],
376		[ 'expectStringExtended', 'a', null, true],
377
378		];
379		}
380
381		/**
382		* @param string $conditionalComment
383		* @param array $tokenTypes
384		* @dataProvider conditionalCommentProvider
385		*/
386		public function testConditionalComment($conditionalComment, array $tokenTypes)
387		{
388		$stream = $this->makeStream($conditionalComment);
389		$token = $stream->nextToken();
390		$expectedType = current($tokenTypes);
391
392		while ($token->type != Token::EOF) {
393		$this->assertEquals($expectedType, $token->type);
394
395		$token = $stream->nextToken();
396		$expectedType = next($tokenTypes);
397		}
398		}
399
400		/**
401		* @return array
402		*/
403		public function conditionalCommentProvider()
404		{
405		return [
406		// [ conditional comment, list of expected token types ]
407		[
408		'/! abcde fghij /',
409		[
410		Token::IDENTIFIER,
411		Token::IDENTIFIER,
412		],
413		],
414		[
415		'/!12345 fghij /',
416		[
417		Token::IDENTIFIER,
418		],
419		],
420		];
421		}
422
423		public function testContextualise()
424		{
425		$sql = <<<EOF
426		CREATE TABLE `foo` (
427		`a` bar DEFAULT NULL
428		);
429		EOF;
430		$expected = <<<EOF
431		data://text/plain;base64,Q1JFQVRFIFRBQkxFIGBmb29gICgKICAgIGBhYCBiYXIgREVGQVVMVCBOVUxMCik7, line 2: unknown datatype 'bar'
432		1: CREATE TABLE `foo` (
433		2: `a` bar<<HERE>> DEFAULT NULL
434		EOF;
435
436		$stream = $this->makeStream($sql);
437
438		// Use reflection to set the internal offset to the place where error is.
439		$reflection = new \ReflectionClass($stream);
440		$property = $reflection->getProperty('offset');
441		$property->setAccessible(true);
442		$property->setValue($stream, 32);
443
444		$message = $stream->contextualise("unknown datatype 'bar'");
445		$this->assertEquals($expected, $message);
446		}
447		}
448

graze / morphism

GitHub Access Token became invalid

Pull Request — master (#42)

TokenStreamTest::testNewFromText() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like