|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
namespace Graze\CsvToken\Test\Unit; |
|
4
|
|
|
|
|
5
|
|
|
use Graze\CsvToken\Csv\CsvConfiguration; |
|
6
|
|
|
use Graze\CsvToken\Csv\CsvConfigurationInterface; |
|
7
|
|
|
use Graze\CsvToken\Test\TestCase; |
|
8
|
|
|
use Graze\CsvToken\Tokeniser\StreamTokeniser; |
|
9
|
|
|
use Graze\CsvToken\Tokeniser\Token; |
|
10
|
|
|
use GuzzleHttp\Psr7\Stream; |
|
11
|
|
|
|
|
12
|
|
|
class StreamTokeniserTest extends TestCase |
|
13
|
|
|
{ |
|
14
|
|
|
/** |
|
15
|
|
|
* @dataProvider tokeniserTestData |
|
16
|
|
|
* |
|
17
|
|
|
* @param CsvConfigurationInterface $config |
|
18
|
|
|
* @param string $csv |
|
19
|
|
|
* @param array $tokens |
|
20
|
|
|
*/ |
|
21
|
|
View Code Duplication |
public function testTokeniser(CsvConfigurationInterface $config, $csv, array $tokens) |
|
|
|
|
|
|
22
|
|
|
{ |
|
23
|
|
|
$tokeniser = new StreamTokeniser($config, $this->getStream($csv)); |
|
24
|
|
|
|
|
25
|
|
|
$actual = iterator_to_array($tokeniser->getTokens()); |
|
26
|
|
|
|
|
27
|
|
|
$tokensOnly = array_map(function (Token $token) { |
|
28
|
|
|
return [$token->getType(), $token->getPosition(), $token->getLength(), $token->getContent()]; |
|
29
|
|
|
}, $actual); |
|
30
|
|
|
|
|
31
|
|
|
static::assertEquals($tokens, $tokensOnly); |
|
32
|
|
|
} |
|
33
|
|
|
|
|
34
|
|
|
/** |
|
35
|
|
|
* @param string $string |
|
36
|
|
|
* |
|
37
|
|
|
* @return Stream |
|
38
|
|
|
*/ |
|
39
|
|
|
private function getStream($string) |
|
40
|
|
|
{ |
|
41
|
|
|
$stream = fopen('php://memory', 'r+'); |
|
42
|
|
|
fwrite($stream, $string); |
|
43
|
|
|
rewind($stream); |
|
44
|
|
|
return new Stream($stream); |
|
45
|
|
|
} |
|
46
|
|
|
|
|
47
|
|
|
/** |
|
48
|
|
|
* @return array |
|
49
|
|
|
*/ |
|
50
|
|
|
public function tokeniserTestData() |
|
51
|
|
|
{ |
|
52
|
|
|
return [ |
|
53
|
|
|
[ |
|
54
|
|
|
new CsvConfiguration(), |
|
55
|
|
|
'"some","test","","data"', |
|
56
|
|
|
[ |
|
57
|
|
|
[Token::T_QUOTE, 0, 1, '"'], |
|
58
|
|
|
[Token::T_CONTENT, 1, 4, 'some'], |
|
59
|
|
|
[Token::T_QUOTE, 5, 1, '"'], |
|
60
|
|
|
[Token::T_DELIMITER, 6, 1, ','], |
|
61
|
|
|
[Token::T_QUOTE, 7, 1, '"'], |
|
62
|
|
|
[Token::T_CONTENT, 8, 4, 'test'], |
|
63
|
|
|
[Token::T_QUOTE, 12, 1, '"'], |
|
64
|
|
|
[Token::T_DELIMITER, 13, 1, ','], |
|
65
|
|
|
[Token::T_QUOTE, 14, 1, '"'], |
|
66
|
|
|
[Token::T_QUOTE, 15, 1, '"'], |
|
67
|
|
|
[Token::T_DELIMITER, 16, 1, ','], |
|
68
|
|
|
[Token::T_QUOTE, 17, 1, '"'], |
|
69
|
|
|
[Token::T_CONTENT, 18, 4, 'data'], |
|
70
|
|
|
[Token::T_QUOTE, 22, 1, '"'], |
|
71
|
|
|
], |
|
72
|
|
|
], |
|
73
|
|
|
[ |
|
74
|
|
|
new CsvConfiguration(), |
|
75
|
|
|
'some', |
|
76
|
|
|
[ |
|
77
|
|
|
[Token::T_CONTENT, 0, 4, 'some'], |
|
78
|
|
|
], |
|
79
|
|
|
], |
|
80
|
|
|
[ |
|
81
|
|
|
new CsvConfiguration(), |
|
82
|
|
|
'"some",test,"with \" escape"', |
|
83
|
|
|
[ |
|
84
|
|
|
[Token::T_QUOTE, 0, 1, '"'], |
|
85
|
|
|
[Token::T_CONTENT, 1, 4, 'some'], |
|
86
|
|
|
[Token::T_QUOTE, 5, 1, '"'], |
|
87
|
|
|
[Token::T_DELIMITER, 6, 1, ','], |
|
88
|
|
|
[Token::T_CONTENT, 7, 4, 'test'], |
|
89
|
|
|
[Token::T_DELIMITER, 11, 1, ','], |
|
90
|
|
|
[Token::T_QUOTE, 12, 1, '"'], |
|
91
|
|
|
[Token::T_CONTENT, 13, 5, 'with '], |
|
92
|
|
|
[Token::T_ESCAPE, 18, 1, '\\'], |
|
93
|
|
|
[Token::T_QUOTE, 19, 1, '"'], |
|
94
|
|
|
[Token::T_CONTENT, 20, 7, ' escape'], |
|
95
|
|
|
[Token::T_QUOTE, 27, 1, '"'], |
|
96
|
|
|
], |
|
97
|
|
|
], |
|
98
|
|
|
[ |
|
99
|
|
|
new CsvConfiguration([ |
|
100
|
|
|
CsvConfiguration::OPTION_DELIMITER => '|', |
|
101
|
|
|
CsvConfiguration::OPTION_QUOTE => "'", |
|
102
|
|
|
CsvConfiguration::OPTION_ESCAPE => '\\', |
|
103
|
|
|
CsvConfiguration::OPTION_DOUBLE_QUOTE => true, |
|
104
|
|
|
CsvConfiguration::OPTION_NEW_LINE => '---', |
|
105
|
|
|
CsvConfiguration::OPTION_NULL => '\\N', |
|
106
|
|
|
]), |
|
107
|
|
|
"'some'|text|'\\'here'|\\N|'with''quotes'---'another'|'line'", |
|
108
|
|
|
[ |
|
109
|
|
|
[Token::T_QUOTE, 0, 1, "'"], |
|
110
|
|
|
[Token::T_CONTENT, 1, 4, 'some'], |
|
111
|
|
|
[Token::T_QUOTE, 5, 1, "'"], |
|
112
|
|
|
[Token::T_DELIMITER, 6, 1, '|'], |
|
113
|
|
|
[Token::T_CONTENT, 7, 4, 'text'], |
|
114
|
|
|
[Token::T_DELIMITER, 11, 1, '|'], |
|
115
|
|
|
[Token::T_QUOTE, 12, 1, "'"], |
|
116
|
|
|
[Token::T_ESCAPE, 13, 1, '\\'], |
|
117
|
|
|
[Token::T_QUOTE, 14, 1, "'"], |
|
118
|
|
|
[Token::T_CONTENT, 15, 4, "here"], |
|
119
|
|
|
[Token::T_QUOTE, 19, 1, "'"], |
|
120
|
|
|
[Token::T_DELIMITER, 20, 1, '|'], |
|
121
|
|
|
[Token::T_NULL, 21, 2, '\\N'], |
|
122
|
|
|
[Token::T_DELIMITER, 23, 1, '|'], |
|
123
|
|
|
[Token::T_QUOTE, 24, 1, "'"], |
|
124
|
|
|
[Token::T_CONTENT, 25, 4, 'with'], |
|
125
|
|
|
[Token::T_DOUBLE_QUOTE, 29, 2, "''"], |
|
126
|
|
|
[Token::T_CONTENT, 31, 6, 'quotes'], |
|
127
|
|
|
[Token::T_QUOTE, 37, 1, "'"], |
|
128
|
|
|
[Token::T_NEW_LINE, 38, 3, '---'], |
|
129
|
|
|
[Token::T_QUOTE, 41, 1, "'"], |
|
130
|
|
|
[Token::T_CONTENT, 42, 7, 'another'], |
|
131
|
|
|
[Token::T_QUOTE, 49, 1, "'"], |
|
132
|
|
|
[Token::T_DELIMITER, 50, 1, '|'], |
|
133
|
|
|
[Token::T_QUOTE, 51, 1, "'"], |
|
134
|
|
|
[Token::T_CONTENT, 52, 4, 'line'], |
|
135
|
|
|
[Token::T_QUOTE, 56, 1, "'"], |
|
136
|
|
|
], |
|
137
|
|
|
], |
|
138
|
|
|
[ |
|
139
|
|
|
new CsvConfiguration(), |
|
140
|
|
|
'"some","new' . "\n" . 'line",with\\' . "\n" . 'escaped,"in\\' . "\n" . 'quotes"', |
|
141
|
|
|
[ |
|
142
|
|
|
[Token::T_QUOTE, 0, 1, '"'], |
|
143
|
|
|
[Token::T_CONTENT, 1, 4, 'some'], |
|
144
|
|
|
[Token::T_QUOTE, 5, 1, '"'], |
|
145
|
|
|
[Token::T_DELIMITER, 6, 1, ','], |
|
146
|
|
|
[Token::T_QUOTE, 7, 1, '"'], |
|
147
|
|
|
[Token::T_CONTENT, 8, 3, 'new'], |
|
148
|
|
|
[Token::T_NEW_LINE, 11, 1, "\n"], |
|
149
|
|
|
[Token::T_CONTENT, 12, 4, 'line'], |
|
150
|
|
|
[Token::T_QUOTE, 16, 1, '"'], |
|
151
|
|
|
[Token::T_DELIMITER, 17, 1, ','], |
|
152
|
|
|
[Token::T_CONTENT, 18, 4, 'with'], |
|
153
|
|
|
[Token::T_ESCAPE, 22, 1, '\\'], |
|
154
|
|
|
[Token::T_NEW_LINE, 23, 1, "\n"], |
|
155
|
|
|
[Token::T_CONTENT, 24, 7, 'escaped'], |
|
156
|
|
|
[Token::T_DELIMITER, 31, 1, ','], |
|
157
|
|
|
[Token::T_QUOTE, 32, 1, '"'], |
|
158
|
|
|
[Token::T_CONTENT, 33, 2, 'in'], |
|
159
|
|
|
[Token::T_ESCAPE, 35, 1, '\\'], |
|
160
|
|
|
[Token::T_NEW_LINE, 36, 1, "\n"], |
|
161
|
|
|
[Token::T_CONTENT, 37, 6, 'quotes'], |
|
162
|
|
|
[Token::T_QUOTE, 43, 1, '"'], |
|
163
|
|
|
], |
|
164
|
|
|
], |
|
165
|
|
|
[ |
|
166
|
|
|
new CsvConfiguration(), |
|
167
|
|
|
'', |
|
168
|
|
|
[], |
|
169
|
|
|
], |
|
170
|
|
|
]; |
|
171
|
|
|
} |
|
172
|
|
|
} |
|
173
|
|
|
|
graze /
csv-token
Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation.
You can also find more detailed suggestions in the “Code” section of your repository.