|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
namespace Graze\CsvToken\Tokeniser\Token; |
|
4
|
|
|
|
|
5
|
|
|
use Graze\CsvToken\Csv\CsvConfigurationInterface; |
|
6
|
|
|
|
|
7
|
|
|
class TokenStore implements TokenStoreInterface |
|
8
|
|
|
{ |
|
9
|
|
|
/** @var int[][] */ |
|
10
|
|
|
private $maskStore = []; |
|
11
|
|
|
/** @var int[] */ |
|
12
|
|
|
private $tokens = []; |
|
13
|
|
|
/** @var string|null */ |
|
14
|
|
|
private $lastEncoding = null; |
|
15
|
|
|
|
|
16
|
|
|
/** |
|
17
|
|
|
* TokenStore constructor. |
|
18
|
|
|
* |
|
19
|
|
|
* @param CsvConfigurationInterface $config |
|
20
|
|
|
*/ |
|
21
|
31 |
|
public function __construct(CsvConfigurationInterface $config) |
|
22
|
|
|
{ |
|
23
|
31 |
|
$this->tokens = $this->buildTokens($config); |
|
24
|
31 |
|
$this->setEncoding($config->getEncoding()); |
|
25
|
31 |
|
$this->sort(); |
|
26
|
31 |
|
} |
|
27
|
|
|
|
|
28
|
|
|
/** |
|
29
|
|
|
* @param int $mask |
|
30
|
|
|
* |
|
31
|
|
|
* @return int[] |
|
32
|
|
|
*/ |
|
33
|
31 |
|
public function getTokens($mask = Token::T_ANY) |
|
34
|
|
|
{ |
|
35
|
31 |
|
if (!array_key_exists($mask, $this->maskStore)) { |
|
36
|
|
|
$this->maskStore[$mask] = array_filter($this->tokens, function ($type) use ($mask) { |
|
37
|
31 |
|
return $type & $mask; |
|
38
|
31 |
|
}); |
|
39
|
31 |
|
} |
|
40
|
|
|
|
|
41
|
31 |
|
return $this->maskStore[$mask]; |
|
42
|
|
|
} |
|
43
|
|
|
|
|
44
|
|
|
/** |
|
45
|
|
|
* @param CsvConfigurationInterface $config |
|
46
|
|
|
* |
|
47
|
|
|
* @return int[] |
|
48
|
|
|
*/ |
|
49
|
31 |
|
private function buildTokens(CsvConfigurationInterface $config) |
|
50
|
|
|
{ |
|
51
|
|
|
$tokens = [ |
|
52
|
31 |
|
$config->getDelimiter() => Token::T_DELIMITER, |
|
53
|
31 |
|
]; |
|
54
|
|
|
|
|
55
|
31 |
|
if ($config->getQuote() != '') { |
|
56
|
29 |
|
$tokens[$config->getQuote()] = Token::T_QUOTE; |
|
57
|
29 |
|
if ($config->useDoubleQuotes()) { |
|
58
|
5 |
|
$tokens[str_repeat($config->getQuote(), 2)] = Token::T_DOUBLE_QUOTE; |
|
59
|
5 |
|
} |
|
60
|
29 |
|
} |
|
61
|
31 |
|
if ($config->getEscape() != '') { |
|
62
|
29 |
|
$tokens[$config->getEscape()] = Token::T_ESCAPE; |
|
63
|
29 |
|
} |
|
64
|
|
|
|
|
65
|
31 |
|
foreach ($config->getNewLines() as $newLine) { |
|
66
|
31 |
|
$tokens[$newLine] = Token::T_NEW_LINE; |
|
67
|
31 |
|
} |
|
68
|
31 |
|
if (!is_null($config->getNullValue())) { |
|
69
|
31 |
|
$tokens[$config->getNullValue()] = Token::T_NULL; |
|
70
|
31 |
|
} |
|
71
|
|
|
|
|
72
|
31 |
|
foreach ($config->getBoms() as $bom) { |
|
73
|
31 |
|
$tokens[$bom] = Token::T_BOM; |
|
74
|
31 |
|
} |
|
75
|
|
|
|
|
76
|
31 |
|
return $tokens; |
|
77
|
|
|
} |
|
78
|
|
|
|
|
79
|
|
|
/** |
|
80
|
|
|
* @param string $encoding |
|
81
|
|
|
*/ |
|
82
|
31 |
|
public function setEncoding($encoding) |
|
83
|
|
|
{ |
|
84
|
31 |
|
if ($encoding != $this->lastEncoding) { |
|
85
|
31 |
|
if (!is_null($this->lastEncoding)) { |
|
86
|
|
|
$changeEncoding = function ($string) use ($encoding) { |
|
87
|
2 |
|
return mb_convert_encoding($string, $encoding, $this->lastEncoding); |
|
88
|
2 |
|
}; |
|
89
|
2 |
|
} else { |
|
90
|
|
|
$changeEncoding = function ($string) use ($encoding) { |
|
91
|
31 |
|
return mb_convert_encoding($string, $encoding); |
|
92
|
31 |
|
}; |
|
93
|
|
|
} |
|
94
|
31 |
|
$tokens = []; |
|
95
|
31 |
|
foreach ($this->tokens as $string => $token) { |
|
96
|
31 |
|
if ($token != Token::T_BOM) { |
|
97
|
31 |
|
$string = $changeEncoding($string); |
|
98
|
31 |
|
} |
|
99
|
31 |
|
$tokens[$string] = $token; |
|
100
|
31 |
|
} |
|
101
|
31 |
|
$this->tokens = $tokens; |
|
102
|
31 |
|
$this->sort(); |
|
103
|
|
|
|
|
104
|
31 |
|
$this->lastEncoding = $encoding; |
|
105
|
31 |
|
$this->maskStore = []; |
|
106
|
31 |
|
} |
|
107
|
31 |
|
} |
|
108
|
|
|
|
|
109
|
|
|
/** |
|
110
|
|
|
* Sort the tokens into reverse key length order |
|
111
|
|
|
*/ |
|
112
|
|
|
private function sort() |
|
113
|
|
|
{ |
|
114
|
|
|
// sort by reverse key length |
|
115
|
31 |
|
uksort($this->tokens, function ($first, $second) { |
|
116
|
31 |
|
return strlen($second) - strlen($first); |
|
117
|
31 |
|
}); |
|
118
|
31 |
|
} |
|
119
|
|
|
|
|
120
|
|
|
/** |
|
121
|
|
|
* Determine if a mask set of tokens has changed |
|
122
|
|
|
* |
|
123
|
|
|
* @param int $mask |
|
124
|
|
|
* |
|
125
|
|
|
* @return bool |
|
126
|
|
|
*/ |
|
127
|
28 |
|
public function hasChanged($mask = Token::T_ANY) |
|
128
|
|
|
{ |
|
129
|
28 |
|
return !isset($this->maskStore[$mask]); |
|
130
|
|
|
} |
|
131
|
|
|
} |
|
132
|
|
|
|
graze /
csv-token
