|
1
|
|
|
<?php |
|
2
|
|
|
/******************************************** |
|
3
|
|
|
* DBF to MySQL Converter |
|
4
|
|
|
* |
|
5
|
|
|
* Author: Chizhov Nikolay <[email protected]> |
|
6
|
|
|
* (c) 2019 CIOB "Inok" |
|
7
|
|
|
********************************************/ |
|
8
|
|
|
|
|
9
|
|
|
namespace Inok\Dbf2mysql; |
|
10
|
|
|
|
|
11
|
|
|
use DirectoryIterator; |
|
12
|
|
|
use \Inok\Dbf\Table; |
|
13
|
|
|
use \Inok\Dbf\Records; |
|
14
|
|
|
use PDO; |
|
15
|
|
|
use PDOException; |
|
16
|
|
|
use RegexIterator; |
|
17
|
|
|
|
|
18
|
|
|
class convert { |
|
19
|
|
|
/** |
|
20
|
|
|
* @var PDO |
|
21
|
|
|
*/ |
|
22
|
|
|
private $db; |
|
23
|
|
|
|
|
24
|
|
|
/** |
|
25
|
|
|
* @var Records |
|
26
|
|
|
*/ |
|
27
|
|
|
private $dbfRecords; |
|
28
|
|
|
|
|
29
|
|
|
private $config, $dbfHeaders, $dbfColumns, $log, $log_replaces; |
|
30
|
|
|
private $timer = [ |
|
31
|
|
|
"start" => null, |
|
32
|
|
|
"tableStart" => null, |
|
33
|
|
|
]; |
|
34
|
|
|
private $colors = [ |
|
35
|
|
|
"default" => "\e[0m", |
|
36
|
|
|
"red" => "\e[31m", |
|
37
|
|
|
"yellow" => "\e[93m", |
|
38
|
|
|
"white" => "\e[97m" |
|
39
|
|
|
]; |
|
40
|
|
|
private $percent = -1; |
|
41
|
|
|
private $column_fixes; |
|
42
|
|
|
|
|
43
|
|
|
public function __construct($config) { |
|
44
|
|
|
ini_set("memory_limit", "2048M"); |
|
45
|
|
|
|
|
46
|
|
|
$this->config = $config; |
|
47
|
|
|
|
|
48
|
|
|
$this->checkConfig(); |
|
49
|
|
|
$this->dbConnect(); |
|
50
|
|
|
$this->timer["start"] = time(); |
|
51
|
|
|
$this->writeLog("Start importing"); |
|
52
|
|
|
$this->convert(); |
|
53
|
|
|
} |
|
54
|
|
|
|
|
55
|
|
|
public function __destruct() { |
|
56
|
|
|
if (!is_null($this->timer["start"])) { |
|
57
|
|
|
$this->writeLog("Finish importing in <red>".(round((time() - $this->timer["start"]) / 60, 2))."<default> minutes"); |
|
58
|
|
|
} |
|
59
|
|
|
fclose($this->log); |
|
60
|
|
|
} |
|
61
|
|
|
|
|
62
|
|
|
private function checkConfig() { |
|
63
|
|
|
//defaults |
|
64
|
|
|
$config_defaults = [ |
|
65
|
|
|
"db_host" => "localhost", |
|
66
|
|
|
"db_port" => 3306, |
|
67
|
|
|
"db_username" => "root", |
|
68
|
|
|
"db_password" => "", |
|
69
|
|
|
"db_charset" => "utf-8", |
|
70
|
|
|
"dbf_list" => null, |
|
71
|
|
|
"dbf_charset" => 866, |
|
72
|
|
|
"table_prefix" => null, |
|
73
|
|
|
"columns_only" => false, |
|
74
|
|
|
"deleted_records" => false, |
|
75
|
|
|
"key_field" => null, |
|
76
|
|
|
"verbose" => true, |
|
77
|
|
|
"log_path" => realpath(dirname(__FILE__)."/..")."/dbf2mysql.log" |
|
78
|
|
|
]; |
|
79
|
|
|
|
|
80
|
|
|
$this->config = $this->config + $config_defaults; |
|
81
|
|
|
|
|
82
|
|
|
$this->config["db_charset2"] = str_replace("-", "", $this->config["db_charset"]); |
|
83
|
|
|
|
|
84
|
|
|
$this->initLog(); |
|
85
|
|
|
|
|
86
|
|
|
//check MySQL |
|
87
|
|
|
if (!is_numeric($this->config["db_port"])) { |
|
88
|
|
|
$this->writeLog("<red>Error in config:<default> MySQL port should be number"); |
|
89
|
|
|
exit; |
|
|
|
|
|
|
90
|
|
|
} |
|
91
|
|
|
if (!isset($this->config["db_name"])) { |
|
92
|
|
|
$this->writeLog("<red>Error in config:<default> MySQL database name not exists"); |
|
93
|
|
|
exit; |
|
|
|
|
|
|
94
|
|
|
} |
|
95
|
|
|
if (!isset($this->config["dbf_path"])) { |
|
96
|
|
|
$this->writeLog("<red>Error in config:<default> DBF-files directory not exists"); |
|
97
|
|
|
exit; |
|
|
|
|
|
|
98
|
|
|
} |
|
99
|
|
|
if (is_null($this->config["table_prefix"])) { |
|
100
|
|
|
$this->config["table_prefix"] = ""; |
|
101
|
|
|
} |
|
102
|
|
|
//check dbf |
|
103
|
|
|
if (!is_numeric($this->config["dbf_charset"])) { |
|
104
|
|
|
$this->writeLog("<red>Error in config:<default> DBF-charset should be number"); |
|
105
|
|
|
exit; |
|
|
|
|
|
|
106
|
|
|
} |
|
107
|
|
|
if (!is_null($this->config["dbf_list"])) { |
|
108
|
|
|
if (is_array($this->config["dbf_list"])) { |
|
109
|
|
|
$this->config["dbf_list"] = array_map("strtolower", $this->config["dbf_list"]); |
|
110
|
|
|
} |
|
111
|
|
|
else { |
|
112
|
|
|
$this->writeLog("<red>Error in config:<default> dbf list should be array or null"); |
|
113
|
|
|
exit; |
|
|
|
|
|
|
114
|
|
|
} |
|
115
|
|
|
} |
|
116
|
|
|
} |
|
117
|
|
|
|
|
118
|
|
|
private function initLog() { |
|
119
|
|
|
$this->log = fopen($this->config["log_path"], "a"); |
|
120
|
|
|
if ($this->log === false) { |
|
121
|
|
|
echo($this->colors["red"]."Error in log:".$this->colors["default"]." Couldn't create log file\n"); |
|
122
|
|
|
exit; |
|
|
|
|
|
|
123
|
|
|
} |
|
124
|
|
|
$this->log_replaces = [ |
|
125
|
|
|
"from" => array_map(function($value) { |
|
126
|
|
|
return "<".$value.">"; |
|
127
|
|
|
}, array_keys($this->colors)), |
|
128
|
|
|
"to" => array_values($this->colors) |
|
129
|
|
|
]; |
|
130
|
|
|
} |
|
131
|
|
|
|
|
132
|
|
|
private function dbConnect() { |
|
133
|
|
|
$db_options = [ |
|
134
|
|
|
PDO::ATTR_ERRMODE => PDO::ERRMODE_WARNING, |
|
135
|
|
|
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC |
|
136
|
|
|
]; |
|
137
|
|
|
try { |
|
138
|
|
|
$this->db = new PDO("mysql:host=".$this->config["db_host"].";port=".$this->config["db_port"].";dbname=" . $this->config["db_name"], |
|
139
|
|
|
$this->config["db_username"], $this->config["db_password"], $db_options); |
|
140
|
|
|
} |
|
141
|
|
|
catch (PDOException $e) { |
|
142
|
|
|
$this->writeLog("<red>Error in MySQL connection:<default> ".$e->getMessage()); |
|
143
|
|
|
exit; |
|
|
|
|
|
|
144
|
|
|
} |
|
145
|
|
|
|
|
146
|
|
|
$this->db->exec("SET NAMES ".$this->config["db_charset2"]); |
|
147
|
|
|
} |
|
148
|
|
|
|
|
149
|
|
|
private function convert() { |
|
150
|
|
|
$dbfs = new RegexIterator(new DirectoryIterator($this->config["dbf_path"]), "/\\.dbf\$/i"); |
|
151
|
|
|
foreach ($dbfs as $file) { |
|
152
|
|
|
if (!is_null($this->config["dbf_list"]) && !in_array(strtolower($file->getBasename(".".$file->getExtension())), $this->config["dbf_list"])) { |
|
153
|
|
|
continue; |
|
154
|
|
|
} |
|
155
|
|
|
$this->timer["tableStart"] = time(); |
|
156
|
|
|
$table = new Table($file->getPathname(), $this->config["dbf_charset"]); |
|
157
|
|
|
$this->dbfHeaders = $table->getHeaders(); |
|
158
|
|
|
if ($table->error) { |
|
159
|
|
|
$this->writeLog("<red>Error in DBF:<default> ".$table->error_info); |
|
160
|
|
|
continue; |
|
161
|
|
|
} |
|
162
|
|
|
$this->dbfHeaders["table"] = $this->config["table_prefix"].$this->dbfHeaders["table"]; |
|
163
|
|
|
$this->dbfColumns = $table->getColumns(); |
|
164
|
|
|
if ($table->error) { |
|
165
|
|
|
$this->writeLog("<red>Error in DBF:<default> ".$table->error_info); |
|
166
|
|
|
continue; |
|
167
|
|
|
} |
|
168
|
|
|
$this->createMySQLColumns(); |
|
169
|
|
|
if (!$this->config["columns_only"]) { |
|
170
|
|
|
$this->dbfRecords = new Records($table, $this->config["db_charset"]); |
|
171
|
|
|
$this->writeRecords(); |
|
172
|
|
|
} |
|
173
|
|
|
$this->setKeyField(); |
|
174
|
|
|
unset($table); |
|
175
|
|
|
} |
|
176
|
|
|
} |
|
177
|
|
|
|
|
178
|
|
|
private function createMySQLColumns() { |
|
179
|
|
|
$line = []; |
|
180
|
|
|
$this->db->exec("DROP TABLE IF EXISTS `".$this->dbfHeaders["table"]."`"); |
|
181
|
|
|
foreach ($this->dbfColumns as $column) { |
|
182
|
|
|
$name = "`".$column["name"]."`"; |
|
183
|
|
|
switch ($column["type"]) { |
|
184
|
|
|
case "I": |
|
185
|
|
|
case "F": |
|
186
|
|
|
case "N": |
|
187
|
|
|
case "Y": |
|
188
|
|
|
case "0": |
|
189
|
|
|
if ($column["decimal"]) { |
|
190
|
|
|
$line[] = $name." decimal(".($column["length"] + $column["decimal"]).", ".$column["decimal"].") NULL DEFAULT 0"; |
|
191
|
|
|
} |
|
192
|
|
|
else { |
|
193
|
|
|
$line[] = $name." bigint(".$column["length"].") NULL DEFAULT 0"; |
|
194
|
|
|
} |
|
195
|
|
|
break; |
|
196
|
|
|
case "D": |
|
197
|
|
|
$line[] = $name." date DEFAULT NULL"; |
|
198
|
|
|
break; |
|
199
|
|
|
case "T": |
|
200
|
|
|
$line[] = $name." datetime DEFAULT NULL"; |
|
201
|
|
|
break; |
|
202
|
|
|
case "L": |
|
203
|
|
|
$line[] = $name." tinyint(1) NULL DEFAULT '0'"; |
|
204
|
|
|
break; |
|
205
|
|
|
case "C": |
|
206
|
|
|
$line[] = $name." varchar(".$column["length"].") NULL DEFAULT ''"; |
|
207
|
|
|
break; |
|
208
|
|
|
case "M": |
|
209
|
|
|
$line[] = $name." text NOT NULL DEFAULT ''"; |
|
210
|
|
|
break; |
|
211
|
|
|
case "P": |
|
212
|
|
|
case "G": |
|
213
|
|
|
$line[] = $name." blob NULL DEFAULT NULL"; |
|
214
|
|
|
break; |
|
215
|
|
|
} |
|
216
|
|
|
} |
|
217
|
|
|
if (count($line)) { |
|
218
|
|
|
if ($this->config["deleted_records"]) { |
|
219
|
|
|
$line[] = "`deleted` tinyint(1) NOT NULL DEFAULT '0'"; |
|
220
|
|
|
} |
|
221
|
|
|
$result = $this->db->exec("CREATE TABLE IF NOT EXISTS `".$this->dbfHeaders["table"]."` (". |
|
222
|
|
|
implode(", ", $line). |
|
223
|
|
|
") ENGINE=InnoDB DEFAULT |
|
224
|
|
|
CHARSET=".$this->config["db_charset2"]." |
|
225
|
|
|
COMMENT='Converted DBF file: ".$this->dbfHeaders["table"].".dbf'"); |
|
226
|
|
|
if ($result !== false) { |
|
227
|
|
|
$this->writeLog("Table <yellow>".$this->dbfHeaders["table"]."<default> successfully created"); |
|
228
|
|
|
} |
|
229
|
|
|
else { |
|
230
|
|
|
$this->writeLog("<red>Error in MySQL:<default> ".print_r($this->db->errorInfo(), true)); |
|
|
|
|
|
|
231
|
|
|
} |
|
232
|
|
|
} |
|
233
|
|
|
unset($line); |
|
234
|
|
|
} |
|
235
|
|
|
|
|
236
|
|
|
private function writeRecords() { |
|
237
|
|
|
if (!count($this->dbfColumns)) { |
|
238
|
|
|
return; |
|
239
|
|
|
} |
|
240
|
|
|
$this->writeLog("Init import records for table <yellow>".$this->dbfHeaders["table"]."<default>"); |
|
241
|
|
|
$i = 0; $recordsPerPosition = $this->dbfHeaders["records"] / 50; |
|
242
|
|
|
$this->column_fixes = []; |
|
243
|
|
|
$sql_keys = []; |
|
244
|
|
|
$sql_values = []; |
|
245
|
|
|
foreach($this->dbfColumns as $column) { |
|
246
|
|
|
$sql_keys[] = "`".$column["name"]."`"; |
|
247
|
|
|
$sql_values[] = ":".$column["name"]; |
|
248
|
|
|
if (in_array($column["type"], ["F", "N", "I", "Y", "0"])) { |
|
249
|
|
|
$this->column_fixes[$column["name"]] = [ |
|
250
|
|
|
"min" => 0, |
|
251
|
|
|
"max" => 0 |
|
252
|
|
|
]; |
|
253
|
|
|
} |
|
254
|
|
|
} |
|
255
|
|
|
if ($this->config["deleted_records"]) { |
|
256
|
|
|
$sql_keys[] = "`deleted`"; |
|
257
|
|
|
$sql_values[] = ":deleted"; |
|
258
|
|
|
} |
|
259
|
|
|
$result = $this->db->prepare("INSERT INTO `".$this->dbfHeaders["table"]."` (".implode(", ", $sql_keys).") |
|
260
|
|
|
VALUES(".implode(", ", $sql_values).")"); |
|
261
|
|
|
$this->db->beginTransaction(); |
|
262
|
|
|
while ($record = $this->dbfRecords->nextRecord()) { |
|
263
|
|
|
$record = $this->prepareRecord($record); |
|
264
|
|
|
$deleted = false; |
|
265
|
|
|
if ($this->config["deleted_records"]) { |
|
266
|
|
|
$result->execute($record); |
|
267
|
|
|
} |
|
268
|
|
|
else { |
|
269
|
|
|
if (!$record["deleted"]) { |
|
270
|
|
|
unset($record["deleted"]); |
|
271
|
|
|
$result->execute($record); |
|
272
|
|
|
} |
|
273
|
|
|
else { |
|
274
|
|
|
$deleted = true; |
|
275
|
|
|
} |
|
276
|
|
|
} |
|
277
|
|
|
|
|
278
|
|
|
if (!$deleted) { |
|
279
|
|
|
foreach ($this->column_fixes as $c_name => &$vals) { |
|
280
|
|
|
if ($vals["min"] > $record[$c_name]) { |
|
281
|
|
|
$vals["min"] = $record[$c_name]; |
|
282
|
|
|
} |
|
283
|
|
|
if ($vals["max"] < $record[$c_name]) { |
|
284
|
|
|
$vals["max"] = $record[$c_name]; |
|
285
|
|
|
} |
|
286
|
|
|
} |
|
287
|
|
|
} |
|
288
|
|
|
|
|
289
|
|
|
$i++; |
|
290
|
|
|
if ($this->config["verbose"]) { |
|
291
|
|
|
$this->drawStatus($i, $recordsPerPosition); |
|
292
|
|
|
} |
|
293
|
|
|
} |
|
294
|
|
|
$this->db->commit(); |
|
295
|
|
|
|
|
296
|
|
|
//Fix max values |
|
297
|
|
|
$this->fixValues(); |
|
298
|
|
|
|
|
299
|
|
|
$this->writeLog("Table <yellow>".$this->dbfHeaders["table"]."<default> successfully imported in <red>". |
|
300
|
|
|
round((time() - $this->timer["tableStart"]) / 60, 2)."<default> minutes"); |
|
301
|
|
|
unset($sql_keys, $sql_values); |
|
302
|
|
|
} |
|
303
|
|
|
|
|
304
|
|
|
private function prepareRecord($record) { |
|
305
|
|
|
foreach ($record as $name => $value) { |
|
306
|
|
|
if (is_bool($value)) { |
|
307
|
|
|
$record[$name] = (int) $value; |
|
308
|
|
|
} |
|
309
|
|
|
} |
|
310
|
|
|
return $record; |
|
311
|
|
|
} |
|
312
|
|
|
|
|
313
|
|
|
private function fixValues() { |
|
314
|
|
|
$this->writeLog("\nCalculate column types for table <yellow>".$this->dbfHeaders["table"]."<default>"); |
|
315
|
|
|
$lines = []; |
|
316
|
|
|
foreach ($this->dbfColumns as $column) { |
|
317
|
|
|
if (in_array($column["type"], ["F", "N", "I", "Y", "0"])) { |
|
318
|
|
|
$result = $this->column_fixes[$column["name"]]; |
|
319
|
|
|
$unsigned = !($result["min"] < 0); |
|
320
|
|
|
if ($unsigned) { |
|
321
|
|
|
if (!$column["decimal"]) { |
|
322
|
|
|
$type = "bigint"; |
|
323
|
|
|
if ($result["max"] > 16777215 && $result["max"] <= 4294967295) { |
|
324
|
|
|
$type = "int"; |
|
325
|
|
|
} elseif ($result["max"] > 65535 && $result["max"] <= 16777215) { |
|
326
|
|
|
$type = "mediumint"; |
|
327
|
|
|
} elseif ($result["max"] > 255 && $result["max"] <= 65535) { |
|
328
|
|
|
$type = "smallint"; |
|
329
|
|
|
} elseif ($result["max"] <= 255) { |
|
330
|
|
|
$type = "tinyint"; |
|
331
|
|
|
} |
|
332
|
|
|
} |
|
333
|
|
|
} |
|
334
|
|
|
else { |
|
335
|
|
|
if (!$column["decimal"]) { |
|
336
|
|
|
$type = "bigint"; |
|
337
|
|
|
if ($result["min"] >= -128 && $result["max"] <= 127) { |
|
338
|
|
|
$type = "tinyint"; |
|
339
|
|
|
} elseif ($result["min"] >= -32768 && $result["max"] <= 32767) { |
|
340
|
|
|
$type = "smallint"; |
|
341
|
|
|
} elseif ($result["min"] >= -8388608 && $result["max"] <= 8388607) { |
|
342
|
|
|
$type = "mediumint"; |
|
343
|
|
|
} elseif ($result["min"] >= -2147483648 && $result["max"] <= 2147483647) { |
|
344
|
|
|
$type = "int"; |
|
345
|
|
|
} |
|
346
|
|
|
} |
|
347
|
|
|
} |
|
348
|
|
|
if ($column["decimal"] && $unsigned) { |
|
349
|
|
|
$lines[] = "CHANGE `".$column["name"]."` `".$column["name"]."` decimal(".($column["length"] + $column["decimal"]).", ".$column["decimal"].") UNSIGNED |
|
350
|
|
|
NULL DEFAULT '0'"; |
|
351
|
|
|
} |
|
352
|
|
|
else { |
|
353
|
|
|
$lines[] = "CHANGE `".$column["name"]."` `".$column["name"]."` ".$type."(".$column["length"].")".($unsigned ? " UNSIGNED" : "")." |
|
|
|
|
|
|
354
|
|
|
NULL DEFAULT '0'"; |
|
355
|
|
|
} |
|
356
|
|
|
} |
|
357
|
|
|
} |
|
358
|
|
|
if (count($lines)) { |
|
359
|
|
|
$this->db->beginTransaction(); |
|
360
|
|
|
$this->db->exec("FLUSH TABLES; ALTER TABLE `".$this->dbfHeaders["table"]."` ".implode(", ", $lines).";"); |
|
361
|
|
|
$this->db->commit(); |
|
362
|
|
|
} |
|
363
|
|
|
unset($lines); |
|
364
|
|
|
} |
|
365
|
|
|
|
|
366
|
|
|
private function setKeyField() { |
|
367
|
|
|
if (!is_null($this->config["key_field"])) { |
|
368
|
|
|
$this->writeLog("Setting up index column for table <yellow>".$this->dbfHeaders["table"]."<default>"); |
|
369
|
|
|
$result = $this->db->prepare("SELECT COLUMN_NAME |
|
370
|
|
|
FROM INFORMATION_SCHEMA.COLUMNS |
|
371
|
|
|
WHERE TABLE_SCHEMA = :db AND TABLE_NAME = :table AND COLUMN_NAME = :column |
|
372
|
|
|
LIMIT 1"); |
|
373
|
|
|
$result->execute(["db" => $this->config["db_name"], |
|
374
|
|
|
"table" => $this->dbfHeaders["table"], |
|
375
|
|
|
"column" => $this->config["key_field"]]); |
|
376
|
|
|
if ($result->rowCount()) { |
|
377
|
|
|
$this->db->exec("ALTER TABLE `" . $this->dbfHeaders["table"] . "` ADD INDEX(`" . $this->config["key_field"] . "`)"); |
|
378
|
|
|
} |
|
379
|
|
|
} |
|
380
|
|
|
} |
|
381
|
|
|
|
|
382
|
|
|
private function drawStatus($position, $perPosition) { |
|
383
|
|
|
$percent = (int) round($position / $perPosition); |
|
384
|
|
|
if ($this->percent <> !$percent) { |
|
385
|
|
|
$this->percent = $percent; |
|
386
|
|
|
$part1 = (($percent < 25) ? |
|
387
|
|
|
$this->colors["white"].str_repeat("H", $percent).$this->colors["default"].$this->colors["red"].str_repeat(".", 25 - $percent) : |
|
388
|
|
|
$this->colors["white"].str_repeat("H", 25)).$this->colors["default"]; |
|
389
|
|
|
if ($percent > 25) { |
|
390
|
|
|
$percent = $percent - 25; |
|
391
|
|
|
$part2 = (($percent < 50) ? |
|
392
|
|
|
$this->colors["white"].str_repeat("H", $percent).$this->colors["default"].$this->colors["red"].str_repeat(".", 25 - $percent) : |
|
393
|
|
|
$this->colors["white"].str_repeat("H", 25)).$this->colors["default"]; |
|
394
|
|
|
} else { |
|
395
|
|
|
$part2 = $this->colors["red"].str_repeat(".", 25).$this->colors["default"]; |
|
396
|
|
|
} |
|
397
|
|
|
echo($this->colors["red"]."[".$this->colors["default"].$part1. |
|
398
|
|
|
$this->colors["red"]."50%".$this->colors["default"]. |
|
399
|
|
|
$part2.$this->colors["red"]."]".$this->colors["default"]."\r"); |
|
400
|
|
|
} |
|
401
|
|
|
} |
|
402
|
|
|
|
|
403
|
|
|
private function writeLog($message) { |
|
404
|
|
|
$message .= "\n"; |
|
405
|
|
|
fwrite($this->log, "[".date("d.m.Y H:i:s")."] ".ltrim(str_replace($this->log_replaces["from"], "", $message), "\n")); |
|
406
|
|
|
if ($this->config["verbose"]) { |
|
407
|
|
|
echo(str_replace($this->log_replaces["from"], $this->log_replaces["to"], $message)); |
|
408
|
|
|
} |
|
409
|
|
|
} |
|
410
|
|
|
} |
|
411
|
|
|
|
nchizhov /
inok-dbf2mysql
Loading history...
In general, usage of exit should be done with care and only when running in a scripting context like a CLI script.