1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
/** |
4
|
|
|
* webtrees: online genealogy |
5
|
|
|
* Copyright (C) 2021 webtrees development team |
6
|
|
|
* This program is free software: you can redistribute it and/or modify |
7
|
|
|
* it under the terms of the GNU General Public License as published by |
8
|
|
|
* the Free Software Foundation, either version 3 of the License, or |
9
|
|
|
* (at your option) any later version. |
10
|
|
|
* This program is distributed in the hope that it will be useful, |
11
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
12
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13
|
|
|
* GNU General Public License for more details. |
14
|
|
|
* You should have received a copy of the GNU General Public License |
15
|
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>. |
16
|
|
|
*/ |
17
|
|
|
|
18
|
|
|
declare(strict_types=1); |
19
|
|
|
|
20
|
|
|
namespace Fisharebest\Webtrees\Http\RequestHandlers; |
21
|
|
|
|
22
|
|
|
use Exception; |
23
|
|
|
use Fisharebest\Webtrees\Encodings\UTF8; |
24
|
|
|
use Fisharebest\Webtrees\Exceptions\GedcomErrorException; |
25
|
|
|
use Fisharebest\Webtrees\Http\ViewResponseTrait; |
26
|
|
|
use Fisharebest\Webtrees\I18N; |
27
|
|
|
use Fisharebest\Webtrees\Services\GedcomImportService; |
28
|
|
|
use Fisharebest\Webtrees\Services\TimeoutService; |
29
|
|
|
use Fisharebest\Webtrees\Services\TreeService; |
30
|
|
|
use Fisharebest\Webtrees\Tree; |
31
|
|
|
use Illuminate\Database\Capsule\Manager as DB; |
32
|
|
|
use Illuminate\Database\DetectsConcurrencyErrors; |
33
|
|
|
use Psr\Http\Message\ResponseInterface; |
34
|
|
|
use Psr\Http\Message\ServerRequestInterface; |
35
|
|
|
use Psr\Http\Server\RequestHandlerInterface; |
36
|
|
|
|
37
|
|
|
use function assert; |
38
|
|
|
use function preg_split; |
39
|
|
|
use function response; |
40
|
|
|
use function str_replace; |
41
|
|
|
use function str_starts_with; |
42
|
|
|
use function strlen; |
43
|
|
|
use function substr; |
44
|
|
|
use function view; |
45
|
|
|
|
46
|
|
|
/** |
47
|
|
|
* Load a chunk of GEDCOM data. |
48
|
|
|
*/ |
49
|
|
|
class GedcomLoad implements RequestHandlerInterface |
50
|
|
|
{ |
51
|
|
|
use ViewResponseTrait; |
52
|
|
|
use DetectsConcurrencyErrors; |
53
|
|
|
|
54
|
|
|
private GedcomImportService $gedcom_import_service; |
55
|
|
|
|
56
|
|
|
private TimeoutService $timeout_service; |
57
|
|
|
|
58
|
|
|
private TreeService $tree_service; |
59
|
|
|
|
60
|
|
|
/** |
61
|
|
|
* GedcomLoad constructor. |
62
|
|
|
* |
63
|
|
|
* @param GedcomImportService $gedcom_import_service |
64
|
|
|
* @param TimeoutService $timeout_service |
65
|
|
|
* @param TreeService $tree_service |
66
|
|
|
*/ |
67
|
|
|
public function __construct( |
68
|
|
|
GedcomImportService $gedcom_import_service, |
69
|
|
|
TimeoutService $timeout_service, |
70
|
|
|
TreeService $tree_service |
71
|
|
|
) { |
72
|
|
|
$this->gedcom_import_service = $gedcom_import_service; |
73
|
|
|
$this->timeout_service = $timeout_service; |
74
|
|
|
$this->tree_service = $tree_service; |
75
|
|
|
} |
76
|
|
|
|
77
|
|
|
/** |
78
|
|
|
* @param ServerRequestInterface $request |
79
|
|
|
* |
80
|
|
|
* @return ResponseInterface |
81
|
|
|
*/ |
82
|
|
|
public function handle(ServerRequestInterface $request): ResponseInterface |
83
|
|
|
{ |
84
|
|
|
$this->layout = 'layouts/ajax'; |
85
|
|
|
|
86
|
|
|
$tree = $request->getAttribute('tree'); |
87
|
|
|
assert($tree instanceof Tree); |
88
|
|
|
|
89
|
|
|
try { |
90
|
|
|
// What is the current import status? |
91
|
|
|
$import_offset = DB::table('gedcom_chunk') |
92
|
|
|
->where('gedcom_id', '=', $tree->id()) |
93
|
|
|
->where('imported', '=', '1') |
94
|
|
|
->count(); |
95
|
|
|
|
96
|
|
|
$import_total = DB::table('gedcom_chunk') |
97
|
|
|
->where('gedcom_id', '=', $tree->id()) |
98
|
|
|
->count(); |
99
|
|
|
|
100
|
|
|
// Finished? |
101
|
|
|
if ($import_offset === $import_total) { |
102
|
|
|
$tree->setPreference('imported', '1'); |
103
|
|
|
|
104
|
|
|
$html = view('admin/import-complete', ['tree' => $tree]); |
105
|
|
|
|
106
|
|
|
return response($html); |
107
|
|
|
} |
108
|
|
|
|
109
|
|
|
// Calculate progress so far |
110
|
|
|
$progress = $import_offset / $import_total; |
111
|
|
|
|
112
|
|
|
$first_time = $import_offset === 0; |
113
|
|
|
|
114
|
|
|
// Collect up any errors, and show them later. |
115
|
|
|
$errors = ''; |
116
|
|
|
|
117
|
|
|
// Run for a short period of time. This keeps the resource requirements low. |
118
|
|
|
do { |
119
|
|
|
$data = DB::table('gedcom_chunk') |
120
|
|
|
->where('gedcom_id', '=', $tree->id()) |
121
|
|
|
->where('imported', '=', '0') |
122
|
|
|
->orderBy('gedcom_chunk_id') |
123
|
|
|
->select(['gedcom_chunk_id', 'chunk_data']) |
124
|
|
|
->first(); |
125
|
|
|
|
126
|
|
|
if ($data === null) { |
127
|
|
|
break; |
128
|
|
|
} |
129
|
|
|
|
130
|
|
|
// Mark the chunk as imported. This will create a row-lock, to prevent other |
131
|
|
|
// processes from reading it until we have finished. |
132
|
|
|
$n = DB::table('gedcom_chunk') |
133
|
|
|
->where('gedcom_chunk_id', '=', $data->gedcom_chunk_id) |
134
|
|
|
->where('imported', '=', '0') |
135
|
|
|
->update(['imported' => 1]); |
136
|
|
|
|
137
|
|
|
// Another process has already imported this data? |
138
|
|
|
if ($n === 0) { |
139
|
|
|
break; |
140
|
|
|
} |
141
|
|
|
|
142
|
|
|
// If we are loading the first (header) record, then delete old data. |
143
|
|
|
if ($first_time) { |
144
|
|
|
$this->tree_service->deleteGenealogyData($tree, (bool) $tree->getPreference('keep_media')); |
145
|
|
|
|
146
|
|
|
// Remove any byte-order-mark |
147
|
|
|
if (str_starts_with($data->chunk_data, UTF8::BYTE_ORDER_MARK)) { |
148
|
|
|
$data->chunk_data = substr($data->chunk_data, strlen(UTF8::BYTE_ORDER_MARK)); |
149
|
|
|
DB::table('gedcom_chunk') |
150
|
|
|
->where('gedcom_chunk_id', '=', $data->gedcom_chunk_id) |
151
|
|
|
->update(['chunk_data' => $data->chunk_data]); |
152
|
|
|
} |
153
|
|
|
|
154
|
|
|
if (!str_starts_with($data->chunk_data, '0 HEAD')) { |
155
|
|
|
return $this->viewResponse('admin/import-fail', [ |
156
|
|
|
'error' => I18N::translate('Invalid GEDCOM file - no header record found.'), |
157
|
|
|
'tree' => $tree, |
158
|
|
|
]); |
159
|
|
|
} |
160
|
|
|
|
161
|
|
|
$first_time = false; |
162
|
|
|
} |
163
|
|
|
|
164
|
|
|
$data->chunk_data = str_replace("\r", "\n", $data->chunk_data); |
165
|
|
|
|
166
|
|
|
// Import all the records in this chunk of data |
167
|
|
|
foreach (preg_split('/\n+(?=0)/', $data->chunk_data) as $rec) { |
168
|
|
|
try { |
169
|
|
|
$this->gedcom_import_service->importRecord($rec, $tree, false); |
170
|
|
|
} catch (GedcomErrorException $exception) { |
171
|
|
|
$errors .= $exception->getMessage(); |
172
|
|
|
} |
173
|
|
|
} |
174
|
|
|
|
175
|
|
|
// Do not need the data any more. |
176
|
|
|
DB::table('gedcom_chunk') |
177
|
|
|
->where('gedcom_chunk_id', '=', $data->gedcom_chunk_id) |
178
|
|
|
->update(['chunk_data' => '']); |
179
|
|
|
} while (!$this->timeout_service->isTimeLimitUp()); |
180
|
|
|
|
181
|
|
|
return $this->viewResponse('admin/import-progress', [ |
182
|
|
|
'errors' => $errors, |
183
|
|
|
'progress' => $progress, |
184
|
|
|
'tree' => $tree, |
185
|
|
|
]); |
186
|
|
|
} catch (Exception $ex) { |
187
|
|
|
DB::connection()->rollBack(); |
188
|
|
|
|
189
|
|
|
// Deadlock? Try again. |
190
|
|
|
if ($this->causedByConcurrencyError($ex)) { |
191
|
|
|
return $this->viewResponse('admin/import-progress', [ |
192
|
|
|
'errors' => '', |
193
|
|
|
'progress' => $progress ?? 0.0, |
194
|
|
|
'tree' => $tree, |
195
|
|
|
]); |
196
|
|
|
} |
197
|
|
|
|
198
|
|
|
return $this->viewResponse('admin/import-fail', [ |
199
|
|
|
'error' => $ex->getMessage(), |
200
|
|
|
'tree' => $tree, |
201
|
|
|
]); |
202
|
|
|
} |
203
|
|
|
} |
204
|
|
|
} |
205
|
|
|
|