1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* This file is part of dispositif/wikibot application |
4
|
|
|
* 2019 : Philippe M. <[email protected]> |
5
|
|
|
* For the full copyright and MIT license information, please view the LICENSE file. |
6
|
|
|
*/ |
7
|
|
|
|
8
|
|
|
declare(strict_types=1); |
9
|
|
|
|
10
|
|
|
namespace App\Application\Examples; |
11
|
|
|
|
12
|
|
|
use App\Application\Bot; |
13
|
|
|
use App\Application\Memory; |
14
|
|
|
use App\Application\QueueInterface; |
15
|
|
|
use App\Domain\Models\Wiki\OuvrageTemplate; |
16
|
|
|
use App\Domain\OuvrageComplete; |
17
|
|
|
use App\Domain\OuvrageFactory; |
18
|
|
|
use App\Domain\OuvrageOptimize; |
19
|
|
|
use App\Domain\Utils\TemplateParser; |
20
|
|
|
use App\Infrastructure\DbAdapter; |
21
|
|
|
use App\Infrastructure\GoogleBooksAdapter; |
22
|
|
|
use Normalizer; |
23
|
|
|
use Throwable; |
24
|
|
|
|
25
|
|
|
include __DIR__.'/../myBootstrap.php'; |
26
|
|
|
|
27
|
|
|
$dirty = new CompleteProcess(new DbAdapter()); |
28
|
|
|
$dirty->run(); |
29
|
|
|
|
30
|
|
|
/** |
31
|
|
|
* Class CompleteProcess |
32
|
|
|
*/ |
33
|
|
|
class CompleteProcess |
34
|
|
|
{ |
35
|
|
|
/** |
36
|
|
|
* @var QueueInterface |
37
|
|
|
*/ |
38
|
|
|
private $queueAdapter; |
39
|
|
|
/** |
40
|
|
|
* @var string |
41
|
|
|
*/ |
42
|
|
|
private $raw = ''; |
43
|
|
|
private $log = []; |
44
|
|
|
private $notCosmetic = false; |
45
|
|
|
private $major = false; |
46
|
|
|
/** |
47
|
|
|
* @var OuvrageTemplate |
48
|
|
|
*/ |
49
|
|
|
private $ouvrage; |
50
|
|
|
private $continue = true; |
|
|
|
|
51
|
|
|
|
52
|
|
|
public function __construct(QueueInterface $queueAdapter) |
53
|
|
|
{ |
54
|
|
|
$this->queueAdapter = $queueAdapter; |
55
|
|
|
} |
56
|
|
|
|
57
|
|
|
public function run(?int $limit = 10000) |
58
|
|
|
{ |
59
|
|
|
$memory = new Memory(); |
60
|
|
|
while ($limit > 0) { |
61
|
|
|
$limit--; |
62
|
|
|
sleep(1); |
63
|
|
|
$this->raw = $this->getNewRaw(); |
64
|
|
|
|
65
|
|
|
echo sprintf( |
66
|
|
|
"-------------------------------\n%s [%s]\n\n%s\n", |
67
|
|
|
date("Y-m-d H:i:s"), |
68
|
|
|
Bot::getGitVersion() ?? '', |
69
|
|
|
$this->raw |
70
|
|
|
); |
71
|
|
|
$memory->echoMemory(true); |
72
|
|
|
|
73
|
|
|
// initialise variables |
74
|
|
|
$this->log = []; |
75
|
|
|
$this->ouvrage = null; |
76
|
|
|
$this->notCosmetic = false; |
77
|
|
|
$this->major = false; |
78
|
|
|
|
79
|
|
|
|
80
|
|
|
try { |
81
|
|
|
$parse = TemplateParser::parseAllTemplateByName('ouvrage', $this->raw); |
82
|
|
|
$origin = $parse['ouvrage'][0]['model'] ?? null; |
83
|
|
|
} catch (Throwable $e) { |
84
|
|
|
echo sprintf("*** ERREUR impossible de transformer en modèle %s \n", $this->raw); |
85
|
|
|
continue; |
86
|
|
|
} |
87
|
|
|
|
88
|
|
|
if (!$origin instanceof OuvrageTemplate) { |
89
|
|
|
echo sprintf("*** ERREUR impossible de transformer en modèle %s \n", $this->raw); |
90
|
|
|
continue; |
91
|
|
|
} |
92
|
|
|
|
93
|
|
|
// Final optimizing (with online predictions) |
94
|
|
|
$optimizer = new OuvrageOptimize($origin, null); |
95
|
|
|
$optimizer->doTasks(); |
96
|
|
|
$this->ouvrage = $optimizer->getOuvrage(); |
97
|
|
|
$this->log = array_merge($this->log, $optimizer->getLog()); |
98
|
|
|
$this->notCosmetic = ($optimizer->notCosmetic || $this->notCosmetic); |
99
|
|
|
|
100
|
|
|
/** |
101
|
|
|
* RECHERCHE ONLINE |
102
|
|
|
*/ |
103
|
|
|
$isbn = $origin->getParam('isbn') ?? null; // avant mise en forme EAN>ISBN |
104
|
|
|
$isbn10 = $origin->getParam('isbn10') ?? null; |
105
|
|
|
if (!empty($isbn) |
106
|
|
|
&& empty($origin->getParam('isbn invalide')) |
107
|
|
|
&& empty($origin->getParam('isbn erroné')) |
108
|
|
|
) { |
109
|
|
|
$this->onlineIsbnSearch($isbn, $isbn10); |
110
|
|
|
} |
111
|
|
|
|
112
|
|
|
$this->sendCompleted(); |
113
|
|
|
unset($optimizer); |
114
|
|
|
unset($parse); |
115
|
|
|
unset($origin); |
116
|
|
|
} // END WHILE |
117
|
|
|
|
118
|
|
|
return true; |
119
|
|
|
} |
120
|
|
|
|
121
|
|
|
/** |
122
|
|
|
* Get raw string to complete from AMQP queue, SQL Select or file reading. |
123
|
|
|
* |
124
|
|
|
* @return string|null |
125
|
|
|
*/ |
126
|
|
|
private function getNewRaw(): ?string |
127
|
|
|
{ |
128
|
|
|
$raw = $this->queueAdapter->getNewRaw(); |
129
|
|
|
if (!$raw) { |
130
|
|
|
echo "STOP: no more queue to process \n"; |
131
|
|
|
exit; |
|
|
|
|
132
|
|
|
} |
133
|
|
|
|
134
|
|
|
return $raw; |
135
|
|
|
} |
136
|
|
|
|
137
|
|
|
private function onlineIsbnSearch(string $isbn, ?string $isbn10 = null) |
138
|
|
|
{ |
139
|
|
|
online: |
140
|
|
|
echo "sleep 20...\n"; |
141
|
|
|
sleep(40); |
142
|
|
|
|
143
|
|
|
try { |
144
|
|
|
dump('BIBLIO NAT FRANCE...'); |
145
|
|
|
// BnF sait pas trouver un vieux livre (10) d'après ISBN-13... FACEPALM ! |
146
|
|
|
if ($isbn10) { |
147
|
|
|
$bnfOuvrage = OuvrageFactory::BnfFromIsbn($isbn10); |
148
|
|
|
sleep(2); |
149
|
|
|
} |
150
|
|
|
if (!$isbn10 || !isset($bnfOuvrage) || empty($bnfOuvrage->getParam('titre'))) { |
151
|
|
|
$bnfOuvrage = OuvrageFactory::BnfFromIsbn($isbn); |
152
|
|
|
} |
153
|
|
|
if (isset($bnfOuvrage) and $bnfOuvrage instanceof OuvrageTemplate) { |
154
|
|
|
$this->completeOuvrage($bnfOuvrage); |
155
|
|
|
} |
156
|
|
|
} catch (Throwable $e) { |
157
|
|
|
echo "*** ERREUR BnF Isbn Search".$e->getMessage()."\n"; |
158
|
|
|
// echo "sleep 5min\n"; |
159
|
|
|
// sleep(60 * 5); |
160
|
|
|
// echo "Wake up\n"; |
161
|
|
|
// goto online; |
162
|
|
|
} |
163
|
|
|
|
164
|
|
|
try { |
165
|
|
|
if (!$bnfOuvrage || !$this->skipGoogle($bnfOuvrage)) { |
|
|
|
|
166
|
|
|
dump('GOOGLE...'); |
167
|
|
|
$googleOuvrage = OuvrageFactory::GoogleFromIsbn($isbn); |
168
|
|
|
$this->completeOuvrage($googleOuvrage); |
169
|
|
|
} |
170
|
|
|
} catch (Throwable $e) { |
171
|
|
|
echo "*** ERREUR GOOGLE Isbn Search ***".$e->getMessage()."\n"; |
172
|
|
|
sleep(10); |
173
|
|
|
if (strpos($e->getMessage(), 'Daily Limit Exceeded') !== false) { |
174
|
|
|
echo "sleep 1h\n"; |
175
|
|
|
sleep(60 * 60); |
176
|
|
|
echo "Wake up\n"; |
177
|
|
|
goto online; |
178
|
|
|
} |
179
|
|
|
} |
180
|
|
|
|
181
|
|
|
|
182
|
|
|
try { |
183
|
|
|
dump('OpenLibrary...'); |
184
|
|
|
$openLibraryOuvrage = OuvrageFactory::OpenLibraryFromIsbn($isbn); |
185
|
|
|
if (!empty($openLibraryOuvrage)) { |
186
|
|
|
$this->completeOuvrage($openLibraryOuvrage); |
187
|
|
|
} |
188
|
|
|
} catch (Throwable $e) { |
189
|
|
|
echo '**** ERREUR OpenLibrary Isbn Search'; |
190
|
|
|
} |
191
|
|
|
} |
192
|
|
|
|
193
|
|
|
private function onlineQuerySearch(string $query) |
|
|
|
|
194
|
|
|
{ |
195
|
|
|
echo "sleep 40..."; |
196
|
|
|
sleep(20); |
197
|
|
|
onlineQuerySearch: |
198
|
|
|
|
199
|
|
|
try { |
200
|
|
|
dump('GOOGLE SEARCH...'); |
201
|
|
|
// $googleOuvrage = OuvrageFactory::GoogleFromIsbn($isbn); |
202
|
|
|
$adapter = new GoogleBooksAdapter(); |
203
|
|
|
$data = $adapter->search('blabla'); |
204
|
|
|
dump($data); |
205
|
|
|
die; |
|
|
|
|
206
|
|
|
// return $import->getOuvrage(); |
207
|
|
|
// $this->completeOuvrage($googleOuvrage); |
208
|
|
|
} catch (Throwable $e) { |
209
|
|
|
echo "*** ERREUR GOOGLE QuerySearch *** ".$e->getMessage()."\n"; |
210
|
|
|
echo "sleep 30min"; |
211
|
|
|
sleep(60 * 30); |
212
|
|
|
echo "Wake up\n"; |
213
|
|
|
goto onlineQuerySearch; |
214
|
|
|
} |
215
|
|
|
} |
216
|
|
|
|
217
|
|
|
private function completeOuvrage(OuvrageTemplate $onlineOuvrage) |
218
|
|
|
{ |
219
|
|
|
dump($onlineOuvrage->serialize(true)); |
220
|
|
|
$optimizer = new OuvrageOptimize($onlineOuvrage); |
221
|
|
|
$onlineOptimized = ($optimizer)->doTasks()->getOuvrage(); |
222
|
|
|
|
223
|
|
|
$completer = new OuvrageComplete($this->ouvrage, $onlineOptimized); |
224
|
|
|
$this->ouvrage = $completer->getResult(); |
225
|
|
|
dump($completer->getLog()); |
226
|
|
|
if ($completer->major) { |
227
|
|
|
$this->major = true; |
228
|
|
|
} |
229
|
|
|
$this->notCosmetic = ($completer->notCosmetic || $this->notCosmetic); |
230
|
|
|
$this->log = array_merge($this->log, $completer->getLog()); |
231
|
|
|
unset($optimizer); |
232
|
|
|
unset($completer); |
233
|
|
|
} |
234
|
|
|
|
235
|
|
|
private function sendCompleted() |
236
|
|
|
{ |
237
|
|
|
$isbn13 = $this->ouvrage->getParam('isbn') ?? null; |
238
|
|
|
|
239
|
|
|
$finalData = [ |
240
|
|
|
// 'page' => |
241
|
|
|
'raw' => $this->raw, |
242
|
|
|
'opti' => $this->serializeFinalOpti(), |
243
|
|
|
'optidate' => date("Y-m-d H:i:s"), |
244
|
|
|
'modifs' => mb_substr(implode(',', $this->log), 0, 250), |
245
|
|
|
'notcosmetic' => ($this->notCosmetic) ? 1 : 0, |
246
|
|
|
'major' => ($this->major) ? 1 : 0, |
247
|
|
|
'isbn' => substr($isbn13, 0, 20), |
248
|
|
|
'version' => Bot::getGitVersion() ?? null, |
249
|
|
|
]; |
250
|
|
|
dump($finalData); |
251
|
|
|
// Json ? |
252
|
|
|
$result = $this->queueAdapter->sendCompletedData($finalData); |
253
|
|
|
dump($result); // bool |
254
|
|
|
} |
255
|
|
|
|
256
|
|
|
/** |
257
|
|
|
* Final serialization of the completed OuvrageTemplate. |
258
|
|
|
* |
259
|
|
|
* @return string |
260
|
|
|
*/ |
261
|
|
|
private function serializeFinalOpti(): string |
262
|
|
|
{ |
263
|
|
|
$finalOpti = $this->ouvrage->serialize(true); |
264
|
|
|
$finalOpti = Normalizer::normalize($finalOpti); |
265
|
|
|
|
266
|
|
|
return $finalOpti; |
267
|
|
|
} |
268
|
|
|
|
269
|
|
|
private function skipGoogle(OuvrageTemplate $bnfOuvrage): bool |
|
|
|
|
270
|
|
|
{ |
271
|
|
|
return false; |
272
|
|
|
} |
273
|
|
|
} |
274
|
|
|
|