Completed
Push — master ( 511693...0349ad )
by Dispositif
04:19
created

EditProcess::initialize()   A

Complexity

Conditions 1
Paths 1

Size

Total Lines 12
Code Lines 8

Duplication

Lines 0
Ratio 0 %

Importance

Changes 1
Bugs 0 Features 0
Metric Value
cc 1
eloc 8
c 1
b 0
f 0
nc 1
nop 0
dl 0
loc 12
rs 10
1
<?php
2
/**
3
 * This file is part of dispositif/wikibot application
4
 * 2019 : Philippe M. <[email protected]>
5
 * For the full copyright and MIT license information, please view the LICENSE file.
6
 */
7
8
declare(strict_types=1);
9
10
namespace App\Application\Examples;
11
12
use App\Application\Bot;
13
use App\Application\Memory;
14
use App\Application\WikiPageAction;
15
use App\Domain\Utils\WikiTextUtil;
16
use App\Infrastructure\DbAdapter;
17
use App\Infrastructure\ServiceFactory;
18
use Exception;
19
use LogicException;
20
use Mediawiki\DataModel\EditInfo;
21
use Normalizer;
22
use Throwable;
23
24
//use App\Application\CLI;
25
26
include __DIR__.'/../myBootstrap.php';
27
28
$process = new EditProcess();
29
$process->run();
30
31
/**
32
 * TODO refac
33
 * Class EditProcess
34
 *
35
 * @package App\Application\Examples
36
 */
37
class EditProcess
38
{
39
    const TASK_NAME        = 'Amélioration bibliographique';
40
    const EDIT_SIGNALEMENT = true;
41
42
    const CITATION_LIMIT         = 150;
43
    const DELAY_BOTFLAG_SECONDS  = 10;
44
    const DELAY_NOBOT_IN_SECONDS = 60;
45
    const ERROR_MSG_TEMPLATE     = __DIR__.'/../templates/message_errors.wiki';
46
47
    private $db;
48
    private $bot;
49
    private $wiki;
50
    private $wikiText;
51
    private $citationSummary;
52
    private $citationVersion = '';
53
    private $errorWarning = [];
54
    private $importantSummary = [];
55
    private $nbRows;
56
57
    // Minor flag on edit
58
    private $minorFlag = true;
59
    // Bot flag on edit
60
    private $botFlag = true;
61
62
    public function __construct()
63
    {
64
        $this->db = new DbAdapter();
65
        $this->bot = new Bot();
66
67
        $this->wikiLogin();
68
    }
69
70
    public function run()
71
    {
72
        $memory = new Memory();
73
        while (true) {
74
            echo "\n-------------------------------------\n\n";
75
            echo date("Y-m-d H:i")."\n";
76
            $memory->echoMemory(true);
77
78
            $this->pageProcess();
79
        }
80
    }
81
82
    private function pageProcess()
83
    {
84
        $this->initialize();
85
86
        // get a random queue line
87
        $json = $this->db->getAllRowsToEdit(self::CITATION_LIMIT);
88
        $data = json_decode($json, true);
89
90
        if (empty($data)) {
91
            echo "SKIP : no rows to process\n";
92
            echo "Sleep 2h.";
93
            sleep(60 * 120);
94
95
            // or exit
96
            return false;
97
        }
98
99
        try {
100
            $title = $data[0]['page'];
101
            echo "$title \n";
102
            $page = new WikiPageAction($this->wiki, $title);
103
        } catch (Exception $e) {
104
            echo "*** WikiPageAction error : $title \n";
105
            sleep(20);
106
107
            return false;
108
        }
109
110
        // TODO : HACK
111
        if (in_array($page->getLastEditor(), [getenv('BOT_NAME'), getenv('BOT_OWNER')])) {
112
            echo "SKIP : édité recemment par bot/dresseur.\n";
113
            $this->db->skipArticle($title);
114
115
            return false;
116
        }
117
        $this->wikiText = $page->getText();
118
119
        if (BOT::isEditionRestricted($this->wikiText)) {
120
            echo "SKIP : protection/3R.\n";
121
            $this->db->skipArticle($title);
122
        }
123
124
        if ($this->bot->minutesSinceLastEdit($title) < 15) {
125
            echo "SKIP : édition humaine dans les dernières 15 minutes.\n";
126
127
            return false;
128
        }
129
130
        // Skip AdQ
131
        if (preg_match('#{{ ?En-tête label#i', $this->wikiText) > 0) {
132
            echo "SKIP : AdQ ou BA.\n";
133
            $this->db->skipArticle($title);
134
135
            return false;
136
        }
137
138
        // GET all article lines from db
139
        echo sprintf(">> %s rows to process\n", count($data));
140
141
        // foreach line
142
        $changed = false;
143
        foreach ($data as $dat) {
144
            // hack temporaire pour éviter articles dont CompleteProcess incomplet
145
            if (empty($dat['opti']) || empty($dat['optidate']) || $dat['optidate'] < DbAdapter::OPTI_VALID_DATE) {
146
                echo "SKIP : Complètement incomplet de l'article \n";
147
148
                return false;
149
            }
150
            $success = $this->dataProcess($dat);
151
            $changed = ($success) ? true : $changed;
152
        }
153
        if (!$changed) {
154
            echo "Rien à changer...\n\n";
155
            $this->db->skipArticle($title);
156
157
            return false;
158
        }
159
160
        // EDIT THE PAGE
161
        if (!$this->wikiText) {
162
            return false;
163
        }
164
165
        $miniSummary = $this->generateSummary();
166
        echo "Edition ?\n".$miniSummary."\n\n";
167
        echo "sleep 20...\n";
168
        sleep(20);
169
170
        pageEdit:
171
172
        try{
173
            $editInfo = new EditInfo($miniSummary, $this->minorFlag, $this->botFlag);
174
            $success = $page->editPage(Normalizer::normalize($this->wikiText), $editInfo);
175
        }catch (\Throwable $e){
176
            // Invalid CSRF token.
177
            if(strpos($e->getMessage(), 'Invalid CSRF token') !== false ) {
178
                echo "*** Invalid CSRF token \n";
179
                sleep(60);
180
                $this->wikiLogin();
181
                goto pageEdit;
182
            }else{
183
                dump($e); // todo log
184
                sleep(60);
185
                return false;
186
            }
187
        }
188
189
        echo ($success) ? "Ok\n" : "***** Erreur edit\n";
190
191
        if ($success) {
192
            // updata DB
193
            foreach ($data as $dat) {
194
                $this->db->sendEditedData(['id' => $dat['id']]);
195
            }
196
197
            try {
198
                if (self::EDIT_SIGNALEMENT) {
199
                    $this->sendErrorMessage($data);
200
                }
201
            } catch (Throwable $e) {
202
                dump($e);
203
                unset($e);
204
            }
205
206
            if (!$this->botFlag) {
207
                echo "sleep ".self::DELAY_NOBOT_IN_SECONDS."\n";
208
                sleep(self::DELAY_NOBOT_IN_SECONDS);
209
            }
210
            echo "sleep ".self::DELAY_BOTFLAG_SECONDS."\n";
211
            sleep(self::DELAY_BOTFLAG_SECONDS);
212
        }
213
214
        return $success;
215
    }
216
217
    private function dataProcess(array $data): bool
218
    {
219
        $origin = $data['raw'];
220
        $completed = $data['opti'];
221
222
        dump($origin, $completed, $data['modifs'], $data['version']);
223
224
        if (WikiTextUtil::isCommented($origin)) {
225
            echo "SKIP: template avec commentaire HTML\n";
226
            $this->db->skipRow(intval($data['id']));
227
228
            return false;
229
        }
230
231
        $find = mb_strpos($this->wikiText, $origin);
232
        if ($find === false) {
233
            echo "String non trouvée. \n\n";
234
            $this->db->skipRow(intval($data['id']));
235
236
            return false;
237
        }
238
239
        $this->checkErrorWarning($data);
240
241
        // Replace text
242
        $newText = WikiPageAction::replaceTemplateInText($this->wikiText, $origin, $completed);
243
244
        if (!$newText || $newText === $this->wikiText) {
245
            echo "newText error\n";
246
247
            return false;
248
        }
249
        $this->wikiText = $newText;
250
        $this->minorFlag = ('1' === $data['major']) ? false : $this->minorFlag;
251
        $this->citationVersion = $data['version'];
252
        $this->citationSummary[] = $data['modifs'];
253
        $this->nbRows++;
254
255
        return true;
256
    }
257
258
    /**
259
     * Generate wiki edition summary.
260
     *
261
     * @return string
262
     */
263
    public function generateSummary(): string
264
    {
265
        // Start summary with "Bot" when using botflag, else "*"
266
        $prefix = ($this->botFlag) ? 'bot' : '☆';
267
        // add "/!\" when errorWarning
268
        $prefix = (!empty($this->errorWarning) && !$this->botFlag) ? '⚠' : $prefix;
269
270
271
        // basic modifs
272
        $citeSummary = implode(' ', $this->citationSummary);
273
        // replace by list of modifs to verify by humans
274
        if (!empty($this->importantSummary)) {
275
            $citeSummary = implode(', ', $this->importantSummary);
276
        }
277
278
        $summary = sprintf(
279
            '%s [%s/%s] %s %s : %s',
280
            $prefix,
281
            str_replace('v', '', $this->bot::getGitVersion()),
282
            str_replace('v0.', '', $this->citationVersion),
283
            self::TASK_NAME,
284
            $this->nbRows,
285
            $citeSummary
286
        );
287
288
        if (!empty($this->importantSummary)) {
289
            $summary .= '...';
290
        }
291
292
        // shrink long summary if no important details to verify
293
        if (empty($this->importantSummary)) {
294
            $length = strlen($summary);
295
            $summary = substr($summary, 0, 80);
296
            $summary .= ($length > strlen($summary)) ? '…' : '';
297
        }
298
299
        return $summary;
300
    }
301
302
    /**
303
     * Vérifie alerte d'erreurs humaines.
304
     *
305
     * @param array $data
306
     *
307
     * @throws Exception
308
     */
309
    private function checkErrorWarning(array $data): void
310
    {
311
        if (!isset($data['opti'])) {
312
            throw new LogicException('Opti NULL');
313
        }
314
315
        // paramètre inconnu
316
        if (preg_match(
317
                "#\|[^|]+<!-- ?(PARAMETRE [^>]+ N'EXISTE PAS|VALEUR SANS NOM DE PARAMETRE) ?-->#",
318
                $data['opti'],
319
                $matches
320
            ) > 0
321
        ) {
322
            $this->addErrorWarning($data['page'], $matches[0]);
323
            $this->botFlag = false;
324
            $this->addSummaryTag('paramètre non corrigé');
325
        }
326
327
        // ISBN invalide
328
        if (preg_match("#isbn invalide ?=[^|}]+#i", $data['opti'], $matches) > 0) {
329
            $this->addErrorWarning($data['page'], $matches[0]);
330
            $this->botFlag = false;
331
            $this->addSummaryTag('ISBN invalide');
332
        }
333
334
        // Edits avec ajout conséquent de donnée
335
        if (preg_match('#distinction des auteurs#', $data['modifs']) > 0) {
336
            $this->botFlag = false;
337
            $this->addSummaryTag('distinction des auteurs');
338
        }
339
        // prédiction paramètre correct
340
        if (preg_match('#[^,]+(=>|⇒)[^,]+#', $data['modifs'], $matches) > 0) {
341
            $this->botFlag = false;
342
            $this->addSummaryTag(sprintf('%s', $matches[0]));
343
        }
344
        if (preg_match('#\+\+sous-titre#', $data['modifs']) > 0) {
345
            $this->botFlag = false;
346
            $this->addSummaryTag('+sous-titre');
347
        }
348
        if (preg_match('#\+lieu#', $data['modifs']) > 0) {
349
            $this->addSummaryTag('+lieu');
350
        }
351
        if (preg_match('#\+éditeur#', $data['modifs']) > 0) {
352
            $this->addSummaryTag('éditeur');
353
        }
354
        //        if (preg_match('#\+langue#', $data['modifs']) > 0) {
355
        //            $this->addSummaryTag('langue');
356
        //        }
357
358
        // mention BnF si ajout donnée + ajout identifiant bnf=
359
        if (!empty($this->importantSummary) && preg_match('#\+bnf#i', $data['modifs'], $matches) > 0) {
360
            $this->addSummaryTag('[[BnF]]');
361
        }
362
    }
363
364
    /**
365
     * Pour éviter les doublons dans signalements d'erreur.
366
     *
367
     * @param string $page
368
     * @param string $text
369
     */
370
    private function addErrorWarning(string $page, string $text): void
371
    {
372
        if (!isset($this->errorWarning[$page]) || !in_array($text, $this->errorWarning[$page])) {
373
            $this->errorWarning[$page][] = $text;
374
        }
375
    }
376
377
    /**
378
     * For substantive or ambiguous modifications done.
379
     *
380
     * @param string $tag
381
     */
382
    private function addSummaryTag(string $tag)
383
    {
384
        if (!in_array($tag, $this->importantSummary)) {
385
            $this->importantSummary[] = $tag;
386
        }
387
    }
388
389
    /**
390
     * @param array $rows Collection of citations
391
     */
392
    private function sendErrorMessage(array $rows): void
393
    {
394
        if (empty($this->errorWarning[$rows[0]['page']])) {
395
            return;
396
        }
397
        echo "** Send Error Message on talk page. Wait 10... \n";
398
        sleep(3);
399
400
        // format wiki message
401
        $errorList = '';
402
        foreach ($this->errorWarning[$rows[0]['page']] as $error) {
403
            $errorList .= sprintf("* <span style=\"background:#FCDFE8\"><nowiki>%s</nowiki></span> \n", $error);
404
        }
405
        $errorMessage = file_get_contents(self::ERROR_MSG_TEMPLATE);
406
        $errorMessage = str_replace('##ERROR LIST##', trim($errorList), $errorMessage);
407
        $errorMessage = str_replace('##ARTICLE##', $rows[0]['page'], $errorMessage);
408
409
        // Edit wiki talk page
410
        try {
411
            $talkPage = new WikiPageAction($this->wiki, 'Discussion:'.$rows[0]['page']);
412
            $editInfo = new EditInfo('Signalement erreur {ouvrage}', false, false);
413
            $talkPage->addToBottomOrCreatePage($errorMessage, $editInfo);
414
        } catch (Throwable $e) {
415
            dump($e);
416
            unset($e);
417
        }
418
    }
419
420
    private function initialize(): void
421
    {
422
        // initialisation vars
423
        $this->botFlag = true;
424
        $this->errorWarning = [];
425
        $this->wikiText = null;
426
        $this->citationSummary = [];
427
        $this->importantSummary = [];
428
        $this->minorFlag = true;
429
        $this->nbRows = 0;
430
431
        $this->bot->checkStopOnTalkpage();
432
    }
433
434
    private function wikiLogin(): void
435
    {
436
        $this->wiki = ServiceFactory::wikiApi();
437
    }
438
439
}
440