|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
/* |
|
4
|
|
|
* This file is part of dispositif/wikibot application (@github) |
|
5
|
|
|
* 2019-2023 © Philippe M./Irønie <[email protected]> |
|
6
|
|
|
* For the full copyright and MIT license information, view the license file. |
|
7
|
|
|
*/ |
|
8
|
|
|
declare(strict_types=1); |
|
9
|
|
|
|
|
10
|
|
|
namespace App\Application\ExternLink; |
|
11
|
|
|
|
|
12
|
|
|
use App\Application\AbstractRefBotWorker; |
|
13
|
|
|
use App\Application\InfrastructurePorts\PageListForAppInterface as PageListInterface; |
|
14
|
|
|
use App\Application\WikiBotConfig; |
|
15
|
|
|
use App\Domain\Exceptions\ConfigException; |
|
16
|
|
|
use App\Domain\ExternLink\ExternRefTransformer; |
|
17
|
|
|
use Mediawiki\Api\MediawikiFactory; |
|
18
|
|
|
use Throwable; |
|
19
|
|
|
|
|
20
|
|
|
/** |
|
21
|
|
|
* TODO add construct arguments for TOR-Enabled |
|
22
|
|
|
*/ |
|
23
|
|
|
class ExternRefWorker extends AbstractRefBotWorker |
|
24
|
|
|
{ |
|
25
|
|
|
public const TOR_ENABLED_FOR_WEB_CRAWL = true; |
|
26
|
|
|
public const TASK_BOT_FLAG = true; |
|
27
|
|
|
public const MAX_REFS_PROCESSED_IN_ARTICLE = 30; |
|
28
|
|
|
public const SLEEP_AFTER_EDITION = 5; // sec |
|
29
|
|
|
public const MINUTES_DELAY_AFTER_LAST_HUMAN_EDIT = 10; // minutes |
|
30
|
|
|
public const CHECK_EDIT_CONFLICT = true; |
|
31
|
|
|
public const ARTICLE_ANALYZED_FILENAME = __DIR__ . '/../resources/article_externRef_edited.txt'; |
|
32
|
|
|
public const SKIP_ADQ = false; |
|
33
|
|
|
public const SKIP_LASTEDIT_BY_BOT = false; |
|
34
|
|
|
public const CITATION_NUMBER_ON_FIRE = 15; |
|
35
|
|
|
public const CITATION_NUMBER_NO_BOTFLAG = 20; |
|
36
|
|
|
public const DEAD_LINK_NO_BOTFLAG = 5; |
|
37
|
|
|
public const SKIP_SITE_BLACKLISTED = true; |
|
38
|
|
|
public const SKIP_ROBOT_NOINDEX = true; |
|
39
|
|
|
protected const STRING_WAYBACK_URL = '://web.archive.org/web/'; |
|
40
|
|
|
protected const STRING_WIKIWIX_URL = 'https://archive.wikiwix.com/cache/'; |
|
41
|
|
|
|
|
42
|
|
|
protected $modeAuto = true; |
|
43
|
|
|
|
|
44
|
|
|
protected ?ExternRefTransformer $transformer; |
|
45
|
|
|
protected array $webArchivers = []; |
|
46
|
|
|
|
|
47
|
|
|
public function __construct( |
|
48
|
|
|
WikiBotConfig $bot, |
|
49
|
|
|
MediawikiFactory $wiki, |
|
50
|
|
|
?PageListInterface $pagesGen = null, |
|
51
|
|
|
ExternRefTransformer $transformer = null |
|
52
|
|
|
) |
|
53
|
|
|
{ |
|
54
|
|
|
if (!$transformer instanceof ExternRefTransformer) { |
|
55
|
|
|
throw new ConfigException('ExternRefTransformer not set'); |
|
56
|
|
|
} |
|
57
|
|
|
$this->transformer = $transformer; |
|
58
|
|
|
$this->transformer->skipSiteBlacklisted = self::SKIP_SITE_BLACKLISTED; |
|
59
|
|
|
$this->transformer->skipRobotNoIndex = self::SKIP_ROBOT_NOINDEX; |
|
60
|
|
|
|
|
61
|
|
|
parent::__construct($bot, $wiki, $pagesGen); |
|
62
|
|
|
} |
|
63
|
|
|
|
|
64
|
|
|
|
|
65
|
|
|
/** |
|
66
|
|
|
* Traite contenu d'une <ref> ou bien lien externe (précédé d'une puce). |
|
67
|
|
|
*/ |
|
68
|
|
|
public function processRefContent(string $refContent): string |
|
69
|
|
|
{ |
|
70
|
|
|
// todo // hack Temporary Skip URL |
|
71
|
|
|
if (preg_match('#books\.google#', $refContent)) { |
|
72
|
|
|
$this->log->stats->increment('externref.skip.booksgoogle'); |
|
|
|
|
|
|
73
|
|
|
return $refContent; |
|
74
|
|
|
} |
|
75
|
|
|
|
|
76
|
|
|
try { |
|
77
|
|
|
$result = $this->transformer->process($refContent, $this->summary); |
|
|
|
|
|
|
78
|
|
|
} catch (Throwable $e) { |
|
79
|
|
|
$this->log->critical( |
|
80
|
|
|
'Error patate34 ' . $e->getMessage() . " " . $e->getFile() . ":" . $e->getLine(), |
|
81
|
|
|
['stats' => 'externref.exception.patate34'] |
|
82
|
|
|
); |
|
83
|
|
|
// TODO : parse $e->message -> variable process, taskName, botflag... |
|
84
|
|
|
|
|
85
|
|
|
return $refContent; |
|
86
|
|
|
} |
|
87
|
|
|
|
|
88
|
|
|
if (trim($result) === trim($refContent)) { |
|
89
|
|
|
$this->log->stats->increment('externref.transform.same'); |
|
90
|
|
|
|
|
91
|
|
|
return $refContent; |
|
92
|
|
|
} |
|
93
|
|
|
|
|
94
|
|
|
// Gestion semi-auto : todo CONDITION POURRI FAUSSE $this->transformer->skipUnauthorised |
|
95
|
|
|
|
|
96
|
|
|
$this->printDiff($refContent, $result, 'echo'); |
|
97
|
|
|
if (!$this->autoOrYesConfirmation('Conserver cette modif ?')) { |
|
98
|
|
|
return $refContent; |
|
99
|
|
|
} |
|
100
|
|
|
|
|
101
|
|
|
|
|
102
|
|
|
if (preg_match('#{{lien brisé#i', $result)) { |
|
103
|
|
|
$this->log->stats->increment('externref.transform.lienbrisé'); |
|
104
|
|
|
$this->summary->memo['count lien brisé'] = 1 + ($this->summary->memo['count lien brisé'] ?? 0); |
|
105
|
|
|
if ($this->summary->memo['count lien brisé'] >= self::DEAD_LINK_NO_BOTFLAG) { |
|
106
|
|
|
$this->summary->setBotFlag(false); |
|
107
|
|
|
} |
|
108
|
|
|
} |
|
109
|
|
|
|
|
110
|
|
|
if (str_contains($result, self::STRING_WIKIWIX_URL)) { |
|
111
|
|
|
$this->log->stats->increment('externref.transform.wikiwix'); |
|
112
|
|
|
$this->summary->memo['wikiwix'] = 1 + ($this->summary->memo['wikiwix'] ?? 0); |
|
113
|
|
|
if ($this->summary->memo['wikiwix'] >= self::DEAD_LINK_NO_BOTFLAG) { |
|
114
|
|
|
$this->summary->setBotFlag(false); |
|
115
|
|
|
} |
|
116
|
|
|
} |
|
117
|
|
|
// not httpS in 2023 |
|
118
|
|
|
if (str_contains($result, self::STRING_WAYBACK_URL)) { |
|
119
|
|
|
$this->log->stats->increment('externref.transform.wayback'); |
|
120
|
|
|
$this->summary->memo['wayback'] = 1 + ($this->summary->memo['wayback'] ?? 0); |
|
121
|
|
|
if ($this->summary->memo['wayback'] >= self::DEAD_LINK_NO_BOTFLAG) { |
|
122
|
|
|
$this->summary->setBotFlag(false); |
|
123
|
|
|
} |
|
124
|
|
|
} |
|
125
|
|
|
|
|
126
|
|
|
if ($this->summary->citationNumber >= self::CITATION_NUMBER_NO_BOTFLAG) { |
|
127
|
|
|
$this->summary->setBotFlag(false); |
|
128
|
|
|
} |
|
129
|
|
|
|
|
130
|
|
|
$this->log->stats->increment('externref.transform.total'); |
|
131
|
|
|
$this->summary->memo['count URL'] = 1 + ($this->summary->memo['count URL'] ?? 0); |
|
132
|
|
|
|
|
133
|
|
|
return $result; |
|
134
|
|
|
} |
|
135
|
|
|
|
|
136
|
|
|
/** |
|
137
|
|
|
* todo move to a Summary child ? |
|
138
|
|
|
* Rewriting default Summary::serialize() |
|
139
|
|
|
* @return string |
|
140
|
|
|
*/ |
|
141
|
|
|
protected function generateSummaryText(): string |
|
142
|
|
|
{ |
|
143
|
|
|
$prefixSummary = ($this->summary->isBotFlag()) ? 'bot ' : ''; |
|
144
|
|
|
$suffix = ''; |
|
145
|
|
|
if (isset($this->summary->memo['count article'])) { |
|
146
|
|
|
$this->log->stats->increment('externref.count.article'); |
|
|
|
|
|
|
147
|
|
|
$suffix .= ' ' . $this->summary->memo['count article'] . 'x {article}'; |
|
148
|
|
|
} |
|
149
|
|
|
if (isset($this->summary->memo['count lien web'])) { |
|
150
|
|
|
$this->log->stats->increment('externref.count.lienweb'); |
|
151
|
|
|
$suffix .= ' ' . $this->summary->memo['count lien web'] . 'x {lien web}'; |
|
152
|
|
|
} |
|
153
|
|
|
if (isset($this->summary->memo['presse'])) { |
|
154
|
|
|
$this->log->stats->increment('externref.count.presse'); |
|
155
|
|
|
$suffix .= ' 🗞️'; // 🗞️ 📰 |
|
156
|
|
|
} |
|
157
|
|
|
if (isset($this->summary->memo['science'])) { |
|
158
|
|
|
$this->log->stats->increment('externref.count.science'); |
|
159
|
|
|
$suffix .= ' 🧪'; // 🧪 🔬 |
|
160
|
|
|
} |
|
161
|
|
|
if (isset($this->summary->memo['count lien brisé'])) { |
|
162
|
|
|
$this->log->stats->increment('externref.count.lienbrisé'); |
|
163
|
|
|
$suffix .= ' ⚠️️️lien brisé'; //⚠️💩 |
|
164
|
|
|
$suffix .= ($this->summary->memo['count lien brisé'] > 1) |
|
165
|
|
|
? ' x' . $this->summary->memo['count lien brisé'] |
|
166
|
|
|
: ''; |
|
167
|
|
|
} |
|
168
|
|
|
if (isset($this->summary->memo['wikiwix'])) { |
|
169
|
|
|
$suffix .= ' '; |
|
170
|
|
|
$suffix .= ($this->summary->memo['wikiwix'] > 1) |
|
171
|
|
|
? $this->summary->memo['wikiwix'] . 'x ' |
|
172
|
|
|
: ''; |
|
173
|
|
|
$suffix .= 'Wikiwix'; |
|
174
|
|
|
} |
|
175
|
|
|
if (isset($this->summary->memo['wayback'])) { |
|
176
|
|
|
$suffix .= ' '; |
|
177
|
|
|
$suffix .= ($this->summary->memo['wayback'] > 1) |
|
178
|
|
|
? $this->summary->memo['wayback'] . 'x ' |
|
179
|
|
|
: ''; |
|
180
|
|
|
$suffix .= 'InternetArchive'; |
|
181
|
|
|
} |
|
182
|
|
|
|
|
183
|
|
|
if (isset($this->summary->memo['accès url non libre'])) { |
|
184
|
|
|
$suffix .= ' 🔒'; |
|
185
|
|
|
} |
|
186
|
|
|
|
|
187
|
|
|
if ($this->summary->citationNumber >= self::CITATION_NUMBER_ON_FIRE) { |
|
188
|
|
|
$suffix .= ' 🔥'; |
|
189
|
|
|
} |
|
190
|
|
|
|
|
191
|
|
|
return $prefixSummary . $this->summary->taskName . $suffix; |
|
192
|
|
|
} |
|
193
|
|
|
} |
|
194
|
|
|
|