|
1
|
|
|
<?php declare(strict_types=1); defined('BASEPATH') OR exit('No direct script access allowed'); |
|
2
|
|
|
|
|
3
|
|
|
/** |
|
4
|
|
|
* Class Tracker_Sites_Model |
|
5
|
|
|
*/ |
|
6
|
|
|
class Tracker_Sites_Model extends CI_Model { |
|
7
|
127 |
|
public function __construct() { |
|
8
|
127 |
|
parent::__construct(); |
|
9
|
127 |
|
} |
|
10
|
|
|
|
|
11
|
|
|
public function __get($name) { |
|
12
|
|
|
//TODO: Is this a good idea? There wasn't a good consensus on if this is good practice or not.. |
|
13
|
|
|
// It's probably a minor speed reduction, but that isn't much of an issue. |
|
14
|
|
|
// An alternate solution would simply have a function which generates a PHP file with code to load each model. Similar to: https://github.com/shish/shimmie2/blob/834bc740a4eeef751f546979e6400fd089db64f8/core/util.inc.php#L1422 |
|
15
|
|
|
$validClasses = [ |
|
16
|
|
|
'Base_Site_Model', |
|
17
|
|
|
'Base_FoolSlide_Site_Model', |
|
18
|
|
|
'Base_myMangaReaderCMS_Site_Model', |
|
19
|
|
|
'Base_GlossyBright_Site_Model', |
|
20
|
|
|
'Base_Roku_Site_Model' |
|
21
|
|
|
]; |
|
22
|
|
|
if(!class_exists($name) || !(in_array(get_parent_class($name), $validClasses))) { |
|
23
|
|
|
return get_instance()->{$name}; |
|
24
|
|
|
} else { |
|
25
|
|
|
$this->loadSite($name); |
|
26
|
|
|
return $this->{$name}; |
|
27
|
|
|
} |
|
28
|
|
|
} |
|
29
|
|
|
|
|
30
|
|
|
private function loadSite(string $siteName) : void { |
|
31
|
|
|
$this->{$siteName} = new $siteName(); |
|
32
|
|
|
} |
|
33
|
|
|
} |
|
34
|
|
|
|
|
35
|
|
|
abstract class Base_Site_Model extends CI_Model { |
|
36
|
|
|
public $site = ''; |
|
37
|
|
|
public $titleFormat = '//'; |
|
38
|
|
|
public $chapterFormat = '//'; |
|
39
|
|
|
public $hasCloudFlare = FALSE; |
|
40
|
|
|
public $userAgent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2824.0 Safari/537.36'; |
|
41
|
|
|
|
|
42
|
|
|
public $baseURL = ''; |
|
43
|
|
|
|
|
44
|
|
|
/** |
|
45
|
|
|
* 0: No custom updater. |
|
46
|
|
|
* 1: Uses following page. |
|
47
|
|
|
* 2: Uses latest releases page. |
|
48
|
|
|
*/ |
|
49
|
|
|
public $customType = 0; |
|
50
|
|
|
|
|
51
|
|
|
public $canHaveNoChapters = FALSE; |
|
52
|
|
|
|
|
53
|
16 |
|
public function __construct() { |
|
54
|
16 |
|
parent::__construct(); |
|
55
|
|
|
|
|
56
|
16 |
|
$this->load->database(); |
|
57
|
|
|
|
|
58
|
16 |
|
$this->site = get_class($this); |
|
59
|
16 |
|
} |
|
60
|
|
|
|
|
61
|
|
|
/** |
|
62
|
|
|
* Generates URL to the title page of the requested series. |
|
63
|
|
|
* |
|
64
|
|
|
* NOTE: In some cases, we are required to store more data in the title_string than is needed to generate the URL. (Namely as the title_string is our unique identifier for that series) |
|
65
|
|
|
* When storing additional data, we use ':--:' as a delimiter to separate the data. Make sure to handle this as needed. |
|
66
|
|
|
* |
|
67
|
|
|
* Example: |
|
68
|
|
|
* return "http://mangafox.me/manga/{$title_url}/"; |
|
69
|
|
|
* |
|
70
|
|
|
* Example (with extra data): |
|
71
|
|
|
* $title_parts = explode(':--:', title_url); |
|
72
|
|
|
* return "https://bato.to/comic/_/comics/-r".$title_parts[0]; |
|
73
|
|
|
* |
|
74
|
|
|
* @param string $title_url |
|
75
|
|
|
* @return string |
|
76
|
|
|
*/ |
|
77
|
|
|
abstract public function getFullTitleURL(string $title_url) : string; |
|
78
|
|
|
|
|
79
|
|
|
/** |
|
80
|
|
|
* Generates chapter data from given $title_url and $chapter. |
|
81
|
|
|
* |
|
82
|
|
|
* Chapter must be in a (v[0-9]+/)?c[0-9]+(\..+)? format. |
|
83
|
|
|
* |
|
84
|
|
|
* NOTE: In some cases, we are required to store the chapter number, and the segment required to generate the chapter URL separately. |
|
85
|
|
|
* Much like when generating the title URL, we use ':--:' as a delimiter to separate the data. Make sure to handle this as needed. |
|
86
|
|
|
* |
|
87
|
|
|
* Example: |
|
88
|
|
|
* return [ |
|
89
|
|
|
* 'url' => $this->getFullTitleURL($title_url).'/'.$chapter, |
|
90
|
|
|
* 'number' => "c{$chapter}" |
|
91
|
|
|
* ]; |
|
92
|
|
|
* |
|
93
|
|
|
* @param string $title_url |
|
94
|
|
|
* @param string $chapter |
|
95
|
|
|
* @return array [url, number] |
|
96
|
|
|
*/ |
|
97
|
|
|
abstract public function getChapterData(string $title_url, string $chapter) : array; |
|
98
|
|
|
|
|
99
|
|
|
/** |
|
100
|
|
|
* Used to get the latest chapter of given $title_url. |
|
101
|
|
|
* |
|
102
|
|
|
* This <should> utilize both get_content and parseTitleDataDOM functions when possible, as these can both reduce a lot of the code required to set this up. |
|
103
|
|
|
* |
|
104
|
|
|
* $titleData params must be set accordingly: |
|
105
|
|
|
* * `title` should always be used with html_entity_decode. |
|
106
|
|
|
* * `latest_chapter` must match $this->chapterFormat. |
|
107
|
|
|
* * `last_updated` should always be in date("Y-m-d H:i:s") format. |
|
108
|
|
|
* * `followed` should never be set within via getTitleData, with the exception of via a array_merge with doCustomFollow. |
|
109
|
|
|
* |
|
110
|
|
|
* $firstGet is set to true when the series is first added to the DB, and is used to follow the series on given site (if possible). |
|
111
|
|
|
* |
|
112
|
|
|
* @param string $title_url |
|
113
|
|
|
* @param bool $firstGet |
|
114
|
|
|
* @return array|null [title,latest_chapter,last_updated,followed?] |
|
115
|
|
|
*/ |
|
116
|
|
|
abstract public function getTitleData(string $title_url, bool $firstGet = FALSE) : ?array; |
|
117
|
|
|
|
|
118
|
|
|
/** |
|
119
|
|
|
* Validates given $title_url against titleFormat. |
|
120
|
|
|
* |
|
121
|
|
|
* Failure to match against titleFormat will stop the series from being added to the DB. |
|
122
|
|
|
* |
|
123
|
|
|
* @param string $title_url |
|
124
|
|
|
* @return bool |
|
125
|
|
|
*/ |
|
126
|
2 |
|
final public function isValidTitleURL(string $title_url) : bool { |
|
127
|
2 |
|
$success = (bool) preg_match($this->titleFormat, $title_url); |
|
128
|
2 |
|
if(!$success) log_message('error', "Invalid Title URL ({$this->site}): {$title_url}"); |
|
129
|
2 |
|
return $success; |
|
130
|
|
|
} |
|
131
|
|
|
|
|
132
|
|
|
/** |
|
133
|
|
|
* Validates given $chapter against chapterFormat. |
|
134
|
|
|
* |
|
135
|
|
|
* Failure to match against chapterFormat will stop the chapter being updated. |
|
136
|
|
|
* |
|
137
|
|
|
* @param string $chapter |
|
138
|
|
|
* @return bool |
|
139
|
|
|
*/ |
|
140
|
2 |
|
final public function isValidChapter(string $chapter) : bool { |
|
141
|
2 |
|
$success = (bool) preg_match($this->chapterFormat, $chapter); |
|
142
|
2 |
|
if(!$success) log_message('error', "Invalid Chapter ({$this->site}): {$chapter}"); |
|
143
|
2 |
|
return $success; |
|
144
|
|
|
} |
|
145
|
|
|
|
|
146
|
|
|
/** |
|
147
|
|
|
* Used by getTitleData (& similar functions) to get the requested page data. |
|
148
|
|
|
* |
|
149
|
|
|
* @param string $url |
|
150
|
|
|
* @param string $cookie_string |
|
151
|
|
|
* @param string $cookiejar_path |
|
152
|
|
|
* @param bool $follow_redirect |
|
153
|
|
|
* @param bool $isPost |
|
154
|
|
|
* @param array $postFields |
|
155
|
|
|
* |
|
156
|
|
|
* @return array|bool |
|
157
|
|
|
*/ |
|
158
|
|
|
final protected function get_content(string $url, string $cookie_string = "", string $cookiejar_path = "", bool $follow_redirect = FALSE, bool $isPost = FALSE, array $postFields = []) { |
|
159
|
|
|
$refresh = TRUE; //For sites that have CloudFlare, we want to loop get_content again. |
|
160
|
|
|
$loops = 0; |
|
161
|
|
|
while($refresh && $loops < 2) { |
|
162
|
|
|
$refresh = FALSE; |
|
163
|
|
|
$loops++; |
|
164
|
|
|
|
|
165
|
|
|
$ch = curl_init(); |
|
166
|
|
|
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); |
|
167
|
|
|
curl_setopt($ch, CURLOPT_ENCODING , "gzip"); |
|
168
|
|
|
//curl_setopt($ch, CURLOPT_VERBOSE, 1); |
|
169
|
|
|
curl_setopt($ch, CURLOPT_HEADER, 1); |
|
170
|
|
|
|
|
171
|
|
|
if($follow_redirect) curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE); |
|
172
|
|
|
|
|
173
|
|
|
if($cookies = $this->cache->get("cloudflare_{$this->site}")) { |
|
174
|
|
|
$cookie_string .= "; {$cookies}"; |
|
175
|
|
|
} |
|
176
|
|
|
|
|
177
|
|
|
if(!empty($cookie_string)) curl_setopt($ch, CURLOPT_COOKIE, $cookie_string); |
|
178
|
|
|
if(!empty($cookiejar_path)) curl_setopt($ch, CURLOPT_COOKIEFILE, $cookiejar_path); |
|
179
|
|
|
|
|
180
|
|
|
//Some sites check the useragent for stuff, use a pre-defined user-agent to avoid stuff. |
|
181
|
|
|
curl_setopt($ch, CURLOPT_USERAGENT, $this->userAgent); |
|
182
|
|
|
|
|
183
|
|
|
//NOTE: This is required for SSL URLs for now. Without it we tend to get error code 60. |
|
184
|
|
|
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE); //FIXME: This isn't safe, but it allows us to grab SSL URLs |
|
185
|
|
|
|
|
186
|
|
|
curl_setopt($ch, CURLOPT_URL, $url); |
|
187
|
|
|
|
|
188
|
|
|
if($isPost) { |
|
189
|
|
|
curl_setopt($ch,CURLOPT_POST, count($postFields)); |
|
190
|
|
|
curl_setopt($ch,CURLOPT_POSTFIELDS, http_build_query($postFields)); |
|
191
|
|
|
} |
|
192
|
|
|
|
|
193
|
|
|
$response = curl_exec($ch); |
|
194
|
|
|
|
|
195
|
|
|
$this->Tracker->admin->incrementRequests(); |
|
196
|
|
|
|
|
197
|
|
|
if($response === FALSE) { |
|
198
|
|
|
log_message('error', "curl failed with error: ".curl_errno($ch)." | ".curl_error($ch)); |
|
199
|
|
|
//FIXME: We don't always account for FALSE return |
|
200
|
|
|
return FALSE; |
|
201
|
|
|
} |
|
202
|
|
|
|
|
203
|
|
|
$status_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); |
|
204
|
|
|
$header_size = curl_getinfo($ch, CURLINFO_HEADER_SIZE); |
|
205
|
|
|
$headers = http_parse_headers(substr($response, 0, $header_size)); |
|
206
|
|
|
$body = substr($response, $header_size); |
|
207
|
|
|
curl_close($ch); |
|
208
|
|
|
|
|
209
|
|
|
if($status_code === 503) $refresh = $this->handleCloudFlare($url, $body); |
|
210
|
|
|
} |
|
211
|
|
|
|
|
212
|
|
|
return [ |
|
213
|
|
|
'headers' => $headers, |
|
|
|
|
|
|
214
|
|
|
'status_code' => $status_code, |
|
|
|
|
|
|
215
|
|
|
'body' => $body |
|
|
|
|
|
|
216
|
|
|
]; |
|
217
|
|
|
} |
|
218
|
|
|
|
|
219
|
|
|
final private function handleCloudFlare(string $url, string $body) : bool { |
|
220
|
|
|
$refresh = FALSE; |
|
221
|
|
|
|
|
222
|
|
|
if((strpos($body, 'DDoS protection by Cloudflare') !== FALSE) || (strpos($body, '<input type="hidden" id="jschl-answer" name="jschl_answer"/>') !== FALSE)) { |
|
223
|
|
|
//print "Cloudflare detected? Grabbing Cookies.\n"; |
|
224
|
|
|
if(!$this->hasCloudFlare) { |
|
|
|
|
|
|
225
|
|
|
//TODO: Site appears to have enabled CloudFlare, disable it and contact admin. |
|
226
|
|
|
// We'll continue to bypass CloudFlare as this may occur in a loop. |
|
227
|
|
|
} |
|
228
|
|
|
|
|
229
|
|
|
$urlData = [ |
|
230
|
|
|
'url' => $url, |
|
231
|
|
|
'user_agent' => $this->userAgent |
|
232
|
|
|
]; |
|
233
|
|
|
//TODO: shell_exec seems bad since the URLs "could" be user inputted? Better way of doing this? |
|
234
|
|
|
$result = shell_exec('python '.APPPATH.'../_scripts/get_cloudflare_cookie.py '.escapeshellarg(json_encode($urlData))); |
|
235
|
|
|
$cookieData = json_decode($result, TRUE); |
|
236
|
|
|
|
|
237
|
|
|
$this->cache->save("cloudflare_{$this->site}", $cookieData['cookies'], 31536000 /* 1 year, or until we renew it */); |
|
238
|
|
|
log_message('debug', "Saving CloudFlare Cookies for {$this->site}"); |
|
239
|
|
|
|
|
240
|
|
|
$refresh = TRUE; |
|
241
|
|
|
} else { |
|
|
|
|
|
|
242
|
|
|
//Either site doesn't have CloudFlare or we have bypassed it. Either is good! |
|
243
|
|
|
} |
|
244
|
|
|
return $refresh; |
|
245
|
|
|
} |
|
246
|
|
|
|
|
247
|
|
|
/** |
|
248
|
|
|
* Used by getTitleData to get the title, latest_chapter & last_updated data from the data returned by get_content. |
|
249
|
|
|
* |
|
250
|
|
|
* parseTitleDataDOM checks if the data returned by get_content is valid via a few simple checks. |
|
251
|
|
|
* * If the request was actually successful, had a valid status code & data wasn't empty. We also do an additional check on an optional $failure_string param, which will throw a failure if it's matched. |
|
252
|
|
|
* |
|
253
|
|
|
* Data is cleaned by cleanTitleDataDOM prior to being passed to DOMDocument. |
|
254
|
|
|
* |
|
255
|
|
|
* All $node_* params must be XPath to the requested node, and must only return 1 result. Anything else will throw a failure. |
|
256
|
|
|
* |
|
257
|
|
|
* @param array $content |
|
258
|
|
|
* @param string $title_url |
|
259
|
|
|
* @param string $node_title_string |
|
260
|
|
|
* @param string $node_row_string |
|
261
|
|
|
* @param string $node_latest_string |
|
262
|
|
|
* @param string $node_chapter_string |
|
263
|
|
|
* @param string $failure_string |
|
264
|
|
|
* @param closure|null $noChaptersCall |
|
265
|
|
|
* @param closure|null $extraCall |
|
266
|
|
|
* |
|
267
|
|
|
* @return DOMElement[]|false [nodes_title,nodes_chapter,nodes_latest] |
|
268
|
|
|
*/ |
|
269
|
|
|
final protected function parseTitleDataDOM( |
|
270
|
|
|
$content, string $title_url, |
|
271
|
|
|
string $node_title_string, string $node_row_string, |
|
272
|
|
|
string $node_latest_string, string $node_chapter_string, |
|
273
|
|
|
string $failure_string = "", closure $noChaptersCall = NULL, closure $extraCall = NULL) { |
|
274
|
|
|
|
|
275
|
|
|
if(!is_array($content)) { |
|
276
|
|
|
log_message('error', "{$this->site} : {$title_url} | Failed to grab URL (See above curl error)"); |
|
277
|
|
|
} else { |
|
278
|
|
|
list('headers' => $headers, 'status_code' => $status_code, 'body' => $data) = $content; |
|
|
|
|
|
|
279
|
|
|
|
|
280
|
|
|
if(!($status_code >= 200 && $status_code < 300)) { |
|
281
|
|
|
if($status_code === 502) { |
|
|
|
|
|
|
282
|
|
|
// Site is overloaded, no need to log this. |
|
283
|
|
|
} else { |
|
284
|
|
|
log_message('error', "{$this->site} : {$title_url} | Bad Status Code ({$status_code})"); |
|
285
|
|
|
} |
|
286
|
|
|
} else if(empty($data)) { |
|
287
|
|
|
log_message('error', "{$this->site} : {$title_url} | Data is empty? (Status code: {$status_code})"); |
|
288
|
|
|
} else if($failure_string !== "" && strpos($data, $failure_string) !== FALSE) { |
|
289
|
|
|
log_message('error', "{$this->site} : {$title_url} | Failure string matched"); |
|
290
|
|
|
} else { |
|
291
|
|
|
$data = $this->cleanTitleDataDOM($data); //This allows us to clean the DOM prior to parsing. It's faster to grab the only part we need THEN parse it. |
|
292
|
|
|
|
|
293
|
|
|
$dom = new DOMDocument(); |
|
294
|
|
|
libxml_use_internal_errors(TRUE); |
|
295
|
|
|
$dom->loadHTML('<?xml encoding="utf-8" ?>' . $data); |
|
296
|
|
|
libxml_use_internal_errors(FALSE); |
|
297
|
|
|
|
|
298
|
|
|
$xpath = new DOMXPath($dom); |
|
299
|
|
|
$nodes_title = $xpath->query($node_title_string); |
|
300
|
|
|
$nodes_row = $xpath->query($node_row_string); |
|
301
|
|
|
if($nodes_title->length === 1) { |
|
302
|
|
|
if($nodes_row->length === 1) { |
|
303
|
|
|
$firstRow = $nodes_row->item(0); |
|
304
|
|
|
$nodes_latest = $xpath->query($node_latest_string, $firstRow); |
|
305
|
|
|
|
|
306
|
|
|
if($node_chapter_string !== '') { |
|
307
|
|
|
$nodes_chapter = $xpath->query($node_chapter_string, $firstRow); |
|
308
|
|
|
} else { |
|
309
|
|
|
$nodes_chapter = $nodes_row; |
|
310
|
|
|
} |
|
311
|
|
|
|
|
312
|
|
|
if($nodes_latest->length === 1 && $nodes_chapter->length === 1) { |
|
313
|
|
|
$returnData = [ |
|
314
|
|
|
'nodes_title' => $nodes_title->item(0), |
|
315
|
|
|
'nodes_latest' => $nodes_latest->item(0), |
|
316
|
|
|
'nodes_chapter' => $nodes_chapter->item(0) |
|
317
|
|
|
]; |
|
318
|
|
|
|
|
319
|
|
|
if(is_callable($extraCall)) $extraCall($xpath, $returnData); |
|
320
|
|
|
|
|
321
|
|
|
return $returnData; |
|
322
|
|
|
} else { |
|
323
|
|
|
log_message('error', "{$this->site} : {$title_url} | Invalid amount of nodes (LATEST: {$nodes_latest->length} | CHAPTER: {$nodes_chapter->length})"); |
|
324
|
|
|
} |
|
325
|
|
|
} elseif($this->canHaveNoChapters && !is_null($noChaptersCall) && is_callable($noChaptersCall)) { |
|
326
|
|
|
$returnData = [ |
|
327
|
|
|
'nodes_title' => $nodes_title->item(0) |
|
328
|
|
|
]; |
|
329
|
|
|
|
|
330
|
|
|
$noChaptersCall($data, $xpath, $returnData); |
|
331
|
|
|
|
|
332
|
|
|
if(is_array($returnData)) { |
|
333
|
|
|
if(is_callable($extraCall) && is_array($returnData)) $extraCall($xpath, $returnData); |
|
334
|
|
|
} else { |
|
335
|
|
|
log_message('error', "{$this->site} : {$title_url} | canHaveNoChapters set, but doesn't match possible checks! XPath is probably broken."); |
|
336
|
|
|
} |
|
337
|
|
|
|
|
338
|
|
|
return $returnData; |
|
339
|
|
|
} else { |
|
340
|
|
|
log_message('error', "{$this->site} : {$title_url} | Invalid amount of nodes (ROW: {$nodes_row->length})"); |
|
341
|
|
|
} |
|
342
|
|
|
} else { |
|
343
|
|
|
log_message('error', "{$this->site} : {$title_url} | Invalid amount of nodes (TITLE: {$nodes_title->length})"); |
|
344
|
|
|
} |
|
345
|
|
|
} |
|
346
|
|
|
} |
|
347
|
|
|
|
|
348
|
|
|
return FALSE; |
|
349
|
|
|
} |
|
350
|
|
|
|
|
351
|
|
|
/** |
|
352
|
|
|
* Used by parseTitleDataDOM to clean the data prior to passing it to DOMDocument & DOMXPath. |
|
353
|
|
|
* This is mostly done as an (assumed) speed improvement due to the reduced amount of DOM to parse, or simply just making it easier to parse with XPath. |
|
354
|
|
|
* |
|
355
|
|
|
* @param string $data |
|
356
|
|
|
* @return string |
|
357
|
|
|
*/ |
|
358
|
|
|
public function cleanTitleDataDOM(string $data) : string { |
|
359
|
|
|
return $data; |
|
360
|
|
|
} |
|
361
|
|
|
|
|
362
|
|
|
/** |
|
363
|
|
|
* Used to follow a series on given site if supported. |
|
364
|
|
|
* |
|
365
|
|
|
* This is called by getTitleData if $firstGet is true (which occurs when the series is first being added to the DB). |
|
366
|
|
|
* |
|
367
|
|
|
* Most of the actual following is done by handleCustomFollow. |
|
368
|
|
|
* |
|
369
|
|
|
* @param string $data |
|
370
|
|
|
* @param array $extra |
|
371
|
|
|
* @return array |
|
372
|
|
|
*/ |
|
373
|
|
|
final public function doCustomFollow(string $data = "", array $extra = []) : array { |
|
374
|
|
|
$titleData = []; |
|
375
|
|
|
$this->handleCustomFollow(function($content, $id, closure $successCallback = NULL) use(&$titleData) { |
|
376
|
|
|
if(is_array($content)) { |
|
377
|
|
|
if(array_key_exists('status_code', $content)) { |
|
378
|
|
|
$statusCode = $content['status_code']; |
|
379
|
|
|
if($statusCode === 200) { |
|
380
|
|
|
$isCallable = is_callable($successCallback); |
|
381
|
|
|
if(($isCallable && $successCallback($content['body'])) || !$isCallable) { |
|
382
|
|
|
$titleData['followed'] = 'Y'; |
|
383
|
|
|
|
|
384
|
|
|
log_message('info', "doCustomFollow succeeded for {$id}"); |
|
385
|
|
|
} else { |
|
386
|
|
|
log_message('error', "doCustomFollow failed (Invalid response?) for {$id}"); |
|
387
|
|
|
} |
|
388
|
|
|
} else { |
|
389
|
|
|
log_message('error', "doCustomFollow failed (Invalid status code ({$statusCode})) for {$id}"); |
|
390
|
|
|
} |
|
391
|
|
|
} else { |
|
392
|
|
|
log_message('error', "doCustomFollow failed (Missing status code?) for {$id}"); |
|
393
|
|
|
} |
|
394
|
|
|
} else { |
|
395
|
|
|
log_message('error', "doCustomFollow failed (Failed request) for {$id}"); |
|
396
|
|
|
} |
|
397
|
|
|
}, $data, $extra); |
|
398
|
|
|
return $titleData; |
|
399
|
|
|
} |
|
400
|
|
|
|
|
401
|
|
|
/** |
|
402
|
|
|
* Used by doCustomFollow to handle following series on sites. |
|
403
|
|
|
* |
|
404
|
|
|
* Uses get_content to get data. |
|
405
|
|
|
* |
|
406
|
|
|
* $callback must return ($content, $id, closure $successCallback = NULL). |
|
407
|
|
|
* * $content is simply just the get_content data. |
|
408
|
|
|
* * $id is the dbID. This should be passed by the $extra arr. |
|
409
|
|
|
* * $successCallback is an optional success check to make sure the series was properly followed. |
|
410
|
|
|
* |
|
411
|
|
|
* @param callable $callback |
|
412
|
|
|
* @param string $data |
|
413
|
|
|
* @param array $extra |
|
414
|
|
|
*/ |
|
415
|
|
|
public function handleCustomFollow(callable $callback, string $data = "", array $extra = []) { |
|
|
|
|
|
|
416
|
|
|
if($this->customType === 2) { |
|
417
|
|
|
$content = ['status_code' => 200]; |
|
418
|
|
|
$callback($content, $extra['id']); |
|
419
|
|
|
} |
|
420
|
|
|
} |
|
421
|
|
|
|
|
422
|
|
|
/** |
|
423
|
|
|
* Used to check the sites following page for new updates (if supported). |
|
424
|
|
|
* This should work much like getTitleData, but instead checks the following page. |
|
425
|
|
|
* |
|
426
|
|
|
* This must return an array containing arrays of each of the chapters data. |
|
427
|
|
|
*/ |
|
428
|
|
|
public function doCustomUpdate() {} |
|
429
|
|
|
|
|
430
|
|
|
/** |
|
431
|
|
|
* Used by the custom updater to check if a chapter looks newer than the current one. |
|
432
|
|
|
* |
|
433
|
|
|
* This calls doCustomCheckCompare which handles the majority of the checking. |
|
434
|
|
|
* NOTE: Depending on the site, you may need to call getChapterData to get the chapter number to be used with this. |
|
435
|
|
|
* |
|
436
|
|
|
* @param string $oldChapterString |
|
437
|
|
|
* @param string $newChapterString |
|
438
|
|
|
* @return bool |
|
439
|
|
|
*/ |
|
440
|
|
|
public function doCustomCheck(?string $oldChapterString, string $newChapterString) : bool { |
|
441
|
|
|
if(!is_null($oldChapterString)) { |
|
442
|
|
|
$oldChapterSegments = explode('/', $this->getChapterData('', $oldChapterString)['number']); |
|
443
|
|
|
$newChapterSegments = explode('/', $this->getChapterData('', $newChapterString)['number']); |
|
444
|
|
|
|
|
445
|
|
|
$status = $this->doCustomCheckCompare($oldChapterSegments, $newChapterSegments); |
|
446
|
|
|
} else { |
|
447
|
|
|
$status = TRUE; |
|
448
|
|
|
} |
|
449
|
|
|
|
|
450
|
|
|
return $status; |
|
451
|
|
|
} |
|
452
|
|
|
|
|
453
|
|
|
/** |
|
454
|
|
|
* Used by doCustomCheck to check if a chapter looks newer than the current one. |
|
455
|
|
|
* Chapter must be in a (v[0-9]+/)?c[0-9]+(\..+)? format. |
|
456
|
|
|
* |
|
457
|
|
|
* To avoid issues with the occasional off case, this will only ever return true if we are 100% sure that the new chapter is newer than the old one. |
|
458
|
|
|
* |
|
459
|
|
|
* @param array $oldChapterSegments |
|
460
|
|
|
* @param array $newChapterSegments |
|
461
|
|
|
* @return bool |
|
462
|
|
|
*/ |
|
463
|
12 |
|
final public function doCustomCheckCompare(array $oldChapterSegments, array $newChapterSegments) : bool { |
|
464
|
|
|
//NOTE: We only need to check against the new chapter here, as that is what is used for confirming update. |
|
465
|
12 |
|
$status = FALSE; |
|
466
|
|
|
|
|
467
|
|
|
//Make sure we have a volume element |
|
468
|
12 |
|
if(count($oldChapterSegments) === 1) array_unshift($oldChapterSegments, 'v0'); |
|
469
|
12 |
|
if(count($newChapterSegments) === 1) array_unshift($newChapterSegments, 'v0'); |
|
470
|
|
|
|
|
471
|
12 |
|
$oldCount = count($oldChapterSegments); |
|
472
|
12 |
|
$newCount = count($newChapterSegments); |
|
473
|
12 |
|
if($newCount === $oldCount) { |
|
474
|
|
|
//Make sure chapter format looks correct. |
|
475
|
|
|
//NOTE: We only need to check newCount as we know oldCount is the same count. |
|
476
|
12 |
|
if($newCount === 2) { |
|
477
|
|
|
//FIXME: Can we loop this? |
|
478
|
12 |
|
$oldVolume = substr(array_shift($oldChapterSegments), 1); |
|
479
|
12 |
|
$newVolume = substr(array_shift($newChapterSegments), 1); |
|
480
|
|
|
|
|
481
|
|
|
//Forcing volume to 0 as TBD might not be the latest (although it can be, but that is covered by other checks) |
|
482
|
12 |
|
if(in_array($oldVolume, ['TBD', 'TBA', 'NA', 'LMT'])) $oldVolume = 0; |
|
483
|
12 |
|
if(in_array($newVolume, ['TBD', 'TBA', 'NA', 'LMT'])) $newVolume = 0; |
|
484
|
|
|
|
|
485
|
12 |
|
$oldVolume = floatval($oldVolume); |
|
486
|
12 |
|
$newVolume = floatval($newVolume); |
|
487
|
|
|
} else { |
|
488
|
|
|
$oldVolume = 0; |
|
489
|
|
|
$newVolume = 0; |
|
490
|
|
|
} |
|
491
|
12 |
|
$oldChapter = floatval(substr(array_shift($oldChapterSegments), 1)); |
|
492
|
12 |
|
$newChapter = floatval(substr(array_shift($newChapterSegments), 1)); |
|
493
|
|
|
|
|
494
|
12 |
|
if($newChapter > $oldChapter && ($oldChapter >= 10 && $newChapter >= 10)) { |
|
495
|
|
|
//$newChapter is higher than $oldChapter AND $oldChapter and $newChapter are both more than 10 |
|
496
|
|
|
//This is intended to cover the /majority/ of valid updates, as we technically shouldn't have to check volumes. |
|
497
|
|
|
|
|
498
|
4 |
|
$status = TRUE; |
|
499
|
8 |
|
} elseif($newVolume > $oldVolume && ($oldChapter < 10 && $newChapter < 10)) { |
|
500
|
|
|
//This is pretty much just to match a one-off case where the site doesn't properly increment chapter numbers across volumes, and instead does something like: v1/c1..v1/c5, v2/c1..v1/c5 (and so on). |
|
501
|
1 |
|
$status = TRUE; |
|
502
|
7 |
|
} elseif($newVolume > $oldVolume && $newChapter >= $oldChapter) { |
|
503
|
|
|
//$newVolume is higher, and chapter is higher so no need to check chapter. |
|
504
|
2 |
|
$status = TRUE; |
|
505
|
5 |
|
} elseif($newChapter > $oldChapter) { |
|
506
|
|
|
//$newVolume isn't higher, but chapter is. |
|
507
|
|
|
$status = TRUE; |
|
508
|
|
|
} |
|
509
|
|
|
} |
|
510
|
|
|
|
|
511
|
12 |
|
return $status; |
|
512
|
|
|
} |
|
513
|
|
|
} |
|
514
|
|
|
|
|
515
|
|
|
abstract class Base_FoolSlide_Site_Model extends Base_Site_Model { |
|
516
|
|
|
public $titleFormat = '/^[a-z0-9_-]+$/'; |
|
517
|
|
|
public $chapterFormat = '/^(?:en(?:-us)?|pt|es)\/[0-9]+(?:\/[0-9]+(?:\/[0-9]+(?:\/[0-9]+)?)?)?$/'; |
|
518
|
|
|
public $customType = 2; |
|
519
|
|
|
|
|
520
|
|
|
public function getFullTitleURL(string $title_url) : string { |
|
521
|
|
|
return "{$this->baseURL}/series/{$title_url}"; |
|
522
|
|
|
} |
|
523
|
|
|
|
|
524
|
|
|
public function getChapterData(string $title_url, string $chapter) : array { |
|
525
|
|
|
$chapter_parts = explode('/', $chapter); //returns #LANG#/#VOLUME#/#CHAPTER#/#CHAPTER_EXTRA#(/#PAGE#/) |
|
526
|
|
|
return [ |
|
527
|
|
|
'url' => $this->getChapterURL($title_url, $chapter), |
|
528
|
|
|
'number' => ($chapter_parts[1] !== '0' ? "v{$chapter_parts[1]}/" : '') . "c{$chapter_parts[2]}" . (isset($chapter_parts[3]) ? ".{$chapter_parts[3]}" : '')/*)*/ |
|
529
|
|
|
]; |
|
530
|
|
|
} |
|
531
|
|
|
public function getChapterURL(string $title_url, string $chapter) : string { |
|
532
|
|
|
return "{$this->baseURL}/read/{$title_url}/{$chapter}/"; |
|
533
|
|
|
} |
|
534
|
|
|
|
|
535
|
|
|
public function getTitleData(string $title_url, bool $firstGet = FALSE) : ?array { |
|
536
|
|
|
$titleData = []; |
|
537
|
|
|
|
|
538
|
|
|
$jsonURL = $this->getJSONTitleURL($title_url); |
|
539
|
|
|
if($content = $this->get_content($jsonURL)) { |
|
540
|
|
|
$json = json_decode($content['body'], TRUE); |
|
541
|
|
|
if($json && isset($json['chapters']) && count($json['chapters']) > 0) { |
|
542
|
|
|
$titleData['title'] = trim($json['comic']['name']); |
|
543
|
|
|
|
|
544
|
|
|
//FoolSlide title API doesn't appear to let you sort (yet every other API method which has chapters does, so we need to sort ourselves.. |
|
545
|
|
|
usort($json['chapters'], function($a, $b) { |
|
546
|
|
|
return floatval("{$b['chapter']['chapter']}.{$b['chapter']['subchapter']}") <=> floatval("{$a['chapter']['chapter']}.{$a['chapter']['subchapter']}"); |
|
547
|
|
|
}); |
|
548
|
|
|
$latestChapter = reset($json['chapters'])['chapter']; |
|
549
|
|
|
|
|
550
|
|
|
$latestChapterString = "{$latestChapter['language']}/{$latestChapter['volume']}/{$latestChapter['chapter']}"; |
|
551
|
|
|
if($latestChapter['subchapter'] !== '0') { |
|
552
|
|
|
$latestChapterString .= "/{$latestChapter['subchapter']}"; |
|
553
|
|
|
} |
|
554
|
|
|
$titleData['latest_chapter'] = $latestChapterString; |
|
555
|
|
|
|
|
556
|
|
|
//No need to use date() here since this is already formatted as such. |
|
557
|
|
|
$titleData['last_updated'] = ($latestChapter['updated'] !== '0000-00-00 00:00:00' ? $latestChapter['updated'] : $latestChapter['created']); |
|
558
|
|
|
} |
|
559
|
|
|
} |
|
560
|
|
|
|
|
561
|
|
|
return (!empty($titleData) ? $titleData : NULL); |
|
562
|
|
|
} |
|
563
|
|
|
|
|
564
|
|
|
public function doCustomUpdate() { |
|
565
|
|
|
$titleDataList = []; |
|
566
|
|
|
|
|
567
|
|
|
$jsonURL = $this->getJSONUpdateURL(); |
|
568
|
|
|
if(($content = $this->get_content($jsonURL)) && $content['status_code'] == 200) { |
|
569
|
|
|
if(($json = json_decode($content['body'], TRUE)) && isset($json['chapters'])) { |
|
570
|
|
|
//This should fix edge cases where chapters are uploaded in bulk in the wrong order (HelveticaScans does this with Mousou Telepathy). |
|
571
|
|
|
usort($json['chapters'], function($a, $b) { |
|
572
|
|
|
$a_date = new DateTime($a['chapter']['updated'] !== '0000-00-00 00:00:00' ? $a['chapter']['updated'] : $a['chapter']['created']); |
|
573
|
|
|
$b_date = new DateTime($b['chapter']['updated'] !== '0000-00-00 00:00:00' ? $b['chapter']['updated'] : $b['chapter']['created']); |
|
574
|
|
|
return $b_date <=> $a_date; |
|
575
|
|
|
}); |
|
576
|
|
|
|
|
577
|
|
|
$parsedTitles = []; |
|
578
|
|
|
foreach($json['chapters'] as $chapterData) { |
|
579
|
|
|
if(!in_array($chapterData['comic']['stub'], $parsedTitles)) { |
|
580
|
|
|
$parsedTitles[] = $chapterData['comic']['stub']; |
|
581
|
|
|
|
|
582
|
|
|
$titleData = []; |
|
583
|
|
|
$titleData['title'] = trim($chapterData['comic']['name']); |
|
584
|
|
|
|
|
585
|
|
|
$latestChapter = $chapterData['chapter']; |
|
586
|
|
|
|
|
587
|
|
|
$latestChapterString = "en/{$latestChapter['volume']}/{$latestChapter['chapter']}"; |
|
588
|
|
|
if($latestChapter['subchapter'] !== '0') { |
|
589
|
|
|
$latestChapterString .= "/{$latestChapter['subchapter']}"; |
|
590
|
|
|
} |
|
591
|
|
|
$titleData['latest_chapter'] = $latestChapterString; |
|
592
|
|
|
|
|
593
|
|
|
//No need to use date() here since this is already formatted as such. |
|
594
|
|
|
$titleData['last_updated'] = ($latestChapter['updated'] !== '0000-00-00 00:00:00' ? $latestChapter['updated'] : $latestChapter['created']); |
|
595
|
|
|
|
|
596
|
|
|
$titleDataList[$chapterData['comic']['stub']] = $titleData; |
|
597
|
|
|
} else { |
|
598
|
|
|
//We already have title data for this title. |
|
599
|
|
|
continue; |
|
600
|
|
|
} |
|
601
|
|
|
} |
|
602
|
|
|
} else { |
|
603
|
|
|
log_message('error', "{$this->site} - Custom updating failed (no chapters arg?) for {$this->baseURL}."); |
|
604
|
|
|
} |
|
605
|
|
|
} else { |
|
606
|
|
|
log_message('error', "{$this->site} - Custom updating failed for {$this->baseURL}."); |
|
607
|
|
|
} |
|
608
|
|
|
|
|
609
|
|
|
return $titleDataList; |
|
610
|
|
|
} |
|
611
|
|
|
|
|
612
|
|
|
public function getJSONTitleURL(string $title_url) : string { |
|
613
|
|
|
return "{$this->baseURL}/api/reader/comic/stub/{$title_url}/format/json"; |
|
614
|
|
|
} |
|
615
|
|
|
public function getJSONUpdateURL() : string { |
|
616
|
|
|
return "{$this->baseURL}/api/reader/chapters/orderby/desc_created/format/json"; |
|
617
|
|
|
} |
|
618
|
|
|
} |
|
619
|
|
|
|
|
620
|
|
|
abstract class Base_myMangaReaderCMS_Site_Model extends Base_Site_Model { |
|
621
|
|
|
public $titleFormat = '/^[a-zA-Z0-9_-]+$/'; |
|
622
|
|
|
public $chapterFormat = '/^(?:oneshot|(?:chapter-)?[0-9\.]+)$/'; |
|
623
|
|
|
public $customType = 2; |
|
624
|
|
|
|
|
625
|
|
|
public function getFullTitleURL(string $title_url) : string { |
|
626
|
|
|
return "{$this->baseURL}/manga/{$title_url}"; |
|
627
|
|
|
} |
|
628
|
|
|
|
|
629
|
|
|
public function getChapterData(string $title_url, string $chapter) : array { |
|
630
|
|
|
$chapterN = (ctype_digit($chapter) ? "c${chapter}" : $chapter); |
|
631
|
|
|
return [ |
|
632
|
|
|
'url' => $this->getChapterURL($title_url, $chapter), |
|
633
|
|
|
'number' => $chapterN |
|
634
|
|
|
]; |
|
635
|
|
|
} |
|
636
|
|
|
public function getChapterURL(string $title_url, string $chapter) : string { |
|
637
|
|
|
return $this->getFullTitleURL($title_url).'/'.$chapter; |
|
638
|
|
|
} |
|
639
|
|
|
|
|
640
|
|
|
public function getTitleData(string $title_url, bool $firstGet = FALSE) : ?array { |
|
641
|
|
|
$titleData = []; |
|
642
|
|
|
|
|
643
|
|
|
$fullURL = $this->getFullTitleURL($title_url); |
|
644
|
|
|
|
|
645
|
|
|
$content = $this->get_content($fullURL); |
|
646
|
|
|
|
|
647
|
|
|
$data = $this->parseTitleDataDOM( |
|
648
|
|
|
$content, |
|
|
|
|
|
|
649
|
|
|
$title_url, |
|
650
|
|
|
"(//h2[@class='widget-title'])[1]", |
|
651
|
|
|
"//ul[contains(@class, 'chapters')]/li[not(contains(@class, 'btn'))][1]", |
|
652
|
|
|
"div[contains(@class, 'action')]/div[@class='date-chapter-title-rtl']", |
|
653
|
|
|
"h5/a[1] | h3/a[1]", |
|
654
|
|
|
"Whoops, looks like something went wrong." |
|
655
|
|
|
); |
|
656
|
|
|
if($data) { |
|
657
|
|
|
$titleData['title'] = trim($data['nodes_title']->textContent); |
|
658
|
|
|
|
|
659
|
|
|
$segments = explode('/', (string) $data['nodes_chapter']->getAttribute('href')); |
|
660
|
|
|
$needle = array_search('manga', array_reverse($segments, TRUE)) + 2; |
|
661
|
|
|
$titleData['latest_chapter'] = $segments[$needle]; |
|
662
|
|
|
|
|
663
|
|
|
$dateString = $data['nodes_latest']->nodeValue; |
|
664
|
|
|
$titleData['last_updated'] = date("Y-m-d H:i:s", strtotime(preg_replace('/ (-|\[A\]).*$/', '', $dateString))); |
|
665
|
|
|
} |
|
666
|
|
|
|
|
667
|
|
|
return (!empty($titleData) ? $titleData : NULL); |
|
668
|
|
|
} |
|
669
|
|
|
|
|
670
|
|
|
public function doCustomUpdate() { |
|
671
|
|
|
$titleDataList = []; |
|
672
|
|
|
|
|
673
|
|
|
$updateURL = "{$this->baseURL}/latest-release"; |
|
674
|
|
|
if(($content = $this->get_content($updateURL)) && $content['status_code'] == 200) { |
|
675
|
|
|
$data = $content['body']; |
|
676
|
|
|
|
|
677
|
|
|
$data = preg_replace('/^[\s\S]+<dl>/', '<dl>', $data); |
|
678
|
|
|
$data = preg_replace('/<\/dl>[\s\S]+$/', '</dl>', $data); |
|
679
|
|
|
|
|
680
|
|
|
$dom = new DOMDocument(); |
|
681
|
|
|
libxml_use_internal_errors(TRUE); |
|
682
|
|
|
$dom->loadHTML($data); |
|
683
|
|
|
libxml_use_internal_errors(FALSE); |
|
684
|
|
|
|
|
685
|
|
|
$xpath = new DOMXPath($dom); |
|
686
|
|
|
$nodes_rows = $xpath->query("//dl/dd | //div[@class='mangalist']/div[@class='manga-item']"); |
|
687
|
|
|
if($nodes_rows->length > 0) { |
|
688
|
|
|
foreach($nodes_rows as $row) { |
|
689
|
|
|
$titleData = []; |
|
690
|
|
|
|
|
691
|
|
|
$nodes_title = $xpath->query("div[@class='events ']/div[@class='events-body']/h3[@class='events-heading']/a | h3/a", $row); |
|
692
|
|
|
$nodes_chapter = $xpath->query("(div[@class='events '][1]/div[@class='events-body'][1] | div[@class='manga-chapter'][1])/h6[@class='events-subtitle'][1]/a[1]", $row); |
|
693
|
|
|
$nodes_latest = $xpath->query("div[@class='time'] | small", $row); |
|
694
|
|
|
|
|
695
|
|
|
if($nodes_title->length === 1 && $nodes_chapter->length === 1 && $nodes_latest->length === 1) { |
|
696
|
|
|
$title = $nodes_title->item(0); |
|
697
|
|
|
|
|
698
|
|
|
preg_match('/(?<url>[^\/]+(?=\/$|$))/', $title->getAttribute('href'), $title_url_arr); |
|
699
|
|
|
$title_url = $title_url_arr['url']; |
|
700
|
|
|
|
|
701
|
|
|
if(!array_key_exists($title_url, $titleDataList)) { |
|
702
|
|
|
$titleData['title'] = trim($title->textContent); |
|
703
|
|
|
|
|
704
|
|
|
$chapter = $nodes_chapter->item(0); |
|
705
|
|
|
preg_match('/(?<chapter>[^\/]+(?=\/$|$))/', $chapter->getAttribute('href'), $chapter_arr); |
|
706
|
|
|
$titleData['latest_chapter'] = $chapter_arr['chapter']; |
|
707
|
|
|
|
|
708
|
|
|
$dateString = str_replace('/', '-', trim($nodes_latest->item(0)->nodeValue)); //NOTE: We replace slashes here as it stops strtotime interpreting the date as US date format. |
|
709
|
|
|
if($dateString == 'T') { |
|
710
|
|
|
$dateString = date("Y-m-d",now()); |
|
711
|
|
|
} |
|
712
|
|
|
$titleData['last_updated'] = date("Y-m-d H:i:s", strtotime($dateString . ' 00:00')); |
|
713
|
|
|
|
|
714
|
|
|
$titleDataList[$title_url] = $titleData; |
|
715
|
|
|
} |
|
716
|
|
|
} else { |
|
717
|
|
|
log_message('error', "{$this->site}/Custom | Invalid amount of nodes (TITLE: {$nodes_title->length} | CHAPTER: {$nodes_chapter->length}) | LATEST: {$nodes_latest->length})"); |
|
718
|
|
|
} |
|
719
|
|
|
} |
|
720
|
|
|
} else { |
|
721
|
|
|
log_message('error', "{$this->site} | Following list is empty?"); |
|
722
|
|
|
} |
|
723
|
|
|
} else { |
|
724
|
|
|
log_message('error', "{$this->site} - Custom updating failed for {$this->baseURL}."); |
|
725
|
|
|
} |
|
726
|
|
|
|
|
727
|
|
|
return $titleDataList; |
|
728
|
|
|
} |
|
729
|
|
|
} |
|
730
|
|
|
|
|
731
|
|
|
abstract class Base_GlossyBright_Site_Model extends Base_Site_Model { |
|
732
|
|
|
public $titleFormat = '/^[a-zA-Z0-9_-]+$/'; |
|
733
|
|
|
public $chapterFormat = '/^[0-9\.]+$/'; |
|
734
|
|
|
|
|
735
|
|
|
public $customType = 2; |
|
736
|
|
|
|
|
737
|
|
|
public function getFullTitleURL(string $title_url) : string { |
|
738
|
|
|
return "{$this->baseURL}/{$title_url}"; |
|
739
|
|
|
} |
|
740
|
|
|
|
|
741
|
|
|
public function getChapterData(string $title_url, string $chapter) : array { |
|
742
|
|
|
return [ |
|
743
|
|
|
'url' => $this->getFullTitleURL($title_url).'/'.$chapter.'/', |
|
744
|
|
|
'number' => "c{$chapter}" |
|
745
|
|
|
]; |
|
746
|
|
|
} |
|
747
|
|
|
|
|
748
|
|
|
public function getTitleData(string $title_url, bool $firstGet = FALSE) : ?array { |
|
749
|
|
|
$titleData = []; |
|
750
|
|
|
|
|
751
|
|
|
$fullURL = "{$this->baseURL}/manga-rss/{$title_url}"; |
|
752
|
|
|
$content = $this->get_content($fullURL); |
|
753
|
|
|
$data = $this->parseTitleDataDOM( |
|
754
|
|
|
$content, |
|
|
|
|
|
|
755
|
|
|
$title_url, |
|
756
|
|
|
"//rss/channel/image/title", |
|
757
|
|
|
"//rss/channel/item[1]", |
|
758
|
|
|
"pubdate", |
|
759
|
|
|
"title" |
|
760
|
|
|
); |
|
761
|
|
|
if($data) { |
|
762
|
|
|
$titleData['title'] = preg_replace('/^Recent chapters of (.*?) manga$/', '$1', trim($data['nodes_title']->textContent)); |
|
763
|
|
|
|
|
764
|
|
|
//For whatever reason, DOMDocument breaks the <link> element we need to grab the chapter, so we have to grab it elsewhere. |
|
765
|
|
|
$titleData['latest_chapter'] = preg_replace('/^.*? - ([0-9\.]+) - .*?$/', '$1', trim($data['nodes_chapter']->textContent)); |
|
766
|
|
|
|
|
767
|
|
|
$titleData['last_updated'] = date("Y-m-d H:i:s", strtotime((string) $data['nodes_latest']->textContent)); |
|
768
|
|
|
} |
|
769
|
|
|
|
|
770
|
|
|
return (!empty($titleData) ? $titleData : NULL); |
|
771
|
|
|
} |
|
772
|
|
|
|
|
773
|
|
|
public function doCustomUpdate() { |
|
774
|
|
|
$titleDataList = []; |
|
775
|
|
|
|
|
776
|
|
|
$baseURLRegex = str_replace('.', '\\.', parse_url($this->baseURL, PHP_URL_HOST)); |
|
777
|
|
|
if(($content = $this->get_content($this->baseURL)) && $content['status_code'] == 200) { |
|
778
|
|
|
$data = $content['body']; |
|
779
|
|
|
|
|
780
|
|
|
$dom = new DOMDocument(); |
|
781
|
|
|
libxml_use_internal_errors(TRUE); |
|
782
|
|
|
$dom->loadHTML($data); |
|
783
|
|
|
libxml_use_internal_errors(FALSE); |
|
784
|
|
|
|
|
785
|
|
|
$xpath = new DOMXPath($dom); |
|
786
|
|
|
$nodes_rows = $xpath->query("//table[@id='wpm_mng_lst']/tr/td | //*[@id='wpm_mng_lst']/li/div"); |
|
787
|
|
|
if($nodes_rows->length > 0) { |
|
788
|
|
|
foreach($nodes_rows as $row) { |
|
789
|
|
|
$titleData = []; |
|
790
|
|
|
|
|
791
|
|
|
$nodes_title = $xpath->query("a[2]", $row); |
|
792
|
|
|
$nodes_chapter = $xpath->query("a[2]", $row); |
|
793
|
|
|
$nodes_latest = $xpath->query("b", $row); |
|
794
|
|
|
|
|
795
|
|
|
if($nodes_latest->length === 0) { |
|
796
|
|
|
$nodes_latest = $xpath->query('text()[last()]', $row); |
|
797
|
|
|
} |
|
798
|
|
|
|
|
799
|
|
|
if($nodes_title->length === 1 && $nodes_chapter->length === 1 && $nodes_latest->length === 1) { |
|
800
|
|
|
$title = $nodes_title->item(0); |
|
801
|
|
|
$chapter = $nodes_chapter->item(0); |
|
802
|
|
|
|
|
803
|
|
|
preg_match('/'.$baseURLRegex.'\/(?<url>.*?)\//', $title->getAttribute('href'), $title_url_arr); |
|
804
|
|
|
$title_url = $title_url_arr['url']; |
|
805
|
|
|
|
|
806
|
|
|
if(!array_key_exists($title_url, $titleDataList)) { |
|
807
|
|
|
$titleData['title'] = trim($title->getAttribute('title')); |
|
808
|
|
|
|
|
809
|
|
|
preg_match('/(?<chapter>[^\/]+(?=\/$|$))/', $chapter->getAttribute('href'), $chapter_arr); |
|
810
|
|
|
$titleData['latest_chapter'] = $chapter_arr['chapter']; |
|
811
|
|
|
|
|
812
|
|
|
$dateString = trim($nodes_latest->item(0)->textContent); |
|
813
|
|
|
switch($dateString) { |
|
814
|
|
|
case 'Today': |
|
|
|
|
|
|
815
|
|
|
$dateString = date("Y-m-d", now()); |
|
816
|
|
|
break; |
|
817
|
|
|
|
|
818
|
|
|
case 'Yesterday': |
|
819
|
|
|
$dateString = date("Y-m-d", strtotime("-1 days")); |
|
820
|
|
|
break; |
|
821
|
|
|
|
|
822
|
|
|
default: |
|
823
|
|
|
//Do nothing |
|
824
|
|
|
break; |
|
825
|
|
|
} |
|
826
|
|
|
$titleData['last_updated'] = date("Y-m-d H:i:s", strtotime($dateString)); |
|
827
|
|
|
|
|
828
|
|
|
$titleDataList[$title_url] = $titleData; |
|
829
|
|
|
} |
|
830
|
|
|
} else { |
|
831
|
|
|
log_message('error', "{$this->site}/Custom | Invalid amount of nodes (TITLE: {$nodes_title->length} | CHAPTER: {$nodes_chapter->length}) | LATEST: {$nodes_latest->length})"); |
|
832
|
|
|
} |
|
833
|
|
|
} |
|
834
|
|
|
} else { |
|
835
|
|
|
log_message('error', "{$this->site} | Following list is empty?"); |
|
836
|
|
|
} |
|
837
|
|
|
} else { |
|
838
|
|
|
log_message('error', "{$this->site} - Custom updating failed."); |
|
839
|
|
|
} |
|
840
|
|
|
|
|
841
|
|
|
return $titleDataList; |
|
842
|
|
|
} |
|
843
|
|
|
} |
|
844
|
|
|
|
|
845
|
|
|
abstract class Base_Roku_Site_Model extends Base_Site_Model { |
|
846
|
|
|
public $titleFormat = '/^[a-zA-Z0-9-]+$/'; |
|
847
|
|
|
public $chapterFormat = '/^[0-9\.]+$/'; |
|
848
|
|
|
|
|
849
|
|
|
public $customType = 2; |
|
850
|
|
|
|
|
851
|
|
|
public function getFullTitleURL(string $title_url) : string { |
|
852
|
|
|
return "{$this->baseURL}/series/{$title_url}"; |
|
853
|
|
|
} |
|
854
|
|
|
public function getChapterData(string $title_url, string $chapter) : array { |
|
855
|
|
|
return [ |
|
856
|
|
|
'url' => "{$this->baseURL}/read/{$title_url}/{$chapter}", |
|
857
|
|
|
'number' => "c{$chapter}" |
|
858
|
|
|
]; |
|
859
|
|
|
} |
|
860
|
|
|
public function getTitleData(string $title_url, bool $firstGet = FALSE) : ?array { |
|
861
|
|
|
$titleData = []; |
|
862
|
|
|
$fullURL = $this->getFullTitleURL($title_url); |
|
863
|
|
|
$content = $this->get_content($fullURL); |
|
864
|
|
|
$data = $this->parseTitleDataDOM( |
|
865
|
|
|
$content, |
|
|
|
|
|
|
866
|
|
|
$title_url, |
|
867
|
|
|
"//div[@id='activity']/descendant::div[@class='media'][1]/descendant::div[@class='media-body']/h2/text()", |
|
868
|
|
|
"//ul[contains(@class, 'media-list')]/li[@class='media'][1]/a", |
|
869
|
|
|
"div[@class='media-body']/span[@class='text-muted']", |
|
870
|
|
|
"" |
|
871
|
|
|
); |
|
872
|
|
|
if($data) { |
|
873
|
|
|
$titleData['title'] = trim(preg_replace('/ Added on .*$/','', $data['nodes_title']->textContent)); |
|
874
|
|
|
$titleData['latest_chapter'] = preg_replace('/^.*\/([0-9\.]+)$/', '$1', (string) $data['nodes_chapter']->getAttribute('href')); |
|
875
|
|
|
|
|
876
|
|
|
$dateString = preg_replace('/^Added (?:on )?/', '',$data['nodes_latest']->textContent); |
|
877
|
|
|
$titleData['last_updated'] = date("Y-m-d H:i:s", strtotime($dateString)); |
|
878
|
|
|
} |
|
879
|
|
|
return (!empty($titleData) ? $titleData : NULL); |
|
880
|
|
|
} |
|
881
|
|
|
|
|
882
|
|
|
|
|
883
|
|
|
public function doCustomUpdate() { |
|
884
|
|
|
$titleDataList = []; |
|
885
|
|
|
|
|
886
|
|
|
$updateURL = "{$this->baseURL}/latest"; |
|
887
|
|
|
if(($content = $this->get_content($updateURL)) && $content['status_code'] == 200) { |
|
888
|
|
|
$data = $content['body']; |
|
889
|
|
|
|
|
890
|
|
|
$dom = new DOMDocument(); |
|
891
|
|
|
libxml_use_internal_errors(TRUE); |
|
892
|
|
|
$dom->loadHTML($data); |
|
893
|
|
|
libxml_use_internal_errors(FALSE); |
|
894
|
|
|
|
|
895
|
|
|
$xpath = new DOMXPath($dom); |
|
896
|
|
|
$nodes_rows = $xpath->query("//div[@class='content-wrapper']/div[@class='row']/div/div"); |
|
897
|
|
|
if($nodes_rows->length > 0) { |
|
898
|
|
|
foreach($nodes_rows as $row) { |
|
899
|
|
|
$titleData = []; |
|
900
|
|
|
|
|
901
|
|
|
$nodes_title = $xpath->query("div[@class='caption']/h6/a", $row); |
|
902
|
|
|
$nodes_chapter = $xpath->query("div[@class='panel-footer no-padding']/a", $row); |
|
903
|
|
|
$nodes_latest = $xpath->query("div[@class='caption']/text()", $row); |
|
904
|
|
|
|
|
905
|
|
|
if($nodes_title->length === 1 && $nodes_chapter->length === 1 && $nodes_latest->length === 1) { |
|
906
|
|
|
$title = $nodes_title->item(0); |
|
907
|
|
|
|
|
908
|
|
|
preg_match('/(?<url>[^\/]+(?=\/$|$))/', $title->getAttribute('href'), $title_url_arr); |
|
909
|
|
|
$title_url = $title_url_arr['url']; |
|
910
|
|
|
|
|
911
|
|
|
if(!array_key_exists($title_url, $titleDataList)) { |
|
912
|
|
|
$titleData['title'] = trim($title->textContent); |
|
913
|
|
|
|
|
914
|
|
|
$chapter = $nodes_chapter->item(0); |
|
915
|
|
|
preg_match('/(?<chapter>[^\/]+(?=\/$|$))/', $chapter->getAttribute('href'), $chapter_arr); |
|
916
|
|
|
$titleData['latest_chapter'] = $chapter_arr['chapter']; |
|
917
|
|
|
|
|
918
|
|
|
$dateString = trim(str_replace('Added ', '', $nodes_latest->item(0)->textContent)); |
|
919
|
|
|
$titleData['last_updated'] = date("Y-m-d H:i:s", strtotime($dateString)); |
|
920
|
|
|
|
|
921
|
|
|
$titleDataList[$title_url] = $titleData; |
|
922
|
|
|
} |
|
923
|
|
|
} else { |
|
924
|
|
|
log_message('error', "{$this->site}/Custom | Invalid amount of nodes (TITLE: {$nodes_title->length} | CHAPTER: {$nodes_chapter->length}) | LATEST: {$nodes_latest->length})"); |
|
925
|
|
|
} |
|
926
|
|
|
} |
|
927
|
|
|
} else { |
|
928
|
|
|
log_message('error', "{$this->site} | Following list is empty?"); |
|
929
|
|
|
} |
|
930
|
|
|
} else { |
|
931
|
|
|
log_message('error', "{$this->site} - Custom updating failed."); |
|
932
|
|
|
} |
|
933
|
|
|
|
|
934
|
|
|
return $titleDataList; |
|
935
|
|
|
} |
|
936
|
|
|
} |
|
937
|
|
|
|
If you define a variable conditionally, it can happen that it is not defined for all execution paths.
Let’s take a look at an example:
In the above example, the variable $x is defined if you pass “foo” or “bar” as argument for $a. However, since the switch statement has no default case statement, if you pass any other value, the variable $x would be undefined.
Available Fixes
Check for existence of the variable explicitly:
Define a default value for the variable:
Add a value for the missing path: