DakuTree /
manga-tracker
Complex classes like Base_Site_Model often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes. You can also have a look at the cohesion graph to spot any un-connected, or weakly-connected components.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use Base_Site_Model, and based on these observations, apply Extract Interface, too.
| 1 | <?php declare(strict_types=1); defined('BASEPATH') OR exit('No direct script access allowed'); |
||
| 35 | abstract class Base_Site_Model extends CI_Model { |
||
| 36 | public $site = ''; |
||
| 37 | public $titleFormat = '//'; |
||
| 38 | public $chapterFormat = '//'; |
||
| 39 | public $hasCloudFlare = FALSE; |
||
| 40 | public $userAgent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2824.0 Safari/537.36'; |
||
| 41 | |||
| 42 | public $baseURL = ''; |
||
| 43 | |||
| 44 | /** |
||
| 45 | * 0: No custom updater. |
||
| 46 | * 1: Uses following page. |
||
| 47 | * 2: Uses latest releases page. |
||
| 48 | */ |
||
| 49 | public $customType = 0; |
||
| 50 | |||
| 51 | public $canHaveNoChapters = FALSE; |
||
| 52 | |||
| 53 | 16 | public function __construct() { |
|
| 54 | 16 | parent::__construct(); |
|
| 55 | |||
| 56 | 16 | $this->load->database(); |
|
| 57 | |||
| 58 | 16 | $this->site = get_class($this); |
|
| 59 | 16 | } |
|
| 60 | |||
| 61 | /** |
||
| 62 | * Generates URL to the title page of the requested series. |
||
| 63 | * |
||
| 64 | * NOTE: In some cases, we are required to store more data in the title_string than is needed to generate the URL. (Namely as the title_string is our unique identifier for that series) |
||
| 65 | * When storing additional data, we use ':--:' as a delimiter to separate the data. Make sure to handle this as needed. |
||
| 66 | * |
||
| 67 | * Example: |
||
| 68 | * return "http://mangafox.me/manga/{$title_url}/"; |
||
| 69 | * |
||
| 70 | * Example (with extra data): |
||
| 71 | * $title_parts = explode(':--:', title_url); |
||
| 72 | * return "https://bato.to/comic/_/comics/-r".$title_parts[0]; |
||
| 73 | * |
||
| 74 | * @param string $title_url |
||
| 75 | * @return string |
||
| 76 | */ |
||
| 77 | abstract public function getFullTitleURL(string $title_url) : string; |
||
| 78 | |||
| 79 | /** |
||
| 80 | * Generates chapter data from given $title_url and $chapter. |
||
| 81 | * |
||
| 82 | * Chapter must be in a (v[0-9]+/)?c[0-9]+(\..+)? format. |
||
| 83 | * |
||
| 84 | * NOTE: In some cases, we are required to store the chapter number, and the segment required to generate the chapter URL separately. |
||
| 85 | * Much like when generating the title URL, we use ':--:' as a delimiter to separate the data. Make sure to handle this as needed. |
||
| 86 | * |
||
| 87 | * Example: |
||
| 88 | * return [ |
||
| 89 | * 'url' => $this->getFullTitleURL($title_url).'/'.$chapter, |
||
| 90 | * 'number' => "c{$chapter}" |
||
| 91 | * ]; |
||
| 92 | * |
||
| 93 | * @param string $title_url |
||
| 94 | * @param string $chapter |
||
| 95 | * @return array [url, number] |
||
| 96 | */ |
||
| 97 | abstract public function getChapterData(string $title_url, string $chapter) : array; |
||
| 98 | |||
| 99 | /** |
||
| 100 | * Used to get the latest chapter of given $title_url. |
||
| 101 | * |
||
| 102 | * This <should> utilize both get_content and parseTitleDataDOM functions when possible, as these can both reduce a lot of the code required to set this up. |
||
| 103 | * |
||
| 104 | * $titleData params must be set accordingly: |
||
| 105 | * * `title` should always be used with html_entity_decode. |
||
| 106 | * * `latest_chapter` must match $this->chapterFormat. |
||
| 107 | * * `last_updated` should always be in date("Y-m-d H:i:s") format. |
||
| 108 | * * `followed` should never be set within via getTitleData, with the exception of via a array_merge with doCustomFollow. |
||
| 109 | * |
||
| 110 | * $firstGet is set to true when the series is first added to the DB, and is used to follow the series on given site (if possible). |
||
| 111 | * |
||
| 112 | * @param string $title_url |
||
| 113 | * @param bool $firstGet |
||
| 114 | * @return array|null [title,latest_chapter,last_updated,followed?] |
||
| 115 | */ |
||
| 116 | abstract public function getTitleData(string $title_url, bool $firstGet = FALSE) : ?array; |
||
| 117 | |||
| 118 | /** |
||
| 119 | * Validates given $title_url against titleFormat. |
||
| 120 | * |
||
| 121 | * Failure to match against titleFormat will stop the series from being added to the DB. |
||
| 122 | * |
||
| 123 | * @param string $title_url |
||
| 124 | * @return bool |
||
| 125 | */ |
||
| 126 | 2 | final public function isValidTitleURL(string $title_url) : bool { |
|
| 127 | 2 | $success = (bool) preg_match($this->titleFormat, $title_url); |
|
| 128 | 2 | if(!$success) log_message('error', "Invalid Title URL ({$this->site}): {$title_url}"); |
|
| 129 | 2 | return $success; |
|
| 130 | } |
||
| 131 | |||
| 132 | /** |
||
| 133 | * Validates given $chapter against chapterFormat. |
||
| 134 | * |
||
| 135 | * Failure to match against chapterFormat will stop the chapter being updated. |
||
| 136 | * |
||
| 137 | * @param string $chapter |
||
| 138 | * @return bool |
||
| 139 | */ |
||
| 140 | 2 | final public function isValidChapter(string $chapter) : bool { |
|
| 141 | 2 | $success = (bool) preg_match($this->chapterFormat, $chapter); |
|
| 142 | 2 | if(!$success) log_message('error', "Invalid Chapter ({$this->site}): {$chapter}"); |
|
| 143 | 2 | return $success; |
|
| 144 | } |
||
| 145 | |||
| 146 | /** |
||
| 147 | * Used by getTitleData (& similar functions) to get the requested page data. |
||
| 148 | * |
||
| 149 | * @param string $url |
||
| 150 | * @param string $cookie_string |
||
| 151 | * @param string $cookiejar_path |
||
| 152 | * @param bool $follow_redirect |
||
| 153 | * @param bool $isPost |
||
| 154 | * @param array $postFields |
||
| 155 | * |
||
| 156 | * @return array|bool |
||
| 157 | */ |
||
| 158 | final protected function get_content(string $url, string $cookie_string = "", string $cookiejar_path = "", bool $follow_redirect = FALSE, bool $isPost = FALSE, array $postFields = []) { |
||
| 159 | $refresh = TRUE; //For sites that have CloudFlare, we want to loop get_content again. |
||
| 160 | $loops = 0; |
||
| 161 | while($refresh && $loops < 2) { |
||
| 162 | $refresh = FALSE; |
||
| 163 | $loops++; |
||
| 164 | |||
| 165 | $ch = curl_init(); |
||
| 166 | curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); |
||
| 167 | curl_setopt($ch, CURLOPT_ENCODING , "gzip"); |
||
| 168 | //curl_setopt($ch, CURLOPT_VERBOSE, 1); |
||
| 169 | curl_setopt($ch, CURLOPT_HEADER, 1); |
||
| 170 | |||
| 171 | if($follow_redirect) curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE); |
||
| 172 | |||
| 173 | if($cookies = $this->cache->get("cloudflare_{$this->site}")) { |
||
| 174 | $cookie_string .= "; {$cookies}"; |
||
| 175 | } |
||
| 176 | |||
| 177 | if(!empty($cookie_string)) curl_setopt($ch, CURLOPT_COOKIE, $cookie_string); |
||
| 178 | if(!empty($cookiejar_path)) curl_setopt($ch, CURLOPT_COOKIEFILE, $cookiejar_path); |
||
| 179 | |||
| 180 | //Some sites check the useragent for stuff, use a pre-defined user-agent to avoid stuff. |
||
| 181 | curl_setopt($ch, CURLOPT_USERAGENT, $this->userAgent); |
||
| 182 | |||
| 183 | //NOTE: This is required for SSL URLs for now. Without it we tend to get error code 60. |
||
| 184 | curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE); //FIXME: This isn't safe, but it allows us to grab SSL URLs |
||
| 185 | |||
| 186 | curl_setopt($ch, CURLOPT_URL, $url); |
||
| 187 | |||
| 188 | if($isPost) { |
||
| 189 | curl_setopt($ch,CURLOPT_POST, count($postFields)); |
||
| 190 | curl_setopt($ch,CURLOPT_POSTFIELDS, http_build_query($postFields)); |
||
| 191 | } |
||
| 192 | |||
| 193 | $response = curl_exec($ch); |
||
| 194 | |||
| 195 | $this->Tracker->admin->incrementRequests(); |
||
| 196 | |||
| 197 | if($response === FALSE) { |
||
| 198 | log_message('error', "curl failed with error: ".curl_errno($ch)." | ".curl_error($ch)); |
||
| 199 | //FIXME: We don't always account for FALSE return |
||
| 200 | return FALSE; |
||
| 201 | } |
||
| 202 | |||
| 203 | $status_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); |
||
| 204 | $header_size = curl_getinfo($ch, CURLINFO_HEADER_SIZE); |
||
| 205 | $headers = http_parse_headers(substr($response, 0, $header_size)); |
||
| 206 | $body = substr($response, $header_size); |
||
| 207 | curl_close($ch); |
||
| 208 | |||
| 209 | if($status_code === 503) $refresh = $this->handleCloudFlare($url, $body); |
||
| 210 | } |
||
| 211 | |||
| 212 | return [ |
||
| 213 | 'headers' => $headers, |
||
|
|
|||
| 214 | 'status_code' => $status_code, |
||
| 215 | 'body' => $body |
||
| 216 | ]; |
||
| 217 | } |
||
| 218 | |||
| 219 | final private function handleCloudFlare(string $url, string $body) : bool { |
||
| 220 | $refresh = FALSE; |
||
| 221 | |||
| 222 | if((strpos($body, 'DDoS protection by Cloudflare') !== FALSE) || (strpos($body, '<input type="hidden" id="jschl-answer" name="jschl_answer"/>') !== FALSE)) { |
||
| 223 | //print "Cloudflare detected? Grabbing Cookies.\n"; |
||
| 224 | if(!$this->hasCloudFlare) { |
||
| 225 | //TODO: Site appears to have enabled CloudFlare, disable it and contact admin. |
||
| 226 | // We'll continue to bypass CloudFlare as this may occur in a loop. |
||
| 227 | } |
||
| 228 | |||
| 229 | $urlData = [ |
||
| 230 | 'url' => $url, |
||
| 231 | 'user_agent' => $this->userAgent |
||
| 232 | ]; |
||
| 233 | //TODO: shell_exec seems bad since the URLs "could" be user inputted? Better way of doing this? |
||
| 234 | $result = shell_exec('python '.APPPATH.'../_scripts/get_cloudflare_cookie.py '.escapeshellarg(json_encode($urlData))); |
||
| 235 | $cookieData = json_decode($result, TRUE); |
||
| 236 | |||
| 237 | $this->cache->save("cloudflare_{$this->site}", $cookieData['cookies'], 31536000 /* 1 year, or until we renew it */); |
||
| 238 | log_message('debug', "Saving CloudFlare Cookies for {$this->site}"); |
||
| 239 | |||
| 240 | $refresh = TRUE; |
||
| 241 | } else { |
||
| 242 | //Either site doesn't have CloudFlare or we have bypassed it. Either is good! |
||
| 243 | } |
||
| 244 | return $refresh; |
||
| 245 | } |
||
| 246 | |||
| 247 | /** |
||
| 248 | * Used by getTitleData to get the title, latest_chapter & last_updated data from the data returned by get_content. |
||
| 249 | * |
||
| 250 | * parseTitleDataDOM checks if the data returned by get_content is valid via a few simple checks. |
||
| 251 | * * If the request was actually successful, had a valid status code & data wasn't empty. We also do an additional check on an optional $failure_string param, which will throw a failure if it's matched. |
||
| 252 | * |
||
| 253 | * Data is cleaned by cleanTitleDataDOM prior to being passed to DOMDocument. |
||
| 254 | * |
||
| 255 | * All $node_* params must be XPath to the requested node, and must only return 1 result. Anything else will throw a failure. |
||
| 256 | * |
||
| 257 | * @param array $content |
||
| 258 | * @param string $title_url |
||
| 259 | * @param string $node_title_string |
||
| 260 | * @param string $node_row_string |
||
| 261 | * @param string $node_latest_string |
||
| 262 | * @param string $node_chapter_string |
||
| 263 | * @param string $failure_string |
||
| 264 | * @param string $no_chapters_string |
||
| 265 | * @return DOMElement[]|false [nodes_title,nodes_chapter,nodes_latest] |
||
| 266 | */ |
||
| 267 | final protected function parseTitleDataDOM( |
||
| 268 | $content, string $title_url, |
||
| 269 | string $node_title_string, string $node_row_string, |
||
| 270 | string $node_latest_string, string $node_chapter_string, |
||
| 271 | string $failure_string = "", string $no_chapters_string = "") { |
||
| 272 | |||
| 273 | if(!is_array($content)) { |
||
| 274 | log_message('error', "{$this->site} : {$title_url} | Failed to grab URL (See above curl error)"); |
||
| 275 | } else { |
||
| 276 | list('headers' => $headers, 'status_code' => $status_code, 'body' => $data) = $content; |
||
| 277 | |||
| 278 | if(!($status_code >= 200 && $status_code < 300)) { |
||
| 279 | log_message('error', "{$this->site} : {$title_url} | Bad Status Code ({$status_code})"); |
||
| 280 | } else if(empty($data)) { |
||
| 281 | log_message('error', "{$this->site} : {$title_url} | Data is empty? (Status code: {$status_code})"); |
||
| 282 | } else if($failure_string !== "" && strpos($data, $failure_string) !== FALSE) { |
||
| 283 | log_message('error', "{$this->site} : {$title_url} | Failure string matched"); |
||
| 284 | } else { |
||
| 285 | $data = $this->cleanTitleDataDOM($data); //This allows us to clean the DOM prior to parsing. It's faster to grab the only part we need THEN parse it. |
||
| 286 | |||
| 287 | $dom = new DOMDocument(); |
||
| 288 | libxml_use_internal_errors(TRUE); |
||
| 289 | $dom->loadHTML('<?xml encoding="utf-8" ?>' . $data); |
||
| 290 | libxml_use_internal_errors(FALSE); |
||
| 291 | |||
| 292 | $xpath = new DOMXPath($dom); |
||
| 293 | $nodes_title = $xpath->query($node_title_string); |
||
| 294 | $nodes_row = $xpath->query($node_row_string); |
||
| 295 | if($nodes_title->length === 1) { |
||
| 296 | if($nodes_row->length === 1) { |
||
| 297 | $firstRow = $nodes_row->item(0); |
||
| 298 | $nodes_latest = $xpath->query($node_latest_string, $firstRow); |
||
| 299 | |||
| 300 | if($node_chapter_string !== '') { |
||
| 301 | $nodes_chapter = $xpath->query($node_chapter_string, $firstRow); |
||
| 302 | } else { |
||
| 303 | $nodes_chapter = $nodes_row; |
||
| 304 | } |
||
| 305 | |||
| 306 | if($nodes_latest->length === 1 && $nodes_chapter->length === 1) { |
||
| 307 | return [ |
||
| 308 | 'nodes_title' => $nodes_title->item(0), |
||
| 309 | 'nodes_latest' => $nodes_latest->item(0), |
||
| 310 | 'nodes_chapter' => $nodes_chapter->item(0) |
||
| 311 | ]; |
||
| 312 | } else { |
||
| 313 | log_message('error', "{$this->site} : {$title_url} | Invalid amount of nodes (LATEST: {$nodes_latest->length} | CHAPTER: {$nodes_chapter->length})"); |
||
| 314 | } |
||
| 315 | } elseif($this->canHaveNoChapters && !empty($no_chapters_string) && strpos($data, $no_chapters_string) !== FALSE) { |
||
| 316 | return [ |
||
| 317 | 'nodes_title' => $nodes_title->item(0) |
||
| 318 | ]; |
||
| 319 | } else { |
||
| 320 | log_message('error', "{$this->site} : {$title_url} | Invalid amount of nodes (ROW: {$nodes_row->length})"); |
||
| 321 | } |
||
| 322 | } else { |
||
| 323 | log_message('error', "{$this->site} : {$title_url} | Invalid amount of nodes (TITLE: {$nodes_title->length})"); |
||
| 324 | } |
||
| 325 | } |
||
| 326 | } |
||
| 327 | |||
| 328 | return FALSE; |
||
| 329 | } |
||
| 330 | |||
| 331 | /** |
||
| 332 | * Used by parseTitleDataDOM to clean the data prior to passing it to DOMDocument & DOMXPath. |
||
| 333 | * This is mostly done as an (assumed) speed improvement due to the reduced amount of DOM to parse, or simply just making it easier to parse with XPath. |
||
| 334 | * |
||
| 335 | * @param string $data |
||
| 336 | * @return string |
||
| 337 | */ |
||
| 338 | public function cleanTitleDataDOM(string $data) : string { |
||
| 339 | return $data; |
||
| 340 | } |
||
| 341 | |||
| 342 | /** |
||
| 343 | * Used to follow a series on given site if supported. |
||
| 344 | * |
||
| 345 | * This is called by getTitleData if $firstGet is true (which occurs when the series is first being added to the DB). |
||
| 346 | * |
||
| 347 | * Most of the actual following is done by handleCustomFollow. |
||
| 348 | * |
||
| 349 | * @param string $data |
||
| 350 | * @param array $extra |
||
| 351 | * @return array |
||
| 352 | */ |
||
| 353 | final public function doCustomFollow(string $data = "", array $extra = []) : array { |
||
| 354 | $titleData = []; |
||
| 355 | $this->handleCustomFollow(function($content, $id, closure $successCallback = NULL) use(&$titleData) { |
||
| 356 | if(is_array($content)) { |
||
| 357 | if(array_key_exists('status_code', $content)) { |
||
| 358 | $statusCode = $content['status_code']; |
||
| 359 | if($statusCode === 200) { |
||
| 360 | $isCallable = is_callable($successCallback); |
||
| 361 | if(($isCallable && $successCallback($content['body'])) || !$isCallable) { |
||
| 362 | $titleData['followed'] = 'Y'; |
||
| 363 | |||
| 364 | log_message('info', "doCustomFollow succeeded for {$id}"); |
||
| 365 | } else { |
||
| 366 | log_message('error', "doCustomFollow failed (Invalid response?) for {$id}"); |
||
| 367 | } |
||
| 368 | } else { |
||
| 369 | log_message('error', "doCustomFollow failed (Invalid status code ({$statusCode})) for {$id}"); |
||
| 370 | } |
||
| 371 | } else { |
||
| 372 | log_message('error', "doCustomFollow failed (Missing status code?) for {$id}"); |
||
| 373 | } |
||
| 374 | } else { |
||
| 375 | log_message('error', "doCustomFollow failed (Failed request) for {$id}"); |
||
| 376 | } |
||
| 377 | }, $data, $extra); |
||
| 378 | return $titleData; |
||
| 379 | } |
||
| 380 | |||
| 381 | /** |
||
| 382 | * Used by doCustomFollow to handle following series on sites. |
||
| 383 | * |
||
| 384 | * Uses get_content to get data. |
||
| 385 | * |
||
| 386 | * $callback must return ($content, $id, closure $successCallback = NULL). |
||
| 387 | * * $content is simply just the get_content data. |
||
| 388 | * * $id is the dbID. This should be passed by the $extra arr. |
||
| 389 | * * $successCallback is an optional success check to make sure the series was properly followed. |
||
| 390 | * |
||
| 391 | * @param callable $callback |
||
| 392 | * @param string $data |
||
| 393 | * @param array $extra |
||
| 394 | */ |
||
| 395 | public function handleCustomFollow(callable $callback, string $data = "", array $extra = []) { |
||
| 396 | if($this->customType === 2) { |
||
| 397 | $content = ['status_code' => 200]; |
||
| 398 | $callback($content, $extra['id']); |
||
| 399 | } |
||
| 400 | } |
||
| 401 | |||
| 402 | /** |
||
| 403 | * Used to check the sites following page for new updates (if supported). |
||
| 404 | * This should work much like getTitleData, but instead checks the following page. |
||
| 405 | * |
||
| 406 | * This must return an array containing arrays of each of the chapters data. |
||
| 407 | */ |
||
| 408 | public function doCustomUpdate() {} |
||
| 409 | |||
| 410 | /** |
||
| 411 | * Used by the custom updater to check if a chapter looks newer than the current one. |
||
| 412 | * |
||
| 413 | * This calls doCustomCheckCompare which handles the majority of the checking. |
||
| 414 | * NOTE: Depending on the site, you may need to call getChapterData to get the chapter number to be used with this. |
||
| 415 | * |
||
| 416 | * @param string $oldChapterString |
||
| 417 | * @param string $newChapterString |
||
| 418 | * @return bool |
||
| 419 | */ |
||
| 420 | public function doCustomCheck(?string $oldChapterString, string $newChapterString) : bool { |
||
| 432 | |||
| 433 | /** |
||
| 434 | * Used by doCustomCheck to check if a chapter looks newer than the current one. |
||
| 435 | * Chapter must be in a (v[0-9]+/)?c[0-9]+(\..+)? format. |
||
| 436 | * |
||
| 437 | * To avoid issues with the occasional off case, this will only ever return true if we are 100% sure that the new chapter is newer than the old one. |
||
| 438 | * |
||
| 439 | * @param array $oldChapterSegments |
||
| 440 | * @param array $newChapterSegments |
||
| 441 | * @return bool |
||
| 442 | */ |
||
| 443 | 12 | final public function doCustomCheckCompare(array $oldChapterSegments, array $newChapterSegments) : bool { |
|
| 493 | } |
||
| 494 | |||
| 917 |
Loading history...
If you define a variable conditionally, it can happen that it is not defined for all execution paths.
Let’s take a look at an example:
In the above example, the variable $x is defined if you pass “foo” or “bar” as argument for $a. However, since the switch statement has no default case statement, if you pass any other value, the variable $x would be undefined.
Available Fixes
Check for existence of the variable explicitly:
Define a default value for the variable:
Add a value for the missing path: